release: 2026-08-01

This commit is contained in:
moaljumaa
2026-08-01 14:01:17 +02:00
commit 7d8b930bc3
137 changed files with 34695 additions and 0 deletions
+107
View File
@@ -0,0 +1,107 @@
const APOLLO_BASE = "https://api.apollo.io/api/v1";
export interface ApolloMatchResult {
apollo_id: string;
linkedin_url: string | null;
headline: string | null;
seniority: string | null;
functions: string[] | null;
departments: string[] | null;
email: string | null;
email_status: string | null;
email_domain_catchall: boolean;
city: string | null;
country: string | null;
time_zone: string | null;
positions_json: string | null;
organization: {
name: string | null;
domain: string | null;
industry: string | null;
estimated_num_employees: number | null;
short_description: string | null;
location: string | null;
linkedin_url: string | null;
website_url: string | null;
founded_year: number | null;
logo_url: string | null;
phone: string | null;
annual_revenue_printed: string | null;
technology_names: string[] | null;
keywords: string[] | null;
city: string | null;
country: string | null;
} | null;
}
export async function matchPerson(
linkedinUrl: string,
apiKey: string
): Promise<ApolloMatchResult | null> {
const res = await fetch(`${APOLLO_BASE}/people/match`, {
method: "POST",
headers: { "Content-Type": "application/json", "X-Api-Key": apiKey },
body: JSON.stringify({ linkedin_url: linkedinUrl, reveal_personal_emails: false }),
});
if (!res.ok) return null;
const data = await res.json();
const p = data.person;
if (!p || !p.id) return null;
const org = p.organization ?? null;
const location =
org
? [org.city, org.country].filter(Boolean).join(", ") || null
: null;
// Map employment_history to positions_json format (same shape as Sales Nav enrichment)
let positions_json: string | null = null;
if (Array.isArray(p.employment_history) && p.employment_history.length > 0) {
const positions = p.employment_history.map((e: Record<string, unknown>) => ({
title: e.title ?? "",
companyName: e.organization_name ?? "",
current: e.current === true,
startDate: e.start_date ?? undefined,
endDate: e.end_date ?? undefined,
}));
positions_json = JSON.stringify(positions);
}
return {
apollo_id: p.id,
linkedin_url: p.linkedin_url ?? null,
headline: p.headline ?? null,
seniority: p.seniority ?? null,
functions: Array.isArray(p.functions) && p.functions.length > 0 ? p.functions : null,
departments: Array.isArray(p.departments) && p.departments.length > 0 ? p.departments : null,
email: p.email ?? null,
email_status: p.email_status ?? null,
email_domain_catchall: p.email_domain_catchall === true,
city: p.city ?? null,
country: p.country ?? null,
time_zone: p.time_zone ?? null,
positions_json,
organization: org
? {
name: org.name ?? null,
domain: org.primary_domain ?? null,
industry: org.industry ?? null,
estimated_num_employees: org.estimated_num_employees ?? null,
short_description: org.short_description ?? null,
location,
linkedin_url: org.linkedin_url ?? null,
website_url: org.website_url ?? null,
founded_year: org.founded_year ?? null,
logo_url: org.logo_url ?? null,
phone: org.phone ?? org.sanitized_phone ?? null,
annual_revenue_printed: org.annual_revenue_printed ?? null,
technology_names: Array.isArray(org.technology_names) && org.technology_names.length > 0 ? org.technology_names : null,
keywords: Array.isArray(org.keywords) && org.keywords.length > 0 ? org.keywords : null,
city: org.city ?? null,
country: org.country ?? null,
}
: null,
};
}
+49
View File
@@ -0,0 +1,49 @@
import { getToken } from "next-auth/jwt";
import type { NextRequest } from "next/server";
const INTERNAL_HEADER = "x-internal-secret";
/**
* Shared request-auth check used by proxy.ts to gate /api/*.
*
* Accepts either:
* - a NextAuth session cookie (browser app) — verified via getToken().
* - the internal service secret (server-to-server loopback calls, e.g. the MCP server's
* tool handlers calling Linki's own /api/* routes — see ee/mcp/server.ts). This secret
* never leaves the host: it's not sent to a browser and never crosses the public ngrok/
* reverse-proxy path, only Node processes on 127.0.0.1 exchange it.
*
* Routes with their own independent auth (Authorization: Bearer, e.g. /api/mcp) are
* excluded from the gate entirely in proxy.ts rather than special-cased here.
*/
export async function isAuthenticated(req: NextRequest): Promise<boolean> {
if (await hasValidInternalSecret(req)) return true;
const token = await getToken({ req, secret: process.env.NEXTAUTH_SECRET });
return token !== null;
}
async function hasValidInternalSecret(req: NextRequest): Promise<boolean> {
const expected = process.env.INTERNAL_API_SECRET;
if (!expected) return false;
const provided = req.headers.get(INTERNAL_HEADER);
if (!provided) return false;
return timingSafeEqual(provided, expected);
}
// Constant-time string comparison via Web Crypto (available on both the Edge and Node.js
// runtimes) so an internal-secret guess can't be timed byte-by-byte.
async function timingSafeEqual(a: string, b: string): Promise<boolean> {
const enc = new TextEncoder();
const [digestA, digestB] = await Promise.all([
crypto.subtle.digest("SHA-256", enc.encode(a)),
crypto.subtle.digest("SHA-256", enc.encode(b)),
]);
const bytesA = new Uint8Array(digestA);
const bytesB = new Uint8Array(digestB);
let diff = 0;
for (let i = 0; i < bytesA.length; i++) diff |= bytesA[i] ^ bytesB[i];
return diff === 0;
}
+43
View File
@@ -0,0 +1,43 @@
import { createCipheriv, createDecipheriv, hkdfSync, randomBytes } from "crypto";
// Encrypts secrets at rest (LinkedIn session cookies, email passwords, API keys) so a
// leaked/copied linki.db doesn't hand those out in plaintext. Key is derived from
// NEXTAUTH_SECRET (already a required, real secret) via HKDF — no new env var needed.
//
// Format: "v1:<iv-base64>:<authTag-base64>:<ciphertext-base64>". decryptSecret() passes
// any value NOT in this format straight through unchanged, so pre-migration plaintext
// rows keep working — see lib/db.ts's encryptLegacySecretsMigration.
const VERSION = "v1";
const ALGORITHM = "aes-256-gcm";
function deriveKey(): Buffer {
const secret = process.env.NEXTAUTH_SECRET;
if (!secret) throw new Error("NEXTAUTH_SECRET must be set to encrypt/decrypt stored secrets");
return Buffer.from(hkdfSync("sha256", secret, "", "linki-secret-encryption", 32));
}
export function encryptSecret(plaintext: string): string {
const key = deriveKey();
const iv = randomBytes(12);
const cipher = createCipheriv(ALGORITHM, key, iv);
const ciphertext = Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]);
const authTag = cipher.getAuthTag();
return [VERSION, iv.toString("base64"), authTag.toString("base64"), ciphertext.toString("base64")].join(":");
}
export function decryptSecret(value: string | null): string | null {
if (value === null) return null;
if (!isEncrypted(value)) return value; // not-yet-migrated plaintext — pass through
const [, ivB64, authTagB64, dataB64] = value.split(":");
const key = deriveKey();
const decipher = createDecipheriv(ALGORITHM, key, Buffer.from(ivB64, "base64"));
decipher.setAuthTag(Buffer.from(authTagB64, "base64"));
const plaintext = Buffer.concat([decipher.update(Buffer.from(dataB64, "base64")), decipher.final()]);
return plaintext.toString("utf8");
}
export function isEncrypted(value: string): boolean {
return value.startsWith(`${VERSION}:`);
}
+169
View File
@@ -0,0 +1,169 @@
import Papa from "papaparse";
import type DatabaseType from "better-sqlite3";
import { randomUUID } from "crypto";
type DB = DatabaseType.Database;
// User-fillable target fields — everything else on `targets` (URNs, JSON blobs,
// enrichment timestamps, apollo/automation internals) is system-owned and not
// importable. Mirrors the PATCH /api/targets/[id] editable set.
const EDITABLE_FIELDS = [
"first_name", "last_name", "title", "company", "location",
"city", "country", "phone", "headline", "summary", "notes",
] as const;
type EditableField = (typeof EDITABLE_FIELDS)[number];
// One template covers every case: a pure LinkedIn list, a pure email list (incl.
// generic inboxes like info@company.com), or an export that already has both
// (e.g. a list exported from another tool). Each row just needs linkedin_url
// and/or email filled in — whatever the contact actually has.
const TEMPLATE_COLUMNS = ["linkedin_url", "sales_nav_url", "email", ...EDITABLE_FIELDS] as const;
const SAMPLE_VALUES: Record<EditableField, string> = {
first_name: "Jane",
last_name: "Doe",
title: "Head of Marketing",
company: "Acme Inc",
location: "Berlin, Germany",
city: "Berlin",
country: "Germany",
phone: "+49 30 1234567",
headline: "Head of Marketing @ Acme Inc",
summary: "10+ years in B2B SaaS marketing.",
notes: "Met at SaaStr 2026",
};
export function buildCsvTemplate(): string {
const sample = TEMPLATE_COLUMNS.map((c) =>
c === "linkedin_url" ? "https://www.linkedin.com/in/example-profile/" :
c === "sales_nav_url" ? "" :
c === "email" ? "jane@acme.com" :
SAMPLE_VALUES[c as EditableField]
);
return Papa.unparse({ fields: [...TEMPLATE_COLUMNS], data: [sample] });
}
export interface CsvImportResult {
imported: number;
updated: number;
skipped: number;
errors: string[];
}
const EMAIL_RE = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
function normalizeLinkedinUrl(raw: string): string | null {
const trimmed = raw.trim();
if (!trimmed || !trimmed.includes("linkedin.com/in/")) return null;
return trimmed;
}
interface ParsedRow {
linkedin_url: string | null;
sales_nav_url: string | null;
email: string | null;
full_name: string | null;
fields: Record<EditableField, string | null>;
}
function get(row: Record<string, string>, key: string): string | null {
const v = row[key];
const t = typeof v === "string" ? v.trim() : "";
return t.length > 0 ? t : null;
}
export function importCsv(db: DB, listId: string, csvText: string): CsvImportResult {
const parsed = Papa.parse<Record<string, string>>(csvText, {
header: true,
skipEmptyLines: true,
transformHeader: (h) => h.trim().toLowerCase().replace(/\s+/g, "_"),
});
const errors: string[] = [];
let imported = 0;
let updated = 0;
let skipped = 0;
const rows: ParsedRow[] = [];
parsed.data.forEach((raw, idx) => {
const rowNum = idx + 2; // header is row 1
const fields = Object.fromEntries(EDITABLE_FIELDS.map((f) => [f, get(raw, f)])) as Record<EditableField, string | null>;
const full_name = [fields.first_name, fields.last_name].filter(Boolean).join(" ") || null;
const rawUrl = get(raw, "linkedin_url");
const linkedin_url = rawUrl ? normalizeLinkedinUrl(rawUrl) : null;
if (rawUrl && !linkedin_url) { errors.push(`Row ${rowNum}: "${rawUrl}" is not a valid linkedin.com/in/ URL`); return; }
const sales_nav_url = get(raw, "sales_nav_url");
const rawEmail = get(raw, "email");
let email: string | null = null;
if (rawEmail) {
if (!EMAIL_RE.test(rawEmail)) { errors.push(`Row ${rowNum}: "${rawEmail}" is not a valid email`); return; }
email = rawEmail.toLowerCase();
}
if (!linkedin_url && !email) {
errors.push(`Row ${rowNum}: needs at least a linkedin_url or an email`);
return;
}
rows.push({ linkedin_url, sales_nav_url, email, full_name, fields });
});
const editableCols = [...EDITABLE_FIELDS, "full_name", "sales_nav_url"] as const;
const insertByLinkedin = db.prepare(`
INSERT INTO targets (id, linkedin_url, email, sales_nav_url, full_name, ${EDITABLE_FIELDS.join(", ")})
VALUES (?, ?, ?, ?, ?, ${EDITABLE_FIELDS.map(() => "?").join(", ")})
ON CONFLICT(linkedin_url) DO UPDATE SET
email = COALESCE(excluded.email, targets.email),
${editableCols.map((c) => `${c} = COALESCE(excluded.${c}, targets.${c})`).join(",\n ")}
`);
const findByLinkedin = db.prepare("SELECT id FROM targets WHERE linkedin_url = ?");
const findByEmail = db.prepare("SELECT id FROM targets WHERE email = ? LIMIT 1");
const insertByEmail = db.prepare(`
INSERT INTO targets (id, email, full_name, ${EDITABLE_FIELDS.join(", ")}, sales_nav_url)
VALUES (?, ?, ?, ${EDITABLE_FIELDS.map(() => "?").join(", ")}, ?)
`);
const updateByEmail = db.prepare(`
UPDATE targets SET
${editableCols.map((c) => `${c} = COALESCE(?, ${c})`).join(",\n ")}
WHERE id = ?
`);
const linkToList = db.prepare("INSERT OR IGNORE INTO list_targets (list_id, target_id) VALUES (?, ?)");
db.transaction(() => {
for (const row of rows) {
let targetId: string;
let isNew: boolean;
const fieldValues = EDITABLE_FIELDS.map((f) => row.fields[f]);
if (row.linkedin_url) {
const existing = findByLinkedin.get(row.linkedin_url) as { id: string } | undefined;
isNew = !existing;
targetId = existing?.id ?? randomUUID();
insertByLinkedin.run(targetId, row.linkedin_url, row.email, row.sales_nav_url, row.full_name, ...fieldValues);
} else {
const existing = findByEmail.get(row.email) as { id: string } | undefined;
isNew = !existing;
if (existing) {
targetId = existing.id;
updateByEmail.run(...fieldValues, row.full_name, row.sales_nav_url, targetId);
} else {
targetId = randomUUID();
insertByEmail.run(targetId, row.email, row.full_name, ...fieldValues, row.sales_nav_url);
}
}
const linkResult = linkToList.run(listId, targetId);
if (linkResult.changes > 0) {
if (isNew) imported++; else updated++;
} else {
skipped++; // already in this list, no changes
}
}
})();
return { imported, updated, skipped, errors };
}
+883
View File
@@ -0,0 +1,883 @@
import Database from "better-sqlite3";
import path from "path";
import { randomUUID } from "crypto";
import { scheduleUpdateCheck } from "@/lib/update-check";
import { encryptSecret, isEncrypted } from "@/lib/crypto";
const DB_PATH = process.env.LINKI_DB_PATH || path.join(process.cwd(), "linki.db");
let db: Database.Database;
export function getDb(): Database.Database {
if (!db) {
db = new Database(DB_PATH);
db.pragma("journal_mode = WAL");
db.pragma("foreign_keys = ON");
initDb(db);
runMigrations(db);
scheduleUpdateCheck();
}
return db;
}
function runParallelTracksMigration(db: Database.Database) {
// This backfill reads the legacy run_profiles.state column. If that column no longer
// exists, dropDeprecatedRunProfileColumns has already run (a prior startup) and this
// migration is moot — skip, otherwise the SELECT rp.state below throws "no such column".
// (Fresh DBs hit this: state is created, dropped on one startup, then this guard would
// otherwise re-enter on the next because no email-tracked steps exist yet.)
try {
const rpCols = db.prepare("PRAGMA table_info(run_profiles)").all() as { name: string }[];
if (!rpCols.some(c => c.name === "state")) return;
} catch { return; }
// Idempotent: skip if run_profile_tracks already has rows or workflow_steps already has email-tracked rows
try {
const alreadyRun = (db.prepare("SELECT COUNT(*) as c FROM run_profile_tracks").get() as { c: number }).c > 0
|| (db.prepare("SELECT COUNT(*) as c FROM workflow_steps WHERE track = 'email'").get() as { c: number }).c > 0;
if (alreadyRun) return;
} catch { return; }
db.transaction(() => {
// 1. Assign email step_type rows to the email track
db.exec("UPDATE workflow_steps SET track = 'email' WHERE step_type = 'email'");
// Also assign delay steps to the email track if the next non-delay step after them is an email step.
// Without this, delays between email steps default to 'linkedin' and create a ghost linkedin track.
db.exec(`
UPDATE workflow_steps SET track = 'email'
WHERE step_type = 'delay'
AND (
SELECT step_type FROM workflow_steps ws2
WHERE ws2.workflow_id = workflow_steps.workflow_id
AND ws2.step_order > workflow_steps.step_order
AND ws2.step_type != 'delay'
ORDER BY ws2.step_order ASC
LIMIT 1
) = 'email'
`);
// 2. Re-number step_order densely within each (workflow_id, track), preserving original order
const stepGroups = db.prepare(
"SELECT id, workflow_id, track, step_order FROM workflow_steps ORDER BY workflow_id, track, step_order"
).all() as Array<{ id: string; workflow_id: string; track: string; step_order: number }>;
// Group by (workflow_id, track) and assign dense 1-based order
const grouped = new Map<string, Array<{ id: string; step_order: number }>>();
for (const row of stepGroups) {
const key = `${row.workflow_id}|${row.track}`;
if (!grouped.has(key)) grouped.set(key, []);
grouped.get(key)!.push({ id: row.id, step_order: row.step_order });
}
const updateStep = db.prepare("UPDATE workflow_steps SET step_order = ? WHERE id = ?");
for (const steps of grouped.values()) {
steps.sort((a, b) => a.step_order - b.step_order);
steps.forEach((s, i) => updateStep.run(i + 1, s.id));
}
// 3. Backfill run_profile_tracks from existing run_profiles
// Only backfill if there are run_profiles rows to process
const allProfiles = db.prepare(
`SELECT rp.id, rp.run_id, rp.state, rp.current_step, rp.next_step_at,
rp.error_message, rp.last_email_subject, rp.last_email_body, rp.last_linkedin_message,
r.workflow_id
FROM run_profiles rp
JOIN runs r ON r.id = rp.run_id`
).all() as Array<{
id: string; run_id: string; state: string; current_step: number;
next_step_at: string | null; error_message: string | null;
last_email_subject: string | null; last_email_body: string | null;
last_linkedin_message: string | null; workflow_id: string;
}>;
if (allProfiles.length === 0) return;
// Load all workflow steps grouped by workflow_id, preserving their original ordering
// We need the ORIGINAL step_order to map legacy current_step (0-based flat index) to tracks.
// After the re-numbering above, step_order is now per-track. We stored the original order as the
// sort key inside stepGroups above. Rebuild a per-workflow flat list from the original ordering.
const workflowStepsOrig = db.prepare(
"SELECT id, workflow_id, track, step_order FROM workflow_steps ORDER BY workflow_id, step_order"
).all() as Array<{ id: string; workflow_id: string; track: string; step_order: number }>;
// For each workflow, build the flat list of steps in their original order (by new step_order within track, then track order linkedin < email)
// BUT: we need the ORIGINAL flat order before re-numbering. Since we already re-numbered, we have to reconstruct.
// Approach: the original flat step_order was cross-track. The re-numbered step_order is per-track and starts at 1.
// We stored the original step_order in stepGroups (before modification). Use that.
const origStepOrderMap = new Map<string, number>(); // step id → original flat step_order
for (const row of stepGroups) {
origStepOrderMap.set(row.id, row.step_order);
}
// Per workflow: flat list of steps sorted by original step_order
const workflowFlatSteps = new Map<string, Array<{ id: string; track: string; orig_order: number }>>();
for (const step of workflowStepsOrig) {
if (!workflowFlatSteps.has(step.workflow_id)) workflowFlatSteps.set(step.workflow_id, []);
workflowFlatSteps.get(step.workflow_id)!.push({
id: step.id,
track: step.track,
orig_order: origStepOrderMap.get(step.id) ?? step.step_order,
});
}
for (const steps of workflowFlatSteps.values()) {
steps.sort((a, b) => a.orig_order - b.orig_order);
}
// Per workflow: which tracks exist
const workflowTracks = new Map<string, Set<string>>();
for (const step of workflowStepsOrig) {
if (!workflowTracks.has(step.workflow_id)) workflowTracks.set(step.workflow_id, new Set());
workflowTracks.get(step.workflow_id)!.add(step.track);
}
const insertTrack = db.prepare(`
INSERT OR IGNORE INTO run_profile_tracks
(id, run_profile_id, track, state, current_step, last_step_at, next_step_at,
error_message, last_email_subject, last_email_body, last_linkedin_message)
VALUES (?, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?)
`);
for (const rp of allProfiles) {
const flatSteps = workflowFlatSteps.get(rp.workflow_id) ?? [];
const tracks = workflowTracks.get(rp.workflow_id) ?? new Set(["linkedin"]);
const terminalStates = new Set(["completed", "failed", "skipped"]);
if (terminalStates.has(rp.state)) {
// Terminal profile — mark all tracks with the same terminal state
for (const track of tracks) {
insertTrack.run(
randomUUID(), rp.id, track, rp.state, 0,
null, rp.error_message ?? null, null, null, null
);
}
continue;
}
// Active profile — compute per-track completed step count
// legacy current_step is 0-based index into the flat step list = steps already completed
const completedCount = rp.current_step;
const completedSteps = flatSteps.slice(0, completedCount);
const trackCompletedCount = new Map<string, number>();
for (const track of tracks) trackCompletedCount.set(track, 0);
for (const s of completedSteps) {
trackCompletedCount.set(s.track, (trackCompletedCount.get(s.track) ?? 0) + 1);
}
// Which track owned the step we were waiting on (index = completedCount)?
const currentFlatStep = flatSteps[completedCount];
const waitingTrack = currentFlatStep?.track ?? null;
for (const track of tracks) {
const trackCurrentStep = trackCompletedCount.get(track) ?? 0;
// next_step_at: only carry over to the track that was waiting; other track starts immediately
const nextStepAt = track === waitingTrack ? rp.next_step_at : null;
const lastEmailSubject = track === "email" ? rp.last_email_subject : null;
const lastEmailBody = track === "email" ? rp.last_email_body : null;
const lastLinkedinMessage = track === "linkedin" ? rp.last_linkedin_message : null;
insertTrack.run(
randomUUID(), rp.id, track, rp.state, trackCurrentStep,
nextStepAt, rp.error_message ?? null,
lastEmailSubject, lastEmailBody, lastLinkedinMessage
);
}
}
})();
}
function dropDeprecatedRunProfileColumns(db: Database.Database) {
// Idempotent: check if state column still exists on run_profiles
try {
const tableInfo = db.prepare("PRAGMA table_info(run_profiles)").all() as { name: string }[];
const hasState = tableInfo.some(c => c.name === "state");
if (!hasState) return; // already dropped
} catch { return; }
// SQLite requires a table rebuild to drop columns
try {
db.exec(`
PRAGMA foreign_keys = OFF;
CREATE TABLE run_profiles_new (
id TEXT PRIMARY KEY,
run_id TEXT REFERENCES runs(id) ON DELETE CASCADE,
target_id TEXT REFERENCES targets(id),
email_account_id TEXT,
created_at TEXT DEFAULT (datetime('now')),
UNIQUE(run_id, target_id)
);
INSERT INTO run_profiles_new (id, run_id, target_id, email_account_id, created_at)
SELECT id, run_id, target_id, email_account_id, created_at FROM run_profiles;
DROP TABLE run_profiles;
ALTER TABLE run_profiles_new RENAME TO run_profiles;
PRAGMA foreign_keys = ON;
`);
} catch { /* ignore — may already be done */ }
}
function runMigrations(db: Database.Database) {
// Add columns introduced after initial schema — safe to run on existing DBs
const migrations = [
"ALTER TABLE targets ADD COLUMN degree INTEGER",
"ALTER TABLE targets ADD COLUMN connection_requested_at TEXT",
"ALTER TABLE targets ADD COLUMN connected_at TEXT",
"ALTER TABLE targets ADD COLUMN message_sent_at TEXT",
"ALTER TABLE targets ADD COLUMN last_replied_at TEXT",
"ALTER TABLE targets ADD COLUMN linkedin_member_urn TEXT",
"ALTER TABLE targets ADD COLUMN sales_nav_url TEXT",
"ALTER TABLE lists ADD COLUMN sales_nav_url TEXT",
"ALTER TABLE accounts ADD COLUMN inbox_synced_at TEXT",
"ALTER TABLE accounts ADD COLUMN active_hours_start INTEGER DEFAULT 9",
"ALTER TABLE accounts ADD COLUMN active_hours_end INTEGER DEFAULT 18",
"ALTER TABLE accounts ADD COLUMN timezone TEXT DEFAULT 'UTC'",
"ALTER TABLE accounts ADD COLUMN working_days TEXT DEFAULT '1,2,3,4,5'",
"ALTER TABLE workflow_steps ADD COLUMN connect_note TEXT",
"ALTER TABLE workflow_steps ADD COLUMN message_body TEXT",
"ALTER TABLE targets ADD COLUMN headline TEXT",
"ALTER TABLE targets ADD COLUMN summary TEXT",
"ALTER TABLE accounts ADD COLUMN accepted_sync_at TEXT",
// Messaging identity (fsd_profile URN, "urn:li:fsd_profile:ACoAA...") — the
// form the messaging GraphQL API returns. It is NOT convertible to the
// numeric urn:li:member we store elsewhere, so we capture it on first reply
// (matched by name) and then join by it directly on every later sync.
"ALTER TABLE targets ADD COLUMN messaging_urn TEXT",
"CREATE INDEX IF NOT EXISTS idx_targets_messaging_urn ON targets(messaging_urn)",
// Boundary for the incremental connections sync: the createdAt of the newest
// connection seen last run. NULL = never synced (first run does a full pass).
"ALTER TABLE accounts ADD COLUMN connections_synced_through_ms INTEGER",
"ALTER TABLE accounts ADD COLUMN li_connections INTEGER",
"ALTER TABLE accounts ADD COLUMN li_pending INTEGER",
"ALTER TABLE accounts ADD COLUMN li_profile_views INTEGER",
"ALTER TABLE accounts ADD COLUMN li_stats_synced_at TEXT",
`CREATE TABLE IF NOT EXISTS workflow_step_templates (
step_id TEXT REFERENCES workflow_steps(id) ON DELETE CASCADE,
template_id TEXT REFERENCES templates(id) ON DELETE CASCADE,
PRIMARY KEY (step_id, template_id)
)`,
// Extended lead data from salesApiLeadSearch
"ALTER TABLE targets ADD COLUMN object_urn TEXT",
"ALTER TABLE targets ADD COLUMN open_link INTEGER DEFAULT 0",
"ALTER TABLE targets ADD COLUMN company_industry TEXT",
"ALTER TABLE targets ADD COLUMN company_location TEXT",
"ALTER TABLE targets ADD COLUMN tenure_months INTEGER",
"ALTER TABLE targets ADD COLUMN spotlight_badges TEXT",
// Profile enrichment — populated by visiting their Sales Nav profile page
"ALTER TABLE targets ADD COLUMN positions_json TEXT",
"ALTER TABLE targets ADD COLUMN skills_json TEXT",
"ALTER TABLE targets ADD COLUMN enriched_profile_at TEXT",
// Email outreach fields
"ALTER TABLE targets ADD COLUMN email TEXT",
"ALTER TABLE targets ADD COLUMN email_replied_at TEXT",
"ALTER TABLE targets ADD COLUMN company_id TEXT",
// Email account on runs (nullable — only needed when workflow has email steps)
"ALTER TABLE runs ADD COLUMN email_account_id TEXT REFERENCES email_accounts(id)",
// Workflow step email fields
"ALTER TABLE workflow_steps ADD COLUMN email_subject TEXT",
"ALTER TABLE workflow_steps ADD COLUMN email_body TEXT",
// Apollo enrichment fields
"ALTER TABLE targets ADD COLUMN apollo_id TEXT",
"ALTER TABLE targets ADD COLUMN seniority TEXT",
"ALTER TABLE targets ADD COLUMN apollo_functions TEXT",
"ALTER TABLE targets ADD COLUMN company_description TEXT",
"ALTER TABLE targets ADD COLUMN company_size INTEGER",
"ALTER TABLE targets ADD COLUMN apollo_enriched_at TEXT",
"ALTER TABLE targets ADD COLUMN email_status TEXT",
// Manual fields
"ALTER TABLE targets ADD COLUMN notes TEXT",
// Apollo extra person fields
"ALTER TABLE targets ADD COLUMN city TEXT",
"ALTER TABLE targets ADD COLUMN country TEXT",
"ALTER TABLE targets ADD COLUMN time_zone TEXT",
"ALTER TABLE targets ADD COLUMN apollo_departments TEXT",
// Apollo extra company fields on companies table
"ALTER TABLE companies ADD COLUMN founded_year INTEGER",
"ALTER TABLE companies ADD COLUMN logo_url TEXT",
"ALTER TABLE companies ADD COLUMN phone TEXT",
"ALTER TABLE companies ADD COLUMN annual_revenue TEXT",
"ALTER TABLE companies ADD COLUMN technology_names TEXT",
"ALTER TABLE companies ADD COLUMN keywords TEXT",
"ALTER TABLE companies ADD COLUMN city TEXT",
"ALTER TABLE companies ADD COLUMN country TEXT",
// Email signature
"ALTER TABLE email_accounts ADD COLUMN signature TEXT",
// Reply-To override — if set, outgoing emails include Reply-To header
"ALTER TABLE email_accounts ADD COLUMN reply_to TEXT",
// Ramp-up: start slow and increase sending volume over time
"ALTER TABLE email_accounts ADD COLUMN ramp_up_enabled INTEGER DEFAULT 1",
"ALTER TABLE email_accounts ADD COLUMN ramp_start_date TEXT",
// Company description and employee count moved from targets to companies
"ALTER TABLE companies ADD COLUMN description TEXT",
"ALTER TABLE companies ADD COLUMN employee_count INTEGER",
// AI agent columns on workflow steps
"ALTER TABLE workflow_steps ADD COLUMN ai_enabled INTEGER DEFAULT 0",
"ALTER TABLE workflow_steps ADD COLUMN ai_model TEXT",
"ALTER TABLE workflow_steps ADD COLUMN ai_prompt TEXT",
"ALTER TABLE workflow_steps ADD COLUMN ai_max_words INTEGER",
// Email step position — which followup number (1 = cold email, 2 = first followup, etc.)
"ALTER TABLE workflow_steps ADD COLUMN email_position INTEGER DEFAULT 1",
// Agent default model (stored on agent_config)
"ALTER TABLE agent_config ADD COLUMN default_model TEXT",
// Email threading — store sent email message-id for reply threading (future use)
"ALTER TABLE run_profiles ADD COLUMN last_email_subject TEXT",
"ALTER TABLE run_profiles ADD COLUMN last_email_body TEXT",
// LinkedIn message follow-up tracking
"ALTER TABLE workflow_steps ADD COLUMN message_position INTEGER DEFAULT 1",
"ALTER TABLE run_profiles ADD COLUMN last_linkedin_message TEXT",
// Language for AI-generated content per step
"ALTER TABLE workflow_steps ADD COLUMN ai_language TEXT DEFAULT 'English'",
// Campaign-level prompt — per-workflow AI context (USP, persona, tone for this campaign)
"ALTER TABLE workflows ADD COLUMN prompt TEXT",
// Email domain invalid flag on companies — set when a bounce is detected for any contact at this company
"ALTER TABLE companies ADD COLUMN email_domain_invalid INTEGER DEFAULT 0",
// Apollo extra person fields
"ALTER TABLE targets ADD COLUMN email_domain_catchall INTEGER DEFAULT 0",
// Per-profile email account assignment (multi-account routing)
"ALTER TABLE run_profiles ADD COLUMN email_account_id TEXT",
// Backfill: copy email_account_id from runs → run_profiles for existing records
`UPDATE run_profiles SET email_account_id = (SELECT email_account_id FROM runs WHERE runs.id = run_profiles.run_id) WHERE email_account_id IS NULL`,
// Import job progress tracking
`CREATE TABLE IF NOT EXISTS list_imports (
id TEXT PRIMARY KEY,
list_id TEXT NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
status TEXT NOT NULL DEFAULT 'running',
phase TEXT,
page INTEGER DEFAULT 0,
total_pages INTEGER DEFAULT 0,
count INTEGER DEFAULT 0,
total INTEGER DEFAULT 0,
imported INTEGER DEFAULT 0,
skipped INTEGER DEFAULT 0,
error TEXT,
started_at TEXT NOT NULL DEFAULT (datetime('now')),
finished_at TEXT
)`,
// Parallel tracks: add track column to workflow_steps
"ALTER TABLE workflow_steps ADD COLUMN track TEXT NOT NULL DEFAULT 'linkedin' CHECK(track IN ('linkedin', 'email'))",
// Parallel tracks: create run_profile_tracks table
`CREATE TABLE IF NOT EXISTS run_profile_tracks (
id TEXT PRIMARY KEY,
run_profile_id TEXT NOT NULL REFERENCES run_profiles(id) ON DELETE CASCADE,
track TEXT NOT NULL CHECK(track IN ('linkedin', 'email')),
state TEXT NOT NULL DEFAULT 'pending' CHECK(state IN ('pending', 'in_progress', 'completed', 'failed', 'skipped')),
current_step INTEGER NOT NULL DEFAULT 0,
last_step_at TEXT,
next_step_at TEXT,
error_message TEXT,
last_email_subject TEXT,
last_email_body TEXT,
last_linkedin_message TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
UNIQUE(run_profile_id, track)
)`,
"CREATE INDEX IF NOT EXISTS idx_run_profile_tracks_run_profile_id ON run_profile_tracks(run_profile_id)",
"CREATE INDEX IF NOT EXISTS idx_run_profile_tracks_state_next ON run_profile_tracks(state, next_step_at)",
// Drop deprecated columns from run_profiles — all consumers now read from run_profile_tracks
// SQLite does not support DROP COLUMN directly before 3.35; handled via table rebuild below
// Separate IMAP credentials for custom mail providers where SMTP ≠ IMAP auth
"ALTER TABLE email_accounts ADD COLUMN imap_username TEXT",
"ALTER TABLE email_accounts ADD COLUMN imap_password TEXT",
"ALTER TABLE workflows ADD COLUMN is_archived INTEGER NOT NULL DEFAULT 0",
// Per-step signature override for email steps (null = use email account default)
"ALTER TABLE workflow_steps ADD COLUMN email_signature TEXT",
// CRM: todos per contact
`CREATE TABLE IF NOT EXISTS todos (
id TEXT PRIMARY KEY,
target_id TEXT NOT NULL REFERENCES targets(id) ON DELETE CASCADE,
title TEXT NOT NULL,
due_date TEXT,
status TEXT NOT NULL DEFAULT 'open' CHECK(status IN ('open', 'done')),
created_at TEXT NOT NULL DEFAULT (datetime('now'))
)`,
"CREATE INDEX IF NOT EXISTS idx_todos_target_id ON todos(target_id)",
"CREATE INDEX IF NOT EXISTS idx_todos_status ON todos(status)",
"ALTER TABLE todos ADD COLUMN description TEXT",
// CRM: activity log per contact
`CREATE TABLE IF NOT EXISTS activity_logs (
id TEXT PRIMARY KEY,
target_id TEXT NOT NULL REFERENCES targets(id) ON DELETE CASCADE,
type TEXT NOT NULL DEFAULT 'note' CHECK(type IN ('call', 'email', 'meeting', 'note', 'other')),
body TEXT NOT NULL,
logged_at TEXT NOT NULL DEFAULT (datetime('now')),
created_at TEXT NOT NULL DEFAULT (datetime('now'))
)`,
"CREATE INDEX IF NOT EXISTS idx_activity_logs_target_id ON activity_logs(target_id)",
// Reply classifier: captured email replies + classifier verdict + dispatcher result
`CREATE TABLE IF NOT EXISTS email_replies (
id TEXT PRIMARY KEY,
target_id TEXT NOT NULL REFERENCES targets(id) ON DELETE CASCADE,
run_id TEXT REFERENCES runs(id) ON DELETE SET NULL,
from_email TEXT NOT NULL,
subject TEXT,
body_text TEXT NOT NULL,
received_at TEXT NOT NULL,
classified_at TEXT,
classification_json TEXT,
classification_error TEXT,
dispatched_at TEXT,
dispatch_result_json TEXT,
manually_edited INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
)`,
"CREATE INDEX IF NOT EXISTS idx_email_replies_target_id ON email_replies(target_id)",
"CREATE INDEX IF NOT EXISTS idx_email_replies_dispatched_at ON email_replies(dispatched_at)",
// Reply classifier verdict stamped on the target — enables reply-rate-by-kind metrics
"ALTER TABLE targets ADD COLUMN reply_kind TEXT",
"ALTER TABLE targets ADD COLUMN inmail_sent_at TEXT",
"ALTER TABLE targets ADD COLUMN posts_json TEXT", // recent posts from visit_profile
"ALTER TABLE targets ADD COLUMN posts_scraped_at TEXT",
// One-shot OOO reply context for the AI follow-up writer — set by the dispatcher,
// read + cleared by the runner on the next email send. Distinct from last_email_body
// (which holds the last email WE sent, used for follow-up threading).
"ALTER TABLE run_profile_tracks ADD COLUMN pending_reply_context TEXT",
// Removed the in-app chat agent (replaced by the hosted MCP endpoint at /api/mcp) — drop its tables.
"DROP TABLE IF EXISTS chat_messages",
"DROP TABLE IF EXISTS chat_sessions",
// Batched/scheduled imports — split large lists across days under a daily cap.
"ALTER TABLE list_imports ADD COLUMN account_id TEXT",
"ALTER TABLE list_imports ADD COLUMN sales_nav_url TEXT",
"ALTER TABLE list_imports ADD COLUMN scheduled_for TEXT", // 'YYYY-MM-DD'; NULL = run now
"ALTER TABLE list_imports ADD COLUMN start_page INTEGER DEFAULT 1",
"ALTER TABLE list_imports ADD COLUMN cap INTEGER", // max contacts for this batch
"ALTER TABLE list_imports ADD COLUMN cancel_requested INTEGER DEFAULT 0",
"ALTER TABLE list_imports ADD COLUMN batch_index INTEGER DEFAULT 1",
"ALTER TABLE list_imports ADD COLUMN enrich INTEGER DEFAULT 0",
"CREATE INDEX IF NOT EXISTS idx_list_imports_scheduled ON list_imports(status, scheduled_for)",
// Simple app-wide key/value settings (e.g. daily_import_cap)
`CREATE TABLE IF NOT EXISTS app_settings (
key TEXT PRIMARY KEY,
value TEXT,
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
)`,
// Sales Nav InMail gets its own daily budget — separate from daily_message_limit,
// since InMail (non-connections) and regular messages (connections) were being
// gated off the same counter, starving one whenever the other was busy.
"ALTER TABLE accounts ADD COLUMN daily_inmail_limit INTEGER DEFAULT 15",
// Sales Nav search: persistent cache of resolved filter values (typeahead
// ids). Dedup on (filter_type, id) — LinkedIn ids are stable (e.g. Berlin's
// geoUrn never changes), so once resolved a value is reused forever with no
// further live typeahead calls. Lives here (not ee/) per the open-core rule:
// all migrations stay in lib/db.ts. Consumed only by the ee/ search feature.
`CREATE TABLE IF NOT EXISTS search_filter_cache (
id TEXT NOT NULL,
filter_type TEXT NOT NULL,
display_value TEXT NOT NULL,
headline TEXT,
query TEXT NOT NULL,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
PRIMARY KEY (filter_type, id)
)`,
"CREATE INDEX IF NOT EXISTS idx_sfc_type_query ON search_filter_cache(filter_type, query)",
"CREATE INDEX IF NOT EXISTS idx_sfc_type_display ON search_filter_cache(filter_type, display_value)",
// CSV import: lists can be flagged as linkedin- or email-only, so the runner/UI
// can warn before enrolling a purpose-mismatched list into a campaign.
"ALTER TABLE lists ADD COLUMN purpose TEXT",
// Manual/CSV-only field — no automation reads or writes this, reference data only.
"ALTER TABLE targets ADD COLUMN phone TEXT",
];
for (const sql of migrations) {
try { db.exec(sql); } catch { /* column already exists */ }
}
// Parallel tracks: assign email steps to email track, re-number step_order, backfill run_profile_tracks
runParallelTracksMigration(db);
// Drop deprecated run_profiles columns (state, current_step, etc.) — consumers now read track-runs
dropDeprecatedRunProfileColumns(db);
// Migrate workflow_steps CHECK constraint to allow 'delay' and 'email' step_types
try {
const tableInfo = db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='workflow_steps'").get() as { sql: string } | undefined;
if (tableInfo && (!tableInfo.sql.includes("'delay'") || !tableInfo.sql.includes("'email'"))) {
db.exec(`
PRAGMA foreign_keys = OFF;
CREATE TABLE workflow_steps_new (
id TEXT PRIMARY KEY,
workflow_id TEXT REFERENCES workflows(id) ON DELETE CASCADE,
step_order INTEGER NOT NULL,
step_type TEXT NOT NULL CHECK(step_type IN ('visit', 'connect', 'message', 'delay', 'email')),
template_id TEXT REFERENCES templates(id),
delay_seconds INTEGER DEFAULT 0,
connect_note TEXT,
message_body TEXT,
email_subject TEXT,
email_body TEXT,
enabled INTEGER DEFAULT 1
);
INSERT INTO workflow_steps_new
SELECT id, workflow_id, step_order, step_type, template_id, delay_seconds,
connect_note, message_body,
NULL, NULL,
enabled
FROM workflow_steps;
DROP TABLE workflow_steps;
ALTER TABLE workflow_steps_new RENAME TO workflow_steps;
PRAGMA foreign_keys = ON;
`);
}
} catch { /* migration already done */ }
// Allow the 'sales_inmail' step_type (Sales Navigator InMail). Rebuilds the
// table preserving EVERY current column (the historical rebuild above only
// copied the original columns — do NOT reuse it). InMail reuses message_body
// for the body and email_subject for the required subject.
try {
const ti = db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='workflow_steps'").get() as { sql: string } | undefined;
if (ti && !ti.sql.includes("'sales_inmail'")) {
const cols = (db.prepare("PRAGMA table_info(workflow_steps)").all() as Array<{ name: string }>).map((c) => c.name);
const colList = cols.join(", ");
db.exec(`
PRAGMA foreign_keys = OFF;
CREATE TABLE workflow_steps_new (
id TEXT PRIMARY KEY,
workflow_id TEXT REFERENCES workflows(id) ON DELETE CASCADE,
step_order INTEGER NOT NULL,
step_type TEXT NOT NULL CHECK(step_type IN ('visit', 'connect', 'message', 'sales_inmail', 'delay', 'email')),
template_id TEXT REFERENCES templates(id),
delay_seconds INTEGER DEFAULT 0,
connect_note TEXT,
message_body TEXT,
email_subject TEXT,
email_body TEXT,
enabled INTEGER DEFAULT 1,
ai_enabled INTEGER DEFAULT 0,
ai_model TEXT,
ai_prompt TEXT,
ai_max_words INTEGER,
email_position INTEGER DEFAULT 1,
message_position INTEGER DEFAULT 1,
ai_language TEXT DEFAULT 'English',
track TEXT NOT NULL DEFAULT 'linkedin' CHECK(track IN ('linkedin', 'email')),
email_signature TEXT
);
INSERT INTO workflow_steps_new (${colList}) SELECT ${colList} FROM workflow_steps;
DROP TABLE workflow_steps;
ALTER TABLE workflow_steps_new RENAME TO workflow_steps;
PRAGMA foreign_keys = ON;
`);
}
} catch { /* migration already done */ }
// CSV import: allow email-only targets (no LinkedIn URL). targets.linkedin_url was
// NOT NULL UNIQUE from the base schema — rebuild to make it nullable (still UNIQUE,
// SQLite allows multiple NULLs under UNIQUE) preserving EVERY current column, same
// pattern as the sales_inmail rebuild above (do not reuse the older historical rebuilds
// that only copied the original columns).
try {
const ti = db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='targets'").get() as { sql: string } | undefined;
if (ti && ti.sql.includes("linkedin_url TEXT NOT NULL")) {
const cols = (db.prepare("PRAGMA table_info(targets)").all() as Array<{ name: string; type: string; notnull: number; dflt_value: string | null }>);
const colDefs = cols.map((c) => {
if (c.name === "linkedin_url") return "linkedin_url TEXT UNIQUE";
if (c.name === "id") return "id TEXT PRIMARY KEY";
const notnull = c.notnull ? " NOT NULL" : "";
// Non-literal defaults (e.g. datetime('now')) must be parenthesized in SQLite DDL.
const isLiteral = c.dflt_value === null || /^-?\d+(\.\d+)?$/.test(c.dflt_value) || /^'.*'$/.test(c.dflt_value);
const dflt = c.dflt_value !== null ? ` DEFAULT ${isLiteral ? c.dflt_value : `(${c.dflt_value})`}` : "";
return `${c.name} ${c.type}${notnull}${dflt}`;
});
const colList = cols.map((c) => c.name).join(", ");
db.exec(`
PRAGMA foreign_keys = OFF;
CREATE TABLE targets_new (
${colDefs.join(",\n ")}
);
INSERT INTO targets_new (${colList}) SELECT ${colList} FROM targets;
DROP TABLE targets;
ALTER TABLE targets_new RENAME TO targets;
PRAGMA foreign_keys = ON;
`);
}
} catch { /* migration already done */ }
// Backfill: move company_description and company_size from targets into companies
try {
db.exec(`
UPDATE companies
SET
description = COALESCE(description, (
SELECT t.company_description FROM targets t
WHERE t.company_id = companies.id AND t.company_description IS NOT NULL
LIMIT 1
)),
employee_count = COALESCE(employee_count, (
SELECT t.company_size FROM targets t
WHERE t.company_id = companies.id AND t.company_size IS NOT NULL
LIMIT 1
))
WHERE description IS NULL OR employee_count IS NULL
`);
} catch { /* ignore */ }
// Backfill: for old records where linkedin_url is a Sales Nav URL, move it to sales_nav_url
try {
db.exec(`
UPDATE targets
SET sales_nav_url = linkedin_url
WHERE linkedin_url LIKE '%/sales/lead/%' AND (sales_nav_url IS NULL OR sales_nav_url = '')
`);
} catch { /* ignore */ }
// Create unique index on run_profiles if not already present (idempotent)
try {
db.exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_run_profiles_unique ON run_profiles(run_id, target_id);");
} catch { /* ignore */ }
encryptLegacySecretsMigration(db);
}
// One-time (per-row) migration: encrypt any plaintext accounts.cookies_json,
// email_accounts.password/imap_password, integrations.api_key left over from before
// encryption-at-rest was added. isEncrypted() lets this run safely on every boot — already
//-encrypted rows (real "v1:" ciphertext) are skipped, so this is idempotent and cheap once
// migrated. See lib/crypto.ts for the format and lib/premium.ts-style boundary notes.
function encryptLegacySecretsMigration(db: Database.Database) {
const accounts = db.prepare("SELECT id, cookies_json FROM accounts WHERE cookies_json IS NOT NULL").all() as
{ id: string; cookies_json: string }[];
for (const row of accounts) {
if (isEncrypted(row.cookies_json)) continue;
db.prepare("UPDATE accounts SET cookies_json = ? WHERE id = ?").run(encryptSecret(row.cookies_json), row.id);
}
const emailAccounts = db
.prepare("SELECT id, password, imap_password FROM email_accounts")
.all() as { id: string; password: string; imap_password: string | null }[];
for (const row of emailAccounts) {
const needsPassword = !isEncrypted(row.password);
const needsImapPassword = row.imap_password !== null && !isEncrypted(row.imap_password);
if (!needsPassword && !needsImapPassword) continue;
db.prepare("UPDATE email_accounts SET password = ?, imap_password = ? WHERE id = ?").run(
needsPassword ? encryptSecret(row.password) : row.password,
needsImapPassword ? encryptSecret(row.imap_password!) : row.imap_password,
row.id
);
}
const integrations = db.prepare("SELECT key, api_key FROM integrations WHERE api_key IS NOT NULL").all() as
{ key: string; api_key: string }[];
for (const row of integrations) {
if (isEncrypted(row.api_key)) continue;
db.prepare("UPDATE integrations SET api_key = ? WHERE key = ?").run(encryptSecret(row.api_key), row.key);
}
}
function initDb(db: Database.Database) {
db.exec(`
CREATE TABLE IF NOT EXISTS accounts (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
email TEXT NOT NULL UNIQUE,
cookies_json TEXT,
is_authenticated INTEGER DEFAULT 0,
daily_connection_limit INTEGER DEFAULT 20,
daily_message_limit INTEGER DEFAULT 50,
daily_inmail_limit INTEGER DEFAULT 15,
active_hours_start INTEGER DEFAULT 9,
active_hours_end INTEGER DEFAULT 18,
timezone TEXT DEFAULT 'UTC',
working_days TEXT DEFAULT '1,2,3,4,5',
created_at TEXT DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS targets (
id TEXT PRIMARY KEY,
linkedin_url TEXT UNIQUE,
sales_nav_url TEXT,
first_name TEXT,
last_name TEXT,
full_name TEXT,
title TEXT,
company TEXT,
location TEXT,
profile_image_url TEXT,
degree INTEGER,
connection_requested_at TEXT,
connected_at TEXT,
message_sent_at TEXT,
last_replied_at TEXT,
linkedin_member_urn TEXT,
enriched_at TEXT,
created_at TEXT DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS lists (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
description TEXT,
sales_nav_url TEXT,
purpose TEXT CHECK(purpose IN ('linkedin', 'email')),
created_at TEXT DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS list_targets (
list_id TEXT REFERENCES lists(id) ON DELETE CASCADE,
target_id TEXT REFERENCES targets(id) ON DELETE CASCADE,
PRIMARY KEY (list_id, target_id)
);
CREATE TABLE IF NOT EXISTS templates (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
body TEXT NOT NULL,
created_at TEXT DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS workflows (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
description TEXT,
created_at TEXT DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS workflow_steps (
id TEXT PRIMARY KEY,
workflow_id TEXT REFERENCES workflows(id) ON DELETE CASCADE,
step_order INTEGER NOT NULL,
step_type TEXT NOT NULL CHECK(step_type IN ('visit', 'connect', 'message', 'delay')),
template_id TEXT REFERENCES templates(id),
delay_seconds INTEGER DEFAULT 0,
connect_note TEXT,
message_body TEXT,
enabled INTEGER DEFAULT 1
);
CREATE TABLE IF NOT EXISTS workflow_step_templates (
step_id TEXT REFERENCES workflow_steps(id) ON DELETE CASCADE,
template_id TEXT REFERENCES templates(id) ON DELETE CASCADE,
PRIMARY KEY (step_id, template_id)
);
CREATE TABLE IF NOT EXISTS runs (
id TEXT PRIMARY KEY,
workflow_id TEXT REFERENCES workflows(id),
list_id TEXT REFERENCES lists(id),
account_id TEXT REFERENCES accounts(id),
status TEXT DEFAULT 'pending' CHECK(status IN ('pending', 'running', 'paused', 'completed', 'failed')),
created_at TEXT DEFAULT (datetime('now')),
started_at TEXT,
completed_at TEXT,
runner_pid INTEGER
);
CREATE TABLE IF NOT EXISTS run_profiles (
id TEXT PRIMARY KEY,
run_id TEXT REFERENCES runs(id) ON DELETE CASCADE,
target_id TEXT REFERENCES targets(id),
state TEXT DEFAULT 'pending' CHECK(state IN ('pending', 'in_progress', 'completed', 'failed', 'skipped')),
current_step INTEGER DEFAULT 0,
last_step_at TEXT,
next_step_at TEXT,
error_message TEXT,
created_at TEXT DEFAULT (datetime('now')),
UNIQUE(run_id, target_id)
);
CREATE TABLE IF NOT EXISTS logs (
id TEXT PRIMARY KEY,
run_id TEXT REFERENCES runs(id) ON DELETE CASCADE,
target_id TEXT REFERENCES targets(id),
level TEXT DEFAULT 'info' CHECK(level IN ('info', 'warn', 'error')),
message TEXT NOT NULL,
created_at TEXT DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS users (
id TEXT PRIMARY KEY,
email TEXT NOT NULL UNIQUE,
password_hash TEXT NOT NULL,
created_at TEXT DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS companies (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
domain TEXT,
industry TEXT,
location TEXT,
linkedin_url TEXT,
website TEXT,
notes TEXT,
created_at TEXT DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS integrations (
key TEXT PRIMARY KEY,
api_key TEXT,
created_at TEXT DEFAULT (datetime('now')),
updated_at TEXT DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS agent_config (
id INTEGER PRIMARY KEY DEFAULT 1,
system_prompt TEXT,
user_prompt TEXT,
email_examples TEXT,
linkedin_examples TEXT,
updated_at TEXT DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS agent_sessions (
id TEXT PRIMARY KEY,
run_id TEXT,
target_id TEXT,
step_id TEXT,
model TEXT,
input_tokens INTEGER,
output_tokens INTEGER,
cost_usd REAL,
prompt TEXT,
generated_text TEXT,
created_at TEXT DEFAULT (datetime('now'))
);
-- Hosted MCP OAuth 2.1 server: clients (DCR), single-use auth codes, access/refresh tokens.
-- Tokens/codes are stored only as sha256 hashes. Reuses NEXTAUTH user identity.
CREATE TABLE IF NOT EXISTS oauth_clients (
client_id TEXT PRIMARY KEY,
client_name TEXT,
redirect_uris TEXT NOT NULL,
created_at TEXT DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS oauth_auth_codes (
code_hash TEXT PRIMARY KEY,
client_id TEXT NOT NULL,
user_id TEXT NOT NULL,
redirect_uri TEXT NOT NULL,
code_challenge TEXT NOT NULL,
scope TEXT,
expires_at TEXT NOT NULL,
created_at TEXT DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS oauth_tokens (
id TEXT PRIMARY KEY,
access_hash TEXT NOT NULL UNIQUE,
refresh_hash TEXT UNIQUE,
client_id TEXT NOT NULL,
user_id TEXT NOT NULL,
scope TEXT,
expires_at TEXT NOT NULL,
created_at TEXT DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS email_accounts (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
from_email TEXT NOT NULL,
from_name TEXT,
smtp_host TEXT NOT NULL,
smtp_port INTEGER DEFAULT 587,
smtp_secure INTEGER DEFAULT 0,
imap_host TEXT,
imap_port INTEGER DEFAULT 993,
username TEXT NOT NULL,
password TEXT NOT NULL,
daily_email_limit INTEGER DEFAULT 50,
active_hours_start INTEGER DEFAULT 9,
active_hours_end INTEGER DEFAULT 18,
timezone TEXT DEFAULT 'UTC',
working_days TEXT DEFAULT '1,2,3,4,5',
is_verified INTEGER DEFAULT 0,
inbox_synced_at TEXT,
created_at TEXT DEFAULT (datetime('now'))
);
`);
}
+1
View File
@@ -0,0 +1 @@
declare module 'imap-simple';
+367
View File
@@ -0,0 +1,367 @@
import Imap from "imap";
import { simpleParser } from "mailparser";
import { randomUUID } from "crypto";
import { getDb } from "@/lib/db";
import { premium } from "@/lib/premium";
import { decryptSecret } from "@/lib/crypto";
const IMAP_POLL_INTERVAL_MS = 2 * 60 * 60 * 1000; // 2 hours
// Jul 2026 incident: all email accounts became "due" in the same tick and synced
// back-to-back (one 20-account burst ran 3 minutes straight, one account alone took
// 85s), which lined up with NocoDB/n8n healthcheck failures on the shared host. A
// deterministic per-account offset spreads accounts across a window so they drift
// apart over successive cycles instead of staying clustered together forever.
const IMAP_POLL_JITTER_MS = 30 * 60 * 1000; // 30 min
function accountJitterMs(accountId: string): number {
let h = 0;
for (let i = 0; i < accountId.length; i++) h = (h * 31 + accountId.charCodeAt(i)) >>> 0;
return h % IMAP_POLL_JITTER_MS;
}
const BOUNCE_SENDER_PATTERNS = [
/mailer-daemon@/i,
/postmaster@/i,
/mail-delivery-subsystem@/i,
/delivery-status@/i,
/amazonses\.com$/i,
];
function extractEmails(text: string): string[] {
return [...text.matchAll(/[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}/g)].map(m => m[0].toLowerCase());
}
function isBounce(fromEmail: string): boolean {
return BOUNCE_SENDER_PATTERNS.some(p => p.test(fromEmail));
}
function parseHeaderValue(raw: string, field: string): string {
const regex = new RegExp(`^${field}:[ \\t]*(.+?)(?=\\r?\\n[^\\s]|$)`, "im");
const m = raw.match(regex);
if (!m) return "";
return m[1].replace(/\r?\n[\t ]+/g, " ").trim();
}
interface EmailAccount {
id: string;
imap_host: string | null;
imap_port: number | null;
username: string;
password: string;
imap_username: string | null;
imap_password: string | null;
inbox_synced_at: string | null;
}
/**
* Fetches the body + headers for a given UID and inserts a row into
* `email_replies` (idempotent — skips if a row with the same target_id +
* received_at already exists). Best-effort: errors are swallowed by the caller.
*/
export function captureReplyBody(
imap: Imap,
db: ReturnType<typeof getDb>,
targetId: string,
fromEmail: string,
uid: number,
): Promise<string | null> {
return new Promise<string | null>((resolve) => {
// Fetch the full raw RFC822 message — mailparser handles MIME multipart,
// base64 / quoted-printable transfer encodings, and charset decoding. The
// old HEADER+TEXT regex approach stored raw base64 / =XX escapes for the
// common German auto-reply formats, feeding the classifier garbage.
const fetch = imap.fetch(uid, { bodies: [""], struct: false });
const chunks: Buffer[] = [];
fetch.on("message", (msg) => {
msg.on("body", (stream) => {
stream.on("data", (c: Buffer) => chunks.push(c));
});
});
fetch.once("error", () => resolve(null));
fetch.once("end", () => {
void (async () => {
try {
const raw = Buffer.concat(chunks);
const parsed = await simpleParser(raw);
const subject = parsed.subject ?? null;
const parsedFrom =
parsed.from?.value?.[0]?.address?.toLowerCase().trim() || fromEmail.toLowerCase();
const receivedAt = parsed.date ? parsed.date.toISOString() : new Date().toISOString();
// Prefer decoded plain text; fall back to stripping the HTML part.
const rawText =
parsed.text ??
(parsed.html ? parsed.html.replace(/<[^>]+>/g, " ") : "");
const bodyText = rawText
.replace(/\r\n/g, "\n")
.replace(/\n{3,}/g, "\n\n")
.replace(/[ \t]{2,}/g, " ")
.trim()
.slice(0, 16_000);
if (!bodyText) { resolve(null); return; }
// Dedup: skip if we already have a row for this target with the same received_at
const existing = db.prepare(
"SELECT id FROM email_replies WHERE target_id = ? AND received_at = ?"
).get(targetId, receivedAt) as { id: string } | undefined;
if (existing) { resolve(null); return; }
// Look up the most recent active run for this target — the dispatcher needs
// it to find the email track to reschedule / enroll a substitute into.
const runRow = db.prepare(
`SELECT r.id FROM runs r
JOIN run_profiles rp ON rp.run_id = r.id
WHERE rp.target_id = ? AND r.status IN ('running', 'paused')
ORDER BY r.created_at DESC LIMIT 1`
).get(targetId) as { id: string } | undefined;
const replyId = randomUUID();
db.prepare(
`INSERT INTO email_replies (id, target_id, run_id, from_email, subject, body_text, received_at)
VALUES (?, ?, ?, ?, ?, ?, ?)`
).run(replyId, targetId, runRow?.id ?? null, parsedFrom, subject, bodyText, receivedAt);
resolve(replyId);
} catch (err) {
console.warn(`[email-inbox] captureReplyBody parse/insert failed:`, err);
resolve(null);
}
})();
});
});
}
export function shouldSyncEmailInbox(emailAccountId: string): boolean {
const db = getDb();
const account = db
.prepare("SELECT inbox_synced_at FROM email_accounts WHERE id = ?")
.get(emailAccountId) as { inbox_synced_at: string | null } | undefined;
if (!account?.inbox_synced_at) return true;
const dueAfterMs = IMAP_POLL_INTERVAL_MS + accountJitterMs(emailAccountId);
return Date.now() - new Date(account.inbox_synced_at).getTime() >= dueAfterMs;
}
/**
* Opens one IMAP connection, then for each lead that has been emailed but
* not yet replied, runs a server-side FROM search. Only touches the mailbox
* index — never downloads message bodies for reply detection.
*
* Also scans the last 50 messages for bounces (mailer-daemon etc.).
*/
export async function syncEmailInbox(emailAccountId: string): Promise<{ replies: number; bounces: number }> {
const db = getDb();
const account = db
.prepare("SELECT id, imap_host, imap_port, username, password, imap_username, imap_password, inbox_synced_at FROM email_accounts WHERE id = ?")
.get(emailAccountId) as EmailAccount | undefined;
if (!account?.imap_host) {
console.warn(`[email-inbox] Account ${emailAccountId} has no IMAP config — skipping`);
return { replies: 0, bounces: 0 };
}
// Leads that were emailed via this account and haven't replied yet
const pendingTargets = db.prepare(`
SELECT DISTINCT t.id, t.email
FROM targets t
JOIN run_profiles rp ON rp.target_id = t.id
JOIN run_profile_tracks rt ON rt.run_profile_id = rp.id
WHERE t.email IS NOT NULL
AND t.email_replied_at IS NULL
AND t.email_status != 'invalid'
AND rt.track = 'email'
AND rt.state NOT IN ('pending')
AND rp.email_account_id = ?
`).all(emailAccountId) as { id: string; email: string }[];
if (pendingTargets.length === 0) {
db.prepare("UPDATE email_accounts SET inbox_synced_at = datetime('now') WHERE id = ?").run(emailAccountId);
return { replies: 0, bounces: 0 };
}
console.log(`[email-inbox] Checking ${pendingTargets.length} leads via IMAP FROM search`);
const imapUser = account.imap_username ?? account.username;
const imapPass = decryptSecret(account.imap_password) ?? decryptSecret(account.password)!;
let replies = 0;
let bounces = 0;
await new Promise<void>((resolve) => {
const imap = new Imap({
host: account.imap_host!,
port: account.imap_port ?? 993,
tls: true,
tlsOptions: { rejectUnauthorized: false },
user: imapUser,
password: imapPass,
authTimeout: 10_000,
connTimeout: 12_000,
});
const done = () => {
try { imap.end(); } catch { /* ignore */ }
resolve();
};
imap.once("error", (err: Error) => {
console.warn(`[email-inbox] IMAP error for account ${emailAccountId}:`, err.message);
done();
});
imap.once("ready", () => {
imap.openBox("INBOX", true, async (err, box) => {
if (err || !box) { console.warn("[email-inbox] openBox failed:", err?.message); done(); return; }
// ── Reply detection: one FROM search per lead ──────────────────────────
// Run sequentially — the imap library serialises commands over one TCP connection
for (const target of pendingTargets) {
await new Promise<void>((resSearch) => {
imap.search([["FROM", target.email]], async (searchErr, uids) => {
if (searchErr) { resSearch(); return; }
if (uids.length > 0) {
console.log(`[email-inbox] Reply detected for ${target.email} (target ${target.id})`);
replies++;
// Capture the body, then let the classifier+dispatcher decide the action.
// We no longer eagerly stamp email_replied_at here — the dispatcher does it
// for skip-buckets (call_task / human_reply / not_interested) but leaves it
// unset for OOO follow-ups so the runner keeps the contact enrolled.
// On any failure the contact stays enrolled (safe fallback, plan §3.2).
try {
const latestUid = uids[uids.length - 1];
const replyId = await captureReplyBody(imap, db, target.id, target.email, latestUid);
// The reply is always STORED (open-core). AI classification + auto-followup
// is a premium feature — skipped cleanly when ee/ is absent.
if (replyId && premium?.replies) {
await premium.replies.classifyAndDispatch(replyId);
}
} catch (err) {
console.warn(`[email-inbox] Failed to capture/dispatch reply for ${target.email}:`, err);
}
}
resSearch();
});
});
}
// ── Bounce detection: scan last 50 messages for mailer-daemon ─────────
if (box.messages.total > 0) {
const total = box.messages.total;
const start = Math.max(1, total - 49);
const range = `${start}:${total}`;
await new Promise<void>((resFetch) => {
const fetch = imap.seq.fetch(range, {
bodies: ["HEADER.FIELDS (FROM TO)", "TEXT"],
struct: false,
});
type RawMsg = { header: string; body: string };
const msgs: RawMsg[] = [];
fetch.on("message", (msg) => {
const entry: RawMsg = { header: "", body: "" };
msg.on("body", (stream, info) => {
const chunks: Buffer[] = [];
stream.on("data", (c: Buffer) => chunks.push(c));
stream.once("end", () => {
const text = Buffer.concat(chunks).toString();
if (info.which.startsWith("HEADER")) entry.header = text;
else entry.body = text.slice(0, 3000);
});
});
msg.once("end", () => msgs.push(entry));
});
fetch.once("error", () => resFetch());
fetch.once("end", () => {
for (const msg of msgs) {
const fromRaw = parseHeaderValue(msg.header, "From");
const toRaw = parseHeaderValue(msg.header, "To");
const emailMatch = fromRaw.match(/<([^>]+)>/) ?? fromRaw.match(/([^\s]+@[^\s]+)/);
const fromEmail = emailMatch?.[1]?.toLowerCase().trim();
if (!fromEmail || !isBounce(fromEmail)) continue;
// Also extract Final-Recipient from DSN bodies (SES bounce format)
const finalRecipient = msg.body.match(/Final-Recipient:\s*rfc822;\s*([^\s\r\n]+)/i)?.[1] ?? "";
const candidates = extractEmails(msg.body + " " + toRaw + " " + finalRecipient);
for (const candidate of candidates) {
if (BOUNCE_SENDER_PATTERNS.some(p => p.test(candidate))) continue;
const target = db
.prepare("SELECT id, email_status, company_id FROM targets WHERE lower(email) = ?")
.get(candidate) as { id: string; email_status: string | null; company_id: string | null } | undefined;
if (!target || target.email_status === "invalid") continue;
const note = `Email bounced on ${new Date().toISOString().slice(0, 10)} — marked invalid`;
db.prepare(`
UPDATE targets SET email_status = 'invalid',
notes = CASE WHEN notes IS NULL OR notes = '' THEN ? ELSE notes || char(10) || ? END
WHERE id = ?
`).run(note, note, target.id);
db.prepare(`
UPDATE run_profile_tracks SET state = 'skipped', error_message = 'Email bounced — invalid address'
WHERE run_profile_id IN (SELECT id FROM run_profiles WHERE target_id = ?)
AND state IN ('pending', 'in_progress')
`).run(target.id);
if (target.company_id) {
const companyNote = `Email domain flagged invalid — bounce for ${candidate} on ${new Date().toISOString().slice(0, 10)}`;
db.prepare(`
UPDATE companies SET email_domain_invalid = 1,
notes = CASE WHEN notes IS NULL OR notes = '' THEN ? ELSE notes || char(10) || ? END
WHERE id = ?
`).run(companyNote, companyNote, target.company_id);
const siblings = db.prepare(`
SELECT id FROM targets WHERE company_id = ? AND id != ? AND email IS NOT NULL AND email_status != 'invalid'
`).all(target.company_id, target.id) as { id: string }[];
for (const sibling of siblings) {
const sibNote = `Email bounced on ${new Date().toISOString().slice(0, 10)} — marked invalid (domain flagged via company)`;
db.prepare(`
UPDATE targets SET email_status = 'invalid',
notes = CASE WHEN notes IS NULL OR notes = '' THEN ? ELSE notes || char(10) || ? END
WHERE id = ?
`).run(sibNote, sibNote, sibling.id);
db.prepare(`
UPDATE run_profile_tracks SET state = 'skipped', error_message = 'Email domain invalid — company flagged'
WHERE run_profile_id IN (SELECT id FROM run_profiles WHERE target_id = ?)
AND state IN ('pending', 'in_progress')
`).run(sibling.id);
}
if (siblings.length > 0) {
console.log(`[email-inbox] Company ${target.company_id} flagged — ${siblings.length} sibling(s) marked invalid`);
}
}
console.log(`[email-inbox] Bounce for ${candidate} (target ${target.id}) — marked invalid`);
bounces++;
break;
}
}
resFetch();
});
});
}
done();
});
});
imap.connect();
});
db.prepare("UPDATE email_accounts SET inbox_synced_at = datetime('now') WHERE id = ?").run(emailAccountId);
return { replies, bounces };
}
+103
View File
@@ -0,0 +1,103 @@
import nodemailer from "nodemailer";
import Imap from "imap";
export interface EmailAccount {
id: string;
from_email: string;
from_name: string | null;
reply_to?: string | null;
smtp_host: string;
smtp_port: number;
smtp_secure: number; // 0 = STARTTLS, 1 = SSL
username: string;
password: string;
}
export async function sendEmail(
account: EmailAccount,
to: string,
subject: string,
body: string
): Promise<void> {
const transporter = nodemailer.createTransport({
host: account.smtp_host,
port: account.smtp_port,
secure: account.smtp_secure === 1,
auth: {
user: account.username,
pass: account.password,
},
// Allow self-signed certs (common in some corp SMTP setups)
tls: { rejectUnauthorized: false },
});
const from = account.from_name
? `"${account.from_name}" <${account.from_email}>`
: account.from_email;
await transporter.sendMail({
from, to, subject, text: body,
...(account.reply_to ? { replyTo: account.reply_to } : {}),
});
}
/**
* Verifies SMTP connectivity — used by the test-connection endpoint.
* Returns null on success, error message string on failure.
*/
export async function testSmtpConnection(account: Omit<EmailAccount, "id">): Promise<string | null> {
try {
const transporter = nodemailer.createTransport({
host: account.smtp_host,
port: account.smtp_port,
secure: account.smtp_secure === 1,
auth: { user: account.username, pass: account.password },
tls: { rejectUnauthorized: false },
connectionTimeout: 10_000,
greetingTimeout: 10_000,
});
await transporter.verify();
return null;
} catch (err) {
return err instanceof Error ? err.message : String(err);
}
}
export interface ImapTestAccount {
imap_host: string;
imap_port: number;
username: string;
password: string;
imap_username: string | null;
imap_password: string | null;
}
/**
* Verifies IMAP connectivity — connects, authenticates, then disconnects.
* Returns null on success, error message string on failure.
*/
export async function testImapConnection(account: ImapTestAccount): Promise<string | null> {
return new Promise((resolve) => {
const imap = new Imap({
host: account.imap_host,
port: account.imap_port,
tls: true,
tlsOptions: { rejectUnauthorized: false },
user: account.imap_username ?? account.username,
password: account.imap_password ?? account.password,
authTimeout: 10_000,
connTimeout: 12_000,
});
imap.once("ready", () => {
try { imap.end(); } catch { /* ignore */ }
resolve(null);
});
imap.once("error", (err: Error) => {
resolve(err.message ?? String(err));
});
imap.connect();
});
}
+289
View File
@@ -0,0 +1,289 @@
import type DatabaseType from "better-sqlite3";
import { getDb } from "@/lib/db";
import { randomUUID } from "crypto";
type DB = DatabaseType.Database;
const PAGE_SIZE = 25;
export const DEFAULT_DAILY_CAP = 1500;
export interface ImportRow {
id: string;
list_id: string;
account_id: string | null;
sales_nav_url: string | null;
status: string;
phase: string | null;
page: number;
total_pages: number;
count: number;
total: number;
imported: number;
skipped: number;
error: string | null;
scheduled_for: string | null;
start_page: number;
cap: number | null;
cancel_requested: number;
batch_index: number;
enrich: number;
started_at: string;
finished_at: string | null;
}
// ─── settings ────────────────────────────────────────────────────────────────
export function getDailyImportCap(db: DB = getDb()): number {
const row = db.prepare("SELECT value FROM app_settings WHERE key = 'daily_import_cap'").get() as
| { value: string }
| undefined;
const n = row ? parseInt(row.value, 10) : NaN;
return Number.isFinite(n) && n > 0 ? n : DEFAULT_DAILY_CAP;
}
export function setDailyImportCap(db: DB, n: number): void {
const v = String(Math.max(1, Math.floor(n)));
db.prepare(
`INSERT INTO app_settings (key, value, updated_at) VALUES ('daily_import_cap', ?, datetime('now'))
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = datetime('now')`
).run(v);
}
// ─── quota ───────────────────────────────────────────────────────────────────
/** Contacts imported across ALL lists today (the global daily budget). */
export function importedToday(db: DB): number {
const row = db
.prepare(
`SELECT COALESCE(SUM(imported), 0) c FROM list_imports
WHERE status IN ('done', 'running') AND date(COALESCE(finished_at, started_at)) = date('now')`
)
.get() as { c: number };
return row.c;
}
function todayStr(): string {
return new Date().toISOString().slice(0, 10);
}
function addDaysStr(base: string, days: number): string {
const d = new Date(base + "T00:00:00Z");
d.setUTCDate(d.getUTCDate() + days);
return d.toISOString().slice(0, 10);
}
// ─── public API ──────────────────────────────────────────────────────────────
/**
* Queue an import for a list. Creates the first batch as 'scheduled' for today;
* the runner's scheduler picks it up (one import at a time). Large lists are
* split across days under the daily cap by runBatch chaining continuations.
*/
export function startImport(
db: DB,
opts: { listId: string; accountId: string; salesNavUrl: string; enrich?: boolean }
): { importId: string } {
cancelImportsForList(db, opts.listId); // supersede any prior import for this list
const importId = randomUUID();
db.prepare(
`INSERT INTO list_imports
(id, list_id, account_id, sales_nav_url, status, scheduled_for, start_page, batch_index, enrich, started_at)
VALUES (?, ?, ?, ?, 'scheduled', ?, 1, 1, ?, datetime('now'))`
).run(importId, opts.listId, opts.accountId, opts.salesNavUrl, todayStr(), opts.enrich ? 1 : 0);
return { importId };
}
export function cancelImportsForList(db: DB, listId: string): void {
db.prepare(
`UPDATE list_imports
SET cancel_requested = 1,
status = CASE WHEN status = 'scheduled' THEN 'canceled' ELSE status END,
finished_at = CASE WHEN status = 'scheduled' THEN datetime('now') ELSE finished_at END
WHERE list_id = ? AND status IN ('scheduled', 'running')`
).run(listId);
}
export function cancelImport(db: DB, importId: string): void {
db.prepare(
`UPDATE list_imports
SET cancel_requested = 1,
status = CASE WHEN status = 'scheduled' THEN 'canceled' ELSE status END,
finished_at = CASE WHEN status = 'scheduled' THEN datetime('now') ELSE finished_at END
WHERE id = ?`
).run(importId);
}
// ─── scheduler + executor ────────────────────────────────────────────────────
let importRunning = false;
/** Runner hook (called each tick): start the next due batch if none is running. */
export async function processScheduledImports(db: DB): Promise<void> {
if (importRunning) return;
const due = db
.prepare(
`SELECT * FROM list_imports
WHERE status = 'scheduled' AND cancel_requested = 0
AND (scheduled_for IS NULL OR scheduled_for <= date('now'))
ORDER BY scheduled_for ASC, batch_index ASC LIMIT 1`
)
.get() as ImportRow | undefined;
if (!due) return;
importRunning = true;
db.prepare("UPDATE list_imports SET status = 'running', started_at = datetime('now') WHERE id = ?").run(due.id);
runBatch(due.id)
.catch((e) => console.error("[import] batch crashed:", e))
.finally(() => { importRunning = false; });
}
async function runBatch(importId: string): Promise<void> {
const db = getDb();
const job = db.prepare("SELECT * FROM list_imports WHERE id = ?").get(importId) as ImportRow | undefined;
if (!job || !job.account_id || !job.sales_nav_url) return;
// List deleted out from under us?
const list = db.prepare("SELECT id FROM lists WHERE id = ?").get(job.list_id);
if (!list) {
db.prepare("UPDATE list_imports SET status = 'canceled', finished_at = datetime('now') WHERE id = ?").run(importId);
return;
}
// Today's remaining budget → max whole pages this run
const cap = getDailyImportCap(db);
const remaining = cap - importedToday(db);
const maxPages = Math.floor(remaining / PAGE_SIZE);
if (maxPages < 1) {
db.prepare("UPDATE list_imports SET status = 'scheduled', scheduled_for = ? WHERE id = ?").run(
addDaysStr(todayStr(), 1),
importId
);
return;
}
console.log(`[import] batch ${importId} (b${job.batch_index}) start_page=${job.start_page} maxPages=${maxPages} cap=${cap}`);
const { getSessionContext } = await import("@/lib/linkedin/session");
const { scrapeNavigatorUrl } = await import("@/lib/linkedin/scraper");
const updateProgress = db.prepare(
"UPDATE list_imports SET phase = ?, page = ?, total_pages = ?, count = ?, total = ? WHERE id = ?"
);
const isCanceled = () => {
const r = db.prepare("SELECT cancel_requested FROM list_imports WHERE id = ?").get(importId) as
| { cancel_requested: number }
| undefined;
return !r || r.cancel_requested === 1; // row deleted (list cascade) or explicit cancel
};
try {
const ctx = await getSessionContext(job.account_id);
const { profiles, lastPage, knownTotal, exhausted } = await scrapeNavigatorUrl(ctx, job.sales_nav_url, {
startPage: job.start_page,
maxPages,
onProgress: (p) => updateProgress.run(p.phase, p.page ?? 0, p.totalPages ?? 0, p.count, p.total, importId),
isCanceled,
});
if (isCanceled()) {
if (db.prepare("SELECT id FROM list_imports WHERE id = ?").get(importId)) {
db.prepare("UPDATE list_imports SET status = 'canceled', finished_at = datetime('now') WHERE id = ?").run(importId);
}
return;
}
const { imported, skipped } = insertProfiles(db, job.list_id, profiles);
console.log(`[import] batch ${importId} inserted ${imported} new, skipped ${skipped} (lastPage=${lastPage}, exhausted=${exhausted})`);
db.prepare(
`UPDATE list_imports
SET status = 'done', imported = ?, skipped = ?, count = ?, total = ?, page = ?, total_pages = ?, finished_at = datetime('now')
WHERE id = ?`
).run(imported, skipped, profiles.length, knownTotal, lastPage, Math.ceil(knownTotal / PAGE_SIZE), importId);
// More of the list left → chain the remainder to the next day
if (!exhausted) {
db.prepare(
`INSERT INTO list_imports
(id, list_id, account_id, sales_nav_url, status, scheduled_for, start_page, batch_index, enrich, total, total_pages, started_at)
VALUES (?, ?, ?, ?, 'scheduled', ?, ?, ?, ?, ?, ?, datetime('now'))`
).run(
randomUUID(),
job.list_id,
job.account_id,
job.sales_nav_url,
addDaysStr(todayStr(), 1),
lastPage + 1,
job.batch_index + 1,
job.enrich,
knownTotal,
Math.ceil(knownTotal / PAGE_SIZE)
);
}
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
console.error("[import] FAILED:", message);
if (db.prepare("SELECT id FROM list_imports WHERE id = ?").get(importId)) {
db.prepare("UPDATE list_imports SET status = 'error', error = ?, finished_at = datetime('now') WHERE id = ?").run(
message,
importId
);
}
// A "no data intercepted / re-authentication" failure means the session died.
if (/re-authentication|No data intercepted/i.test(message) && job.account_id) {
try {
const { markNeedsReauth } = await import("@/lib/linkedin/session");
await markNeedsReauth(job.account_id);
} catch { /* ignore */ }
}
}
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
function insertProfiles(db: DB, listId: string, profiles: any[]): { imported: number; skipped: number } {
const insertTarget = db.prepare(
`INSERT INTO targets (
id, linkedin_url, sales_nav_url, first_name, last_name, full_name,
title, company, location, degree,
object_urn, summary, open_link, company_industry, company_location,
tenure_months, spotlight_badges
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(linkedin_url) DO UPDATE SET
sales_nav_url = excluded.sales_nav_url,
first_name = excluded.first_name,
last_name = excluded.last_name,
full_name = excluded.full_name,
title = excluded.title,
company = excluded.company,
location = excluded.location,
degree = excluded.degree,
object_urn = excluded.object_urn,
summary = excluded.summary,
open_link = excluded.open_link,
company_industry = excluded.company_industry,
company_location = excluded.company_location,
tenure_months = excluded.tenure_months,
spotlight_badges = excluded.spotlight_badges`
);
const insertLink = db.prepare("INSERT OR IGNORE INTO list_targets (list_id, target_id) VALUES (?, ?)");
const findTarget = db.prepare("SELECT id FROM targets WHERE linkedin_url = ?");
let imported = 0;
let skipped = 0;
db.transaction(() => {
for (const p of profiles) {
const url = p.linkedinUrl ?? p.salesNavUrl;
insertTarget.run(
randomUUID(), url, p.salesNavUrl,
p.firstName, p.lastName, p.fullName,
p.title, p.company, p.location, p.degree,
p.objectUrn, p.summary, p.openLink ? 1 : 0,
p.companyIndustry, p.companyLocation,
p.tenureMonths, p.spotlightBadges
);
const target = findTarget.get(url) as { id: string };
const result = insertLink.run(listId, target.id);
if (result.changes > 0) imported++;
else skipped++;
}
})();
return { imported, skipped };
}
+89
View File
@@ -0,0 +1,89 @@
import type { Page } from "playwright";
export class WeeklyLimitError extends Error {}
export class AlreadyConnectedError extends Error {}
export class PendingInviteError extends Error {}
/**
* Sends a LinkedIn connection request without a note.
* Navigates to the profile page and clicks the Connect button.
* Throws WeeklyLimitError if the weekly limit popup appears.
* Throws AlreadyConnectedError / PendingInviteError if already in that state.
*/
export async function sendConnectionRequest(page: Page, linkedinUrl: string): Promise<void> {
await page.goto(linkedinUrl, { waitUntil: "domcontentloaded", timeout: 30000 });
await page.waitForTimeout(2000 + Math.random() * 1000);
// Already connected? Primary signal: presence of the profile's "Message" link
// (only shown to 1st-degree connections) — an href attribute, not a CSS class
// or translated text, so it survives LinkedIn's class-name hashing and non-
// English UI languages. Falls back to the old text-scrape for accounts/layouts
// where that link isn't found as a plain <a href>.
//
// MUST be scoped to the visited person's own intro/top card — a page-wide
// search also matches "Message" links / "1st" badges belonging to OTHER
// people rendered elsewhere on the page (sidebar modules like "People also
// viewed"). For a non-connected target this false-positived AlreadyConnected,
// which skipped the real invite AND (via the message step that follows) sent
// a message to a random unrelated 1st-degree connection instead of the
// target — see CLAUDE.md / memory for the Jul 2026 incident. The top card is
// identified structurally as the section containing the page's own <h1> name
// heading, robust to class-name hashing. Mirrors the same fix in visit.ts.
const topCard = page.locator("main section").filter({ has: page.locator("h1") }).first();
const hasMessageLink = await topCard.locator('a[href*="/messaging/compose"]').first().count() > 0;
if (hasMessageLink) throw new AlreadyConnectedError("Already connected");
const pageText = await topCard.innerText().catch(() => "");
if (/\b1st\b/.test(pageText)) throw new AlreadyConnectedError("Already connected");
// Pending?
if (/\bPending\b/.test(pageText)) throw new PendingInviteError("Invitation already pending");
const pendingBtn = page.locator('button[aria-label*="Pending"]:visible');
if (await pendingBtn.count() > 0) throw new PendingInviteError("Invitation already pending");
// Case 1: Direct Connect link (primary CTA) — navigate to its href directly.
// Clicking fails because the Sales Nav overlay SVG intercepts pointer events.
const directConnect = page.locator('a[aria-label*="Invite"][aria-label*="to connect"]:visible, a[href*="custom-invite"]:visible').first();
if (await directConnect.count() > 0) {
const href = await directConnect.getAttribute("href");
if (!href) throw new Error("Connect link has no href");
const inviteUrl = href.startsWith("http") ? href : `https://www.linkedin.com${href}`;
await page.goto(inviteUrl, { waitUntil: "domcontentloaded", timeout: 30000 });
await page.waitForTimeout(1000);
} else {
// Case 2: Connect is inside the "..." More menu
// LinkedIn has two "More" buttons on page: [0] = nav bar, [1] = profile card
const moreBtn = page.locator('button[aria-label="More"]:visible').nth(1);
await moreBtn.click();
await page.waitForTimeout(800);
// Check for Pending in the menu — means invite was already sent
const pendingMenuItem = page.locator('[role="menuitem"]:has-text("Pending"):visible');
if (await pendingMenuItem.count() > 0) throw new PendingInviteError("Invitation already pending (found in More menu)");
const connectOption = page.locator('[role="menuitem"]:has-text("Connect"):visible');
if (await connectOption.count() === 0) throw new Error("Connect option not found in More menu");
await connectOption.first().click();
}
await page.waitForTimeout(1000);
// Click "Send without a note" / "Send now"
const sendBtn = page.locator(
'button:has-text("Send now"), button[aria-label*="Send without"], button[aria-label*="Send invitation"]:not([aria-label*="note"])'
);
if (await sendBtn.count() > 0) {
await sendBtn.first().click({ force: true });
await page.waitForTimeout(1500);
}
// Check for weekly limit popup
const limitPopup = page.locator('div[class*="ip-fuse-limit-alert__warning"]');
if (await limitPopup.count() > 0) throw new WeeklyLimitError("Weekly connection limit reached");
// Check for error toast
const errorToast = page.locator('div[data-test-artdeco-toast-item-type="error"]:visible');
if (await errorToast.count() > 0) {
const msg = await errorToast.innerText();
throw new Error(`Connection error: ${msg.trim()}`);
}
}
+161
View File
@@ -0,0 +1,161 @@
/**
* Profile enrichment via Sales Navigator profile page intercept.
*
* Strategy: navigate to the lead's sales_nav_url. Sales Nav automatically
* fires a salesApiProfiles call that returns the full profile — headline,
* all positions with descriptions, degree, flagshipProfileUrl, summary.
* We intercept that response instead of making extra API calls.
*
* Weight: ~5-8s per profile (one real browser page load). Run fire-and-forget
* after import — do not await in API routes.
*/
import type { BrowserContext } from "playwright";
import { getDb } from "@/lib/db";
interface EnrichedPosition {
title: string;
companyName: string;
current: boolean;
startedOn?: { year?: number; month?: number };
endedOn?: { year?: number; month?: number };
description?: string;
}
interface SalesNavProfileData {
entityUrn?: string;
firstName?: string;
lastName?: string;
fullName?: string;
headline?: string;
summary?: string;
location?: string;
degree?: number;
flagshipProfileUrl?: string;
objectUrn?: string;
positions?: EnrichedPosition[];
skills?: Array<{ name?: string }>;
}
interface InterceptedResponse {
data?: SalesNavProfileData;
[key: string]: unknown;
}
export async function enrichProfile(
ctx: BrowserContext,
target: { id: string; sales_nav_url: string; full_name: string }
): Promise<boolean> {
const db = getDb();
const page = await ctx.newPage();
try {
// Box it so the async response listener can mutate it (TS can't narrow across async callbacks)
const box: { data: SalesNavProfileData | null } = { data: null };
// Intercept the salesApiProfiles call that Sales Nav fires automatically on profile load.
// Two calls fire per page load:
// 1st — flat object with entityUrn, headline, positions, flagshipProfileUrl (what we want)
// 2nd — flat object with educations, skills, numOfConnections (supplementary)
// Both are flat — no .data wrapper. Accept the first one that has entityUrn.
page.on("response", async (resp) => {
if (box.data) return;
if (!resp.url().includes("salesApiProfiles")) return;
if (resp.status() !== 200) return;
try {
const json = await resp.json() as Record<string, unknown>;
if ("entityUrn" in json && "firstName" in json) {
box.data = json as unknown as SalesNavProfileData;
}
} catch { /* ignore */ }
});
await page.goto(target.sales_nav_url, { waitUntil: "domcontentloaded", timeout: 30000 });
await page.waitForTimeout(8000);
const intercepted = box.data;
if (!intercepted) {
console.log(`[enrich] No profile data intercepted for ${target.full_name} — skipping`);
return false;
}
// Extract positions — include all, not just current
const positions: EnrichedPosition[] = (intercepted.positions ?? []).map((p) => ({
title: p.title ?? "",
companyName: p.companyName ?? "",
current: p.current ?? false,
startedOn: p.startedOn,
endedOn: p.endedOn,
description: p.description ?? undefined,
}));
// Skills come from a separate decoration call — if not in intercept, skip for now
// (the page-load intercept does not include skills; they'd need a second fetch)
const skills: string[] = (intercepted.skills ?? [])
.map((s) => s.name)
.filter((n): n is string => !!n);
db.prepare(`
UPDATE targets SET
headline = COALESCE(?, headline),
summary = COALESCE(?, summary),
positions_json = ?,
skills_json = CASE WHEN ? IS NOT NULL THEN ? ELSE skills_json END,
enriched_profile_at = datetime('now')
WHERE id = ?
`).run(
intercepted.headline ?? null,
intercepted.summary ?? null,
positions.length > 0 ? JSON.stringify(positions) : null,
skills.length > 0 ? "1" : null,
skills.length > 0 ? JSON.stringify(skills) : null,
target.id
);
console.log(`[enrich] ${target.full_name} — ${positions.length} positions, ${skills.length} skills, headline: ${!!intercepted.headline}`);
return true;
} catch (err) {
console.error(`[enrich] Error enriching ${target.full_name}:`, err instanceof Error ? err.message : err);
return false;
} finally {
await page.close();
}
}
export async function enrichList(
ctx: BrowserContext,
listId: string,
delayMs = 2000,
onProgress?: (count: number, total: number) => void
): Promise<{ enriched: number; failed: number }> {
const db = getDb();
// Only targets with a sales_nav_url that haven't been enriched yet
const targets = db.prepare(`
SELECT t.id, t.sales_nav_url, t.full_name
FROM targets t
JOIN list_targets lt ON lt.target_id = t.id
WHERE lt.list_id = ?
AND t.sales_nav_url IS NOT NULL
AND t.enriched_profile_at IS NULL
ORDER BY t.created_at ASC
`).all(listId) as Array<{ id: string; sales_nav_url: string; full_name: string }>;
const total = targets.length;
console.log(`[enrich] Starting enrichment for ${total} profiles in list ${listId}`);
let enriched = 0;
let failed = 0;
for (let i = 0; i < targets.length; i++) {
const target = targets[i];
console.log(`[enrich] ${i + 1}/${total} — ${target.full_name}`);
const ok = await enrichProfile(ctx, target);
if (ok) enriched++; else failed++;
console.log(`[enrich] ${i + 1}/${total} done (${enriched} ok, ${failed} failed)`);
onProgress?.(i + 1, total);
if (delayMs > 0) await new Promise((r) => setTimeout(r, delayMs));
}
console.log(`[enrich] Done — ${enriched} enriched, ${failed} failed`);
return { enriched, failed };
}
+60
View File
@@ -0,0 +1,60 @@
import type { Page } from "playwright";
export interface LinkedInStats {
connections: number;
pending: number;
profile_views: number;
}
function parseNum(str: string): number {
return parseInt(str.replace(/[^0-9]/g, ""), 10) || 0;
}
export async function scrapeLinkedInStats(page: Page): Promise<LinkedInStats> {
// ── Connections ───────────────────────────────────────────
await page.goto("https://www.linkedin.com/mynetwork/invite-connect/connections/", {
waitUntil: "domcontentloaded",
timeout: 30000,
});
await page.waitForTimeout(2500);
const connectionsTexts = await page.evaluate(() =>
[...document.querySelectorAll("h1,h2,h3,span,p")]
.map(el => ((el as HTMLElement).innerText ?? "").trim())
.filter(t => /\d.*connection/i.test(t))
);
const connections = parseNum(connectionsTexts[0] ?? "0");
// ── Pending sent ──────────────────────────────────────────
await page.goto("https://www.linkedin.com/mynetwork/invitation-manager/sent/", {
waitUntil: "domcontentloaded",
timeout: 30000,
});
await page.waitForTimeout(2500);
const pendingTexts = await page.evaluate(() =>
[...document.querySelectorAll("button,a,span,h1,h2,h3")]
.map(el => ((el as HTMLElement).innerText ?? "").trim())
.filter(t => /People\s*\(\d+\)/i.test(t))
);
const pendingMatch = (pendingTexts[0] ?? "").match(/\((\d+)\)/);
const pending = pendingMatch ? parseInt(pendingMatch[1], 10) : 0;
// ── Profile views ─────────────────────────────────────────
await page.goto("https://www.linkedin.com/analytics/profile-views/", {
waitUntil: "domcontentloaded",
timeout: 30000,
});
await page.waitForTimeout(3000);
const profileViewsTexts = await page.evaluate(() =>
[...document.querySelectorAll("*")]
.map(el => ((el as HTMLElement).innerText ?? "").trim())
.filter(t => /Profile viewers/i.test(t) && t.length < 100)
);
// The block looks like "185\n\nProfile viewers\n\n30% previous week"
const pvBlock = profileViewsTexts[0] ?? "";
const profile_views = parseNum(pvBlock.split("\n")[0]);
return { connections, pending, profile_views };
}
+138
View File
@@ -0,0 +1,138 @@
import type { Page } from "playwright";
import { visitProfile } from "./visit";
export class NotConnectedError extends Error {}
export interface SendMessageResult {
messagingUrn: string | null;
isFirstDegree: boolean;
}
/**
* Sends a message to a LinkedIn 1st-degree connection.
*
* Self-contained URN resolution — does NOT depend on a prior 'visit' workflow
* step. If messagingUrn is already cached, it's used directly (no extra page
* load). Otherwise this does its own live profile check (the same top-card-
* scoped logic as the 'visit' step, see visit.ts) to fetch a fresh URN and
* verify the target is still actually connected, immediately before sending.
* Only if that live check finds no URN despite confirming 1st-degree does it
* fall back to the connections-only name-search typeahead — a rare last
* resort now, not the default path for every contact that lacks a cached URN.
*
* Throws NotConnectedError if the live check finds the target is NOT 1st-
* degree, instead of guessing via typeahead search (which can silently hit
* an unrelated connection with a similar/truncated name — see CLAUDE.md /
* memory for the Jul 2026 incident this replaced).
*
* Returns the resolved { messagingUrn, isFirstDegree } so the caller can
* persist it to the target record, same as the 'visit' step does.
*/
export async function sendMessage(
page: Page,
fullName: string,
text: string,
linkedinUrl: string,
messagingUrn?: string | null
): Promise<SendMessageResult> {
if (messagingUrn) {
const opened = await openComposeByUrn(page, messagingUrn);
if (opened) {
await sendFromComposeBox(page, text);
return { messagingUrn, isFirstDegree: true };
}
}
const resolved = await visitProfile(page, linkedinUrl);
if (resolved.messagingUrn) {
const opened = await openComposeByUrn(page, resolved.messagingUrn);
if (opened) {
await sendFromComposeBox(page, text);
return resolved;
}
}
if (!resolved.isFirstDegree) {
throw new NotConnectedError(`${fullName} is not a 1st-degree connection — refusing to message`);
}
// Connected, but no message link could be resolved live (unusual layout) —
// last-resort fallback to name search.
await sendMessageViaTypeahead(page, fullName, text);
return resolved;
}
async function openComposeByUrn(page: Page, messagingUrn: string): Promise<boolean> {
try {
const recipientId = messagingUrn.split(":").pop();
const composeUrl = `https://www.linkedin.com/messaging/compose/?profileUrn=${encodeURIComponent(messagingUrn)}&recipient=${recipientId}`;
await page.goto(composeUrl, { waitUntil: "domcontentloaded", timeout: 30000 });
await page.waitForTimeout(1500 + Math.random() * 1000);
const msgInput = page.locator("div.msg-form__contenteditable").first();
await msgInput.waitFor({ timeout: 8000 });
return true;
} catch {
return false;
}
}
async function sendMessageViaTypeahead(page: Page, fullName: string, text: string): Promise<void> {
await page.goto("https://www.linkedin.com/messaging/thread/new/", {
waitUntil: "domcontentloaded",
timeout: 30000,
});
await page.waitForTimeout(1500 + Math.random() * 1000);
// Search for recipient by name
const searchField = page.locator("input.msg-connections-typeahead__search-field").first();
await searchField.waitFor({ timeout: 10000 });
await searchField.click();
await searchField.type(fullName, { delay: 60 + Math.random() * 40 });
await page.waitForTimeout(1500);
// Select first result — but verify it's actually the intended person first.
// This search only returns 1st-degree connections; if the real target isn't
// connected, LinkedIn will still happily return a same/similar-named
// connection as the top result, and we'd silently message a stranger.
const firstResult = page.locator('div[class*="msg-connections-typeahead__search-result-row"]').first();
await firstResult.waitFor({ timeout: 8000 });
const resultText = (await firstResult.innerText().catch(() => "")).trim();
if (!resultNameMatches(resultText, fullName)) {
throw new Error(
`Typeahead search for "${fullName}" returned a non-matching result ("${resultText.replace(/\s+/g, " ")}") — refusing to send to avoid messaging the wrong person`
);
}
await firstResult.click({ delay: 100 });
await page.waitForTimeout(800);
await sendFromComposeBox(page, text);
}
function resultNameMatches(resultText: string, fullName: string): boolean {
const normalize = (s: string) =>
s.toLowerCase().normalize("NFKD").replace(/[̀-ͯ]/g, "").replace(/[^a-z\s]/g, " ").replace(/\s+/g, " ").trim();
const target = normalize(fullName);
if (!target) return false;
return normalize(resultText).includes(target);
}
async function sendFromComposeBox(page: Page, text: string): Promise<void> {
// Paste message into compose area
const msgInput = page.locator("div.msg-form__contenteditable").first();
await msgInput.waitFor({ timeout: 8000 });
await msgInput.click();
try {
await page.evaluate((t) => navigator.clipboard.writeText(t), text);
await page.waitForTimeout(300);
await msgInput.press("Control+V");
} catch {
// Clipboard blocked in headless — fall back to keyboard typing
await msgInput.pressSequentially(text, { delay: 20 });
}
await page.waitForTimeout(500);
// Send
const sendBtn = page.locator("button.msg-form__send-button:visible").first();
await sendBtn.waitFor({ timeout: 5000 });
await sendBtn.click({ delay: 100 });
await page.waitForTimeout(2000);
}
+61
View File
@@ -0,0 +1,61 @@
import type { Page } from "playwright";
/**
* Scrapes the full list of vanity names from the sent invitations page.
* Returns a Set of lowercase vanity names that are STILL pending on LinkedIn.
*
* Strategy: navigate to invitation-manager/sent/, scroll the #workspace container
* until the count stabilises, then extract all /in/ links.
*
* Works headless — the scrollable element is <main id="workspace">, not window.
*/
export async function scrapePendingInvitationVanityNames(page: Page): Promise<Set<string>> {
await page.goto("https://www.linkedin.com/mynetwork/invitation-manager/sent/", {
waitUntil: "domcontentloaded",
timeout: 30000,
});
await page.waitForTimeout(3000);
// Scroll #workspace until link count stabilises
let lastCount = 0;
let stableRounds = 0;
for (let round = 0; round < 40; round++) {
await page.evaluate(() => {
const workspace = document.getElementById("workspace");
if (workspace) {
workspace.scrollTop = workspace.scrollHeight;
} else {
// Fallback: scroll window + body
window.scrollTo(0, document.body.scrollHeight);
document.documentElement.scrollTop = document.documentElement.scrollHeight;
}
});
await page.waitForTimeout(1200);
const currentCount = await page.evaluate(() =>
document.querySelectorAll("a[href*='/in/']").length
);
if (currentCount === lastCount) {
stableRounds++;
if (stableRounds >= 3) break;
} else {
stableRounds = 0;
lastCount = currentCount;
}
}
// Extract unique vanity names from all /in/ links
const vanityNames = await page.evaluate(() => {
const seen = new Set<string>();
for (const link of document.querySelectorAll<HTMLAnchorElement>("a[href*='/in/']")) {
const href = link.getAttribute("href") ?? "";
const match = href.match(/\/in\/([^/?#]+)/);
if (match) seen.add(match[1].toLowerCase());
}
return [...seen];
});
return new Set(vanityNames);
}
+298
View File
@@ -0,0 +1,298 @@
/**
* Live profile scrape → clean structured JSON for an LLM.
*
* Two confirmed sources (probed against prod, Jun 2026 — see docs §18):
* 1. Sales Nav `salesApiProfiles` intercept (navigate the lead's sales_nav_url):
* two flat responses give headline, summary, positions, educations, skills,
* languages, connections, contactInfo, inmailRestriction, memberBadges, degree.
* 2. Voyager `feed/updates?q=memberShareFeed` on the /in/ publicId:
* the 10 most recent authored posts (text + media type + engagement counts)
* plus the comments left on those posts. (q=memberComments → 400, not used.)
*
* Read-only. ~10-12s per profile (one Sales Nav page load + one Voyager fetch).
*/
import type { BrowserContext } from "playwright";
export interface ProfilePost {
activityUrn: string;
text: string;
postedAt: string | null; // ISO, when derivable from the activity id
mediaType: "video" | "image" | "article" | "text";
reactions: number;
comments: number;
shares: number;
}
export interface ProfileScrape {
full_name: string | null;
first_name: string | null;
last_name: string | null;
headline: string | null;
summary: string | null;
location: string | null;
degree: number | null; // network distance (1/2/3)
linkedin_url: string | null; // real /in/ URL
num_connections: number | null;
num_shared_connections: number | null;
is_premium: boolean;
is_open_profile: boolean; // openLink — can be messaged free (note: API often under-reports)
can_inmail: boolean; // inmailRestriction === NO_RESTRICTION
pending_invitation: boolean;
websites: string[];
positions: Array<{
title: string;
company: string;
current: boolean;
started: { year?: number; month?: number } | null;
ended: { year?: number; month?: number } | null;
description: string | null;
}>;
educations: Array<{ school: string | null; degree: string | null; field: string | null }>;
skills: string[];
languages: string[];
recent_posts: ProfilePost[]; // up to 10, most recent first
}
// ─── helpers ──────────────────────────────────────────────────────────────────
function textOf(v: unknown): string | null {
if (typeof v === "string") return v;
if (v && typeof v === "object" && "text" in v && typeof (v as { text: unknown }).text === "string") {
return (v as { text: string }).text;
}
return null;
}
// LinkedIn activity/ugcPost ids encode the creation epoch-ms in the high 41 bits
// (the low 22 bits are a sequence). Shift right 22 = divide by 2^22, via float
// math to avoid BigInt (tsconfig targets < ES2020).
const SHIFT_22 = 4194304; // 2^22
function timeFromActivityUrn(urn: string): string | null {
const m = urn.match(/(\d{18,})/);
if (!m) return null;
const ms = Math.floor(Number(m[1]) / SHIFT_22);
return ms > 1_000_000_000_000 ? new Date(ms).toISOString() : null;
}
interface SalesPosition {
title?: string; companyName?: string; current?: boolean;
startedOn?: { year?: number; month?: number };
endedOn?: { year?: number; month?: number };
description?: string;
}
interface SalesProfileFlat {
entityUrn?: string; firstName?: string; lastName?: string; fullName?: string;
headline?: string; summary?: string; location?: string; degree?: number;
flagshipProfileUrl?: string; pendingInvitation?: boolean;
positions?: SalesPosition[];
contactInfo?: { websites?: Array<{ url?: string }> };
educations?: Array<{ schoolName?: string; degreeName?: string; fieldOfStudy?: string }>;
skills?: Array<{ name?: string }>;
languages?: Array<{ name?: string }>;
numOfConnections?: number; numOfSharedConnections?: number;
inmailRestriction?: string;
memberBadges?: { premium?: boolean; openLink?: boolean };
}
// ─── Sales Nav career ─────────────────────────────────────────────────────────
async function scrapeSalesCareer(
ctx: BrowserContext,
salesNavUrl: string
): Promise<Partial<ProfileScrape>> {
const page = await ctx.newPage();
const responses: SalesProfileFlat[] = [];
page.on("response", async (resp) => {
if (!resp.url().includes("salesApiProfiles") || resp.status() !== 200) return;
try {
const j = (await resp.json()) as Record<string, unknown>;
// Only the two flat profile responses (have entityUrn); ignore the data/included one.
if ("entityUrn" in j && ("firstName" in j || "educations" in j || "skills" in j)) {
responses.push(j as unknown as SalesProfileFlat);
}
} catch { /* ignore */ }
});
try {
await page.goto(salesNavUrl, { waitUntil: "domcontentloaded", timeout: 35000 });
await page.waitForTimeout(9000);
} finally {
await page.close();
}
// Merge the (typically two) flat responses — first carries the core profile,
// second carries educations/skills/languages/connections/inmailRestriction.
const core = responses.find((r) => r.firstName || r.positions) ?? {};
const extra = responses.find((r) => r.educations || r.skills || r.numOfConnections != null) ?? {};
const positions = (core.positions ?? []).map((p) => ({
title: p.title ?? "",
company: p.companyName ?? "",
current: p.current ?? false,
started: p.startedOn ?? null,
ended: p.endedOn ?? null,
description: p.description?.trim() ? p.description : null,
}));
return {
full_name: core.fullName ?? null,
first_name: core.firstName ?? null,
last_name: core.lastName ?? null,
headline: core.headline ?? null,
summary: core.summary ?? null,
location: core.location ?? null,
degree: core.degree ?? null,
linkedin_url: core.flagshipProfileUrl ?? null,
pending_invitation: core.pendingInvitation ?? false,
websites: (core.contactInfo?.websites ?? []).map((w) => w.url).filter((u): u is string => !!u),
positions,
num_connections: extra.numOfConnections ?? null,
num_shared_connections: extra.numOfSharedConnections ?? null,
is_premium: extra.memberBadges?.premium ?? false,
is_open_profile: extra.memberBadges?.openLink ?? false,
can_inmail: (extra.inmailRestriction ?? core.inmailRestriction) === "NO_RESTRICTION",
educations: (extra.educations ?? []).map((e) => ({
school: e.schoolName ?? null, degree: e.degreeName ?? null, field: e.fieldOfStudy ?? null,
})),
skills: (extra.skills ?? []).map((s) => s.name).filter((n): n is string => !!n),
languages: (extra.languages ?? []).map((l) => l.name).filter((n): n is string => !!n),
};
}
// ─── Voyager posts ────────────────────────────────────────────────────────────
function publicIdFromUrl(url: string | null | undefined): string | null {
const m = url && url.match(/\/in\/([^/?#]+)/);
return m ? decodeURIComponent(m[1]) : null;
}
async function scrapePosts(
ctx: BrowserContext,
publicId: string
): Promise<ProfilePost[]> {
const page = await ctx.newPage();
try {
await page.goto("https://www.linkedin.com/feed/", { waitUntil: "domcontentloaded", timeout: 25000 });
const cookies = await ctx.cookies();
const csrf = (cookies.find((c) => c.name === "JSESSIONID")?.value || "").replace(/"/g, "");
if (!csrf) return [];
const url =
`https://www.linkedin.com/voyager/api/feed/updates` +
`?profileId=${encodeURIComponent(publicId)}&q=memberShareFeed&count=10&start=0`;
const raw = await page.evaluate(
async ({ url, csrf }: { url: string; csrf: string }) => {
const r = await fetch(url, {
headers: {
accept: "application/vnd.linkedin.normalized+json+2.1",
"csrf-token": csrf,
"x-restli-protocol-version": "2.0.0",
},
credentials: "include",
});
return r.status === 200 ? r.text() : "";
},
{ url, csrf }
);
if (!raw) return [];
const j = JSON.parse(raw) as { included?: Array<Record<string, unknown>> };
const inc = j.included ?? [];
// Engagement counts keyed by the post's ugcPost/share urn.
const countsByUrn = new Map<string, { reactions: number; comments: number; shares: number }>();
for (const x of inc) {
if (!String(x["$type"]).endsWith("SocialActivityCounts")) continue;
const urn = String(x.urn ?? "");
if (!urn.includes("ugcPost") && !urn.includes("activity")) continue;
countsByUrn.set(urn, {
reactions: Number(x.numLikes ?? 0),
comments: Number(x.numComments ?? 0),
shares: Number(x.numShares ?? 0),
});
}
const posts: ProfilePost[] = [];
for (const x of inc) {
if (!String(x["$type"]).endsWith("UpdateV2")) continue;
const commentary = x.commentary as { text?: unknown } | undefined;
const text = commentary ? textOf(commentary.text) : null;
if (!text) continue;
const meta = x.updateMetadata as { urn?: string } | undefined;
const activityUrn = meta?.urn ?? "";
// Find the matching counts (the SocialActivityCounts urn references the backing ugcPost).
let counts = { reactions: 0, comments: 0, shares: 0 };
for (const [urn, c] of countsByUrn) {
const idActivity = activityUrn.match(/(\d{18,})/)?.[1];
if (idActivity && urn.includes(idActivity)) { counts = c; break; }
}
const content = x.content as Record<string, unknown> | undefined;
const ctype = content ? String(Object.keys(content)[0] ?? "").toLowerCase() : "";
const mediaType: ProfilePost["mediaType"] =
ctype.includes("video") ? "video" :
ctype.includes("image") ? "image" :
ctype.includes("article") ? "article" : "text";
posts.push({
activityUrn,
text,
postedAt: timeFromActivityUrn(activityUrn),
mediaType,
reactions: counts.reactions,
comments: counts.comments,
shares: counts.shares,
});
}
// Most recent first (by encoded time when available).
posts.sort((a, b) => (b.postedAt ?? "").localeCompare(a.postedAt ?? ""));
return posts.slice(0, 10);
} catch {
return [];
} finally {
await page.close();
}
}
// ─── public ───────────────────────────────────────────────────────────────────
export async function scrapeProfile(
ctx: BrowserContext,
target: { sales_nav_url: string | null; linkedin_url: string | null }
): Promise<ProfileScrape> {
if (!target.sales_nav_url) {
throw new Error("Contact has no sales_nav_url — cannot scrape the Sales Nav profile.");
}
const career = await scrapeSalesCareer(ctx, target.sales_nav_url);
// Prefer the flagship /in/ URL from the career scrape; fall back to the stored one.
const inUrl = career.linkedin_url ?? target.linkedin_url ?? null;
const publicId = publicIdFromUrl(inUrl);
const recent_posts = publicId ? await scrapePosts(ctx, publicId) : [];
return {
full_name: career.full_name ?? null,
first_name: career.first_name ?? null,
last_name: career.last_name ?? null,
headline: career.headline ?? null,
summary: career.summary ?? null,
location: career.location ?? null,
degree: career.degree ?? null,
linkedin_url: inUrl,
num_connections: career.num_connections ?? null,
num_shared_connections: career.num_shared_connections ?? null,
is_premium: career.is_premium ?? false,
is_open_profile: career.is_open_profile ?? false,
can_inmail: career.can_inmail ?? false,
pending_invitation: career.pending_invitation ?? false,
websites: career.websites ?? [],
positions: career.positions ?? [],
educations: career.educations ?? [],
skills: career.skills ?? [],
languages: career.languages ?? [],
recent_posts,
};
}
+50
View File
@@ -0,0 +1,50 @@
import type DatabaseType from "better-sqlite3";
type DB = DatabaseType.Database;
export interface ResolvedAccount { id: string; email: string }
/**
* Resolve which authenticated LinkedIn account to act through for a contact.
* Order: explicit id → the contact's most recent run assignment → the sole
* authenticated account. Only ever returns an `is_authenticated = 1` account
* so callers never drive a dead session.
*/
export function resolveLinkedInAccount(db: DB, targetId: string, explicitId?: string): ResolvedAccount | null {
const byId = (aid: string) =>
db.prepare("SELECT id, email FROM accounts WHERE id = ? AND is_authenticated = 1").get(aid) as
| ResolvedAccount
| undefined;
if (explicitId) return byId(explicitId) ?? null;
const assigned = db.prepare(`
SELECT r.account_id FROM run_profiles rp
JOIN runs r ON r.id = rp.run_id
WHERE rp.target_id = ?
ORDER BY rp.created_at DESC LIMIT 1
`).get(targetId) as { account_id: string } | undefined;
if (assigned?.account_id) {
const a = byId(assigned.account_id);
if (a) return a;
}
const all = db.prepare("SELECT id, email FROM accounts WHERE is_authenticated = 1").all() as ResolvedAccount[];
return all.length === 1 ? all[0] : null;
}
/**
* Resolve an authenticated LinkedIn account when there's no target/contact to
* anchor the lookup to (e.g. Sales Navigator search, which runs before any
* lead exists locally). Order: explicit id → the sole authenticated account.
*/
export function resolveAnyAuthenticatedAccount(db: DB, explicitId?: string): ResolvedAccount | null {
const byId = (aid: string) =>
db.prepare("SELECT id, email FROM accounts WHERE id = ? AND is_authenticated = 1").get(aid) as
| ResolvedAccount
| undefined;
if (explicitId) return byId(explicitId) ?? null;
const all = db.prepare("SELECT id, email FROM accounts WHERE is_authenticated = 1").all() as ResolvedAccount[];
return all.length === 1 ? all[0] : null;
}
File diff suppressed because it is too large Load Diff
+453
View File
@@ -0,0 +1,453 @@
/**
* Sales Navigator list scraper using the internal Sales API.
* Uses Playwright's browser context request (inherits browser TLS fingerprint).
*
* Response format discovery (from capture scripts):
* - ctx.request returns normalized JSON: { data: { metadata: { totalDisplayCount: "106" } }, included: [...] }
* - Profiles are in `included` filtered by entityUrn containing "salesProfile"
* - Browser-intercepted first page returns flat: { elements: [...], paging: { total } }
* - Query format: parentheses/commas unencoded, colons in URN encoded as %3A
*
* IMPORTANT — vanityName is gone from salesApiPeopleSearch (dropped by LinkedIn ~March 2026):
* The list API no longer returns vanityName. To get the real /in/ URL we call
* salesApiProfiles per batch of 25 after scraping, using flagshipProfileUrl.
* See docs/linkedin-api-learnings.md for the full investigation.
*/
import type { BrowserContext, Page } from "playwright";
export interface ScrapedProfile {
salesNavUrn: string;
salesNavUrl: string;
linkedinUrl: string | null; // regular /in/ URL
fullName: string | null;
firstName: string | null;
lastName: string | null;
title: string | null;
company: string | null;
location: string | null;
degree: number | null;
// Extended fields
objectUrn: string | null; // urn:li:member:XXXX — stable LinkedIn member ID
summary: string | null; // About / bio text
openLink: boolean; // can message without connecting
companyIndustry: string | null;
companyLocation: string | null; // company HQ location
tenureMonths: number | null; // months in current role
spotlightBadges: string | null; // JSON array of badge displayValues
}
interface SalesProfile {
entityUrn: string;
objectUrn?: string;
firstName?: string;
lastName?: string;
fullName?: string;
vanityName?: string;
geoRegion?: string;
degree?: number;
summary?: string;
openLink?: boolean;
premium?: boolean;
pendingInvitation?: boolean;
currentPositions?: Array<{
title?: string;
companyName?: string;
companyUrn?: string;
current?: boolean;
description?: string;
startedOn?: { year?: number; month?: number };
tenureAtPosition?: { numYears?: number; numMonths?: number };
tenureAtCompany?: { numYears?: number; numMonths?: number };
companyUrnResolutionResult?: {
name?: string;
industry?: string;
location?: string;
entityUrn?: string;
};
}>;
leadAssociatedAccount?: { name?: string } | null;
spotlightBadges?: Array<{ displayValue?: string; id?: string }>;
}
// Flat response intercepted from browser
interface FlatResponse {
elements?: SalesProfile[];
paging?: { total: number; count: number; start: number };
}
// salesApiProfiles single-profile response
interface ProfileDetailResponse {
entityUrn?: string;
flagshipProfileUrl?: string;
fullName?: string;
}
function extractListId(url: string): string | null {
const match = url.match(/\/sales\/lists\/people\/(\d+)/);
return match ? match[1] : null;
}
function extractSavedSearchId(url: string): string | null {
const match = url.match(/[?&]savedSearchId=(\d+)/);
return match ? match[1] : null;
}
function urnToSalesNavUrl(urn: string): string {
const match = urn.match(/\(([^)]+)\)/);
if (!match) return "";
return `https://www.linkedin.com/sales/lead/${match[1]}`;
}
/**
* Parse profileId, authType, authToken out of a salesProfile URN.
* urn:li:fs_salesProfile:(ACwAAEIY-4YB25mKP6R5AKfkFjhO9isSbvsVlag,NAME_SEARCH,22wq)
*/
function parseUrn(entityUrn: string): { profileId: string; authType: string; authToken: string } | null {
const match = entityUrn.match(/\(([^,]+),([^,]+),([^)]+)\)/);
if (!match) return null;
return { profileId: match[1], authType: match[2], authToken: match[3] };
}
function profileToResult(el: SalesProfile, linkedinUrl: string | null): ScrapedProfile {
const currentPos = el.currentPositions?.find((p) => p.current) ?? el.currentPositions?.[0];
const company = currentPos?.companyUrnResolutionResult ?? null;
// Tenure in months at current position
let tenureMonths: number | null = null;
const t = currentPos?.tenureAtPosition;
if (t) tenureMonths = (t.numYears ?? 0) * 12 + (t.numMonths ?? 0);
// Spotlight badge labels as a compact JSON array e.g. ["Changed jobs", "Mentioned in news"]
const badges = (el.spotlightBadges ?? [])
.map((b) => b.displayValue)
.filter(Boolean) as string[];
return {
salesNavUrn: el.entityUrn,
salesNavUrl: urnToSalesNavUrl(el.entityUrn),
linkedinUrl,
fullName: el.fullName ?? null,
firstName: el.firstName ?? null,
lastName: el.lastName ?? null,
title: currentPos?.title ?? null,
company: el.leadAssociatedAccount?.name ?? company?.name ?? currentPos?.companyName ?? null,
location: el.geoRegion ?? null,
degree: el.degree ?? null,
objectUrn: el.objectUrn ?? null,
summary: el.summary ?? null,
openLink: el.openLink ?? false,
companyIndustry: company?.industry ?? null,
companyLocation: company?.location ?? null,
tenureMonths,
spotlightBadges: badges.length > 0 ? JSON.stringify(badges) : null,
};
}
/**
* Enrich profiles with their real /in/ URL via salesApiProfiles (flagshipProfileUrl).
*
* Uses page.evaluate fetch with JSESSIONID csrf-token. On 429 (rate limit),
* backs off for 30s and retries. Delay between calls: 2s normally, 30s after 429.
*
* For 100 profiles at 2s/call = ~3.5 minutes. Happens once at import time.
*/
async function enrichWithFlagshipUrls(
page: Page,
profiles: SalesProfile[],
onProgress?: (p: ImportProgress) => void
): Promise<Map<string, string>> {
const urlMap = new Map<string, string>(); // entityUrn → flagshipProfileUrl
const cookies = await page.context().cookies("https://www.linkedin.com");
const jsessionid = cookies.find(c => c.name === "JSESSIONID")?.value?.replace(/"/g, "") ?? "";
if (!jsessionid) {
console.log("[scraper] JSESSIONID not found — skipping flagshipProfileUrl enrichment");
return urlMap;
}
let done = 0;
for (const el of profiles) {
const parts = parseUrn(el.entityUrn);
if (!parts) continue;
const { profileId, authType, authToken } = parts;
const apiUrl = `https://www.linkedin.com/sales-api/salesApiProfiles/(profileId:${profileId},authType:${authType},authToken:${authToken})?decoration=%28entityUrn%2CflagshipProfileUrl%29`;
let attempts = 0;
while (attempts < 3) {
const evaluatePromise = page.evaluate(async ({ url, csrfToken }: { url: string; csrfToken: string }) => {
try {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), 12000);
const resp = await fetch(url, {
credentials: "include",
signal: controller.signal,
headers: {
"csrf-token": csrfToken,
"x-restli-protocol-version": "2.0.0",
"accept": "application/json",
},
});
clearTimeout(timer);
return { status: resp.status, body: await resp.text() };
} catch (e: unknown) {
return { status: -1, body: (e as Error).message };
}
}, { url: apiUrl, csrfToken: jsessionid });
const timeoutPromise = new Promise<{ status: number; body: string }>((resolve) =>
setTimeout(() => resolve({ status: -1, body: 'timeout' }), 15000)
);
const result = await Promise.race([evaluatePromise, timeoutPromise]);
if (result.status === 200) {
try {
const data = JSON.parse(result.body) as ProfileDetailResponse;
if (data.flagshipProfileUrl) {
const normalized = data.flagshipProfileUrl.endsWith("/")
? data.flagshipProfileUrl
: data.flagshipProfileUrl + "/";
urlMap.set(el.entityUrn, normalized);
}
} catch { /* ignore parse errors */ }
break;
} else if (result.status === 429) {
attempts++;
console.log(`[scraper] 429 rate limit — waiting 30s before retry (attempt ${attempts}/3)`);
await page.waitForTimeout(30000);
} else {
const reason = result.body === 'timeout' ? 'timeout' : result.status;
console.log(`[scraper] enrichment ${reason} for ${el.fullName ?? el.entityUrn.substring(0, 30)} — skipping`);
break;
}
}
// 2s between calls — slow enough to stay under rate limits
await page.waitForTimeout(2000);
done++;
console.log(`[scraper] flagship URLs: ${done}/${profiles.length} resolved`);
onProgress?.({ phase: 'enriching', count: done, total: profiles.length });
}
console.log(`[scraper] enrichment done: ${urlMap.size}/${profiles.length} profiles resolved`);
return urlMap;
}
export interface ImportProgress {
phase: 'scraping' | 'enriching' | 'visiting';
page?: number;
totalPages?: number;
count: number;
total: number;
}
export interface ScrapeOptions {
/** 1-based page to begin this window at (for batched imports). Default 1. */
startPage?: number;
/** Max pages to fetch in THIS window, starting at startPage. Default 50. */
maxPages?: number;
onProgress?: (p: ImportProgress) => void;
/** Polled between pages; return true to stop early (cancel / deleted list). */
isCanceled?: () => boolean | Promise<boolean>;
}
export interface WindowedScrapeResult {
profiles: ScrapedProfile[];
/** Last page actually fetched in this window. */
lastPage: number;
/** Total profiles available in the whole search. */
knownTotal: number;
/** True if this window reached the end of the search (nothing left to batch). */
exhausted: boolean;
}
export async function scrapeNavigatorList(
ctx: BrowserContext,
salesNavUrl: string,
opts: ScrapeOptions = {}
): Promise<WindowedScrapeResult> {
const { startPage = 1, maxPages = 50, onProgress, isCanceled } = opts;
const listId = extractListId(salesNavUrl);
if (!listId) throw new Error(`Invalid Sales Navigator URL: ${salesNavUrl}`);
const allElements: SalesProfile[] = [];
const seen = new Set<string>();
const PAGE_SIZE = 25;
const buildUrl = (n: number) =>
`https://www.linkedin.com/sales/lists/people/${listId}?${n > 1 ? `page=${n}&` : ""}sortCriteria=CREATED_TIME&sortOrder=DESCENDING`;
const page = await ctx.newPage();
let knownTotal = 0;
let intercepted: FlatResponse | null = null;
const waitForIntercept = async (url: string, waitMs: number): Promise<FlatResponse | null> => {
intercepted = null;
page.removeAllListeners("response");
page.on("response", async (response) => {
if (intercepted) return;
if (response.url().includes("salesApiPeopleSearch") && response.status() === 200) {
try { intercepted = await response.json() as FlatResponse; } catch { /* ignore */ }
}
});
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
await page.waitForTimeout(waitMs);
return intercepted;
};
// First page of the window
const firstData = await waitForIntercept(buildUrl(startPage), 15000);
if (!firstData) {
const finalUrl = page.url();
console.error(`[scraper] no intercept after 15s. Final URL: ${finalUrl}`);
await page.close();
throw new Error("No data intercepted from Sales Nav — session may need re-authentication");
}
knownTotal = firstData.paging?.total ?? 0;
for (const el of firstData.elements ?? []) {
if (!el.entityUrn || seen.has(el.entityUrn)) continue;
seen.add(el.entityUrn);
allElements.push(el);
}
console.log(`[scraper] page ${startPage}: ${allElements.length} elements, total=${knownTotal}`);
const totalPages = Math.ceil(knownTotal / PAGE_SIZE);
const endPage = Math.min(totalPages, startPage + maxPages - 1);
let lastPage = startPage;
onProgress?.({ phase: 'scraping', page: startPage, totalPages: endPage, count: allElements.length, total: knownTotal });
for (let pageNum = startPage + 1; pageNum <= endPage; pageNum++) {
if (isCanceled && (await isCanceled())) break;
const delayMs = 60000 + Math.random() * 60000;
console.log(`[scraper] waiting ${Math.round(delayMs / 1000)}s before page ${pageNum}...`);
await page.waitForTimeout(delayMs);
if (isCanceled && (await isCanceled())) break;
let pageData = await waitForIntercept(buildUrl(pageNum), 15000);
if (!pageData || (pageData.elements?.length ?? 0) === 0) {
console.log(`[scraper] page ${pageNum} empty on first try — retrying with 15s wait`);
pageData = await waitForIntercept(buildUrl(pageNum), 15000);
}
if (pageData) {
for (const el of pageData.elements ?? []) {
if (!el.entityUrn || seen.has(el.entityUrn)) continue;
seen.add(el.entityUrn);
allElements.push(el);
}
}
lastPage = pageNum;
console.log(`[scraper] page ${pageNum}/${endPage}: ${allElements.length} (total ${knownTotal})`);
onProgress?.({ phase: 'scraping', page: pageNum, totalPages: endPage, count: allElements.length, total: knownTotal });
}
await page.close();
return {
profiles: allElements.map(el => profileToResult(el, null)),
lastPage,
knownTotal,
exhausted: lastPage >= totalPages,
};
}
export async function scrapeSavedSearch(
ctx: BrowserContext,
savedSearchUrl: string,
opts: ScrapeOptions = {}
): Promise<WindowedScrapeResult> {
const { startPage = 1, maxPages = 50, onProgress, isCanceled } = opts;
const savedSearchId = extractSavedSearchId(savedSearchUrl);
if (!savedSearchId) throw new Error(`Invalid Sales Navigator saved search URL: ${savedSearchUrl}`);
const allElements: SalesProfile[] = [];
const seen = new Set<string>();
const PAGE_SIZE = 25;
const buildUrl = (n: number) =>
`https://www.linkedin.com/sales/search/people?savedSearchId=${savedSearchId}${n > 1 ? `&page=${n}` : ""}`;
const page = await ctx.newPage();
let knownTotal = 0;
const waitForIntercept = async (url: string, waitMs: number): Promise<FlatResponse | null> => {
let intercepted: FlatResponse | null = null;
page.removeAllListeners("response");
page.on("response", async (response) => {
if (intercepted) return;
if (response.url().includes("salesApiLeadSearch") && response.status() === 200) {
try { intercepted = await response.json() as FlatResponse; } catch { /* ignore */ }
}
});
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
await page.waitForTimeout(waitMs);
return intercepted;
};
// First page of the window
const firstData = await waitForIntercept(buildUrl(startPage), 15000);
if (!firstData) {
await page.close();
throw new Error("No data intercepted from saved search — session may need re-authentication");
}
knownTotal = firstData.paging?.total ?? 0;
for (const el of firstData.elements ?? []) {
if (!el.entityUrn || seen.has(el.entityUrn)) continue;
seen.add(el.entityUrn);
allElements.push(el);
}
console.log(`[scraper:saved-search] page ${startPage}: ${allElements.length} elements, total=${knownTotal}`);
const totalPages = Math.ceil(knownTotal / PAGE_SIZE);
const endPage = Math.min(totalPages, startPage + maxPages - 1);
let lastPage = startPage;
onProgress?.({ phase: 'scraping', page: startPage, totalPages: endPage, count: allElements.length, total: knownTotal });
for (let pageNum = startPage + 1; pageNum <= endPage; pageNum++) {
if (isCanceled && (await isCanceled())) break;
const delayMs = 60000 + Math.random() * 60000;
console.log(`[scraper:saved-search] waiting ${Math.round(delayMs / 1000)}s before page ${pageNum}...`);
await page.waitForTimeout(delayMs);
if (isCanceled && (await isCanceled())) break;
let pageData = await waitForIntercept(buildUrl(pageNum), 15000);
if (!pageData || (pageData.elements?.length ?? 0) === 0) {
console.log(`[scraper:saved-search] page ${pageNum} empty on first try — retrying with 15s wait`);
pageData = await waitForIntercept(buildUrl(pageNum), 15000);
}
if (pageData) {
for (const el of pageData.elements ?? []) {
if (!el.entityUrn || seen.has(el.entityUrn)) continue;
seen.add(el.entityUrn);
allElements.push(el);
}
}
lastPage = pageNum;
console.log(`[scraper:saved-search] page ${pageNum}/${endPage}: ${allElements.length} (total ${knownTotal})`);
onProgress?.({ phase: 'scraping', page: pageNum, totalPages: endPage, count: allElements.length, total: knownTotal });
}
await page.close();
return {
profiles: allElements.map(el => profileToResult(el, null)),
lastPage,
knownTotal,
exhausted: lastPage >= totalPages,
};
}
/**
* Dispatcher — accepts either a lead list URL or a saved search URL.
* Callers don't need to know which type they're dealing with.
*/
export async function scrapeNavigatorUrl(
ctx: BrowserContext,
url: string,
opts: ScrapeOptions = {}
): Promise<WindowedScrapeResult> {
if (extractSavedSearchId(url)) {
return scrapeSavedSearch(ctx, url, opts);
}
if (extractListId(url)) {
return scrapeNavigatorList(ctx, url, opts);
}
throw new Error(`Unrecognized Sales Navigator URL. Expected a list URL (/sales/lists/people/...) or saved search URL (?savedSearchId=...)`);
}
+519
View File
@@ -0,0 +1,519 @@
import { chromium } from "playwright-extra";
import type { Browser, BrowserContext, Page } from "playwright";
import StealthPlugin from "puppeteer-extra-plugin-stealth";
import { getDb } from "@/lib/db";
import { encryptSecret, decryptSecret } from "@/lib/crypto";
chromium.use(StealthPlugin());
let browser: Browser | null = null;
const contexts: Map<string, BrowserContext> = new Map();
const HEADLESS = process.env.HEADLESS !== "false";
const CHROMIUM_PATH = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH;
const LAUNCH_ARGS = [
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-dev-shm-usage",
"--disable-gpu",
];
/**
* Shared browser-context fingerprint. Login and runtime MUST use the identical
* options so the LinkedIn session is BORN under the exact fingerprint it will
* later be used with — a mismatch (or a drift) triggers a forced re-auth.
*/
function contextOptions(storageState?: object) {
return {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
storageState: storageState as any,
viewport: { width: 1920, height: 1080 },
userAgent:
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
locale: "en-US",
timezoneId: "America/New_York",
permissions: ["clipboard-read", "clipboard-write"] as ("clipboard-read" | "clipboard-write")[],
};
}
async function getBrowser(headless = HEADLESS): Promise<Browser> {
// B1: if the cached browser is disconnected, CLOSE it before relaunching.
// Without this, a dead-but-not-reaped chromium process tree is orphaned on
// every relaunch (the leak behind the Jun 2026 zombie pile-up).
if (browser && !browser.isConnected()) {
try { await browser.close(); } catch { /* already gone */ }
browser = null;
}
if (!browser) {
browser = await chromium.launch({
headless,
executablePath: CHROMIUM_PATH,
args: LAUNCH_ARGS,
});
}
return browser;
}
async function getOrCreateContext(accountId: string): Promise<BrowserContext> {
const db = getDb();
const account = db.prepare("SELECT * FROM accounts WHERE id = ?").get(accountId) as
| { cookies_json: string | null; email: string }
| undefined;
if (!account) throw new Error(`Account ${accountId} not found`);
if (!contexts.has(accountId)) {
const b = await getBrowser();
let storageState: object | undefined;
if (account.cookies_json) {
try {
storageState = JSON.parse(decryptSecret(account.cookies_json)!);
} catch {
// Invalid storage state — will need re-auth
}
}
const ctx = await b.newContext(contextOptions(storageState));
// Auto-evict from map when context closes for any reason (crash, session expiry, etc.)
ctx.on("close", () => { if (contexts.get(accountId) === ctx) contexts.delete(accountId); });
contexts.set(accountId, ctx);
}
return contexts.get(accountId)!;
}
/** Returns the BrowserContext for an account (for API calls via ctx.request) */
export async function getSessionContext(accountId: string): Promise<BrowserContext> {
try {
return await getOrCreateContext(accountId);
} catch {
// First attempt failed — evict and retry once with a fresh context
contexts.delete(accountId);
return getOrCreateContext(accountId);
}
}
// B6 (Jul 2026 CPU-spike incident): closing a Playwright page doesn't mean the
// underlying Chromium renderer OS process has actually exited — under CPU
// contention on the 2-vCPU prod box, teardown was observed lagging 60-90s
// behind page.close(). If the runner loop (or a concurrent MCP/API call)
// opens its next page immediately after, two renderer processes end up alive
// at once, which is enough to peg both cores and starve sibling containers'
// healthchecks (NocoDB/Chatwoot flapping). Fix: serialize ALL page opens
// app-wide through one queue, and hold the queue for a teardown buffer after
// each page.close() before letting the next one through. PAGE_MAX_HOLD_MS is
// a safety valve so a caller that forgets to close its page can't wedge the
// whole app's browser access forever.
const PAGE_TEARDOWN_GAP_MS = 3000;
const PAGE_MAX_HOLD_MS = 120_000;
let pageQueueTail: Promise<void> = Promise.resolve();
/** Returns a new Page from the account's browser context */
export async function getSessionPage(accountId: string): Promise<Page> {
let releaseTurn!: () => void;
const myTurn = new Promise<void>(r => { releaseTurn = r; });
const previousTail = pageQueueTail;
pageQueueTail = myTurn;
await previousTail;
let released = false;
const release = () => {
if (released) return;
released = true;
releaseTurn();
};
const safetyTimer = setTimeout(release, PAGE_MAX_HOLD_MS);
let page: Page;
try {
const ctx = await getOrCreateContext(accountId);
try {
page = await ctx.newPage();
} catch {
// B2: context was dead — CLOSE it before recreating so its underlying
// browser process isn't left orphaned, then retry once with a fresh one.
try { await ctx.close(); } catch { /* already gone */ }
contexts.delete(accountId);
const freshCtx = await getOrCreateContext(accountId);
page = await freshCtx.newPage();
}
} catch (err) {
clearTimeout(safetyTimer);
release();
throw err;
}
const originalClose = page.close.bind(page);
page.close = (async (options?: Parameters<Page["close"]>[0]) => {
try {
return await originalClose(options);
} finally {
clearTimeout(safetyTimer);
setTimeout(release, PAGE_TEARDOWN_GAP_MS);
}
}) as Page["close"];
return page;
}
export async function saveSessionState(accountId: string): Promise<void> {
const ctx = contexts.get(accountId);
if (!ctx) return;
const db = getDb();
const state = await ctx.storageState();
db.prepare("UPDATE accounts SET cookies_json = ?, is_authenticated = 1 WHERE id = ?").run(
encryptSecret(JSON.stringify(state)),
accountId
);
}
export async function closeSession(accountId: string): Promise<void> {
const ctx = contexts.get(accountId);
if (ctx) {
await ctx.close();
contexts.delete(accountId);
}
}
/**
* B4: flag an account as logged out / needing re-auth. Clears is_authenticated
* so the runner stops working a dead session (no more 30s-timeout fail-loop),
* and drops the live context. The user re-authenticates from Settings.
*/
export async function markNeedsReauth(accountId: string): Promise<void> {
const db = getDb();
db.prepare("UPDATE accounts SET is_authenticated = 0 WHERE id = ?").run(accountId);
try { await closeSession(accountId); } catch { /* ignore */ }
console.warn(`[session] account ${accountId} flagged needs-reauth (session logged out)`);
}
/**
* Opens a visible browser, navigates to LinkedIn login, and waits for the user
* to complete login manually. Returns when the user reaches /feed.
* Saves the full storage state to DB and marks account as authenticated.
*/
export async function authenticateAccount(accountId: string): Promise<void> {
const db = getDb();
const account = db.prepare("SELECT * FROM accounts WHERE id = ?").get(accountId) as
| { email: string }
| undefined;
if (!account) throw new Error(`Account ${accountId} not found`);
// Close any existing context for this account — start fresh
await closeSession(accountId);
// Always launch a VISIBLE browser for manual login
const visibleBrowser = await chromium.launch({
headless: false,
executablePath: CHROMIUM_PATH,
args: [
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-dev-shm-usage",
"--disable-gpu",
],
});
try {
const ctx = await visibleBrowser.newContext({
viewport: { width: 1440, height: 900 },
userAgent:
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
locale: "en-US",
timezoneId: "America/New_York",
});
const page = await ctx.newPage();
await page.goto("https://www.linkedin.com/login");
// Pre-fill email to save the user a step
try {
await page.waitForSelector("input#username", { timeout: 5000 });
await page.fill("input#username", account.email);
} catch {
// Input not found — page may have redirected already
}
// Wait up to 3 minutes for the user to complete login and reach /feed
await page.waitForURL("**/feed/**", { timeout: 180_000 });
// Save full storage state (cookies + localStorage) to DB
const state = await ctx.storageState();
db.prepare("UPDATE accounts SET cookies_json = ?, is_authenticated = 1 WHERE id = ?").run(
encryptSecret(JSON.stringify(state)),
accountId
);
await ctx.close();
} finally {
await visibleBrowser.close();
}
}
// ─── Server-side headless login ───────────────────────────────────────────────
// Logs in directly on the server (no screen) so the session is born under the
// SAME pinned Chromium fingerprint the runner uses, and so ALL cookies — incl.
// httpOnly ones like li_ep_auth_context (the Sales Nav seat cookie that
// document.cookie cannot read) — are captured. A login from a datacenter IP
// almost always triggers an email/SMS PIN, so it's a two-step flow: start
// (email+password) → maybe a challenge → verify (code).
export type LoginResult =
| { status: "authenticated" }
| { status: "challenge"; kind: "otp" | "app" | "captcha" | "unknown"; message: string }
| { status: "error"; message: string };
type PendingLogin = { ctx: BrowserContext; page: Page; createdAt: number };
const pendingLogins: Map<string, PendingLogin> = new Map();
const PENDING_TTL_MS = 10 * 60_000;
// PIN/verification-code input. Named ids first, then type-based fallbacks for
// LinkedIn's React checkpoint pages (which use dynamic ids). Safe: the login
// page itself has no tel/one-time-code input to misfire on.
const PIN_SELECTOR =
"input[name='pin'], #input__email_verification_pin, input[autocomplete='one-time-code']:visible, input[type='tel']:visible";
async function clearPendingLogin(accountId: string): Promise<void> {
const p = pendingLogins.get(accountId);
if (p) {
pendingLogins.delete(accountId);
try { await p.ctx.close(); } catch { /* already gone */ }
}
}
function sweepPendingLogins(): void {
const now = Date.now();
for (const [id, p] of pendingLogins) {
if (now - p.createdAt > PENDING_TTL_MS) void clearPendingLogin(id);
}
}
/**
* Warm the freshly-authenticated session by loading Sales Navigator once, THEN
* persist. The bare login POST lands on /feed and only mints the ~11 core
* LinkedIn cookies — it does NOT yet include the Sales Nav SEAT cookie
* (li_ep_auth_context) nor the secondary auth tokens (li_a, liap) and
* localStorage. Those are only issued once the browser actually enters Sales
* Navigator. Without the seat cookie every Sales Nav API call returns nothing
* ("no intercept after 15s" → import fails → account wrongly flagged
* needs-reauth). So we navigate to /sales/ and wait for it to settle before
* calling storageState(), capturing the FULL session the runner needs.
* Best-effort: if the account has no Sales Nav seat the nav simply doesn't add
* the seat cookie — the rest of the (regular-LinkedIn) session is still saved.
*/
async function persistLogin(accountId: string, ctx: BrowserContext, page?: Page): Promise<void> {
if (page) {
try {
await page.goto("https://www.linkedin.com/sales/home", { waitUntil: "domcontentloaded", timeout: 30_000 });
// Let Sales Nav's bootstrap requests fire so li_ep_auth_context is set.
await page.waitForTimeout(4_000);
} catch {
// Non-fatal — a missing seat / slow load must not fail the whole login.
}
}
const db = getDb();
const state = await ctx.storageState();
db.prepare("UPDATE accounts SET cookies_json = ?, is_authenticated = 1 WHERE id = ?").run(
encryptSecret(JSON.stringify(state)),
accountId
);
// Drop any stale runtime context so the runner reloads the fresh cookies.
await closeSession(accountId);
}
/**
* Inspect the page after a login/verify submit and classify the outcome.
* LinkedIn varies its challenge per attempt — email/SMS code OR device (app)
* approval — so we detect both: a visible code input = otp; a checkpoint page
* with no code input and no captcha = device approval.
*/
async function classifyLoginState(page: Page): Promise<LoginResult> {
const start = Date.now();
const deadline = start + 20_000;
while (Date.now() < deadline) {
const url = page.url();
if (/\/feed\//.test(url) || /linkedin\.com\/sales\//.test(url)) {
return { status: "authenticated" };
}
// Email/SMS PIN entry
const pin = page.locator(PIN_SELECTOR).first();
if ((await pin.count()) > 0 && (await pin.isVisible().catch(() => false))) {
return {
status: "challenge",
kind: "otp",
message: "LinkedIn sent you a verification code (email or SMS). Enter it below.",
};
}
if (/checkpoint\/challenge/.test(url)) {
// CAPTCHA (Arkose / FunCaptcha) — cannot be solved headlessly
const cap = page.locator("iframe[src*='arkoselabs'], iframe[title*='captcha'], #captcha-internal");
if ((await cap.count().catch(() => 0)) > 0) {
return {
status: "challenge",
kind: "captcha",
message: "LinkedIn requires a CAPTCHA, which can't be solved on the server. Use cookie paste instead.",
};
}
// Device/app approval: a settled checkpoint with no code input and no captcha
if (Date.now() - start > 4_000) {
return {
status: "challenge",
kind: "app",
message: "LinkedIn sent a sign-in request to your LinkedIn mobile app. Approve it there, then click Continue.",
};
}
}
// Wrong credentials
const wrongPw = await page
.getByText(/that.?s not the right password|please enter a valid|couldn.?t find a linkedin account/i)
.count()
.catch(() => 0);
if (wrongPw > 0) return { status: "error", message: "Wrong email or password." };
await page.waitForTimeout(800);
}
if (/checkpoint/.test(page.url())) {
return {
status: "challenge",
kind: "unknown",
message: "LinkedIn presented a security checkpoint. If you got a code enter it; if it's an app request, approve it and click Continue.",
};
}
return { status: "error", message: `Login did not complete. Current page: ${page.url()}` };
}
export async function startHeadlessLogin(
accountId: string,
email: string,
password: string
): Promise<LoginResult> {
sweepPendingLogins();
await clearPendingLogin(accountId);
const b = await getBrowser(true);
const ctx = await b.newContext(contextOptions());
const page = await ctx.newPage();
try {
await page.goto("https://www.linkedin.com/login", { waitUntil: "domcontentloaded", timeout: 30_000 });
// LinkedIn's React login page uses dynamic ids — target by type+visibility
// (old #username/#password kept as a fallback for the legacy layout).
const emailInput = page.locator("input#username, input[type='email']:visible").first();
await emailInput.waitFor({ state: "visible", timeout: 20_000 });
await emailInput.fill(email);
const passwordInput = page.locator("input#password, input[type='password']:visible").first();
await passwordInput.fill(password);
await Promise.all([
page.waitForLoadState("domcontentloaded").catch(() => {}),
passwordInput.press("Enter"),
]);
const result = await classifyLoginState(page);
console.log(`[login] start account=${accountId} -> ${result.status}${"kind" in result ? "/" + result.kind : ""} url=${page.url()}`);
if (result.status === "authenticated") {
await persistLogin(accountId, ctx, page);
await ctx.close();
return result;
}
if (result.status === "challenge" && result.kind !== "captcha") {
pendingLogins.set(accountId, { ctx, page, createdAt: Date.now() });
return result;
}
await ctx.close();
return result;
} catch (e) {
console.log(`[login] start account=${accountId} ERROR ${(e as Error).message} url=${page.url()}`);
try { await ctx.close(); } catch { /* ignore */ }
return { status: "error", message: (e as Error).message };
}
}
export async function submitLoginChallenge(accountId: string, code: string): Promise<LoginResult> {
const p = pendingLogins.get(accountId);
if (!p) return { status: "error", message: "No login in progress (it may have timed out — start again)." };
const { ctx, page } = p;
try {
const pin = page.locator(PIN_SELECTOR).first();
await pin.waitFor({ state: "visible", timeout: 15_000 });
await pin.fill(code);
await Promise.all([
page.waitForLoadState("domcontentloaded").catch(() => {}),
pin.press("Enter"),
]);
const result = await classifyLoginState(page);
console.log(`[login] verify account=${accountId} -> ${result.status}${"kind" in result ? "/" + result.kind : ""} url=${page.url()}`);
if (result.status === "authenticated") {
await persistLogin(accountId, ctx, page);
await clearPendingLogin(accountId);
return result;
}
if (result.status === "challenge" && result.kind !== "captcha") {
p.createdAt = Date.now(); // keep the session alive for another step
return result;
}
await clearPendingLogin(accountId);
return result.status === "error"
? result
: { status: "error", message: "Code rejected or login failed." };
} catch (e) {
await clearPendingLogin(accountId);
return { status: "error", message: (e as Error).message };
}
}
/**
* Wait for a device/app-approval challenge to clear. Called after the user
* approves the sign-in in their LinkedIn mobile app — the checkpoint page then
* auto-advances to the feed. Also dismisses a possible "remember this browser?"
* interstitial. If still pending, returns the challenge so the user can retry.
*/
export async function awaitLoginApproval(accountId: string): Promise<LoginResult> {
const p = pendingLogins.get(accountId);
if (!p) return { status: "error", message: "No login in progress (it may have timed out — start again)." };
const { ctx, page } = p;
p.createdAt = Date.now();
try {
const reachedFeed = await page
.waitForURL(/\/feed\/|linkedin\.com\/sales\//, { timeout: 50_000 })
.then(() => true)
.catch(() => false);
if (!reachedFeed) {
// Possible post-approval interstitial (e.g. "remember this browser?")
const btn = page
.locator("button[type=submit]:visible, button:has-text('Yes'):visible, button:has-text('Ja'):visible")
.first();
if ((await btn.count().catch(() => 0)) > 0) {
await btn.click().catch(() => {});
await page.waitForURL(/\/feed\/|linkedin\.com\/sales\//, { timeout: 20_000 }).catch(() => {});
}
}
const result = await classifyLoginState(page);
console.log(`[login] await account=${accountId} -> ${result.status}${"kind" in result ? "/" + result.kind : ""} url=${page.url()}`);
if (result.status === "authenticated") {
await persistLogin(accountId, ctx, page);
await clearPendingLogin(accountId);
return result;
}
if (result.status === "challenge" && result.kind !== "captcha") {
p.createdAt = Date.now();
return result;
}
await clearPendingLogin(accountId);
return result;
} catch (e) {
await clearPendingLogin(accountId);
return { status: "error", message: (e as Error).message };
}
}
+256
View File
@@ -0,0 +1,256 @@
import type { Page } from "playwright";
import { getDb } from "@/lib/db";
import { getSessionPage, saveSessionState, markNeedsReauth } from "@/lib/linkedin/session";
/**
* Accepted-connection sync via the authoritative Voyager connections API.
*
* Replaces the old absence-inference (scroll the SENT list, treat a vanished
* invite as "accepted"). That was measurably wrong: of 1,600 contacts it had
* marked degree=1, only 325 were genuinely connected — 1,275 were phantoms
* (invites that expired/withdrew/were ignored, not accepted). See docs §19.
*
* Sources — all proved against prod (Jul 2026), NO scrolling:
* - DATA: GET /voyager/api/relationships/dash/connections
* ?decorationId=…ConnectionListWithProfile-16&q=search
* &sortType=RECENTLY_ADDED&start=N&count=100
* → data["*elements"] (newest-first) + included[]: Connection { createdAt(ms),
* connectedMember } and Profile { publicIdentifier (vanity), entityUrn }.
* The API paginates to the end (30 pages for ~2947) in ~60s. It does NOT
* expose a grand total.
* - TOTAL (checksum): the connections PAGE header "<N> connections" <p>. Read
* once per full pass via a single navigation (NO scroll). A full pass is
* "verified complete" only when unique-pulled == declared total (matched ±0
* in prod). Presence in the list = 100% proof of a 1st-degree connection.
*
* Behaviour:
* - First run (boundary NULL) → FULL pass: page to the end, stamp degree=1 +
* connected_at from createdAt for every matched contact. If the pass is
* checksum-verified complete, ALSO un-mark phantom degree=1 contacts whose
* vanity is absent from the authoritative list (resets degree/connected_at).
* - Later runs → incremental: page from start=0 and STOP at the first
* connection older than the stored boundary (minus a 24h overlap margin).
* Incremental/incomplete passes are ADD-ONLY — they never un-mark (a partial
* pull must not wipe real accepts). Un-marking happens ONLY on a verified
* full pass.
* - Reuses the runner's shared browser (getSessionPage) — never a 2nd browser.
*/
const ACCEPTED_SYNC_INTERVAL_MS = 8 * 60 * 60 * 1000; // 8h — 3x per day
const PAGE_SIZE = 100;
const MAX_PAGES = 60; // safety cap (60 * 100 = 6000)
const OVERLAP_MARGIN_MS = 24 * 60 * 60 * 1000; // re-check a day of overlap (idempotent)
const DECORATION = "com.linkedin.voyager.dash.deco.web.mynetwork.ConnectionListWithProfile-16";
export function shouldSyncAccepted(accountId: string): boolean {
const db = getDb();
const row = db.prepare("SELECT accepted_sync_at FROM accounts WHERE id = ?").get(accountId) as
| { accepted_sync_at: string | null }
| undefined;
if (!row?.accepted_sync_at) return true;
return Date.now() - new Date(row.accepted_sync_at).getTime() >= ACCEPTED_SYNC_INTERVAL_MS;
}
interface ApiConnection {
vanity: string | null;
createdAt: number; // epoch ms
}
export async function syncAcceptedConnections(accountId: string): Promise<number> {
const db = getDb();
const page = await getSessionPage(accountId);
let stamped = 0;
try {
const boundaryRow = db.prepare("SELECT connections_synced_through_ms FROM accounts WHERE id = ?").get(accountId) as
| { connections_synced_through_ms: number | null }
| undefined;
const boundary = boundaryRow?.connections_synced_through_ms ?? null;
const isFullPass = boundary === null;
const stopBefore = boundary === null ? null : boundary - OVERLAP_MARGIN_MS;
// Read the declared total from the connections PAGE header (single nav, NO
// scroll). This is also our login-wall check and the completeness checksum.
await page.goto("https://www.linkedin.com/mynetwork/invite-connect/connections/", {
waitUntil: "domcontentloaded",
timeout: 35000,
});
await page.waitForTimeout(3500 + Math.random() * 1500);
if (/\/login|\/authwall|\/checkpoint|\/uas\//.test(page.url())) {
console.warn(`[sync-accepted] Session looks logged out (${page.url()}) — skipping`);
return 0;
}
const declaredTotal = await page.evaluate(() => {
const m = document.body.innerText.match(/([\d.,]+)\s+connections?/i);
return m ? parseInt(m[1].replace(/[.,]/g, ""), 10) : null;
});
const findByVanity = db.prepare(
`SELECT id, full_name, connected_at, degree FROM targets
WHERE linkedin_url LIKE ? AND connection_requested_at IS NOT NULL`
);
const stampAccepted = db.prepare(
"UPDATE targets SET degree = 1, connected_at = COALESCE(connected_at, ?) WHERE id = ?"
);
const seenVanities = new Set<string>(); // full-pass phantom check
let uniquePulled = 0;
let newestSeen: number | null = null;
let reachedBoundary = false;
for (let pageIdx = 0; pageIdx < MAX_PAGES; pageIdx++) {
const conns = await fetchConnectionsPage(page, pageIdx * PAGE_SIZE, PAGE_SIZE);
if (conns === null) {
console.warn(`[sync-accepted] connections API failed at start=${pageIdx * PAGE_SIZE} — stopping`);
break;
}
if (conns.length === 0) break; // end of list
for (const c of conns) {
uniquePulled++;
if (c.vanity) seenVanities.add(c.vanity);
if (newestSeen === null || c.createdAt > newestSeen) newestSeen = c.createdAt;
// Incremental early-exit (list is newest-first).
if (stopBefore !== null && c.createdAt < stopBefore) {
reachedBoundary = true;
break;
}
if (!c.vanity) continue;
for (const m of findByVanity.all(`%/in/${c.vanity}/%`) as Array<{
id: string; full_name: string | null; connected_at: string | null; degree: number | null;
}>) {
if (m.degree === 1 && m.connected_at) continue; // already correct
stampAccepted.run(msToSqlite(c.createdAt), m.id);
console.log(`[sync-accepted] Accepted: ${m.full_name ?? c.vanity}`);
stamped++;
}
}
if (reachedBoundary) break;
await page.waitForTimeout(900 + Math.random() * 700); // gentle, API-only
}
// Completeness checksum: a full pass is trustworthy only if what we pulled
// matches LinkedIn's own declared total (±5 slack for live churn).
const verifiedComplete =
isFullPass && declaredTotal !== null && Math.abs(uniquePulled - declaredTotal) <= 5;
// Correction: ONLY on a verified-complete full pass, un-mark phantom
// degree=1 contacts (present nowhere in the authoritative list). Never on an
// incremental/incomplete pass — a partial pull must not wipe real accepts.
let unmarked = 0;
if (verifiedComplete) {
const deg1 = db.prepare(
"SELECT id, full_name, linkedin_url FROM targets WHERE degree = 1 AND linkedin_url LIKE '%/in/%'"
).all() as Array<{ id: string; full_name: string | null; linkedin_url: string }>;
const unmark = db.prepare("UPDATE targets SET degree = NULL, connected_at = NULL WHERE id = ?");
const tx = db.transaction((rows: typeof deg1) => {
for (const t of rows) {
const mm = t.linkedin_url.match(/\/in\/([^/?#]+)/);
const v = mm ? decodeURIComponent(mm[1]).toLowerCase() : null;
if (!v || !seenVanities.has(v)) {
unmark.run(t.id);
unmarked++;
}
}
});
tx(deg1);
console.log(`[sync-accepted] Verified full pass (pulled ${uniquePulled} == declared ${declaredTotal}). Un-marked ${unmarked} phantom degree=1.`);
} else if (isFullPass) {
console.warn(`[sync-accepted] Full pass NOT verified complete (pulled ${uniquePulled}, declared ${declaredTotal}) — add-only, no un-marking.`);
}
// Advance the boundary to the newest connection seen this run.
if (newestSeen !== null) {
db.prepare("UPDATE accounts SET connections_synced_through_ms = ? WHERE id = ?").run(newestSeen, accountId);
}
// Store the declared total for visibility (Settings shows LinkedIn's count).
if (declaredTotal !== null) {
db.prepare("UPDATE accounts SET li_connections = ? WHERE id = ?").run(declaredTotal, accountId);
}
console.log(`[sync-accepted] Stamped ${stamped} accepted, un-marked ${unmarked} phantom (boundary=${newestSeen}).`);
} finally {
// B5 safety: only persist the session if still on a valid page.
let url = "";
try { url = page.url(); } catch { /* gone */ }
try { await page.close(); } catch { /* ignore */ }
if (/\/login|\/authwall|\/checkpoint|\/uas\//.test(url)) {
console.warn(`[sync-accepted] Ended on a wall (${url}) — not persisting; flagging re-auth`);
try { await markNeedsReauth(accountId); } catch { /* ignore */ }
} else {
try { await saveSessionState(accountId); } catch { /* ignore */ }
}
db.prepare("UPDATE accounts SET accepted_sync_at = datetime('now') WHERE id = ?").run(accountId);
}
return stamped;
}
// ── helpers ──────────────────────────────────────────────────────────────────
function msToSqlite(ms: number): string {
return new Date(ms).toISOString().replace("T", " ").slice(0, 19);
}
async function fetchConnectionsPage(page: Page, start: number, count: number): Promise<ApiConnection[] | null> {
return page.evaluate(
async ({ start, count, decoration }): Promise<ApiConnection[] | null> => {
const cookies = document.cookie.split("; ").reduce((a: Record<string, string>, c) => {
const i = c.indexOf("=");
if (i > 0) a[c.slice(0, i)] = c.slice(i + 1);
return a;
}, {});
const csrf = (cookies["JSESSIONID"] || "").replace(/"/g, "");
const url =
`https://www.linkedin.com/voyager/api/relationships/dash/connections` +
`?decorationId=${decoration}&count=${count}&q=search&sortType=RECENTLY_ADDED&start=${start}`;
let json: {
included?: Array<{
$type?: string;
entityUrn?: string;
createdAt?: number;
connectedMember?: string;
publicIdentifier?: string;
}>;
};
try {
const r = await fetch(url, {
headers: {
"csrf-token": csrf,
"accept": "application/vnd.linkedin.normalized+json+2.1",
"x-restli-protocol-version": "2.0.0",
"x-li-lang": "en_US",
},
credentials: "include",
});
if (!r.ok) return null;
json = await r.json();
} catch {
return null;
}
const included = json.included || [];
const vanityByUrn: Record<string, string> = {};
for (const x of included) {
if ((x.$type || "").includes("identity.profile.Profile") && x.entityUrn && x.publicIdentifier) {
vanityByUrn[x.entityUrn] = x.publicIdentifier;
}
}
const out: ApiConnection[] = [];
for (const x of included) {
if ((x.$type || "").includes("relationships.Connection") && typeof x.createdAt === "number") {
const memberUrn = x.connectedMember || null;
out.push({
createdAt: x.createdAt,
vanity: memberUrn ? vanityByUrn[memberUrn] ?? null : null,
});
}
}
out.sort((a, b) => b.createdAt - a.createdAt);
return out;
},
{ start, count, decoration: DECORATION }
);
}
+48
View File
@@ -0,0 +1,48 @@
import type { Page } from "playwright";
/**
* Visits a LinkedIn profile page. This registers as a profile view on LinkedIn.
* Navigates and waits, then reports whether the page shows a 1st-degree badge —
* lets the runner backfill degree=1 for contacts that were already connected
* before Linki ever sent them a connection request (e.g. manually added leads).
*
* Primary-degree signal: presence of the profile's primary "Message" link
* (a[href*="/messaging/compose"]) — only shown to 1st-degree connections, reads
* an href attribute rather than a CSS class or translated text, so it survives
* both LinkedIn's periodic class-name hashing and non-English UI languages.
*
* MUST be scoped to the visited person's own intro/top card — a page-wide
* search also matches "Message" links belonging to OTHER people rendered
* elsewhere on the page (sidebar modules like "People also viewed" / suggested
* connections). For a non-connected target, THEIR page has no such link, but
* the sidebar still does — a page-wide `.first()` silently grabbed a random
* unrelated 1st-degree connection's link, wrongly marked the target degree=1,
* and stored that stranger's messaging URN, causing messages to go to the
* wrong person entirely (incident: Jul 2026, see CLAUDE.md/memory). The top
* card is identified structurally as the section containing the page's own
* <h1> name heading, which is robust to LinkedIn's class-name hashing.
*
* Falls back to the old text-scrape (/\b1st\b/ within that same top card)
* when the link isn't found, in case the current account's profile layout
* doesn't render it as a plain link (e.g. buried behind a click/menu).
*
* The same link's href carries the messaging URN (urn:li:fsd_profile:ACoAA...)
* needed to message this person directly later without a name-search typeahead
* — see lib/linkedin/message.ts. Returned as messagingUrn when found.
*/
export async function visitProfile(page: Page, linkedinUrl: string): Promise<{ isFirstDegree: boolean; messagingUrn: string | null }> {
await page.goto(linkedinUrl, { waitUntil: "domcontentloaded", timeout: 30000 });
await page.waitForTimeout(3000 + Math.random() * 2000);
const topCard = page.locator("main section").filter({ has: page.locator("h1") }).first();
const messageLink = topCard.locator('a[href*="/messaging/compose"]').first();
const messageHref = (await messageLink.count()) > 0 ? await messageLink.getAttribute("href").catch(() => null) : null;
const urnMatch = messageHref?.match(/profileUrn=([^&]+)/);
const messagingUrn = urnMatch ? decodeURIComponent(urnMatch[1]) : null;
if (messageHref) return { isFirstDegree: true, messagingUrn };
const pageText = await topCard.innerText().catch(() => "");
return { isFirstDegree: /\b1st\b/.test(pageText), messagingUrn: null };
}
+68
View File
@@ -0,0 +1,68 @@
// lib/premium.ts — the ONE bridge from open-core code to commercial (ee/) features.
//
// This file lives in open-core and is present in BOTH the private and public builds.
// It loads ee/ if the folder exists, otherwise degrades to `null`. Open-core code must
// call premium features ONLY through the `premium` object exported here — never with a
// direct `import ... from "@/ee/..."` anywhere else. That one rule is what makes the
// public build (ee/ stripped) compile and run with premium cleanly absent.
//
// Full strategy + rules: docs/OPEN_CORE.md
//
// IMPORTANT: the surface type below is declared HERE, in open-core, on purpose. If we
// imported the type from @/ee, the public build (where @/ee does not exist) would fail
// to typecheck. ee/index.ts must SATISFY this type instead. Extend this as features land.
// Minimal structural shapes for what OPEN-CORE consumes from premium features. These are
// declared here (not imported from @/ee) so the public build typechecks with ee/ stripped.
// They are intentionally loose at the boundary — ee/ owns the rich domain types and must
// SATISFY these. Only add a field here when open-core actually reads it. Kept deliberately
// permissive (no index signatures) so the richer ee/ return types remain assignable.
/* eslint-disable @typescript-eslint/no-explicit-any */
export interface AiSurface {
// Returns at least { default_model } — ee's AgentConfig is a superset.
getAgentConfig(): { default_model: string | null };
// Bundle is passed straight back into the writers; open-core never inspects its innards.
getContactWithCompany(targetId: string): { contact: any; company: any } | null;
// Writers take a large param object (ee owns its full type); open-core builds it inline.
writeEmail(params: any): Promise<{ subject: string; body: string }>;
writeLinkedInMessage(params: any): Promise<{ body: string }>;
writeSalesInMail(params: any): Promise<{ subject: string; body: string }>;
}
export interface RepliesSurface {
// AI classify + auto-followup for one stored reply (email or LinkedIn).
classifyAndDispatch(replyId: string): Promise<void>;
// LinkedIn inbox sync (reply detection), driven by the runner loop.
shouldSyncInbox(accountId: string): boolean;
syncAccountInbox(accountId: string): Promise<number>;
}
export interface InMailSurface {
// Sends a Sales Nav InMail via a live browser page (playwright Page type is open-core).
sendInMail(page: any, salesNavUrl: string, subject: string, body: string): Promise<void>;
}
export interface PremiumSurface {
ai?: AiSurface;
replies?: RepliesSurface;
inmail?: InMailSurface;
[key: string]: unknown;
}
function loadPremium(): PremiumSurface | null {
try {
// Loaded by real path. The ee/ folder is imported IN PLACE (not copied) so its own
// relative imports resolve normally. In the public build this require throws → null.
// eslint-disable-next-line @typescript-eslint/no-require-imports
const mod = require("@/ee");
return (mod?.default ?? mod) as PremiumSurface;
} catch {
return null; // public build: ee/ has been stripped
}
}
export const premium: PremiumSurface | null = loadPremium();
/** True in the private/commercial build, false in the public open-source build. */
export const hasPremium: boolean = premium !== null;
+53
View File
@@ -0,0 +1,53 @@
// Minimal shape both a real NextApiRequest and NextAuth's authorize() RequestInternal
// satisfy — socket is optional since RequestInternal doesn't expose one.
type IpSource = {
headers?: { [key: string]: string | string[] | undefined };
socket?: { remoteAddress?: string };
};
// In-memory sliding-window rate limiter, per IP. No Redis/external service (self-hosted,
// single process — see CLAUDE.md). Resets on server restart; acceptable for a single-user
// tool defending against brute force on login/signup, not a distributed-attack mitigation.
const hits = new Map<string, number[]>();
// Periodically drop stale keys so this doesn't grow unbounded over a long-running process.
const SWEEP_INTERVAL_MS = 10 * 60 * 1000;
let lastSweep = Date.now();
export function isRateLimited(req: IpSource, key: string, limit: number, windowMs: number): boolean {
const ip = clientIp(req);
const mapKey = `${key}:${ip}`;
const now = Date.now();
sweepIfDue(now, windowMs);
const timestamps = (hits.get(mapKey) ?? []).filter(t => now - t < windowMs);
if (timestamps.length >= limit) {
hits.set(mapKey, timestamps);
return true;
}
timestamps.push(now);
hits.set(mapKey, timestamps);
return false;
}
function sweepIfDue(now: number, windowMs: number) {
if (now - lastSweep < SWEEP_INTERVAL_MS) return;
lastSweep = now;
for (const [mapKey, timestamps] of hits) {
const fresh = timestamps.filter(t => now - t < windowMs);
if (fresh.length === 0) hits.delete(mapKey);
else hits.set(mapKey, fresh);
}
}
function clientIp(req: IpSource): string {
const xRealIp = req.headers?.["x-real-ip"];
if (typeof xRealIp === "string" && xRealIp) return xRealIp;
const xForwardedFor = req.headers?.["x-forwarded-for"];
if (typeof xForwardedFor === "string" && xForwardedFor) return xForwardedFor.split(",")[0].trim();
return req.socket?.remoteAddress || "unknown";
}
+302
View File
@@ -0,0 +1,302 @@
// lib/tour.ts — product tour engine (driver.js). Open-core: available in both builds.
// Each page gets its own short, independent tour keyed by a "page" id. A tour is shown
// once automatically (tracked server-side via /api/tour) and can be replayed manually
// from Settings → General ("Restart tour"), which never touches the "seen" state.
"use client";
import { driver, type DriveStep } from "driver.js";
import "driver.js/dist/driver.css";
export type TourPage =
| "dashboard"
| "lists"
| "contacts"
| "companies"
| "workflows"
| "inbox"
| "settings";
// Steps whose `element` selector isn't in the DOM are silently skipped by driver.js —
// relied on here so a step referencing a premium-only anchor (Todos nav, InMail, MCP)
// just no-ops cleanly in the open-core build instead of needing its own guard.
const TOURS: Record<TourPage, DriveStep[]> = {
dashboard: [
{
element: '[data-tour="nav-dashboard"]',
popover: {
title: "Welcome to Linki",
description: "This is your outreach dashboard. Everything in Linki runs on one background process at a human pace — nothing fires the instant you click a button, so don't expect instant numbers here either.",
},
},
{
element: '[data-tour="dashboard-filters"]',
popover: {
title: "LinkedIn and email, side by side",
description: "The two channels are tracked completely separately — a campaign can run both at once, and this page reports them independently rather than blended.",
},
},
{
element: '[data-tour="dashboard-funnel"]',
popover: {
title: "Your outreach funnel",
description: "See exactly where leads drop off — from total targets down to replies. This is the \"is my campaign actually working\" view.",
},
},
{
element: '[data-tour="dashboard-chart"]',
popover: {
title: "Activity over time",
description: "Visits, connections, messages and emails per day. Click a series label to toggle it off, or change the day range to reshape the whole page.",
},
},
{
element: '[data-tour="nav-settings"]',
popover: {
title: "First: connect a LinkedIn account",
description: "Nothing else works until an account is authenticated in Settings → LinkedIn — that's the one true prerequisite before importing or running anything.",
},
},
{
element: '[data-tour="nav-lists"]',
popover: {
title: "Then: import a list",
description: "Paste a Sales Navigator search URL — that's how leads get into Linki.",
},
},
{
element: '[data-tour="nav-workflows"]',
popover: {
title: "Then: build a campaign",
description: "Chain steps like visit, connect, message and email into a sequence, pick the list and account, and launch.",
},
},
],
lists: [
{
element: '[data-tour="lists-new"]',
popover: {
title: "Create a list",
description: "Paste a LinkedIn Sales Navigator search URL to scrape leads automatically.",
},
},
{
element: '[data-tour="lists-new"]',
popover: {
title: "Imports run in the background",
description: "A Sales Navigator import doesn't happen instantly — it's queued and scraped in small batches with human-like delays, specifically to avoid LinkedIn flagging the account.",
},
},
{
element: '[data-tour="lists-jobs"]',
popover: {
title: "Import jobs",
description: "There's a shared daily import cap across every list (configurable in Settings → General) — a large list automatically spills into the next day. Track progress and cancel batches here.",
},
},
{
element: '[data-tour="lists-new"]',
popover: {
title: "A fresh import is intentionally thin",
description: "Newly imported leads only have basic data (name, title, company). Full profile and email enrichment happen later, automatically, right before a campaign actually needs them — you don't need to enrich everything up front.",
},
},
],
contacts: [
{
element: '[data-tour="nav-contacts"]',
popover: {
title: "Every lead, in one place",
description: "This is the single, deduplicated view of everyone you've ever imported — the same person can belong to several lists, and shows up here just once regardless.",
},
},
{
element: '[data-tour="contacts-filters"]',
popover: {
title: "Search and filter",
description: "Filter by list, title, connection status, seniority, or email verification to build a targeted sub-audience without re-importing anything.",
},
},
{
element: '[data-tour="contacts-select"]',
popover: {
title: "Bulk-select, then add to a list",
description: "Select multiple contacts and add them all to a list at once — an easy way to turn a filtered view (e.g. everyone who replied) into a new targeted list. Contacts already in that list are skipped.",
},
},
],
companies: [
{
element: '[data-tour="nav-companies"]',
popover: {
title: "Companies",
description: "Enriched organization records — industry, size, tech stack, revenue — that populate automatically as contacts get enriched. You can also add or edit one by hand.",
},
},
],
workflows: [
{
element: '[data-tour="workflows-new"]',
popover: {
title: "Build a campaign",
description: "A campaign chains steps — visit, connect, message, email — into a repeatable sequence, then runs it against a list of contacts through one LinkedIn account.",
},
},
{
element: '[data-tour="workflows-new"]',
popover: {
title: "LinkedIn and email run in parallel",
description: "LinkedIn steps and email steps are two independent tracks for the same contact. A reply on either channel stops that person's outreach on both — you don't need to manage them separately.",
},
},
{
element: '[data-tour="workflows-new"]',
popover: {
title: "A LinkedIn account is always required",
description: "Even an email-only campaign needs one authenticated LinkedIn account selected — set that up first in Settings if you haven't.",
},
},
{
element: '[data-tour="workflows-new"]',
popover: {
title: "One active run at a time",
description: "A campaign can only have one run going at once. To reach more people later, use \"Add contacts\" on the running campaign instead of starting a second run.",
},
},
{
element: '[data-tour="workflows-new"]',
popover: {
title: "It runs at a human pace",
description: "Launching doesn't message everyone instantly — the runner works through the list gradually in the background, respecting daily limits and working hours per account.",
},
},
],
inbox: [
{
element: '[data-tour="nav-inbox"]',
popover: {
title: "Every reply, one place",
description: "Replies from LinkedIn and email land here automatically — no need to check either inbox by hand. A reply also automatically stops that contact's outreach.",
},
},
{
element: '[data-tour="nav-inbox"]',
popover: {
title: "Not every reply means interest",
description: "Reply counts can be misleadingly high — many are auto-responders (out-of-office, \"contact my colleague\") rather than genuine interest. Open one to see the full thread and judge for yourself.",
},
},
],
settings: [
{
element: '[data-tour="settings-tab-linkedin"]',
popover: {
title: "Connect a LinkedIn account first",
description: "An account must be authenticated here before it can be used in any campaign — this is the single most important setup step in Linki.",
},
},
{
element: '[data-tour="settings-tab-email"]',
popover: {
title: "Email accounts, with ramp-up",
description: "Configure SMTP/IMAP for email steps. A new account can ramp up sending volume gradually instead of blasting at full volume from day one, which helps it avoid getting flagged as spam.",
},
},
{
element: '[data-tour="settings-tab-integrations"]',
popover: {
title: "Apollo enrichment",
description: "Add an Apollo API key here to enrich leads with verified emails, seniority, and company data.",
},
},
{
element: '[data-tour="settings-tab-general"]',
popover: {
title: "Connect an AI agent",
description: "The General tab has a copyable MCP URL to hook up Claude Code, Cursor, or any MCP-compatible agent to manage Linki on your behalf — plus the daily import cap and this tour picker.",
},
},
],
};
function markSeen(page: TourPage) {
fetch("/api/tour", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ page }),
}).catch(() => {});
}
function buildAndRun(page: TourPage, persist: boolean) {
const steps = TOURS[page];
if (!steps || steps.length === 0) return;
const d = driver({
showProgress: true,
overlayColor: "#000",
overlayOpacity: 0.65,
stagePadding: 6,
stageRadius: 8,
popoverOffset: 12,
allowClose: true,
popoverClass: "linki-tour-popover",
nextBtnText: "Next",
prevBtnText: "Back",
doneBtnText: "Done",
steps,
onDestroyed: () => {
if (persist) markSeen(page);
},
});
d.drive();
}
export function startPageTour(page: TourPage) {
buildAndRun(page, true);
}
export function replayPageTour(page: TourPage) {
buildAndRun(page, false);
}
export async function getSeenTours(): Promise<Set<TourPage>> {
try {
const res = await fetch("/api/tour");
if (!res.ok) return new Set();
const data = await res.json();
return new Set((data.seen ?? []) as TourPage[]);
} catch {
return new Set();
}
}
export const ALL_TOUR_PAGES: TourPage[] = ["dashboard", "lists", "contacts", "companies", "workflows", "inbox", "settings"];
export const TOUR_PAGE_LABELS: Record<TourPage, string> = {
dashboard: "Dashboard",
lists: "Lists",
contacts: "Contacts",
companies: "Companies",
workflows: "Campaigns",
inbox: "Inbox",
settings: "Settings",
};
// Maps a pathname to its tour page id, or null if the current page has no tour
// (e.g. a detail route like /lists/[id] — the index page's tour covers the entry point).
export function pathToTourPage(pathname: string): TourPage | null {
if (pathname === "/") return "dashboard";
if (pathname === "/lists") return "lists";
if (pathname === "/contacts") return "contacts";
if (pathname === "/companies") return "companies";
if (pathname === "/workflows") return "workflows";
if (pathname === "/inbox") return "inbox";
if (pathname === "/settings") return "settings";
return null;
}
+88
View File
@@ -0,0 +1,88 @@
/**
* Update checker — polls Docker Hub tags to detect if a newer version of the
* image has been published. Runs once on startup, then every 12 hours.
*
* State is kept in memory (reset on restart, which is fine — it re-checks immediately).
*/
const DOCKER_HUB_TAGS_URL =
"https://hub.docker.com/v2/repositories/moaljumaa/linki/tags?page_size=50&ordering=last_updated";
const POLL_INTERVAL_MS = 12 * 60 * 60 * 1000; // 12 hours
export interface UpdateState {
current: string;
latest: string | null;
updateAvailable: boolean;
checkedAt: string | null;
}
const state: UpdateState = {
current: process.env.APP_VERSION ?? "dev",
latest: null,
updateAvailable: false,
checkedAt: null,
};
export function getUpdateState(): UpdateState {
return { ...state };
}
/** Parse semver string like "1.2.3" → [1, 2, 3]. Returns null if not semver. */
function parseSemver(v: string): [number, number, number] | null {
const m = v.match(/^(\d+)\.(\d+)\.(\d+)$/);
if (!m) return null;
return [parseInt(m[1]), parseInt(m[2]), parseInt(m[3])];
}
/** Returns true if b is strictly greater than a */
function isNewer(a: string, b: string): boolean {
const pa = parseSemver(a);
const pb = parseSemver(b);
if (!pa || !pb) return false;
if (pb[0] !== pa[0]) return pb[0] > pa[0];
if (pb[1] !== pa[1]) return pb[1] > pa[1];
return pb[2] > pa[2];
}
async function checkForUpdate() {
try {
const res = await fetch(DOCKER_HUB_TAGS_URL);
if (!res.ok) return;
const data = await res.json() as { results?: { name: string }[] };
const tags: string[] = (data.results ?? [])
.map((t) => t.name)
.filter((n) => parseSemver(n) !== null);
if (tags.length === 0) return;
// Find highest semver tag
const latest = tags.reduce((best, tag) => {
return isNewer(best, tag) ? tag : best;
}, tags[0]);
state.latest = latest;
state.updateAvailable = state.current !== "dev" && isNewer(state.current, latest);
state.checkedAt = new Date().toISOString();
if (state.updateAvailable) {
console.log(`[update-check] New version available: ${latest} (running ${state.current})`);
}
} catch {
// Non-fatal — silently ignore network errors
}
}
const g = global as typeof global & { __updateCheckScheduled?: boolean };
export function scheduleUpdateCheck() {
if (g.__updateCheckScheduled) return;
g.__updateCheckScheduled = true;
// Run immediately on startup (non-blocking)
checkForUpdate();
// Then every 12 hours
setInterval(checkForUpdate, POLL_INTERVAL_MS);
}