release: 2026-08-01
This commit is contained in:
+107
@@ -0,0 +1,107 @@
|
||||
const APOLLO_BASE = "https://api.apollo.io/api/v1";
|
||||
|
||||
export interface ApolloMatchResult {
|
||||
apollo_id: string;
|
||||
linkedin_url: string | null;
|
||||
headline: string | null;
|
||||
seniority: string | null;
|
||||
functions: string[] | null;
|
||||
departments: string[] | null;
|
||||
email: string | null;
|
||||
email_status: string | null;
|
||||
email_domain_catchall: boolean;
|
||||
city: string | null;
|
||||
country: string | null;
|
||||
time_zone: string | null;
|
||||
positions_json: string | null;
|
||||
organization: {
|
||||
name: string | null;
|
||||
domain: string | null;
|
||||
industry: string | null;
|
||||
estimated_num_employees: number | null;
|
||||
short_description: string | null;
|
||||
location: string | null;
|
||||
linkedin_url: string | null;
|
||||
website_url: string | null;
|
||||
founded_year: number | null;
|
||||
logo_url: string | null;
|
||||
phone: string | null;
|
||||
annual_revenue_printed: string | null;
|
||||
technology_names: string[] | null;
|
||||
keywords: string[] | null;
|
||||
city: string | null;
|
||||
country: string | null;
|
||||
} | null;
|
||||
}
|
||||
|
||||
export async function matchPerson(
|
||||
linkedinUrl: string,
|
||||
apiKey: string
|
||||
): Promise<ApolloMatchResult | null> {
|
||||
const res = await fetch(`${APOLLO_BASE}/people/match`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", "X-Api-Key": apiKey },
|
||||
body: JSON.stringify({ linkedin_url: linkedinUrl, reveal_personal_emails: false }),
|
||||
});
|
||||
|
||||
if (!res.ok) return null;
|
||||
|
||||
const data = await res.json();
|
||||
const p = data.person;
|
||||
if (!p || !p.id) return null;
|
||||
|
||||
const org = p.organization ?? null;
|
||||
const location =
|
||||
org
|
||||
? [org.city, org.country].filter(Boolean).join(", ") || null
|
||||
: null;
|
||||
|
||||
// Map employment_history to positions_json format (same shape as Sales Nav enrichment)
|
||||
let positions_json: string | null = null;
|
||||
if (Array.isArray(p.employment_history) && p.employment_history.length > 0) {
|
||||
const positions = p.employment_history.map((e: Record<string, unknown>) => ({
|
||||
title: e.title ?? "",
|
||||
companyName: e.organization_name ?? "",
|
||||
current: e.current === true,
|
||||
startDate: e.start_date ?? undefined,
|
||||
endDate: e.end_date ?? undefined,
|
||||
}));
|
||||
positions_json = JSON.stringify(positions);
|
||||
}
|
||||
|
||||
return {
|
||||
apollo_id: p.id,
|
||||
linkedin_url: p.linkedin_url ?? null,
|
||||
headline: p.headline ?? null,
|
||||
seniority: p.seniority ?? null,
|
||||
functions: Array.isArray(p.functions) && p.functions.length > 0 ? p.functions : null,
|
||||
departments: Array.isArray(p.departments) && p.departments.length > 0 ? p.departments : null,
|
||||
email: p.email ?? null,
|
||||
email_status: p.email_status ?? null,
|
||||
email_domain_catchall: p.email_domain_catchall === true,
|
||||
city: p.city ?? null,
|
||||
country: p.country ?? null,
|
||||
time_zone: p.time_zone ?? null,
|
||||
positions_json,
|
||||
organization: org
|
||||
? {
|
||||
name: org.name ?? null,
|
||||
domain: org.primary_domain ?? null,
|
||||
industry: org.industry ?? null,
|
||||
estimated_num_employees: org.estimated_num_employees ?? null,
|
||||
short_description: org.short_description ?? null,
|
||||
location,
|
||||
linkedin_url: org.linkedin_url ?? null,
|
||||
website_url: org.website_url ?? null,
|
||||
founded_year: org.founded_year ?? null,
|
||||
logo_url: org.logo_url ?? null,
|
||||
phone: org.phone ?? org.sanitized_phone ?? null,
|
||||
annual_revenue_printed: org.annual_revenue_printed ?? null,
|
||||
technology_names: Array.isArray(org.technology_names) && org.technology_names.length > 0 ? org.technology_names : null,
|
||||
keywords: Array.isArray(org.keywords) && org.keywords.length > 0 ? org.keywords : null,
|
||||
city: org.city ?? null,
|
||||
country: org.country ?? null,
|
||||
}
|
||||
: null,
|
||||
};
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
import { getToken } from "next-auth/jwt";
|
||||
import type { NextRequest } from "next/server";
|
||||
|
||||
const INTERNAL_HEADER = "x-internal-secret";
|
||||
|
||||
/**
|
||||
* Shared request-auth check used by proxy.ts to gate /api/*.
|
||||
*
|
||||
* Accepts either:
|
||||
* - a NextAuth session cookie (browser app) — verified via getToken().
|
||||
* - the internal service secret (server-to-server loopback calls, e.g. the MCP server's
|
||||
* tool handlers calling Linki's own /api/* routes — see ee/mcp/server.ts). This secret
|
||||
* never leaves the host: it's not sent to a browser and never crosses the public ngrok/
|
||||
* reverse-proxy path, only Node processes on 127.0.0.1 exchange it.
|
||||
*
|
||||
* Routes with their own independent auth (Authorization: Bearer, e.g. /api/mcp) are
|
||||
* excluded from the gate entirely in proxy.ts rather than special-cased here.
|
||||
*/
|
||||
export async function isAuthenticated(req: NextRequest): Promise<boolean> {
|
||||
if (await hasValidInternalSecret(req)) return true;
|
||||
|
||||
const token = await getToken({ req, secret: process.env.NEXTAUTH_SECRET });
|
||||
return token !== null;
|
||||
}
|
||||
|
||||
async function hasValidInternalSecret(req: NextRequest): Promise<boolean> {
|
||||
const expected = process.env.INTERNAL_API_SECRET;
|
||||
if (!expected) return false;
|
||||
|
||||
const provided = req.headers.get(INTERNAL_HEADER);
|
||||
if (!provided) return false;
|
||||
|
||||
return timingSafeEqual(provided, expected);
|
||||
}
|
||||
|
||||
// Constant-time string comparison via Web Crypto (available on both the Edge and Node.js
|
||||
// runtimes) so an internal-secret guess can't be timed byte-by-byte.
|
||||
async function timingSafeEqual(a: string, b: string): Promise<boolean> {
|
||||
const enc = new TextEncoder();
|
||||
const [digestA, digestB] = await Promise.all([
|
||||
crypto.subtle.digest("SHA-256", enc.encode(a)),
|
||||
crypto.subtle.digest("SHA-256", enc.encode(b)),
|
||||
]);
|
||||
const bytesA = new Uint8Array(digestA);
|
||||
const bytesB = new Uint8Array(digestB);
|
||||
let diff = 0;
|
||||
for (let i = 0; i < bytesA.length; i++) diff |= bytesA[i] ^ bytesB[i];
|
||||
return diff === 0;
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { createCipheriv, createDecipheriv, hkdfSync, randomBytes } from "crypto";
|
||||
|
||||
// Encrypts secrets at rest (LinkedIn session cookies, email passwords, API keys) so a
|
||||
// leaked/copied linki.db doesn't hand those out in plaintext. Key is derived from
|
||||
// NEXTAUTH_SECRET (already a required, real secret) via HKDF — no new env var needed.
|
||||
//
|
||||
// Format: "v1:<iv-base64>:<authTag-base64>:<ciphertext-base64>". decryptSecret() passes
|
||||
// any value NOT in this format straight through unchanged, so pre-migration plaintext
|
||||
// rows keep working — see lib/db.ts's encryptLegacySecretsMigration.
|
||||
|
||||
const VERSION = "v1";
|
||||
const ALGORITHM = "aes-256-gcm";
|
||||
|
||||
function deriveKey(): Buffer {
|
||||
const secret = process.env.NEXTAUTH_SECRET;
|
||||
if (!secret) throw new Error("NEXTAUTH_SECRET must be set to encrypt/decrypt stored secrets");
|
||||
return Buffer.from(hkdfSync("sha256", secret, "", "linki-secret-encryption", 32));
|
||||
}
|
||||
|
||||
export function encryptSecret(plaintext: string): string {
|
||||
const key = deriveKey();
|
||||
const iv = randomBytes(12);
|
||||
const cipher = createCipheriv(ALGORITHM, key, iv);
|
||||
const ciphertext = Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]);
|
||||
const authTag = cipher.getAuthTag();
|
||||
return [VERSION, iv.toString("base64"), authTag.toString("base64"), ciphertext.toString("base64")].join(":");
|
||||
}
|
||||
|
||||
export function decryptSecret(value: string | null): string | null {
|
||||
if (value === null) return null;
|
||||
if (!isEncrypted(value)) return value; // not-yet-migrated plaintext — pass through
|
||||
|
||||
const [, ivB64, authTagB64, dataB64] = value.split(":");
|
||||
const key = deriveKey();
|
||||
const decipher = createDecipheriv(ALGORITHM, key, Buffer.from(ivB64, "base64"));
|
||||
decipher.setAuthTag(Buffer.from(authTagB64, "base64"));
|
||||
const plaintext = Buffer.concat([decipher.update(Buffer.from(dataB64, "base64")), decipher.final()]);
|
||||
return plaintext.toString("utf8");
|
||||
}
|
||||
|
||||
export function isEncrypted(value: string): boolean {
|
||||
return value.startsWith(`${VERSION}:`);
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
import Papa from "papaparse";
|
||||
import type DatabaseType from "better-sqlite3";
|
||||
import { randomUUID } from "crypto";
|
||||
|
||||
type DB = DatabaseType.Database;
|
||||
|
||||
// User-fillable target fields — everything else on `targets` (URNs, JSON blobs,
|
||||
// enrichment timestamps, apollo/automation internals) is system-owned and not
|
||||
// importable. Mirrors the PATCH /api/targets/[id] editable set.
|
||||
const EDITABLE_FIELDS = [
|
||||
"first_name", "last_name", "title", "company", "location",
|
||||
"city", "country", "phone", "headline", "summary", "notes",
|
||||
] as const;
|
||||
type EditableField = (typeof EDITABLE_FIELDS)[number];
|
||||
|
||||
// One template covers every case: a pure LinkedIn list, a pure email list (incl.
|
||||
// generic inboxes like info@company.com), or an export that already has both
|
||||
// (e.g. a list exported from another tool). Each row just needs linkedin_url
|
||||
// and/or email filled in — whatever the contact actually has.
|
||||
const TEMPLATE_COLUMNS = ["linkedin_url", "sales_nav_url", "email", ...EDITABLE_FIELDS] as const;
|
||||
|
||||
const SAMPLE_VALUES: Record<EditableField, string> = {
|
||||
first_name: "Jane",
|
||||
last_name: "Doe",
|
||||
title: "Head of Marketing",
|
||||
company: "Acme Inc",
|
||||
location: "Berlin, Germany",
|
||||
city: "Berlin",
|
||||
country: "Germany",
|
||||
phone: "+49 30 1234567",
|
||||
headline: "Head of Marketing @ Acme Inc",
|
||||
summary: "10+ years in B2B SaaS marketing.",
|
||||
notes: "Met at SaaStr 2026",
|
||||
};
|
||||
|
||||
export function buildCsvTemplate(): string {
|
||||
const sample = TEMPLATE_COLUMNS.map((c) =>
|
||||
c === "linkedin_url" ? "https://www.linkedin.com/in/example-profile/" :
|
||||
c === "sales_nav_url" ? "" :
|
||||
c === "email" ? "jane@acme.com" :
|
||||
SAMPLE_VALUES[c as EditableField]
|
||||
);
|
||||
return Papa.unparse({ fields: [...TEMPLATE_COLUMNS], data: [sample] });
|
||||
}
|
||||
|
||||
export interface CsvImportResult {
|
||||
imported: number;
|
||||
updated: number;
|
||||
skipped: number;
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
const EMAIL_RE = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
|
||||
|
||||
function normalizeLinkedinUrl(raw: string): string | null {
|
||||
const trimmed = raw.trim();
|
||||
if (!trimmed || !trimmed.includes("linkedin.com/in/")) return null;
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
interface ParsedRow {
|
||||
linkedin_url: string | null;
|
||||
sales_nav_url: string | null;
|
||||
email: string | null;
|
||||
full_name: string | null;
|
||||
fields: Record<EditableField, string | null>;
|
||||
}
|
||||
|
||||
function get(row: Record<string, string>, key: string): string | null {
|
||||
const v = row[key];
|
||||
const t = typeof v === "string" ? v.trim() : "";
|
||||
return t.length > 0 ? t : null;
|
||||
}
|
||||
|
||||
export function importCsv(db: DB, listId: string, csvText: string): CsvImportResult {
|
||||
const parsed = Papa.parse<Record<string, string>>(csvText, {
|
||||
header: true,
|
||||
skipEmptyLines: true,
|
||||
transformHeader: (h) => h.trim().toLowerCase().replace(/\s+/g, "_"),
|
||||
});
|
||||
|
||||
const errors: string[] = [];
|
||||
let imported = 0;
|
||||
let updated = 0;
|
||||
let skipped = 0;
|
||||
|
||||
const rows: ParsedRow[] = [];
|
||||
parsed.data.forEach((raw, idx) => {
|
||||
const rowNum = idx + 2; // header is row 1
|
||||
const fields = Object.fromEntries(EDITABLE_FIELDS.map((f) => [f, get(raw, f)])) as Record<EditableField, string | null>;
|
||||
const full_name = [fields.first_name, fields.last_name].filter(Boolean).join(" ") || null;
|
||||
|
||||
const rawUrl = get(raw, "linkedin_url");
|
||||
const linkedin_url = rawUrl ? normalizeLinkedinUrl(rawUrl) : null;
|
||||
if (rawUrl && !linkedin_url) { errors.push(`Row ${rowNum}: "${rawUrl}" is not a valid linkedin.com/in/ URL`); return; }
|
||||
|
||||
const sales_nav_url = get(raw, "sales_nav_url");
|
||||
|
||||
const rawEmail = get(raw, "email");
|
||||
let email: string | null = null;
|
||||
if (rawEmail) {
|
||||
if (!EMAIL_RE.test(rawEmail)) { errors.push(`Row ${rowNum}: "${rawEmail}" is not a valid email`); return; }
|
||||
email = rawEmail.toLowerCase();
|
||||
}
|
||||
|
||||
if (!linkedin_url && !email) {
|
||||
errors.push(`Row ${rowNum}: needs at least a linkedin_url or an email`);
|
||||
return;
|
||||
}
|
||||
|
||||
rows.push({ linkedin_url, sales_nav_url, email, full_name, fields });
|
||||
});
|
||||
|
||||
const editableCols = [...EDITABLE_FIELDS, "full_name", "sales_nav_url"] as const;
|
||||
|
||||
const insertByLinkedin = db.prepare(`
|
||||
INSERT INTO targets (id, linkedin_url, email, sales_nav_url, full_name, ${EDITABLE_FIELDS.join(", ")})
|
||||
VALUES (?, ?, ?, ?, ?, ${EDITABLE_FIELDS.map(() => "?").join(", ")})
|
||||
ON CONFLICT(linkedin_url) DO UPDATE SET
|
||||
email = COALESCE(excluded.email, targets.email),
|
||||
${editableCols.map((c) => `${c} = COALESCE(excluded.${c}, targets.${c})`).join(",\n ")}
|
||||
`);
|
||||
const findByLinkedin = db.prepare("SELECT id FROM targets WHERE linkedin_url = ?");
|
||||
const findByEmail = db.prepare("SELECT id FROM targets WHERE email = ? LIMIT 1");
|
||||
const insertByEmail = db.prepare(`
|
||||
INSERT INTO targets (id, email, full_name, ${EDITABLE_FIELDS.join(", ")}, sales_nav_url)
|
||||
VALUES (?, ?, ?, ${EDITABLE_FIELDS.map(() => "?").join(", ")}, ?)
|
||||
`);
|
||||
const updateByEmail = db.prepare(`
|
||||
UPDATE targets SET
|
||||
${editableCols.map((c) => `${c} = COALESCE(?, ${c})`).join(",\n ")}
|
||||
WHERE id = ?
|
||||
`);
|
||||
const linkToList = db.prepare("INSERT OR IGNORE INTO list_targets (list_id, target_id) VALUES (?, ?)");
|
||||
|
||||
db.transaction(() => {
|
||||
for (const row of rows) {
|
||||
let targetId: string;
|
||||
let isNew: boolean;
|
||||
const fieldValues = EDITABLE_FIELDS.map((f) => row.fields[f]);
|
||||
|
||||
if (row.linkedin_url) {
|
||||
const existing = findByLinkedin.get(row.linkedin_url) as { id: string } | undefined;
|
||||
isNew = !existing;
|
||||
targetId = existing?.id ?? randomUUID();
|
||||
insertByLinkedin.run(targetId, row.linkedin_url, row.email, row.sales_nav_url, row.full_name, ...fieldValues);
|
||||
} else {
|
||||
const existing = findByEmail.get(row.email) as { id: string } | undefined;
|
||||
isNew = !existing;
|
||||
if (existing) {
|
||||
targetId = existing.id;
|
||||
updateByEmail.run(...fieldValues, row.full_name, row.sales_nav_url, targetId);
|
||||
} else {
|
||||
targetId = randomUUID();
|
||||
insertByEmail.run(targetId, row.email, row.full_name, ...fieldValues, row.sales_nav_url);
|
||||
}
|
||||
}
|
||||
|
||||
const linkResult = linkToList.run(listId, targetId);
|
||||
if (linkResult.changes > 0) {
|
||||
if (isNew) imported++; else updated++;
|
||||
} else {
|
||||
skipped++; // already in this list, no changes
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
return { imported, updated, skipped, errors };
|
||||
}
|
||||
@@ -0,0 +1,883 @@
|
||||
import Database from "better-sqlite3";
|
||||
import path from "path";
|
||||
import { randomUUID } from "crypto";
|
||||
import { scheduleUpdateCheck } from "@/lib/update-check";
|
||||
import { encryptSecret, isEncrypted } from "@/lib/crypto";
|
||||
|
||||
const DB_PATH = process.env.LINKI_DB_PATH || path.join(process.cwd(), "linki.db");
|
||||
|
||||
let db: Database.Database;
|
||||
|
||||
export function getDb(): Database.Database {
|
||||
if (!db) {
|
||||
db = new Database(DB_PATH);
|
||||
db.pragma("journal_mode = WAL");
|
||||
db.pragma("foreign_keys = ON");
|
||||
initDb(db);
|
||||
runMigrations(db);
|
||||
scheduleUpdateCheck();
|
||||
}
|
||||
return db;
|
||||
}
|
||||
|
||||
function runParallelTracksMigration(db: Database.Database) {
|
||||
// This backfill reads the legacy run_profiles.state column. If that column no longer
|
||||
// exists, dropDeprecatedRunProfileColumns has already run (a prior startup) and this
|
||||
// migration is moot — skip, otherwise the SELECT rp.state below throws "no such column".
|
||||
// (Fresh DBs hit this: state is created, dropped on one startup, then this guard would
|
||||
// otherwise re-enter on the next because no email-tracked steps exist yet.)
|
||||
try {
|
||||
const rpCols = db.prepare("PRAGMA table_info(run_profiles)").all() as { name: string }[];
|
||||
if (!rpCols.some(c => c.name === "state")) return;
|
||||
} catch { return; }
|
||||
|
||||
// Idempotent: skip if run_profile_tracks already has rows or workflow_steps already has email-tracked rows
|
||||
try {
|
||||
const alreadyRun = (db.prepare("SELECT COUNT(*) as c FROM run_profile_tracks").get() as { c: number }).c > 0
|
||||
|| (db.prepare("SELECT COUNT(*) as c FROM workflow_steps WHERE track = 'email'").get() as { c: number }).c > 0;
|
||||
if (alreadyRun) return;
|
||||
} catch { return; }
|
||||
|
||||
db.transaction(() => {
|
||||
// 1. Assign email step_type rows to the email track
|
||||
db.exec("UPDATE workflow_steps SET track = 'email' WHERE step_type = 'email'");
|
||||
// Also assign delay steps to the email track if the next non-delay step after them is an email step.
|
||||
// Without this, delays between email steps default to 'linkedin' and create a ghost linkedin track.
|
||||
db.exec(`
|
||||
UPDATE workflow_steps SET track = 'email'
|
||||
WHERE step_type = 'delay'
|
||||
AND (
|
||||
SELECT step_type FROM workflow_steps ws2
|
||||
WHERE ws2.workflow_id = workflow_steps.workflow_id
|
||||
AND ws2.step_order > workflow_steps.step_order
|
||||
AND ws2.step_type != 'delay'
|
||||
ORDER BY ws2.step_order ASC
|
||||
LIMIT 1
|
||||
) = 'email'
|
||||
`);
|
||||
|
||||
// 2. Re-number step_order densely within each (workflow_id, track), preserving original order
|
||||
const stepGroups = db.prepare(
|
||||
"SELECT id, workflow_id, track, step_order FROM workflow_steps ORDER BY workflow_id, track, step_order"
|
||||
).all() as Array<{ id: string; workflow_id: string; track: string; step_order: number }>;
|
||||
|
||||
// Group by (workflow_id, track) and assign dense 1-based order
|
||||
const grouped = new Map<string, Array<{ id: string; step_order: number }>>();
|
||||
for (const row of stepGroups) {
|
||||
const key = `${row.workflow_id}|${row.track}`;
|
||||
if (!grouped.has(key)) grouped.set(key, []);
|
||||
grouped.get(key)!.push({ id: row.id, step_order: row.step_order });
|
||||
}
|
||||
const updateStep = db.prepare("UPDATE workflow_steps SET step_order = ? WHERE id = ?");
|
||||
for (const steps of grouped.values()) {
|
||||
steps.sort((a, b) => a.step_order - b.step_order);
|
||||
steps.forEach((s, i) => updateStep.run(i + 1, s.id));
|
||||
}
|
||||
|
||||
// 3. Backfill run_profile_tracks from existing run_profiles
|
||||
// Only backfill if there are run_profiles rows to process
|
||||
const allProfiles = db.prepare(
|
||||
`SELECT rp.id, rp.run_id, rp.state, rp.current_step, rp.next_step_at,
|
||||
rp.error_message, rp.last_email_subject, rp.last_email_body, rp.last_linkedin_message,
|
||||
r.workflow_id
|
||||
FROM run_profiles rp
|
||||
JOIN runs r ON r.id = rp.run_id`
|
||||
).all() as Array<{
|
||||
id: string; run_id: string; state: string; current_step: number;
|
||||
next_step_at: string | null; error_message: string | null;
|
||||
last_email_subject: string | null; last_email_body: string | null;
|
||||
last_linkedin_message: string | null; workflow_id: string;
|
||||
}>;
|
||||
|
||||
if (allProfiles.length === 0) return;
|
||||
|
||||
// Load all workflow steps grouped by workflow_id, preserving their original ordering
|
||||
// We need the ORIGINAL step_order to map legacy current_step (0-based flat index) to tracks.
|
||||
// After the re-numbering above, step_order is now per-track. We stored the original order as the
|
||||
// sort key inside stepGroups above. Rebuild a per-workflow flat list from the original ordering.
|
||||
const workflowStepsOrig = db.prepare(
|
||||
"SELECT id, workflow_id, track, step_order FROM workflow_steps ORDER BY workflow_id, step_order"
|
||||
).all() as Array<{ id: string; workflow_id: string; track: string; step_order: number }>;
|
||||
|
||||
// For each workflow, build the flat list of steps in their original order (by new step_order within track, then track order linkedin < email)
|
||||
// BUT: we need the ORIGINAL flat order before re-numbering. Since we already re-numbered, we have to reconstruct.
|
||||
// Approach: the original flat step_order was cross-track. The re-numbered step_order is per-track and starts at 1.
|
||||
// We stored the original step_order in stepGroups (before modification). Use that.
|
||||
const origStepOrderMap = new Map<string, number>(); // step id → original flat step_order
|
||||
for (const row of stepGroups) {
|
||||
origStepOrderMap.set(row.id, row.step_order);
|
||||
}
|
||||
|
||||
// Per workflow: flat list of steps sorted by original step_order
|
||||
const workflowFlatSteps = new Map<string, Array<{ id: string; track: string; orig_order: number }>>();
|
||||
for (const step of workflowStepsOrig) {
|
||||
if (!workflowFlatSteps.has(step.workflow_id)) workflowFlatSteps.set(step.workflow_id, []);
|
||||
workflowFlatSteps.get(step.workflow_id)!.push({
|
||||
id: step.id,
|
||||
track: step.track,
|
||||
orig_order: origStepOrderMap.get(step.id) ?? step.step_order,
|
||||
});
|
||||
}
|
||||
for (const steps of workflowFlatSteps.values()) {
|
||||
steps.sort((a, b) => a.orig_order - b.orig_order);
|
||||
}
|
||||
|
||||
// Per workflow: which tracks exist
|
||||
const workflowTracks = new Map<string, Set<string>>();
|
||||
for (const step of workflowStepsOrig) {
|
||||
if (!workflowTracks.has(step.workflow_id)) workflowTracks.set(step.workflow_id, new Set());
|
||||
workflowTracks.get(step.workflow_id)!.add(step.track);
|
||||
}
|
||||
|
||||
const insertTrack = db.prepare(`
|
||||
INSERT OR IGNORE INTO run_profile_tracks
|
||||
(id, run_profile_id, track, state, current_step, last_step_at, next_step_at,
|
||||
error_message, last_email_subject, last_email_body, last_linkedin_message)
|
||||
VALUES (?, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
for (const rp of allProfiles) {
|
||||
const flatSteps = workflowFlatSteps.get(rp.workflow_id) ?? [];
|
||||
const tracks = workflowTracks.get(rp.workflow_id) ?? new Set(["linkedin"]);
|
||||
const terminalStates = new Set(["completed", "failed", "skipped"]);
|
||||
|
||||
if (terminalStates.has(rp.state)) {
|
||||
// Terminal profile — mark all tracks with the same terminal state
|
||||
for (const track of tracks) {
|
||||
insertTrack.run(
|
||||
randomUUID(), rp.id, track, rp.state, 0,
|
||||
null, rp.error_message ?? null, null, null, null
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Active profile — compute per-track completed step count
|
||||
// legacy current_step is 0-based index into the flat step list = steps already completed
|
||||
const completedCount = rp.current_step;
|
||||
const completedSteps = flatSteps.slice(0, completedCount);
|
||||
|
||||
const trackCompletedCount = new Map<string, number>();
|
||||
for (const track of tracks) trackCompletedCount.set(track, 0);
|
||||
for (const s of completedSteps) {
|
||||
trackCompletedCount.set(s.track, (trackCompletedCount.get(s.track) ?? 0) + 1);
|
||||
}
|
||||
|
||||
// Which track owned the step we were waiting on (index = completedCount)?
|
||||
const currentFlatStep = flatSteps[completedCount];
|
||||
const waitingTrack = currentFlatStep?.track ?? null;
|
||||
|
||||
for (const track of tracks) {
|
||||
const trackCurrentStep = trackCompletedCount.get(track) ?? 0;
|
||||
// next_step_at: only carry over to the track that was waiting; other track starts immediately
|
||||
const nextStepAt = track === waitingTrack ? rp.next_step_at : null;
|
||||
const lastEmailSubject = track === "email" ? rp.last_email_subject : null;
|
||||
const lastEmailBody = track === "email" ? rp.last_email_body : null;
|
||||
const lastLinkedinMessage = track === "linkedin" ? rp.last_linkedin_message : null;
|
||||
|
||||
insertTrack.run(
|
||||
randomUUID(), rp.id, track, rp.state, trackCurrentStep,
|
||||
nextStepAt, rp.error_message ?? null,
|
||||
lastEmailSubject, lastEmailBody, lastLinkedinMessage
|
||||
);
|
||||
}
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
function dropDeprecatedRunProfileColumns(db: Database.Database) {
|
||||
// Idempotent: check if state column still exists on run_profiles
|
||||
try {
|
||||
const tableInfo = db.prepare("PRAGMA table_info(run_profiles)").all() as { name: string }[];
|
||||
const hasState = tableInfo.some(c => c.name === "state");
|
||||
if (!hasState) return; // already dropped
|
||||
} catch { return; }
|
||||
|
||||
// SQLite requires a table rebuild to drop columns
|
||||
try {
|
||||
db.exec(`
|
||||
PRAGMA foreign_keys = OFF;
|
||||
CREATE TABLE run_profiles_new (
|
||||
id TEXT PRIMARY KEY,
|
||||
run_id TEXT REFERENCES runs(id) ON DELETE CASCADE,
|
||||
target_id TEXT REFERENCES targets(id),
|
||||
email_account_id TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
UNIQUE(run_id, target_id)
|
||||
);
|
||||
INSERT INTO run_profiles_new (id, run_id, target_id, email_account_id, created_at)
|
||||
SELECT id, run_id, target_id, email_account_id, created_at FROM run_profiles;
|
||||
DROP TABLE run_profiles;
|
||||
ALTER TABLE run_profiles_new RENAME TO run_profiles;
|
||||
PRAGMA foreign_keys = ON;
|
||||
`);
|
||||
} catch { /* ignore — may already be done */ }
|
||||
}
|
||||
|
||||
function runMigrations(db: Database.Database) {
|
||||
// Add columns introduced after initial schema — safe to run on existing DBs
|
||||
const migrations = [
|
||||
"ALTER TABLE targets ADD COLUMN degree INTEGER",
|
||||
"ALTER TABLE targets ADD COLUMN connection_requested_at TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN connected_at TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN message_sent_at TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN last_replied_at TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN linkedin_member_urn TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN sales_nav_url TEXT",
|
||||
"ALTER TABLE lists ADD COLUMN sales_nav_url TEXT",
|
||||
"ALTER TABLE accounts ADD COLUMN inbox_synced_at TEXT",
|
||||
"ALTER TABLE accounts ADD COLUMN active_hours_start INTEGER DEFAULT 9",
|
||||
"ALTER TABLE accounts ADD COLUMN active_hours_end INTEGER DEFAULT 18",
|
||||
"ALTER TABLE accounts ADD COLUMN timezone TEXT DEFAULT 'UTC'",
|
||||
"ALTER TABLE accounts ADD COLUMN working_days TEXT DEFAULT '1,2,3,4,5'",
|
||||
"ALTER TABLE workflow_steps ADD COLUMN connect_note TEXT",
|
||||
"ALTER TABLE workflow_steps ADD COLUMN message_body TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN headline TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN summary TEXT",
|
||||
"ALTER TABLE accounts ADD COLUMN accepted_sync_at TEXT",
|
||||
// Messaging identity (fsd_profile URN, "urn:li:fsd_profile:ACoAA...") — the
|
||||
// form the messaging GraphQL API returns. It is NOT convertible to the
|
||||
// numeric urn:li:member we store elsewhere, so we capture it on first reply
|
||||
// (matched by name) and then join by it directly on every later sync.
|
||||
"ALTER TABLE targets ADD COLUMN messaging_urn TEXT",
|
||||
"CREATE INDEX IF NOT EXISTS idx_targets_messaging_urn ON targets(messaging_urn)",
|
||||
// Boundary for the incremental connections sync: the createdAt of the newest
|
||||
// connection seen last run. NULL = never synced (first run does a full pass).
|
||||
"ALTER TABLE accounts ADD COLUMN connections_synced_through_ms INTEGER",
|
||||
"ALTER TABLE accounts ADD COLUMN li_connections INTEGER",
|
||||
"ALTER TABLE accounts ADD COLUMN li_pending INTEGER",
|
||||
"ALTER TABLE accounts ADD COLUMN li_profile_views INTEGER",
|
||||
"ALTER TABLE accounts ADD COLUMN li_stats_synced_at TEXT",
|
||||
`CREATE TABLE IF NOT EXISTS workflow_step_templates (
|
||||
step_id TEXT REFERENCES workflow_steps(id) ON DELETE CASCADE,
|
||||
template_id TEXT REFERENCES templates(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (step_id, template_id)
|
||||
)`,
|
||||
// Extended lead data from salesApiLeadSearch
|
||||
"ALTER TABLE targets ADD COLUMN object_urn TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN open_link INTEGER DEFAULT 0",
|
||||
"ALTER TABLE targets ADD COLUMN company_industry TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN company_location TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN tenure_months INTEGER",
|
||||
"ALTER TABLE targets ADD COLUMN spotlight_badges TEXT",
|
||||
// Profile enrichment — populated by visiting their Sales Nav profile page
|
||||
"ALTER TABLE targets ADD COLUMN positions_json TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN skills_json TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN enriched_profile_at TEXT",
|
||||
// Email outreach fields
|
||||
"ALTER TABLE targets ADD COLUMN email TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN email_replied_at TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN company_id TEXT",
|
||||
// Email account on runs (nullable — only needed when workflow has email steps)
|
||||
"ALTER TABLE runs ADD COLUMN email_account_id TEXT REFERENCES email_accounts(id)",
|
||||
// Workflow step email fields
|
||||
"ALTER TABLE workflow_steps ADD COLUMN email_subject TEXT",
|
||||
"ALTER TABLE workflow_steps ADD COLUMN email_body TEXT",
|
||||
// Apollo enrichment fields
|
||||
"ALTER TABLE targets ADD COLUMN apollo_id TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN seniority TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN apollo_functions TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN company_description TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN company_size INTEGER",
|
||||
"ALTER TABLE targets ADD COLUMN apollo_enriched_at TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN email_status TEXT",
|
||||
// Manual fields
|
||||
"ALTER TABLE targets ADD COLUMN notes TEXT",
|
||||
// Apollo extra person fields
|
||||
"ALTER TABLE targets ADD COLUMN city TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN country TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN time_zone TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN apollo_departments TEXT",
|
||||
// Apollo extra company fields on companies table
|
||||
"ALTER TABLE companies ADD COLUMN founded_year INTEGER",
|
||||
"ALTER TABLE companies ADD COLUMN logo_url TEXT",
|
||||
"ALTER TABLE companies ADD COLUMN phone TEXT",
|
||||
"ALTER TABLE companies ADD COLUMN annual_revenue TEXT",
|
||||
"ALTER TABLE companies ADD COLUMN technology_names TEXT",
|
||||
"ALTER TABLE companies ADD COLUMN keywords TEXT",
|
||||
"ALTER TABLE companies ADD COLUMN city TEXT",
|
||||
"ALTER TABLE companies ADD COLUMN country TEXT",
|
||||
// Email signature
|
||||
"ALTER TABLE email_accounts ADD COLUMN signature TEXT",
|
||||
// Reply-To override — if set, outgoing emails include Reply-To header
|
||||
"ALTER TABLE email_accounts ADD COLUMN reply_to TEXT",
|
||||
// Ramp-up: start slow and increase sending volume over time
|
||||
"ALTER TABLE email_accounts ADD COLUMN ramp_up_enabled INTEGER DEFAULT 1",
|
||||
"ALTER TABLE email_accounts ADD COLUMN ramp_start_date TEXT",
|
||||
// Company description and employee count moved from targets to companies
|
||||
"ALTER TABLE companies ADD COLUMN description TEXT",
|
||||
"ALTER TABLE companies ADD COLUMN employee_count INTEGER",
|
||||
// AI agent columns on workflow steps
|
||||
"ALTER TABLE workflow_steps ADD COLUMN ai_enabled INTEGER DEFAULT 0",
|
||||
"ALTER TABLE workflow_steps ADD COLUMN ai_model TEXT",
|
||||
"ALTER TABLE workflow_steps ADD COLUMN ai_prompt TEXT",
|
||||
"ALTER TABLE workflow_steps ADD COLUMN ai_max_words INTEGER",
|
||||
// Email step position — which followup number (1 = cold email, 2 = first followup, etc.)
|
||||
"ALTER TABLE workflow_steps ADD COLUMN email_position INTEGER DEFAULT 1",
|
||||
// Agent default model (stored on agent_config)
|
||||
"ALTER TABLE agent_config ADD COLUMN default_model TEXT",
|
||||
// Email threading — store sent email message-id for reply threading (future use)
|
||||
"ALTER TABLE run_profiles ADD COLUMN last_email_subject TEXT",
|
||||
"ALTER TABLE run_profiles ADD COLUMN last_email_body TEXT",
|
||||
// LinkedIn message follow-up tracking
|
||||
"ALTER TABLE workflow_steps ADD COLUMN message_position INTEGER DEFAULT 1",
|
||||
"ALTER TABLE run_profiles ADD COLUMN last_linkedin_message TEXT",
|
||||
// Language for AI-generated content per step
|
||||
"ALTER TABLE workflow_steps ADD COLUMN ai_language TEXT DEFAULT 'English'",
|
||||
// Campaign-level prompt — per-workflow AI context (USP, persona, tone for this campaign)
|
||||
"ALTER TABLE workflows ADD COLUMN prompt TEXT",
|
||||
// Email domain invalid flag on companies — set when a bounce is detected for any contact at this company
|
||||
"ALTER TABLE companies ADD COLUMN email_domain_invalid INTEGER DEFAULT 0",
|
||||
// Apollo extra person fields
|
||||
"ALTER TABLE targets ADD COLUMN email_domain_catchall INTEGER DEFAULT 0",
|
||||
// Per-profile email account assignment (multi-account routing)
|
||||
"ALTER TABLE run_profiles ADD COLUMN email_account_id TEXT",
|
||||
// Backfill: copy email_account_id from runs → run_profiles for existing records
|
||||
`UPDATE run_profiles SET email_account_id = (SELECT email_account_id FROM runs WHERE runs.id = run_profiles.run_id) WHERE email_account_id IS NULL`,
|
||||
// Import job progress tracking
|
||||
`CREATE TABLE IF NOT EXISTS list_imports (
|
||||
id TEXT PRIMARY KEY,
|
||||
list_id TEXT NOT NULL REFERENCES lists(id) ON DELETE CASCADE,
|
||||
status TEXT NOT NULL DEFAULT 'running',
|
||||
phase TEXT,
|
||||
page INTEGER DEFAULT 0,
|
||||
total_pages INTEGER DEFAULT 0,
|
||||
count INTEGER DEFAULT 0,
|
||||
total INTEGER DEFAULT 0,
|
||||
imported INTEGER DEFAULT 0,
|
||||
skipped INTEGER DEFAULT 0,
|
||||
error TEXT,
|
||||
started_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
finished_at TEXT
|
||||
)`,
|
||||
// Parallel tracks: add track column to workflow_steps
|
||||
"ALTER TABLE workflow_steps ADD COLUMN track TEXT NOT NULL DEFAULT 'linkedin' CHECK(track IN ('linkedin', 'email'))",
|
||||
// Parallel tracks: create run_profile_tracks table
|
||||
`CREATE TABLE IF NOT EXISTS run_profile_tracks (
|
||||
id TEXT PRIMARY KEY,
|
||||
run_profile_id TEXT NOT NULL REFERENCES run_profiles(id) ON DELETE CASCADE,
|
||||
track TEXT NOT NULL CHECK(track IN ('linkedin', 'email')),
|
||||
state TEXT NOT NULL DEFAULT 'pending' CHECK(state IN ('pending', 'in_progress', 'completed', 'failed', 'skipped')),
|
||||
current_step INTEGER NOT NULL DEFAULT 0,
|
||||
last_step_at TEXT,
|
||||
next_step_at TEXT,
|
||||
error_message TEXT,
|
||||
last_email_subject TEXT,
|
||||
last_email_body TEXT,
|
||||
last_linkedin_message TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
UNIQUE(run_profile_id, track)
|
||||
)`,
|
||||
"CREATE INDEX IF NOT EXISTS idx_run_profile_tracks_run_profile_id ON run_profile_tracks(run_profile_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_run_profile_tracks_state_next ON run_profile_tracks(state, next_step_at)",
|
||||
// Drop deprecated columns from run_profiles — all consumers now read from run_profile_tracks
|
||||
// SQLite does not support DROP COLUMN directly before 3.35; handled via table rebuild below
|
||||
// Separate IMAP credentials for custom mail providers where SMTP ≠ IMAP auth
|
||||
"ALTER TABLE email_accounts ADD COLUMN imap_username TEXT",
|
||||
"ALTER TABLE email_accounts ADD COLUMN imap_password TEXT",
|
||||
"ALTER TABLE workflows ADD COLUMN is_archived INTEGER NOT NULL DEFAULT 0",
|
||||
// Per-step signature override for email steps (null = use email account default)
|
||||
"ALTER TABLE workflow_steps ADD COLUMN email_signature TEXT",
|
||||
// CRM: todos per contact
|
||||
`CREATE TABLE IF NOT EXISTS todos (
|
||||
id TEXT PRIMARY KEY,
|
||||
target_id TEXT NOT NULL REFERENCES targets(id) ON DELETE CASCADE,
|
||||
title TEXT NOT NULL,
|
||||
due_date TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'open' CHECK(status IN ('open', 'done')),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
)`,
|
||||
"CREATE INDEX IF NOT EXISTS idx_todos_target_id ON todos(target_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_todos_status ON todos(status)",
|
||||
"ALTER TABLE todos ADD COLUMN description TEXT",
|
||||
// CRM: activity log per contact
|
||||
`CREATE TABLE IF NOT EXISTS activity_logs (
|
||||
id TEXT PRIMARY KEY,
|
||||
target_id TEXT NOT NULL REFERENCES targets(id) ON DELETE CASCADE,
|
||||
type TEXT NOT NULL DEFAULT 'note' CHECK(type IN ('call', 'email', 'meeting', 'note', 'other')),
|
||||
body TEXT NOT NULL,
|
||||
logged_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
)`,
|
||||
"CREATE INDEX IF NOT EXISTS idx_activity_logs_target_id ON activity_logs(target_id)",
|
||||
// Reply classifier: captured email replies + classifier verdict + dispatcher result
|
||||
`CREATE TABLE IF NOT EXISTS email_replies (
|
||||
id TEXT PRIMARY KEY,
|
||||
target_id TEXT NOT NULL REFERENCES targets(id) ON DELETE CASCADE,
|
||||
run_id TEXT REFERENCES runs(id) ON DELETE SET NULL,
|
||||
from_email TEXT NOT NULL,
|
||||
subject TEXT,
|
||||
body_text TEXT NOT NULL,
|
||||
received_at TEXT NOT NULL,
|
||||
classified_at TEXT,
|
||||
classification_json TEXT,
|
||||
classification_error TEXT,
|
||||
dispatched_at TEXT,
|
||||
dispatch_result_json TEXT,
|
||||
manually_edited INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
)`,
|
||||
"CREATE INDEX IF NOT EXISTS idx_email_replies_target_id ON email_replies(target_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_email_replies_dispatched_at ON email_replies(dispatched_at)",
|
||||
// Reply classifier verdict stamped on the target — enables reply-rate-by-kind metrics
|
||||
"ALTER TABLE targets ADD COLUMN reply_kind TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN inmail_sent_at TEXT",
|
||||
"ALTER TABLE targets ADD COLUMN posts_json TEXT", // recent posts from visit_profile
|
||||
"ALTER TABLE targets ADD COLUMN posts_scraped_at TEXT",
|
||||
// One-shot OOO reply context for the AI follow-up writer — set by the dispatcher,
|
||||
// read + cleared by the runner on the next email send. Distinct from last_email_body
|
||||
// (which holds the last email WE sent, used for follow-up threading).
|
||||
"ALTER TABLE run_profile_tracks ADD COLUMN pending_reply_context TEXT",
|
||||
// Removed the in-app chat agent (replaced by the hosted MCP endpoint at /api/mcp) — drop its tables.
|
||||
"DROP TABLE IF EXISTS chat_messages",
|
||||
"DROP TABLE IF EXISTS chat_sessions",
|
||||
// Batched/scheduled imports — split large lists across days under a daily cap.
|
||||
"ALTER TABLE list_imports ADD COLUMN account_id TEXT",
|
||||
"ALTER TABLE list_imports ADD COLUMN sales_nav_url TEXT",
|
||||
"ALTER TABLE list_imports ADD COLUMN scheduled_for TEXT", // 'YYYY-MM-DD'; NULL = run now
|
||||
"ALTER TABLE list_imports ADD COLUMN start_page INTEGER DEFAULT 1",
|
||||
"ALTER TABLE list_imports ADD COLUMN cap INTEGER", // max contacts for this batch
|
||||
"ALTER TABLE list_imports ADD COLUMN cancel_requested INTEGER DEFAULT 0",
|
||||
"ALTER TABLE list_imports ADD COLUMN batch_index INTEGER DEFAULT 1",
|
||||
"ALTER TABLE list_imports ADD COLUMN enrich INTEGER DEFAULT 0",
|
||||
"CREATE INDEX IF NOT EXISTS idx_list_imports_scheduled ON list_imports(status, scheduled_for)",
|
||||
// Simple app-wide key/value settings (e.g. daily_import_cap)
|
||||
`CREATE TABLE IF NOT EXISTS app_settings (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT,
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
)`,
|
||||
// Sales Nav InMail gets its own daily budget — separate from daily_message_limit,
|
||||
// since InMail (non-connections) and regular messages (connections) were being
|
||||
// gated off the same counter, starving one whenever the other was busy.
|
||||
"ALTER TABLE accounts ADD COLUMN daily_inmail_limit INTEGER DEFAULT 15",
|
||||
// Sales Nav search: persistent cache of resolved filter values (typeahead
|
||||
// ids). Dedup on (filter_type, id) — LinkedIn ids are stable (e.g. Berlin's
|
||||
// geoUrn never changes), so once resolved a value is reused forever with no
|
||||
// further live typeahead calls. Lives here (not ee/) per the open-core rule:
|
||||
// all migrations stay in lib/db.ts. Consumed only by the ee/ search feature.
|
||||
`CREATE TABLE IF NOT EXISTS search_filter_cache (
|
||||
id TEXT NOT NULL,
|
||||
filter_type TEXT NOT NULL,
|
||||
display_value TEXT NOT NULL,
|
||||
headline TEXT,
|
||||
query TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
PRIMARY KEY (filter_type, id)
|
||||
)`,
|
||||
"CREATE INDEX IF NOT EXISTS idx_sfc_type_query ON search_filter_cache(filter_type, query)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_sfc_type_display ON search_filter_cache(filter_type, display_value)",
|
||||
// CSV import: lists can be flagged as linkedin- or email-only, so the runner/UI
|
||||
// can warn before enrolling a purpose-mismatched list into a campaign.
|
||||
"ALTER TABLE lists ADD COLUMN purpose TEXT",
|
||||
// Manual/CSV-only field — no automation reads or writes this, reference data only.
|
||||
"ALTER TABLE targets ADD COLUMN phone TEXT",
|
||||
];
|
||||
for (const sql of migrations) {
|
||||
try { db.exec(sql); } catch { /* column already exists */ }
|
||||
}
|
||||
|
||||
// Parallel tracks: assign email steps to email track, re-number step_order, backfill run_profile_tracks
|
||||
runParallelTracksMigration(db);
|
||||
// Drop deprecated run_profiles columns (state, current_step, etc.) — consumers now read track-runs
|
||||
dropDeprecatedRunProfileColumns(db);
|
||||
|
||||
// Migrate workflow_steps CHECK constraint to allow 'delay' and 'email' step_types
|
||||
try {
|
||||
const tableInfo = db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='workflow_steps'").get() as { sql: string } | undefined;
|
||||
if (tableInfo && (!tableInfo.sql.includes("'delay'") || !tableInfo.sql.includes("'email'"))) {
|
||||
db.exec(`
|
||||
PRAGMA foreign_keys = OFF;
|
||||
CREATE TABLE workflow_steps_new (
|
||||
id TEXT PRIMARY KEY,
|
||||
workflow_id TEXT REFERENCES workflows(id) ON DELETE CASCADE,
|
||||
step_order INTEGER NOT NULL,
|
||||
step_type TEXT NOT NULL CHECK(step_type IN ('visit', 'connect', 'message', 'delay', 'email')),
|
||||
template_id TEXT REFERENCES templates(id),
|
||||
delay_seconds INTEGER DEFAULT 0,
|
||||
connect_note TEXT,
|
||||
message_body TEXT,
|
||||
email_subject TEXT,
|
||||
email_body TEXT,
|
||||
enabled INTEGER DEFAULT 1
|
||||
);
|
||||
INSERT INTO workflow_steps_new
|
||||
SELECT id, workflow_id, step_order, step_type, template_id, delay_seconds,
|
||||
connect_note, message_body,
|
||||
NULL, NULL,
|
||||
enabled
|
||||
FROM workflow_steps;
|
||||
DROP TABLE workflow_steps;
|
||||
ALTER TABLE workflow_steps_new RENAME TO workflow_steps;
|
||||
PRAGMA foreign_keys = ON;
|
||||
`);
|
||||
}
|
||||
} catch { /* migration already done */ }
|
||||
|
||||
// Allow the 'sales_inmail' step_type (Sales Navigator InMail). Rebuilds the
|
||||
// table preserving EVERY current column (the historical rebuild above only
|
||||
// copied the original columns — do NOT reuse it). InMail reuses message_body
|
||||
// for the body and email_subject for the required subject.
|
||||
try {
|
||||
const ti = db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='workflow_steps'").get() as { sql: string } | undefined;
|
||||
if (ti && !ti.sql.includes("'sales_inmail'")) {
|
||||
const cols = (db.prepare("PRAGMA table_info(workflow_steps)").all() as Array<{ name: string }>).map((c) => c.name);
|
||||
const colList = cols.join(", ");
|
||||
db.exec(`
|
||||
PRAGMA foreign_keys = OFF;
|
||||
CREATE TABLE workflow_steps_new (
|
||||
id TEXT PRIMARY KEY,
|
||||
workflow_id TEXT REFERENCES workflows(id) ON DELETE CASCADE,
|
||||
step_order INTEGER NOT NULL,
|
||||
step_type TEXT NOT NULL CHECK(step_type IN ('visit', 'connect', 'message', 'sales_inmail', 'delay', 'email')),
|
||||
template_id TEXT REFERENCES templates(id),
|
||||
delay_seconds INTEGER DEFAULT 0,
|
||||
connect_note TEXT,
|
||||
message_body TEXT,
|
||||
email_subject TEXT,
|
||||
email_body TEXT,
|
||||
enabled INTEGER DEFAULT 1,
|
||||
ai_enabled INTEGER DEFAULT 0,
|
||||
ai_model TEXT,
|
||||
ai_prompt TEXT,
|
||||
ai_max_words INTEGER,
|
||||
email_position INTEGER DEFAULT 1,
|
||||
message_position INTEGER DEFAULT 1,
|
||||
ai_language TEXT DEFAULT 'English',
|
||||
track TEXT NOT NULL DEFAULT 'linkedin' CHECK(track IN ('linkedin', 'email')),
|
||||
email_signature TEXT
|
||||
);
|
||||
INSERT INTO workflow_steps_new (${colList}) SELECT ${colList} FROM workflow_steps;
|
||||
DROP TABLE workflow_steps;
|
||||
ALTER TABLE workflow_steps_new RENAME TO workflow_steps;
|
||||
PRAGMA foreign_keys = ON;
|
||||
`);
|
||||
}
|
||||
} catch { /* migration already done */ }
|
||||
|
||||
// CSV import: allow email-only targets (no LinkedIn URL). targets.linkedin_url was
|
||||
// NOT NULL UNIQUE from the base schema — rebuild to make it nullable (still UNIQUE,
|
||||
// SQLite allows multiple NULLs under UNIQUE) preserving EVERY current column, same
|
||||
// pattern as the sales_inmail rebuild above (do not reuse the older historical rebuilds
|
||||
// that only copied the original columns).
|
||||
try {
|
||||
const ti = db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='targets'").get() as { sql: string } | undefined;
|
||||
if (ti && ti.sql.includes("linkedin_url TEXT NOT NULL")) {
|
||||
const cols = (db.prepare("PRAGMA table_info(targets)").all() as Array<{ name: string; type: string; notnull: number; dflt_value: string | null }>);
|
||||
const colDefs = cols.map((c) => {
|
||||
if (c.name === "linkedin_url") return "linkedin_url TEXT UNIQUE";
|
||||
if (c.name === "id") return "id TEXT PRIMARY KEY";
|
||||
const notnull = c.notnull ? " NOT NULL" : "";
|
||||
// Non-literal defaults (e.g. datetime('now')) must be parenthesized in SQLite DDL.
|
||||
const isLiteral = c.dflt_value === null || /^-?\d+(\.\d+)?$/.test(c.dflt_value) || /^'.*'$/.test(c.dflt_value);
|
||||
const dflt = c.dflt_value !== null ? ` DEFAULT ${isLiteral ? c.dflt_value : `(${c.dflt_value})`}` : "";
|
||||
return `${c.name} ${c.type}${notnull}${dflt}`;
|
||||
});
|
||||
const colList = cols.map((c) => c.name).join(", ");
|
||||
db.exec(`
|
||||
PRAGMA foreign_keys = OFF;
|
||||
CREATE TABLE targets_new (
|
||||
${colDefs.join(",\n ")}
|
||||
);
|
||||
INSERT INTO targets_new (${colList}) SELECT ${colList} FROM targets;
|
||||
DROP TABLE targets;
|
||||
ALTER TABLE targets_new RENAME TO targets;
|
||||
PRAGMA foreign_keys = ON;
|
||||
`);
|
||||
}
|
||||
} catch { /* migration already done */ }
|
||||
|
||||
// Backfill: move company_description and company_size from targets into companies
|
||||
try {
|
||||
db.exec(`
|
||||
UPDATE companies
|
||||
SET
|
||||
description = COALESCE(description, (
|
||||
SELECT t.company_description FROM targets t
|
||||
WHERE t.company_id = companies.id AND t.company_description IS NOT NULL
|
||||
LIMIT 1
|
||||
)),
|
||||
employee_count = COALESCE(employee_count, (
|
||||
SELECT t.company_size FROM targets t
|
||||
WHERE t.company_id = companies.id AND t.company_size IS NOT NULL
|
||||
LIMIT 1
|
||||
))
|
||||
WHERE description IS NULL OR employee_count IS NULL
|
||||
`);
|
||||
} catch { /* ignore */ }
|
||||
|
||||
// Backfill: for old records where linkedin_url is a Sales Nav URL, move it to sales_nav_url
|
||||
try {
|
||||
db.exec(`
|
||||
UPDATE targets
|
||||
SET sales_nav_url = linkedin_url
|
||||
WHERE linkedin_url LIKE '%/sales/lead/%' AND (sales_nav_url IS NULL OR sales_nav_url = '')
|
||||
`);
|
||||
} catch { /* ignore */ }
|
||||
|
||||
// Create unique index on run_profiles if not already present (idempotent)
|
||||
try {
|
||||
db.exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_run_profiles_unique ON run_profiles(run_id, target_id);");
|
||||
} catch { /* ignore */ }
|
||||
|
||||
encryptLegacySecretsMigration(db);
|
||||
}
|
||||
|
||||
// One-time (per-row) migration: encrypt any plaintext accounts.cookies_json,
|
||||
// email_accounts.password/imap_password, integrations.api_key left over from before
|
||||
// encryption-at-rest was added. isEncrypted() lets this run safely on every boot — already
|
||||
//-encrypted rows (real "v1:" ciphertext) are skipped, so this is idempotent and cheap once
|
||||
// migrated. See lib/crypto.ts for the format and lib/premium.ts-style boundary notes.
|
||||
function encryptLegacySecretsMigration(db: Database.Database) {
|
||||
const accounts = db.prepare("SELECT id, cookies_json FROM accounts WHERE cookies_json IS NOT NULL").all() as
|
||||
{ id: string; cookies_json: string }[];
|
||||
for (const row of accounts) {
|
||||
if (isEncrypted(row.cookies_json)) continue;
|
||||
db.prepare("UPDATE accounts SET cookies_json = ? WHERE id = ?").run(encryptSecret(row.cookies_json), row.id);
|
||||
}
|
||||
|
||||
const emailAccounts = db
|
||||
.prepare("SELECT id, password, imap_password FROM email_accounts")
|
||||
.all() as { id: string; password: string; imap_password: string | null }[];
|
||||
for (const row of emailAccounts) {
|
||||
const needsPassword = !isEncrypted(row.password);
|
||||
const needsImapPassword = row.imap_password !== null && !isEncrypted(row.imap_password);
|
||||
if (!needsPassword && !needsImapPassword) continue;
|
||||
db.prepare("UPDATE email_accounts SET password = ?, imap_password = ? WHERE id = ?").run(
|
||||
needsPassword ? encryptSecret(row.password) : row.password,
|
||||
needsImapPassword ? encryptSecret(row.imap_password!) : row.imap_password,
|
||||
row.id
|
||||
);
|
||||
}
|
||||
|
||||
const integrations = db.prepare("SELECT key, api_key FROM integrations WHERE api_key IS NOT NULL").all() as
|
||||
{ key: string; api_key: string }[];
|
||||
for (const row of integrations) {
|
||||
if (isEncrypted(row.api_key)) continue;
|
||||
db.prepare("UPDATE integrations SET api_key = ? WHERE key = ?").run(encryptSecret(row.api_key), row.key);
|
||||
}
|
||||
}
|
||||
|
||||
function initDb(db: Database.Database) {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS accounts (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
email TEXT NOT NULL UNIQUE,
|
||||
cookies_json TEXT,
|
||||
is_authenticated INTEGER DEFAULT 0,
|
||||
daily_connection_limit INTEGER DEFAULT 20,
|
||||
daily_message_limit INTEGER DEFAULT 50,
|
||||
daily_inmail_limit INTEGER DEFAULT 15,
|
||||
active_hours_start INTEGER DEFAULT 9,
|
||||
active_hours_end INTEGER DEFAULT 18,
|
||||
timezone TEXT DEFAULT 'UTC',
|
||||
working_days TEXT DEFAULT '1,2,3,4,5',
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS targets (
|
||||
id TEXT PRIMARY KEY,
|
||||
linkedin_url TEXT UNIQUE,
|
||||
sales_nav_url TEXT,
|
||||
first_name TEXT,
|
||||
last_name TEXT,
|
||||
full_name TEXT,
|
||||
title TEXT,
|
||||
company TEXT,
|
||||
location TEXT,
|
||||
profile_image_url TEXT,
|
||||
degree INTEGER,
|
||||
connection_requested_at TEXT,
|
||||
connected_at TEXT,
|
||||
message_sent_at TEXT,
|
||||
last_replied_at TEXT,
|
||||
linkedin_member_urn TEXT,
|
||||
enriched_at TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS lists (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
sales_nav_url TEXT,
|
||||
purpose TEXT CHECK(purpose IN ('linkedin', 'email')),
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS list_targets (
|
||||
list_id TEXT REFERENCES lists(id) ON DELETE CASCADE,
|
||||
target_id TEXT REFERENCES targets(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (list_id, target_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS templates (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
body TEXT NOT NULL,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS workflows (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS workflow_steps (
|
||||
id TEXT PRIMARY KEY,
|
||||
workflow_id TEXT REFERENCES workflows(id) ON DELETE CASCADE,
|
||||
step_order INTEGER NOT NULL,
|
||||
step_type TEXT NOT NULL CHECK(step_type IN ('visit', 'connect', 'message', 'delay')),
|
||||
template_id TEXT REFERENCES templates(id),
|
||||
delay_seconds INTEGER DEFAULT 0,
|
||||
connect_note TEXT,
|
||||
message_body TEXT,
|
||||
enabled INTEGER DEFAULT 1
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS workflow_step_templates (
|
||||
step_id TEXT REFERENCES workflow_steps(id) ON DELETE CASCADE,
|
||||
template_id TEXT REFERENCES templates(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (step_id, template_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS runs (
|
||||
id TEXT PRIMARY KEY,
|
||||
workflow_id TEXT REFERENCES workflows(id),
|
||||
list_id TEXT REFERENCES lists(id),
|
||||
account_id TEXT REFERENCES accounts(id),
|
||||
status TEXT DEFAULT 'pending' CHECK(status IN ('pending', 'running', 'paused', 'completed', 'failed')),
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
started_at TEXT,
|
||||
completed_at TEXT,
|
||||
runner_pid INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS run_profiles (
|
||||
id TEXT PRIMARY KEY,
|
||||
run_id TEXT REFERENCES runs(id) ON DELETE CASCADE,
|
||||
target_id TEXT REFERENCES targets(id),
|
||||
state TEXT DEFAULT 'pending' CHECK(state IN ('pending', 'in_progress', 'completed', 'failed', 'skipped')),
|
||||
current_step INTEGER DEFAULT 0,
|
||||
last_step_at TEXT,
|
||||
next_step_at TEXT,
|
||||
error_message TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
UNIQUE(run_id, target_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS logs (
|
||||
id TEXT PRIMARY KEY,
|
||||
run_id TEXT REFERENCES runs(id) ON DELETE CASCADE,
|
||||
target_id TEXT REFERENCES targets(id),
|
||||
level TEXT DEFAULT 'info' CHECK(level IN ('info', 'warn', 'error')),
|
||||
message TEXT NOT NULL,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id TEXT PRIMARY KEY,
|
||||
email TEXT NOT NULL UNIQUE,
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS companies (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
domain TEXT,
|
||||
industry TEXT,
|
||||
location TEXT,
|
||||
linkedin_url TEXT,
|
||||
website TEXT,
|
||||
notes TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS integrations (
|
||||
key TEXT PRIMARY KEY,
|
||||
api_key TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
updated_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS agent_config (
|
||||
id INTEGER PRIMARY KEY DEFAULT 1,
|
||||
system_prompt TEXT,
|
||||
user_prompt TEXT,
|
||||
email_examples TEXT,
|
||||
linkedin_examples TEXT,
|
||||
updated_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS agent_sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
run_id TEXT,
|
||||
target_id TEXT,
|
||||
step_id TEXT,
|
||||
model TEXT,
|
||||
input_tokens INTEGER,
|
||||
output_tokens INTEGER,
|
||||
cost_usd REAL,
|
||||
prompt TEXT,
|
||||
generated_text TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
-- Hosted MCP OAuth 2.1 server: clients (DCR), single-use auth codes, access/refresh tokens.
|
||||
-- Tokens/codes are stored only as sha256 hashes. Reuses NEXTAUTH user identity.
|
||||
CREATE TABLE IF NOT EXISTS oauth_clients (
|
||||
client_id TEXT PRIMARY KEY,
|
||||
client_name TEXT,
|
||||
redirect_uris TEXT NOT NULL,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS oauth_auth_codes (
|
||||
code_hash TEXT PRIMARY KEY,
|
||||
client_id TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
redirect_uri TEXT NOT NULL,
|
||||
code_challenge TEXT NOT NULL,
|
||||
scope TEXT,
|
||||
expires_at TEXT NOT NULL,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS oauth_tokens (
|
||||
id TEXT PRIMARY KEY,
|
||||
access_hash TEXT NOT NULL UNIQUE,
|
||||
refresh_hash TEXT UNIQUE,
|
||||
client_id TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
scope TEXT,
|
||||
expires_at TEXT NOT NULL,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS email_accounts (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
from_email TEXT NOT NULL,
|
||||
from_name TEXT,
|
||||
smtp_host TEXT NOT NULL,
|
||||
smtp_port INTEGER DEFAULT 587,
|
||||
smtp_secure INTEGER DEFAULT 0,
|
||||
imap_host TEXT,
|
||||
imap_port INTEGER DEFAULT 993,
|
||||
username TEXT NOT NULL,
|
||||
password TEXT NOT NULL,
|
||||
daily_email_limit INTEGER DEFAULT 50,
|
||||
active_hours_start INTEGER DEFAULT 9,
|
||||
active_hours_end INTEGER DEFAULT 18,
|
||||
timezone TEXT DEFAULT 'UTC',
|
||||
working_days TEXT DEFAULT '1,2,3,4,5',
|
||||
is_verified INTEGER DEFAULT 0,
|
||||
inbox_synced_at TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
`);
|
||||
}
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
declare module 'imap-simple';
|
||||
@@ -0,0 +1,367 @@
|
||||
import Imap from "imap";
|
||||
import { simpleParser } from "mailparser";
|
||||
import { randomUUID } from "crypto";
|
||||
import { getDb } from "@/lib/db";
|
||||
import { premium } from "@/lib/premium";
|
||||
import { decryptSecret } from "@/lib/crypto";
|
||||
|
||||
const IMAP_POLL_INTERVAL_MS = 2 * 60 * 60 * 1000; // 2 hours
|
||||
// Jul 2026 incident: all email accounts became "due" in the same tick and synced
|
||||
// back-to-back (one 20-account burst ran 3 minutes straight, one account alone took
|
||||
// 85s), which lined up with NocoDB/n8n healthcheck failures on the shared host. A
|
||||
// deterministic per-account offset spreads accounts across a window so they drift
|
||||
// apart over successive cycles instead of staying clustered together forever.
|
||||
const IMAP_POLL_JITTER_MS = 30 * 60 * 1000; // 30 min
|
||||
|
||||
function accountJitterMs(accountId: string): number {
|
||||
let h = 0;
|
||||
for (let i = 0; i < accountId.length; i++) h = (h * 31 + accountId.charCodeAt(i)) >>> 0;
|
||||
return h % IMAP_POLL_JITTER_MS;
|
||||
}
|
||||
|
||||
const BOUNCE_SENDER_PATTERNS = [
|
||||
/mailer-daemon@/i,
|
||||
/postmaster@/i,
|
||||
/mail-delivery-subsystem@/i,
|
||||
/delivery-status@/i,
|
||||
/amazonses\.com$/i,
|
||||
];
|
||||
|
||||
function extractEmails(text: string): string[] {
|
||||
return [...text.matchAll(/[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}/g)].map(m => m[0].toLowerCase());
|
||||
}
|
||||
|
||||
function isBounce(fromEmail: string): boolean {
|
||||
return BOUNCE_SENDER_PATTERNS.some(p => p.test(fromEmail));
|
||||
}
|
||||
|
||||
function parseHeaderValue(raw: string, field: string): string {
|
||||
const regex = new RegExp(`^${field}:[ \\t]*(.+?)(?=\\r?\\n[^\\s]|$)`, "im");
|
||||
const m = raw.match(regex);
|
||||
if (!m) return "";
|
||||
return m[1].replace(/\r?\n[\t ]+/g, " ").trim();
|
||||
}
|
||||
|
||||
interface EmailAccount {
|
||||
id: string;
|
||||
imap_host: string | null;
|
||||
imap_port: number | null;
|
||||
username: string;
|
||||
password: string;
|
||||
imap_username: string | null;
|
||||
imap_password: string | null;
|
||||
inbox_synced_at: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the body + headers for a given UID and inserts a row into
|
||||
* `email_replies` (idempotent — skips if a row with the same target_id +
|
||||
* received_at already exists). Best-effort: errors are swallowed by the caller.
|
||||
*/
|
||||
export function captureReplyBody(
|
||||
imap: Imap,
|
||||
db: ReturnType<typeof getDb>,
|
||||
targetId: string,
|
||||
fromEmail: string,
|
||||
uid: number,
|
||||
): Promise<string | null> {
|
||||
return new Promise<string | null>((resolve) => {
|
||||
// Fetch the full raw RFC822 message — mailparser handles MIME multipart,
|
||||
// base64 / quoted-printable transfer encodings, and charset decoding. The
|
||||
// old HEADER+TEXT regex approach stored raw base64 / =XX escapes for the
|
||||
// common German auto-reply formats, feeding the classifier garbage.
|
||||
const fetch = imap.fetch(uid, { bodies: [""], struct: false });
|
||||
|
||||
const chunks: Buffer[] = [];
|
||||
|
||||
fetch.on("message", (msg) => {
|
||||
msg.on("body", (stream) => {
|
||||
stream.on("data", (c: Buffer) => chunks.push(c));
|
||||
});
|
||||
});
|
||||
|
||||
fetch.once("error", () => resolve(null));
|
||||
fetch.once("end", () => {
|
||||
void (async () => {
|
||||
try {
|
||||
const raw = Buffer.concat(chunks);
|
||||
const parsed = await simpleParser(raw);
|
||||
|
||||
const subject = parsed.subject ?? null;
|
||||
const parsedFrom =
|
||||
parsed.from?.value?.[0]?.address?.toLowerCase().trim() || fromEmail.toLowerCase();
|
||||
const receivedAt = parsed.date ? parsed.date.toISOString() : new Date().toISOString();
|
||||
|
||||
// Prefer decoded plain text; fall back to stripping the HTML part.
|
||||
const rawText =
|
||||
parsed.text ??
|
||||
(parsed.html ? parsed.html.replace(/<[^>]+>/g, " ") : "");
|
||||
const bodyText = rawText
|
||||
.replace(/\r\n/g, "\n")
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.replace(/[ \t]{2,}/g, " ")
|
||||
.trim()
|
||||
.slice(0, 16_000);
|
||||
|
||||
if (!bodyText) { resolve(null); return; }
|
||||
|
||||
// Dedup: skip if we already have a row for this target with the same received_at
|
||||
const existing = db.prepare(
|
||||
"SELECT id FROM email_replies WHERE target_id = ? AND received_at = ?"
|
||||
).get(targetId, receivedAt) as { id: string } | undefined;
|
||||
|
||||
if (existing) { resolve(null); return; }
|
||||
|
||||
// Look up the most recent active run for this target — the dispatcher needs
|
||||
// it to find the email track to reschedule / enroll a substitute into.
|
||||
const runRow = db.prepare(
|
||||
`SELECT r.id FROM runs r
|
||||
JOIN run_profiles rp ON rp.run_id = r.id
|
||||
WHERE rp.target_id = ? AND r.status IN ('running', 'paused')
|
||||
ORDER BY r.created_at DESC LIMIT 1`
|
||||
).get(targetId) as { id: string } | undefined;
|
||||
|
||||
const replyId = randomUUID();
|
||||
db.prepare(
|
||||
`INSERT INTO email_replies (id, target_id, run_id, from_email, subject, body_text, received_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`
|
||||
).run(replyId, targetId, runRow?.id ?? null, parsedFrom, subject, bodyText, receivedAt);
|
||||
resolve(replyId);
|
||||
} catch (err) {
|
||||
console.warn(`[email-inbox] captureReplyBody parse/insert failed:`, err);
|
||||
resolve(null);
|
||||
}
|
||||
})();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function shouldSyncEmailInbox(emailAccountId: string): boolean {
|
||||
const db = getDb();
|
||||
const account = db
|
||||
.prepare("SELECT inbox_synced_at FROM email_accounts WHERE id = ?")
|
||||
.get(emailAccountId) as { inbox_synced_at: string | null } | undefined;
|
||||
if (!account?.inbox_synced_at) return true;
|
||||
const dueAfterMs = IMAP_POLL_INTERVAL_MS + accountJitterMs(emailAccountId);
|
||||
return Date.now() - new Date(account.inbox_synced_at).getTime() >= dueAfterMs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens one IMAP connection, then for each lead that has been emailed but
|
||||
* not yet replied, runs a server-side FROM search. Only touches the mailbox
|
||||
* index — never downloads message bodies for reply detection.
|
||||
*
|
||||
* Also scans the last 50 messages for bounces (mailer-daemon etc.).
|
||||
*/
|
||||
export async function syncEmailInbox(emailAccountId: string): Promise<{ replies: number; bounces: number }> {
|
||||
const db = getDb();
|
||||
|
||||
const account = db
|
||||
.prepare("SELECT id, imap_host, imap_port, username, password, imap_username, imap_password, inbox_synced_at FROM email_accounts WHERE id = ?")
|
||||
.get(emailAccountId) as EmailAccount | undefined;
|
||||
|
||||
if (!account?.imap_host) {
|
||||
console.warn(`[email-inbox] Account ${emailAccountId} has no IMAP config — skipping`);
|
||||
return { replies: 0, bounces: 0 };
|
||||
}
|
||||
|
||||
// Leads that were emailed via this account and haven't replied yet
|
||||
const pendingTargets = db.prepare(`
|
||||
SELECT DISTINCT t.id, t.email
|
||||
FROM targets t
|
||||
JOIN run_profiles rp ON rp.target_id = t.id
|
||||
JOIN run_profile_tracks rt ON rt.run_profile_id = rp.id
|
||||
WHERE t.email IS NOT NULL
|
||||
AND t.email_replied_at IS NULL
|
||||
AND t.email_status != 'invalid'
|
||||
AND rt.track = 'email'
|
||||
AND rt.state NOT IN ('pending')
|
||||
AND rp.email_account_id = ?
|
||||
`).all(emailAccountId) as { id: string; email: string }[];
|
||||
|
||||
if (pendingTargets.length === 0) {
|
||||
db.prepare("UPDATE email_accounts SET inbox_synced_at = datetime('now') WHERE id = ?").run(emailAccountId);
|
||||
return { replies: 0, bounces: 0 };
|
||||
}
|
||||
|
||||
console.log(`[email-inbox] Checking ${pendingTargets.length} leads via IMAP FROM search`);
|
||||
|
||||
const imapUser = account.imap_username ?? account.username;
|
||||
const imapPass = decryptSecret(account.imap_password) ?? decryptSecret(account.password)!;
|
||||
|
||||
let replies = 0;
|
||||
let bounces = 0;
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
const imap = new Imap({
|
||||
host: account.imap_host!,
|
||||
port: account.imap_port ?? 993,
|
||||
tls: true,
|
||||
tlsOptions: { rejectUnauthorized: false },
|
||||
user: imapUser,
|
||||
password: imapPass,
|
||||
authTimeout: 10_000,
|
||||
connTimeout: 12_000,
|
||||
});
|
||||
|
||||
const done = () => {
|
||||
try { imap.end(); } catch { /* ignore */ }
|
||||
resolve();
|
||||
};
|
||||
|
||||
imap.once("error", (err: Error) => {
|
||||
console.warn(`[email-inbox] IMAP error for account ${emailAccountId}:`, err.message);
|
||||
done();
|
||||
});
|
||||
|
||||
imap.once("ready", () => {
|
||||
imap.openBox("INBOX", true, async (err, box) => {
|
||||
if (err || !box) { console.warn("[email-inbox] openBox failed:", err?.message); done(); return; }
|
||||
|
||||
// ── Reply detection: one FROM search per lead ──────────────────────────
|
||||
// Run sequentially — the imap library serialises commands over one TCP connection
|
||||
for (const target of pendingTargets) {
|
||||
await new Promise<void>((resSearch) => {
|
||||
imap.search([["FROM", target.email]], async (searchErr, uids) => {
|
||||
if (searchErr) { resSearch(); return; }
|
||||
if (uids.length > 0) {
|
||||
console.log(`[email-inbox] Reply detected for ${target.email} (target ${target.id})`);
|
||||
replies++;
|
||||
|
||||
// Capture the body, then let the classifier+dispatcher decide the action.
|
||||
// We no longer eagerly stamp email_replied_at here — the dispatcher does it
|
||||
// for skip-buckets (call_task / human_reply / not_interested) but leaves it
|
||||
// unset for OOO follow-ups so the runner keeps the contact enrolled.
|
||||
// On any failure the contact stays enrolled (safe fallback, plan §3.2).
|
||||
try {
|
||||
const latestUid = uids[uids.length - 1];
|
||||
const replyId = await captureReplyBody(imap, db, target.id, target.email, latestUid);
|
||||
// The reply is always STORED (open-core). AI classification + auto-followup
|
||||
// is a premium feature — skipped cleanly when ee/ is absent.
|
||||
if (replyId && premium?.replies) {
|
||||
await premium.replies.classifyAndDispatch(replyId);
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn(`[email-inbox] Failed to capture/dispatch reply for ${target.email}:`, err);
|
||||
}
|
||||
}
|
||||
resSearch();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// ── Bounce detection: scan last 50 messages for mailer-daemon ─────────
|
||||
if (box.messages.total > 0) {
|
||||
const total = box.messages.total;
|
||||
const start = Math.max(1, total - 49);
|
||||
const range = `${start}:${total}`;
|
||||
|
||||
await new Promise<void>((resFetch) => {
|
||||
const fetch = imap.seq.fetch(range, {
|
||||
bodies: ["HEADER.FIELDS (FROM TO)", "TEXT"],
|
||||
struct: false,
|
||||
});
|
||||
|
||||
type RawMsg = { header: string; body: string };
|
||||
const msgs: RawMsg[] = [];
|
||||
|
||||
fetch.on("message", (msg) => {
|
||||
const entry: RawMsg = { header: "", body: "" };
|
||||
msg.on("body", (stream, info) => {
|
||||
const chunks: Buffer[] = [];
|
||||
stream.on("data", (c: Buffer) => chunks.push(c));
|
||||
stream.once("end", () => {
|
||||
const text = Buffer.concat(chunks).toString();
|
||||
if (info.which.startsWith("HEADER")) entry.header = text;
|
||||
else entry.body = text.slice(0, 3000);
|
||||
});
|
||||
});
|
||||
msg.once("end", () => msgs.push(entry));
|
||||
});
|
||||
|
||||
fetch.once("error", () => resFetch());
|
||||
fetch.once("end", () => {
|
||||
for (const msg of msgs) {
|
||||
const fromRaw = parseHeaderValue(msg.header, "From");
|
||||
const toRaw = parseHeaderValue(msg.header, "To");
|
||||
|
||||
const emailMatch = fromRaw.match(/<([^>]+)>/) ?? fromRaw.match(/([^\s]+@[^\s]+)/);
|
||||
const fromEmail = emailMatch?.[1]?.toLowerCase().trim();
|
||||
if (!fromEmail || !isBounce(fromEmail)) continue;
|
||||
|
||||
// Also extract Final-Recipient from DSN bodies (SES bounce format)
|
||||
const finalRecipient = msg.body.match(/Final-Recipient:\s*rfc822;\s*([^\s\r\n]+)/i)?.[1] ?? "";
|
||||
const candidates = extractEmails(msg.body + " " + toRaw + " " + finalRecipient);
|
||||
for (const candidate of candidates) {
|
||||
if (BOUNCE_SENDER_PATTERNS.some(p => p.test(candidate))) continue;
|
||||
|
||||
const target = db
|
||||
.prepare("SELECT id, email_status, company_id FROM targets WHERE lower(email) = ?")
|
||||
.get(candidate) as { id: string; email_status: string | null; company_id: string | null } | undefined;
|
||||
|
||||
if (!target || target.email_status === "invalid") continue;
|
||||
|
||||
const note = `Email bounced on ${new Date().toISOString().slice(0, 10)} — marked invalid`;
|
||||
db.prepare(`
|
||||
UPDATE targets SET email_status = 'invalid',
|
||||
notes = CASE WHEN notes IS NULL OR notes = '' THEN ? ELSE notes || char(10) || ? END
|
||||
WHERE id = ?
|
||||
`).run(note, note, target.id);
|
||||
|
||||
db.prepare(`
|
||||
UPDATE run_profile_tracks SET state = 'skipped', error_message = 'Email bounced — invalid address'
|
||||
WHERE run_profile_id IN (SELECT id FROM run_profiles WHERE target_id = ?)
|
||||
AND state IN ('pending', 'in_progress')
|
||||
`).run(target.id);
|
||||
|
||||
if (target.company_id) {
|
||||
const companyNote = `Email domain flagged invalid — bounce for ${candidate} on ${new Date().toISOString().slice(0, 10)}`;
|
||||
db.prepare(`
|
||||
UPDATE companies SET email_domain_invalid = 1,
|
||||
notes = CASE WHEN notes IS NULL OR notes = '' THEN ? ELSE notes || char(10) || ? END
|
||||
WHERE id = ?
|
||||
`).run(companyNote, companyNote, target.company_id);
|
||||
|
||||
const siblings = db.prepare(`
|
||||
SELECT id FROM targets WHERE company_id = ? AND id != ? AND email IS NOT NULL AND email_status != 'invalid'
|
||||
`).all(target.company_id, target.id) as { id: string }[];
|
||||
|
||||
for (const sibling of siblings) {
|
||||
const sibNote = `Email bounced on ${new Date().toISOString().slice(0, 10)} — marked invalid (domain flagged via company)`;
|
||||
db.prepare(`
|
||||
UPDATE targets SET email_status = 'invalid',
|
||||
notes = CASE WHEN notes IS NULL OR notes = '' THEN ? ELSE notes || char(10) || ? END
|
||||
WHERE id = ?
|
||||
`).run(sibNote, sibNote, sibling.id);
|
||||
db.prepare(`
|
||||
UPDATE run_profile_tracks SET state = 'skipped', error_message = 'Email domain invalid — company flagged'
|
||||
WHERE run_profile_id IN (SELECT id FROM run_profiles WHERE target_id = ?)
|
||||
AND state IN ('pending', 'in_progress')
|
||||
`).run(sibling.id);
|
||||
}
|
||||
|
||||
if (siblings.length > 0) {
|
||||
console.log(`[email-inbox] Company ${target.company_id} flagged — ${siblings.length} sibling(s) marked invalid`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[email-inbox] Bounce for ${candidate} (target ${target.id}) — marked invalid`);
|
||||
bounces++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
resFetch();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
imap.connect();
|
||||
});
|
||||
|
||||
db.prepare("UPDATE email_accounts SET inbox_synced_at = datetime('now') WHERE id = ?").run(emailAccountId);
|
||||
return { replies, bounces };
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
import nodemailer from "nodemailer";
|
||||
import Imap from "imap";
|
||||
|
||||
export interface EmailAccount {
|
||||
id: string;
|
||||
from_email: string;
|
||||
from_name: string | null;
|
||||
reply_to?: string | null;
|
||||
smtp_host: string;
|
||||
smtp_port: number;
|
||||
smtp_secure: number; // 0 = STARTTLS, 1 = SSL
|
||||
username: string;
|
||||
password: string;
|
||||
}
|
||||
|
||||
export async function sendEmail(
|
||||
account: EmailAccount,
|
||||
to: string,
|
||||
subject: string,
|
||||
body: string
|
||||
): Promise<void> {
|
||||
const transporter = nodemailer.createTransport({
|
||||
host: account.smtp_host,
|
||||
port: account.smtp_port,
|
||||
secure: account.smtp_secure === 1,
|
||||
auth: {
|
||||
user: account.username,
|
||||
pass: account.password,
|
||||
},
|
||||
// Allow self-signed certs (common in some corp SMTP setups)
|
||||
tls: { rejectUnauthorized: false },
|
||||
});
|
||||
|
||||
const from = account.from_name
|
||||
? `"${account.from_name}" <${account.from_email}>`
|
||||
: account.from_email;
|
||||
|
||||
await transporter.sendMail({
|
||||
from, to, subject, text: body,
|
||||
...(account.reply_to ? { replyTo: account.reply_to } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies SMTP connectivity — used by the test-connection endpoint.
|
||||
* Returns null on success, error message string on failure.
|
||||
*/
|
||||
export async function testSmtpConnection(account: Omit<EmailAccount, "id">): Promise<string | null> {
|
||||
try {
|
||||
const transporter = nodemailer.createTransport({
|
||||
host: account.smtp_host,
|
||||
port: account.smtp_port,
|
||||
secure: account.smtp_secure === 1,
|
||||
auth: { user: account.username, pass: account.password },
|
||||
tls: { rejectUnauthorized: false },
|
||||
connectionTimeout: 10_000,
|
||||
greetingTimeout: 10_000,
|
||||
});
|
||||
await transporter.verify();
|
||||
return null;
|
||||
} catch (err) {
|
||||
return err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
}
|
||||
|
||||
export interface ImapTestAccount {
|
||||
imap_host: string;
|
||||
imap_port: number;
|
||||
username: string;
|
||||
password: string;
|
||||
imap_username: string | null;
|
||||
imap_password: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies IMAP connectivity — connects, authenticates, then disconnects.
|
||||
* Returns null on success, error message string on failure.
|
||||
*/
|
||||
export async function testImapConnection(account: ImapTestAccount): Promise<string | null> {
|
||||
return new Promise((resolve) => {
|
||||
const imap = new Imap({
|
||||
host: account.imap_host,
|
||||
port: account.imap_port,
|
||||
tls: true,
|
||||
tlsOptions: { rejectUnauthorized: false },
|
||||
user: account.imap_username ?? account.username,
|
||||
password: account.imap_password ?? account.password,
|
||||
authTimeout: 10_000,
|
||||
connTimeout: 12_000,
|
||||
});
|
||||
|
||||
imap.once("ready", () => {
|
||||
try { imap.end(); } catch { /* ignore */ }
|
||||
resolve(null);
|
||||
});
|
||||
|
||||
imap.once("error", (err: Error) => {
|
||||
resolve(err.message ?? String(err));
|
||||
});
|
||||
|
||||
imap.connect();
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
import type DatabaseType from "better-sqlite3";
|
||||
import { getDb } from "@/lib/db";
|
||||
import { randomUUID } from "crypto";
|
||||
|
||||
type DB = DatabaseType.Database;
|
||||
|
||||
const PAGE_SIZE = 25;
|
||||
export const DEFAULT_DAILY_CAP = 1500;
|
||||
|
||||
export interface ImportRow {
|
||||
id: string;
|
||||
list_id: string;
|
||||
account_id: string | null;
|
||||
sales_nav_url: string | null;
|
||||
status: string;
|
||||
phase: string | null;
|
||||
page: number;
|
||||
total_pages: number;
|
||||
count: number;
|
||||
total: number;
|
||||
imported: number;
|
||||
skipped: number;
|
||||
error: string | null;
|
||||
scheduled_for: string | null;
|
||||
start_page: number;
|
||||
cap: number | null;
|
||||
cancel_requested: number;
|
||||
batch_index: number;
|
||||
enrich: number;
|
||||
started_at: string;
|
||||
finished_at: string | null;
|
||||
}
|
||||
|
||||
// ─── settings ────────────────────────────────────────────────────────────────
|
||||
|
||||
export function getDailyImportCap(db: DB = getDb()): number {
|
||||
const row = db.prepare("SELECT value FROM app_settings WHERE key = 'daily_import_cap'").get() as
|
||||
| { value: string }
|
||||
| undefined;
|
||||
const n = row ? parseInt(row.value, 10) : NaN;
|
||||
return Number.isFinite(n) && n > 0 ? n : DEFAULT_DAILY_CAP;
|
||||
}
|
||||
|
||||
export function setDailyImportCap(db: DB, n: number): void {
|
||||
const v = String(Math.max(1, Math.floor(n)));
|
||||
db.prepare(
|
||||
`INSERT INTO app_settings (key, value, updated_at) VALUES ('daily_import_cap', ?, datetime('now'))
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = datetime('now')`
|
||||
).run(v);
|
||||
}
|
||||
|
||||
// ─── quota ───────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Contacts imported across ALL lists today (the global daily budget). */
|
||||
export function importedToday(db: DB): number {
|
||||
const row = db
|
||||
.prepare(
|
||||
`SELECT COALESCE(SUM(imported), 0) c FROM list_imports
|
||||
WHERE status IN ('done', 'running') AND date(COALESCE(finished_at, started_at)) = date('now')`
|
||||
)
|
||||
.get() as { c: number };
|
||||
return row.c;
|
||||
}
|
||||
|
||||
function todayStr(): string {
|
||||
return new Date().toISOString().slice(0, 10);
|
||||
}
|
||||
function addDaysStr(base: string, days: number): string {
|
||||
const d = new Date(base + "T00:00:00Z");
|
||||
d.setUTCDate(d.getUTCDate() + days);
|
||||
return d.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
// ─── public API ──────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Queue an import for a list. Creates the first batch as 'scheduled' for today;
|
||||
* the runner's scheduler picks it up (one import at a time). Large lists are
|
||||
* split across days under the daily cap by runBatch chaining continuations.
|
||||
*/
|
||||
export function startImport(
|
||||
db: DB,
|
||||
opts: { listId: string; accountId: string; salesNavUrl: string; enrich?: boolean }
|
||||
): { importId: string } {
|
||||
cancelImportsForList(db, opts.listId); // supersede any prior import for this list
|
||||
const importId = randomUUID();
|
||||
db.prepare(
|
||||
`INSERT INTO list_imports
|
||||
(id, list_id, account_id, sales_nav_url, status, scheduled_for, start_page, batch_index, enrich, started_at)
|
||||
VALUES (?, ?, ?, ?, 'scheduled', ?, 1, 1, ?, datetime('now'))`
|
||||
).run(importId, opts.listId, opts.accountId, opts.salesNavUrl, todayStr(), opts.enrich ? 1 : 0);
|
||||
return { importId };
|
||||
}
|
||||
|
||||
export function cancelImportsForList(db: DB, listId: string): void {
|
||||
db.prepare(
|
||||
`UPDATE list_imports
|
||||
SET cancel_requested = 1,
|
||||
status = CASE WHEN status = 'scheduled' THEN 'canceled' ELSE status END,
|
||||
finished_at = CASE WHEN status = 'scheduled' THEN datetime('now') ELSE finished_at END
|
||||
WHERE list_id = ? AND status IN ('scheduled', 'running')`
|
||||
).run(listId);
|
||||
}
|
||||
|
||||
export function cancelImport(db: DB, importId: string): void {
|
||||
db.prepare(
|
||||
`UPDATE list_imports
|
||||
SET cancel_requested = 1,
|
||||
status = CASE WHEN status = 'scheduled' THEN 'canceled' ELSE status END,
|
||||
finished_at = CASE WHEN status = 'scheduled' THEN datetime('now') ELSE finished_at END
|
||||
WHERE id = ?`
|
||||
).run(importId);
|
||||
}
|
||||
|
||||
// ─── scheduler + executor ────────────────────────────────────────────────────
|
||||
|
||||
let importRunning = false;
|
||||
|
||||
/** Runner hook (called each tick): start the next due batch if none is running. */
|
||||
export async function processScheduledImports(db: DB): Promise<void> {
|
||||
if (importRunning) return;
|
||||
const due = db
|
||||
.prepare(
|
||||
`SELECT * FROM list_imports
|
||||
WHERE status = 'scheduled' AND cancel_requested = 0
|
||||
AND (scheduled_for IS NULL OR scheduled_for <= date('now'))
|
||||
ORDER BY scheduled_for ASC, batch_index ASC LIMIT 1`
|
||||
)
|
||||
.get() as ImportRow | undefined;
|
||||
if (!due) return;
|
||||
|
||||
importRunning = true;
|
||||
db.prepare("UPDATE list_imports SET status = 'running', started_at = datetime('now') WHERE id = ?").run(due.id);
|
||||
runBatch(due.id)
|
||||
.catch((e) => console.error("[import] batch crashed:", e))
|
||||
.finally(() => { importRunning = false; });
|
||||
}
|
||||
|
||||
async function runBatch(importId: string): Promise<void> {
|
||||
const db = getDb();
|
||||
const job = db.prepare("SELECT * FROM list_imports WHERE id = ?").get(importId) as ImportRow | undefined;
|
||||
if (!job || !job.account_id || !job.sales_nav_url) return;
|
||||
|
||||
// List deleted out from under us?
|
||||
const list = db.prepare("SELECT id FROM lists WHERE id = ?").get(job.list_id);
|
||||
if (!list) {
|
||||
db.prepare("UPDATE list_imports SET status = 'canceled', finished_at = datetime('now') WHERE id = ?").run(importId);
|
||||
return;
|
||||
}
|
||||
|
||||
// Today's remaining budget → max whole pages this run
|
||||
const cap = getDailyImportCap(db);
|
||||
const remaining = cap - importedToday(db);
|
||||
const maxPages = Math.floor(remaining / PAGE_SIZE);
|
||||
if (maxPages < 1) {
|
||||
db.prepare("UPDATE list_imports SET status = 'scheduled', scheduled_for = ? WHERE id = ?").run(
|
||||
addDaysStr(todayStr(), 1),
|
||||
importId
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[import] batch ${importId} (b${job.batch_index}) start_page=${job.start_page} maxPages=${maxPages} cap=${cap}`);
|
||||
const { getSessionContext } = await import("@/lib/linkedin/session");
|
||||
const { scrapeNavigatorUrl } = await import("@/lib/linkedin/scraper");
|
||||
|
||||
const updateProgress = db.prepare(
|
||||
"UPDATE list_imports SET phase = ?, page = ?, total_pages = ?, count = ?, total = ? WHERE id = ?"
|
||||
);
|
||||
const isCanceled = () => {
|
||||
const r = db.prepare("SELECT cancel_requested FROM list_imports WHERE id = ?").get(importId) as
|
||||
| { cancel_requested: number }
|
||||
| undefined;
|
||||
return !r || r.cancel_requested === 1; // row deleted (list cascade) or explicit cancel
|
||||
};
|
||||
|
||||
try {
|
||||
const ctx = await getSessionContext(job.account_id);
|
||||
const { profiles, lastPage, knownTotal, exhausted } = await scrapeNavigatorUrl(ctx, job.sales_nav_url, {
|
||||
startPage: job.start_page,
|
||||
maxPages,
|
||||
onProgress: (p) => updateProgress.run(p.phase, p.page ?? 0, p.totalPages ?? 0, p.count, p.total, importId),
|
||||
isCanceled,
|
||||
});
|
||||
|
||||
if (isCanceled()) {
|
||||
if (db.prepare("SELECT id FROM list_imports WHERE id = ?").get(importId)) {
|
||||
db.prepare("UPDATE list_imports SET status = 'canceled', finished_at = datetime('now') WHERE id = ?").run(importId);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const { imported, skipped } = insertProfiles(db, job.list_id, profiles);
|
||||
console.log(`[import] batch ${importId} inserted ${imported} new, skipped ${skipped} (lastPage=${lastPage}, exhausted=${exhausted})`);
|
||||
|
||||
db.prepare(
|
||||
`UPDATE list_imports
|
||||
SET status = 'done', imported = ?, skipped = ?, count = ?, total = ?, page = ?, total_pages = ?, finished_at = datetime('now')
|
||||
WHERE id = ?`
|
||||
).run(imported, skipped, profiles.length, knownTotal, lastPage, Math.ceil(knownTotal / PAGE_SIZE), importId);
|
||||
|
||||
// More of the list left → chain the remainder to the next day
|
||||
if (!exhausted) {
|
||||
db.prepare(
|
||||
`INSERT INTO list_imports
|
||||
(id, list_id, account_id, sales_nav_url, status, scheduled_for, start_page, batch_index, enrich, total, total_pages, started_at)
|
||||
VALUES (?, ?, ?, ?, 'scheduled', ?, ?, ?, ?, ?, ?, datetime('now'))`
|
||||
).run(
|
||||
randomUUID(),
|
||||
job.list_id,
|
||||
job.account_id,
|
||||
job.sales_nav_url,
|
||||
addDaysStr(todayStr(), 1),
|
||||
lastPage + 1,
|
||||
job.batch_index + 1,
|
||||
job.enrich,
|
||||
knownTotal,
|
||||
Math.ceil(knownTotal / PAGE_SIZE)
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.error("[import] FAILED:", message);
|
||||
if (db.prepare("SELECT id FROM list_imports WHERE id = ?").get(importId)) {
|
||||
db.prepare("UPDATE list_imports SET status = 'error', error = ?, finished_at = datetime('now') WHERE id = ?").run(
|
||||
message,
|
||||
importId
|
||||
);
|
||||
}
|
||||
// A "no data intercepted / re-authentication" failure means the session died.
|
||||
if (/re-authentication|No data intercepted/i.test(message) && job.account_id) {
|
||||
try {
|
||||
const { markNeedsReauth } = await import("@/lib/linkedin/session");
|
||||
await markNeedsReauth(job.account_id);
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
function insertProfiles(db: DB, listId: string, profiles: any[]): { imported: number; skipped: number } {
|
||||
const insertTarget = db.prepare(
|
||||
`INSERT INTO targets (
|
||||
id, linkedin_url, sales_nav_url, first_name, last_name, full_name,
|
||||
title, company, location, degree,
|
||||
object_urn, summary, open_link, company_industry, company_location,
|
||||
tenure_months, spotlight_badges
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(linkedin_url) DO UPDATE SET
|
||||
sales_nav_url = excluded.sales_nav_url,
|
||||
first_name = excluded.first_name,
|
||||
last_name = excluded.last_name,
|
||||
full_name = excluded.full_name,
|
||||
title = excluded.title,
|
||||
company = excluded.company,
|
||||
location = excluded.location,
|
||||
degree = excluded.degree,
|
||||
object_urn = excluded.object_urn,
|
||||
summary = excluded.summary,
|
||||
open_link = excluded.open_link,
|
||||
company_industry = excluded.company_industry,
|
||||
company_location = excluded.company_location,
|
||||
tenure_months = excluded.tenure_months,
|
||||
spotlight_badges = excluded.spotlight_badges`
|
||||
);
|
||||
const insertLink = db.prepare("INSERT OR IGNORE INTO list_targets (list_id, target_id) VALUES (?, ?)");
|
||||
const findTarget = db.prepare("SELECT id FROM targets WHERE linkedin_url = ?");
|
||||
|
||||
let imported = 0;
|
||||
let skipped = 0;
|
||||
db.transaction(() => {
|
||||
for (const p of profiles) {
|
||||
const url = p.linkedinUrl ?? p.salesNavUrl;
|
||||
insertTarget.run(
|
||||
randomUUID(), url, p.salesNavUrl,
|
||||
p.firstName, p.lastName, p.fullName,
|
||||
p.title, p.company, p.location, p.degree,
|
||||
p.objectUrn, p.summary, p.openLink ? 1 : 0,
|
||||
p.companyIndustry, p.companyLocation,
|
||||
p.tenureMonths, p.spotlightBadges
|
||||
);
|
||||
const target = findTarget.get(url) as { id: string };
|
||||
const result = insertLink.run(listId, target.id);
|
||||
if (result.changes > 0) imported++;
|
||||
else skipped++;
|
||||
}
|
||||
})();
|
||||
return { imported, skipped };
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
import type { Page } from "playwright";
|
||||
|
||||
export class WeeklyLimitError extends Error {}
|
||||
export class AlreadyConnectedError extends Error {}
|
||||
export class PendingInviteError extends Error {}
|
||||
|
||||
/**
|
||||
* Sends a LinkedIn connection request without a note.
|
||||
* Navigates to the profile page and clicks the Connect button.
|
||||
* Throws WeeklyLimitError if the weekly limit popup appears.
|
||||
* Throws AlreadyConnectedError / PendingInviteError if already in that state.
|
||||
*/
|
||||
export async function sendConnectionRequest(page: Page, linkedinUrl: string): Promise<void> {
|
||||
await page.goto(linkedinUrl, { waitUntil: "domcontentloaded", timeout: 30000 });
|
||||
await page.waitForTimeout(2000 + Math.random() * 1000);
|
||||
|
||||
// Already connected? Primary signal: presence of the profile's "Message" link
|
||||
// (only shown to 1st-degree connections) — an href attribute, not a CSS class
|
||||
// or translated text, so it survives LinkedIn's class-name hashing and non-
|
||||
// English UI languages. Falls back to the old text-scrape for accounts/layouts
|
||||
// where that link isn't found as a plain <a href>.
|
||||
//
|
||||
// MUST be scoped to the visited person's own intro/top card — a page-wide
|
||||
// search also matches "Message" links / "1st" badges belonging to OTHER
|
||||
// people rendered elsewhere on the page (sidebar modules like "People also
|
||||
// viewed"). For a non-connected target this false-positived AlreadyConnected,
|
||||
// which skipped the real invite AND (via the message step that follows) sent
|
||||
// a message to a random unrelated 1st-degree connection instead of the
|
||||
// target — see CLAUDE.md / memory for the Jul 2026 incident. The top card is
|
||||
// identified structurally as the section containing the page's own <h1> name
|
||||
// heading, robust to class-name hashing. Mirrors the same fix in visit.ts.
|
||||
const topCard = page.locator("main section").filter({ has: page.locator("h1") }).first();
|
||||
const hasMessageLink = await topCard.locator('a[href*="/messaging/compose"]').first().count() > 0;
|
||||
if (hasMessageLink) throw new AlreadyConnectedError("Already connected");
|
||||
const pageText = await topCard.innerText().catch(() => "");
|
||||
if (/\b1st\b/.test(pageText)) throw new AlreadyConnectedError("Already connected");
|
||||
|
||||
// Pending?
|
||||
if (/\bPending\b/.test(pageText)) throw new PendingInviteError("Invitation already pending");
|
||||
const pendingBtn = page.locator('button[aria-label*="Pending"]:visible');
|
||||
if (await pendingBtn.count() > 0) throw new PendingInviteError("Invitation already pending");
|
||||
|
||||
// Case 1: Direct Connect link (primary CTA) — navigate to its href directly.
|
||||
// Clicking fails because the Sales Nav overlay SVG intercepts pointer events.
|
||||
const directConnect = page.locator('a[aria-label*="Invite"][aria-label*="to connect"]:visible, a[href*="custom-invite"]:visible').first();
|
||||
if (await directConnect.count() > 0) {
|
||||
const href = await directConnect.getAttribute("href");
|
||||
if (!href) throw new Error("Connect link has no href");
|
||||
const inviteUrl = href.startsWith("http") ? href : `https://www.linkedin.com${href}`;
|
||||
await page.goto(inviteUrl, { waitUntil: "domcontentloaded", timeout: 30000 });
|
||||
await page.waitForTimeout(1000);
|
||||
} else {
|
||||
// Case 2: Connect is inside the "..." More menu
|
||||
// LinkedIn has two "More" buttons on page: [0] = nav bar, [1] = profile card
|
||||
const moreBtn = page.locator('button[aria-label="More"]:visible').nth(1);
|
||||
await moreBtn.click();
|
||||
await page.waitForTimeout(800);
|
||||
|
||||
// Check for Pending in the menu — means invite was already sent
|
||||
const pendingMenuItem = page.locator('[role="menuitem"]:has-text("Pending"):visible');
|
||||
if (await pendingMenuItem.count() > 0) throw new PendingInviteError("Invitation already pending (found in More menu)");
|
||||
|
||||
const connectOption = page.locator('[role="menuitem"]:has-text("Connect"):visible');
|
||||
if (await connectOption.count() === 0) throw new Error("Connect option not found in More menu");
|
||||
await connectOption.first().click();
|
||||
}
|
||||
|
||||
await page.waitForTimeout(1000);
|
||||
|
||||
// Click "Send without a note" / "Send now"
|
||||
const sendBtn = page.locator(
|
||||
'button:has-text("Send now"), button[aria-label*="Send without"], button[aria-label*="Send invitation"]:not([aria-label*="note"])'
|
||||
);
|
||||
if (await sendBtn.count() > 0) {
|
||||
await sendBtn.first().click({ force: true });
|
||||
await page.waitForTimeout(1500);
|
||||
}
|
||||
|
||||
// Check for weekly limit popup
|
||||
const limitPopup = page.locator('div[class*="ip-fuse-limit-alert__warning"]');
|
||||
if (await limitPopup.count() > 0) throw new WeeklyLimitError("Weekly connection limit reached");
|
||||
|
||||
// Check for error toast
|
||||
const errorToast = page.locator('div[data-test-artdeco-toast-item-type="error"]:visible');
|
||||
if (await errorToast.count() > 0) {
|
||||
const msg = await errorToast.innerText();
|
||||
throw new Error(`Connection error: ${msg.trim()}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
/**
|
||||
* Profile enrichment via Sales Navigator profile page intercept.
|
||||
*
|
||||
* Strategy: navigate to the lead's sales_nav_url. Sales Nav automatically
|
||||
* fires a salesApiProfiles call that returns the full profile — headline,
|
||||
* all positions with descriptions, degree, flagshipProfileUrl, summary.
|
||||
* We intercept that response instead of making extra API calls.
|
||||
*
|
||||
* Weight: ~5-8s per profile (one real browser page load). Run fire-and-forget
|
||||
* after import — do not await in API routes.
|
||||
*/
|
||||
import type { BrowserContext } from "playwright";
|
||||
import { getDb } from "@/lib/db";
|
||||
|
||||
interface EnrichedPosition {
|
||||
title: string;
|
||||
companyName: string;
|
||||
current: boolean;
|
||||
startedOn?: { year?: number; month?: number };
|
||||
endedOn?: { year?: number; month?: number };
|
||||
description?: string;
|
||||
}
|
||||
|
||||
interface SalesNavProfileData {
|
||||
entityUrn?: string;
|
||||
firstName?: string;
|
||||
lastName?: string;
|
||||
fullName?: string;
|
||||
headline?: string;
|
||||
summary?: string;
|
||||
location?: string;
|
||||
degree?: number;
|
||||
flagshipProfileUrl?: string;
|
||||
objectUrn?: string;
|
||||
positions?: EnrichedPosition[];
|
||||
skills?: Array<{ name?: string }>;
|
||||
}
|
||||
|
||||
interface InterceptedResponse {
|
||||
data?: SalesNavProfileData;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export async function enrichProfile(
|
||||
ctx: BrowserContext,
|
||||
target: { id: string; sales_nav_url: string; full_name: string }
|
||||
): Promise<boolean> {
|
||||
const db = getDb();
|
||||
const page = await ctx.newPage();
|
||||
|
||||
try {
|
||||
// Box it so the async response listener can mutate it (TS can't narrow across async callbacks)
|
||||
const box: { data: SalesNavProfileData | null } = { data: null };
|
||||
|
||||
// Intercept the salesApiProfiles call that Sales Nav fires automatically on profile load.
|
||||
// Two calls fire per page load:
|
||||
// 1st — flat object with entityUrn, headline, positions, flagshipProfileUrl (what we want)
|
||||
// 2nd — flat object with educations, skills, numOfConnections (supplementary)
|
||||
// Both are flat — no .data wrapper. Accept the first one that has entityUrn.
|
||||
page.on("response", async (resp) => {
|
||||
if (box.data) return;
|
||||
if (!resp.url().includes("salesApiProfiles")) return;
|
||||
if (resp.status() !== 200) return;
|
||||
try {
|
||||
const json = await resp.json() as Record<string, unknown>;
|
||||
if ("entityUrn" in json && "firstName" in json) {
|
||||
box.data = json as unknown as SalesNavProfileData;
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
});
|
||||
|
||||
await page.goto(target.sales_nav_url, { waitUntil: "domcontentloaded", timeout: 30000 });
|
||||
await page.waitForTimeout(8000);
|
||||
|
||||
const intercepted = box.data;
|
||||
if (!intercepted) {
|
||||
console.log(`[enrich] No profile data intercepted for ${target.full_name} — skipping`);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Extract positions — include all, not just current
|
||||
const positions: EnrichedPosition[] = (intercepted.positions ?? []).map((p) => ({
|
||||
title: p.title ?? "",
|
||||
companyName: p.companyName ?? "",
|
||||
current: p.current ?? false,
|
||||
startedOn: p.startedOn,
|
||||
endedOn: p.endedOn,
|
||||
description: p.description ?? undefined,
|
||||
}));
|
||||
|
||||
// Skills come from a separate decoration call — if not in intercept, skip for now
|
||||
// (the page-load intercept does not include skills; they'd need a second fetch)
|
||||
const skills: string[] = (intercepted.skills ?? [])
|
||||
.map((s) => s.name)
|
||||
.filter((n): n is string => !!n);
|
||||
|
||||
db.prepare(`
|
||||
UPDATE targets SET
|
||||
headline = COALESCE(?, headline),
|
||||
summary = COALESCE(?, summary),
|
||||
positions_json = ?,
|
||||
skills_json = CASE WHEN ? IS NOT NULL THEN ? ELSE skills_json END,
|
||||
enriched_profile_at = datetime('now')
|
||||
WHERE id = ?
|
||||
`).run(
|
||||
intercepted.headline ?? null,
|
||||
intercepted.summary ?? null,
|
||||
positions.length > 0 ? JSON.stringify(positions) : null,
|
||||
skills.length > 0 ? "1" : null,
|
||||
skills.length > 0 ? JSON.stringify(skills) : null,
|
||||
target.id
|
||||
);
|
||||
|
||||
console.log(`[enrich] ${target.full_name} — ${positions.length} positions, ${skills.length} skills, headline: ${!!intercepted.headline}`);
|
||||
return true;
|
||||
} catch (err) {
|
||||
console.error(`[enrich] Error enriching ${target.full_name}:`, err instanceof Error ? err.message : err);
|
||||
return false;
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function enrichList(
|
||||
ctx: BrowserContext,
|
||||
listId: string,
|
||||
delayMs = 2000,
|
||||
onProgress?: (count: number, total: number) => void
|
||||
): Promise<{ enriched: number; failed: number }> {
|
||||
const db = getDb();
|
||||
|
||||
// Only targets with a sales_nav_url that haven't been enriched yet
|
||||
const targets = db.prepare(`
|
||||
SELECT t.id, t.sales_nav_url, t.full_name
|
||||
FROM targets t
|
||||
JOIN list_targets lt ON lt.target_id = t.id
|
||||
WHERE lt.list_id = ?
|
||||
AND t.sales_nav_url IS NOT NULL
|
||||
AND t.enriched_profile_at IS NULL
|
||||
ORDER BY t.created_at ASC
|
||||
`).all(listId) as Array<{ id: string; sales_nav_url: string; full_name: string }>;
|
||||
|
||||
const total = targets.length;
|
||||
console.log(`[enrich] Starting enrichment for ${total} profiles in list ${listId}`);
|
||||
|
||||
let enriched = 0;
|
||||
let failed = 0;
|
||||
|
||||
for (let i = 0; i < targets.length; i++) {
|
||||
const target = targets[i];
|
||||
console.log(`[enrich] ${i + 1}/${total} — ${target.full_name}`);
|
||||
const ok = await enrichProfile(ctx, target);
|
||||
if (ok) enriched++; else failed++;
|
||||
console.log(`[enrich] ${i + 1}/${total} done (${enriched} ok, ${failed} failed)`);
|
||||
onProgress?.(i + 1, total);
|
||||
if (delayMs > 0) await new Promise((r) => setTimeout(r, delayMs));
|
||||
}
|
||||
|
||||
console.log(`[enrich] Done — ${enriched} enriched, ${failed} failed`);
|
||||
return { enriched, failed };
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import type { Page } from "playwright";
|
||||
|
||||
export interface LinkedInStats {
|
||||
connections: number;
|
||||
pending: number;
|
||||
profile_views: number;
|
||||
}
|
||||
|
||||
function parseNum(str: string): number {
|
||||
return parseInt(str.replace(/[^0-9]/g, ""), 10) || 0;
|
||||
}
|
||||
|
||||
export async function scrapeLinkedInStats(page: Page): Promise<LinkedInStats> {
|
||||
// ── Connections ───────────────────────────────────────────
|
||||
await page.goto("https://www.linkedin.com/mynetwork/invite-connect/connections/", {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: 30000,
|
||||
});
|
||||
await page.waitForTimeout(2500);
|
||||
|
||||
const connectionsTexts = await page.evaluate(() =>
|
||||
[...document.querySelectorAll("h1,h2,h3,span,p")]
|
||||
.map(el => ((el as HTMLElement).innerText ?? "").trim())
|
||||
.filter(t => /\d.*connection/i.test(t))
|
||||
);
|
||||
const connections = parseNum(connectionsTexts[0] ?? "0");
|
||||
|
||||
// ── Pending sent ──────────────────────────────────────────
|
||||
await page.goto("https://www.linkedin.com/mynetwork/invitation-manager/sent/", {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: 30000,
|
||||
});
|
||||
await page.waitForTimeout(2500);
|
||||
|
||||
const pendingTexts = await page.evaluate(() =>
|
||||
[...document.querySelectorAll("button,a,span,h1,h2,h3")]
|
||||
.map(el => ((el as HTMLElement).innerText ?? "").trim())
|
||||
.filter(t => /People\s*\(\d+\)/i.test(t))
|
||||
);
|
||||
const pendingMatch = (pendingTexts[0] ?? "").match(/\((\d+)\)/);
|
||||
const pending = pendingMatch ? parseInt(pendingMatch[1], 10) : 0;
|
||||
|
||||
// ── Profile views ─────────────────────────────────────────
|
||||
await page.goto("https://www.linkedin.com/analytics/profile-views/", {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: 30000,
|
||||
});
|
||||
await page.waitForTimeout(3000);
|
||||
|
||||
const profileViewsTexts = await page.evaluate(() =>
|
||||
[...document.querySelectorAll("*")]
|
||||
.map(el => ((el as HTMLElement).innerText ?? "").trim())
|
||||
.filter(t => /Profile viewers/i.test(t) && t.length < 100)
|
||||
);
|
||||
// The block looks like "185\n\nProfile viewers\n\n30% previous week"
|
||||
const pvBlock = profileViewsTexts[0] ?? "";
|
||||
const profile_views = parseNum(pvBlock.split("\n")[0]);
|
||||
|
||||
return { connections, pending, profile_views };
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
import type { Page } from "playwright";
|
||||
import { visitProfile } from "./visit";
|
||||
|
||||
export class NotConnectedError extends Error {}
|
||||
|
||||
export interface SendMessageResult {
|
||||
messagingUrn: string | null;
|
||||
isFirstDegree: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends a message to a LinkedIn 1st-degree connection.
|
||||
*
|
||||
* Self-contained URN resolution — does NOT depend on a prior 'visit' workflow
|
||||
* step. If messagingUrn is already cached, it's used directly (no extra page
|
||||
* load). Otherwise this does its own live profile check (the same top-card-
|
||||
* scoped logic as the 'visit' step, see visit.ts) to fetch a fresh URN and
|
||||
* verify the target is still actually connected, immediately before sending.
|
||||
* Only if that live check finds no URN despite confirming 1st-degree does it
|
||||
* fall back to the connections-only name-search typeahead — a rare last
|
||||
* resort now, not the default path for every contact that lacks a cached URN.
|
||||
*
|
||||
* Throws NotConnectedError if the live check finds the target is NOT 1st-
|
||||
* degree, instead of guessing via typeahead search (which can silently hit
|
||||
* an unrelated connection with a similar/truncated name — see CLAUDE.md /
|
||||
* memory for the Jul 2026 incident this replaced).
|
||||
*
|
||||
* Returns the resolved { messagingUrn, isFirstDegree } so the caller can
|
||||
* persist it to the target record, same as the 'visit' step does.
|
||||
*/
|
||||
export async function sendMessage(
|
||||
page: Page,
|
||||
fullName: string,
|
||||
text: string,
|
||||
linkedinUrl: string,
|
||||
messagingUrn?: string | null
|
||||
): Promise<SendMessageResult> {
|
||||
if (messagingUrn) {
|
||||
const opened = await openComposeByUrn(page, messagingUrn);
|
||||
if (opened) {
|
||||
await sendFromComposeBox(page, text);
|
||||
return { messagingUrn, isFirstDegree: true };
|
||||
}
|
||||
}
|
||||
|
||||
const resolved = await visitProfile(page, linkedinUrl);
|
||||
if (resolved.messagingUrn) {
|
||||
const opened = await openComposeByUrn(page, resolved.messagingUrn);
|
||||
if (opened) {
|
||||
await sendFromComposeBox(page, text);
|
||||
return resolved;
|
||||
}
|
||||
}
|
||||
if (!resolved.isFirstDegree) {
|
||||
throw new NotConnectedError(`${fullName} is not a 1st-degree connection — refusing to message`);
|
||||
}
|
||||
|
||||
// Connected, but no message link could be resolved live (unusual layout) —
|
||||
// last-resort fallback to name search.
|
||||
await sendMessageViaTypeahead(page, fullName, text);
|
||||
return resolved;
|
||||
}
|
||||
|
||||
async function openComposeByUrn(page: Page, messagingUrn: string): Promise<boolean> {
|
||||
try {
|
||||
const recipientId = messagingUrn.split(":").pop();
|
||||
const composeUrl = `https://www.linkedin.com/messaging/compose/?profileUrn=${encodeURIComponent(messagingUrn)}&recipient=${recipientId}`;
|
||||
await page.goto(composeUrl, { waitUntil: "domcontentloaded", timeout: 30000 });
|
||||
await page.waitForTimeout(1500 + Math.random() * 1000);
|
||||
const msgInput = page.locator("div.msg-form__contenteditable").first();
|
||||
await msgInput.waitFor({ timeout: 8000 });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function sendMessageViaTypeahead(page: Page, fullName: string, text: string): Promise<void> {
|
||||
await page.goto("https://www.linkedin.com/messaging/thread/new/", {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: 30000,
|
||||
});
|
||||
await page.waitForTimeout(1500 + Math.random() * 1000);
|
||||
|
||||
// Search for recipient by name
|
||||
const searchField = page.locator("input.msg-connections-typeahead__search-field").first();
|
||||
await searchField.waitFor({ timeout: 10000 });
|
||||
await searchField.click();
|
||||
await searchField.type(fullName, { delay: 60 + Math.random() * 40 });
|
||||
await page.waitForTimeout(1500);
|
||||
|
||||
// Select first result — but verify it's actually the intended person first.
|
||||
// This search only returns 1st-degree connections; if the real target isn't
|
||||
// connected, LinkedIn will still happily return a same/similar-named
|
||||
// connection as the top result, and we'd silently message a stranger.
|
||||
const firstResult = page.locator('div[class*="msg-connections-typeahead__search-result-row"]').first();
|
||||
await firstResult.waitFor({ timeout: 8000 });
|
||||
const resultText = (await firstResult.innerText().catch(() => "")).trim();
|
||||
if (!resultNameMatches(resultText, fullName)) {
|
||||
throw new Error(
|
||||
`Typeahead search for "${fullName}" returned a non-matching result ("${resultText.replace(/\s+/g, " ")}") — refusing to send to avoid messaging the wrong person`
|
||||
);
|
||||
}
|
||||
await firstResult.click({ delay: 100 });
|
||||
await page.waitForTimeout(800);
|
||||
|
||||
await sendFromComposeBox(page, text);
|
||||
}
|
||||
|
||||
function resultNameMatches(resultText: string, fullName: string): boolean {
|
||||
const normalize = (s: string) =>
|
||||
s.toLowerCase().normalize("NFKD").replace(/[̀-ͯ]/g, "").replace(/[^a-z\s]/g, " ").replace(/\s+/g, " ").trim();
|
||||
const target = normalize(fullName);
|
||||
if (!target) return false;
|
||||
return normalize(resultText).includes(target);
|
||||
}
|
||||
|
||||
async function sendFromComposeBox(page: Page, text: string): Promise<void> {
|
||||
// Paste message into compose area
|
||||
const msgInput = page.locator("div.msg-form__contenteditable").first();
|
||||
await msgInput.waitFor({ timeout: 8000 });
|
||||
await msgInput.click();
|
||||
try {
|
||||
await page.evaluate((t) => navigator.clipboard.writeText(t), text);
|
||||
await page.waitForTimeout(300);
|
||||
await msgInput.press("Control+V");
|
||||
} catch {
|
||||
// Clipboard blocked in headless — fall back to keyboard typing
|
||||
await msgInput.pressSequentially(text, { delay: 20 });
|
||||
}
|
||||
await page.waitForTimeout(500);
|
||||
|
||||
// Send
|
||||
const sendBtn = page.locator("button.msg-form__send-button:visible").first();
|
||||
await sendBtn.waitFor({ timeout: 5000 });
|
||||
await sendBtn.click({ delay: 100 });
|
||||
await page.waitForTimeout(2000);
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import type { Page } from "playwright";
|
||||
|
||||
/**
|
||||
* Scrapes the full list of vanity names from the sent invitations page.
|
||||
* Returns a Set of lowercase vanity names that are STILL pending on LinkedIn.
|
||||
*
|
||||
* Strategy: navigate to invitation-manager/sent/, scroll the #workspace container
|
||||
* until the count stabilises, then extract all /in/ links.
|
||||
*
|
||||
* Works headless — the scrollable element is <main id="workspace">, not window.
|
||||
*/
|
||||
export async function scrapePendingInvitationVanityNames(page: Page): Promise<Set<string>> {
|
||||
await page.goto("https://www.linkedin.com/mynetwork/invitation-manager/sent/", {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: 30000,
|
||||
});
|
||||
await page.waitForTimeout(3000);
|
||||
|
||||
// Scroll #workspace until link count stabilises
|
||||
let lastCount = 0;
|
||||
let stableRounds = 0;
|
||||
|
||||
for (let round = 0; round < 40; round++) {
|
||||
await page.evaluate(() => {
|
||||
const workspace = document.getElementById("workspace");
|
||||
if (workspace) {
|
||||
workspace.scrollTop = workspace.scrollHeight;
|
||||
} else {
|
||||
// Fallback: scroll window + body
|
||||
window.scrollTo(0, document.body.scrollHeight);
|
||||
document.documentElement.scrollTop = document.documentElement.scrollHeight;
|
||||
}
|
||||
});
|
||||
await page.waitForTimeout(1200);
|
||||
|
||||
const currentCount = await page.evaluate(() =>
|
||||
document.querySelectorAll("a[href*='/in/']").length
|
||||
);
|
||||
|
||||
if (currentCount === lastCount) {
|
||||
stableRounds++;
|
||||
if (stableRounds >= 3) break;
|
||||
} else {
|
||||
stableRounds = 0;
|
||||
lastCount = currentCount;
|
||||
}
|
||||
}
|
||||
|
||||
// Extract unique vanity names from all /in/ links
|
||||
const vanityNames = await page.evaluate(() => {
|
||||
const seen = new Set<string>();
|
||||
for (const link of document.querySelectorAll<HTMLAnchorElement>("a[href*='/in/']")) {
|
||||
const href = link.getAttribute("href") ?? "";
|
||||
const match = href.match(/\/in\/([^/?#]+)/);
|
||||
if (match) seen.add(match[1].toLowerCase());
|
||||
}
|
||||
return [...seen];
|
||||
});
|
||||
|
||||
return new Set(vanityNames);
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
/**
|
||||
* Live profile scrape → clean structured JSON for an LLM.
|
||||
*
|
||||
* Two confirmed sources (probed against prod, Jun 2026 — see docs §18):
|
||||
* 1. Sales Nav `salesApiProfiles` intercept (navigate the lead's sales_nav_url):
|
||||
* two flat responses give headline, summary, positions, educations, skills,
|
||||
* languages, connections, contactInfo, inmailRestriction, memberBadges, degree.
|
||||
* 2. Voyager `feed/updates?q=memberShareFeed` on the /in/ publicId:
|
||||
* the 10 most recent authored posts (text + media type + engagement counts)
|
||||
* plus the comments left on those posts. (q=memberComments → 400, not used.)
|
||||
*
|
||||
* Read-only. ~10-12s per profile (one Sales Nav page load + one Voyager fetch).
|
||||
*/
|
||||
import type { BrowserContext } from "playwright";
|
||||
|
||||
export interface ProfilePost {
|
||||
activityUrn: string;
|
||||
text: string;
|
||||
postedAt: string | null; // ISO, when derivable from the activity id
|
||||
mediaType: "video" | "image" | "article" | "text";
|
||||
reactions: number;
|
||||
comments: number;
|
||||
shares: number;
|
||||
}
|
||||
|
||||
export interface ProfileScrape {
|
||||
full_name: string | null;
|
||||
first_name: string | null;
|
||||
last_name: string | null;
|
||||
headline: string | null;
|
||||
summary: string | null;
|
||||
location: string | null;
|
||||
degree: number | null; // network distance (1/2/3)
|
||||
linkedin_url: string | null; // real /in/ URL
|
||||
num_connections: number | null;
|
||||
num_shared_connections: number | null;
|
||||
is_premium: boolean;
|
||||
is_open_profile: boolean; // openLink — can be messaged free (note: API often under-reports)
|
||||
can_inmail: boolean; // inmailRestriction === NO_RESTRICTION
|
||||
pending_invitation: boolean;
|
||||
websites: string[];
|
||||
positions: Array<{
|
||||
title: string;
|
||||
company: string;
|
||||
current: boolean;
|
||||
started: { year?: number; month?: number } | null;
|
||||
ended: { year?: number; month?: number } | null;
|
||||
description: string | null;
|
||||
}>;
|
||||
educations: Array<{ school: string | null; degree: string | null; field: string | null }>;
|
||||
skills: string[];
|
||||
languages: string[];
|
||||
recent_posts: ProfilePost[]; // up to 10, most recent first
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function textOf(v: unknown): string | null {
|
||||
if (typeof v === "string") return v;
|
||||
if (v && typeof v === "object" && "text" in v && typeof (v as { text: unknown }).text === "string") {
|
||||
return (v as { text: string }).text;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// LinkedIn activity/ugcPost ids encode the creation epoch-ms in the high 41 bits
|
||||
// (the low 22 bits are a sequence). Shift right 22 = divide by 2^22, via float
|
||||
// math to avoid BigInt (tsconfig targets < ES2020).
|
||||
const SHIFT_22 = 4194304; // 2^22
|
||||
function timeFromActivityUrn(urn: string): string | null {
|
||||
const m = urn.match(/(\d{18,})/);
|
||||
if (!m) return null;
|
||||
const ms = Math.floor(Number(m[1]) / SHIFT_22);
|
||||
return ms > 1_000_000_000_000 ? new Date(ms).toISOString() : null;
|
||||
}
|
||||
|
||||
interface SalesPosition {
|
||||
title?: string; companyName?: string; current?: boolean;
|
||||
startedOn?: { year?: number; month?: number };
|
||||
endedOn?: { year?: number; month?: number };
|
||||
description?: string;
|
||||
}
|
||||
interface SalesProfileFlat {
|
||||
entityUrn?: string; firstName?: string; lastName?: string; fullName?: string;
|
||||
headline?: string; summary?: string; location?: string; degree?: number;
|
||||
flagshipProfileUrl?: string; pendingInvitation?: boolean;
|
||||
positions?: SalesPosition[];
|
||||
contactInfo?: { websites?: Array<{ url?: string }> };
|
||||
educations?: Array<{ schoolName?: string; degreeName?: string; fieldOfStudy?: string }>;
|
||||
skills?: Array<{ name?: string }>;
|
||||
languages?: Array<{ name?: string }>;
|
||||
numOfConnections?: number; numOfSharedConnections?: number;
|
||||
inmailRestriction?: string;
|
||||
memberBadges?: { premium?: boolean; openLink?: boolean };
|
||||
}
|
||||
|
||||
// ─── Sales Nav career ─────────────────────────────────────────────────────────
|
||||
|
||||
async function scrapeSalesCareer(
|
||||
ctx: BrowserContext,
|
||||
salesNavUrl: string
|
||||
): Promise<Partial<ProfileScrape>> {
|
||||
const page = await ctx.newPage();
|
||||
const responses: SalesProfileFlat[] = [];
|
||||
page.on("response", async (resp) => {
|
||||
if (!resp.url().includes("salesApiProfiles") || resp.status() !== 200) return;
|
||||
try {
|
||||
const j = (await resp.json()) as Record<string, unknown>;
|
||||
// Only the two flat profile responses (have entityUrn); ignore the data/included one.
|
||||
if ("entityUrn" in j && ("firstName" in j || "educations" in j || "skills" in j)) {
|
||||
responses.push(j as unknown as SalesProfileFlat);
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
});
|
||||
|
||||
try {
|
||||
await page.goto(salesNavUrl, { waitUntil: "domcontentloaded", timeout: 35000 });
|
||||
await page.waitForTimeout(9000);
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
|
||||
// Merge the (typically two) flat responses — first carries the core profile,
|
||||
// second carries educations/skills/languages/connections/inmailRestriction.
|
||||
const core = responses.find((r) => r.firstName || r.positions) ?? {};
|
||||
const extra = responses.find((r) => r.educations || r.skills || r.numOfConnections != null) ?? {};
|
||||
|
||||
const positions = (core.positions ?? []).map((p) => ({
|
||||
title: p.title ?? "",
|
||||
company: p.companyName ?? "",
|
||||
current: p.current ?? false,
|
||||
started: p.startedOn ?? null,
|
||||
ended: p.endedOn ?? null,
|
||||
description: p.description?.trim() ? p.description : null,
|
||||
}));
|
||||
|
||||
return {
|
||||
full_name: core.fullName ?? null,
|
||||
first_name: core.firstName ?? null,
|
||||
last_name: core.lastName ?? null,
|
||||
headline: core.headline ?? null,
|
||||
summary: core.summary ?? null,
|
||||
location: core.location ?? null,
|
||||
degree: core.degree ?? null,
|
||||
linkedin_url: core.flagshipProfileUrl ?? null,
|
||||
pending_invitation: core.pendingInvitation ?? false,
|
||||
websites: (core.contactInfo?.websites ?? []).map((w) => w.url).filter((u): u is string => !!u),
|
||||
positions,
|
||||
num_connections: extra.numOfConnections ?? null,
|
||||
num_shared_connections: extra.numOfSharedConnections ?? null,
|
||||
is_premium: extra.memberBadges?.premium ?? false,
|
||||
is_open_profile: extra.memberBadges?.openLink ?? false,
|
||||
can_inmail: (extra.inmailRestriction ?? core.inmailRestriction) === "NO_RESTRICTION",
|
||||
educations: (extra.educations ?? []).map((e) => ({
|
||||
school: e.schoolName ?? null, degree: e.degreeName ?? null, field: e.fieldOfStudy ?? null,
|
||||
})),
|
||||
skills: (extra.skills ?? []).map((s) => s.name).filter((n): n is string => !!n),
|
||||
languages: (extra.languages ?? []).map((l) => l.name).filter((n): n is string => !!n),
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Voyager posts ────────────────────────────────────────────────────────────
|
||||
|
||||
function publicIdFromUrl(url: string | null | undefined): string | null {
|
||||
const m = url && url.match(/\/in\/([^/?#]+)/);
|
||||
return m ? decodeURIComponent(m[1]) : null;
|
||||
}
|
||||
|
||||
async function scrapePosts(
|
||||
ctx: BrowserContext,
|
||||
publicId: string
|
||||
): Promise<ProfilePost[]> {
|
||||
const page = await ctx.newPage();
|
||||
try {
|
||||
await page.goto("https://www.linkedin.com/feed/", { waitUntil: "domcontentloaded", timeout: 25000 });
|
||||
const cookies = await ctx.cookies();
|
||||
const csrf = (cookies.find((c) => c.name === "JSESSIONID")?.value || "").replace(/"/g, "");
|
||||
if (!csrf) return [];
|
||||
|
||||
const url =
|
||||
`https://www.linkedin.com/voyager/api/feed/updates` +
|
||||
`?profileId=${encodeURIComponent(publicId)}&q=memberShareFeed&count=10&start=0`;
|
||||
const raw = await page.evaluate(
|
||||
async ({ url, csrf }: { url: string; csrf: string }) => {
|
||||
const r = await fetch(url, {
|
||||
headers: {
|
||||
accept: "application/vnd.linkedin.normalized+json+2.1",
|
||||
"csrf-token": csrf,
|
||||
"x-restli-protocol-version": "2.0.0",
|
||||
},
|
||||
credentials: "include",
|
||||
});
|
||||
return r.status === 200 ? r.text() : "";
|
||||
},
|
||||
{ url, csrf }
|
||||
);
|
||||
if (!raw) return [];
|
||||
|
||||
const j = JSON.parse(raw) as { included?: Array<Record<string, unknown>> };
|
||||
const inc = j.included ?? [];
|
||||
|
||||
// Engagement counts keyed by the post's ugcPost/share urn.
|
||||
const countsByUrn = new Map<string, { reactions: number; comments: number; shares: number }>();
|
||||
for (const x of inc) {
|
||||
if (!String(x["$type"]).endsWith("SocialActivityCounts")) continue;
|
||||
const urn = String(x.urn ?? "");
|
||||
if (!urn.includes("ugcPost") && !urn.includes("activity")) continue;
|
||||
countsByUrn.set(urn, {
|
||||
reactions: Number(x.numLikes ?? 0),
|
||||
comments: Number(x.numComments ?? 0),
|
||||
shares: Number(x.numShares ?? 0),
|
||||
});
|
||||
}
|
||||
|
||||
const posts: ProfilePost[] = [];
|
||||
for (const x of inc) {
|
||||
if (!String(x["$type"]).endsWith("UpdateV2")) continue;
|
||||
const commentary = x.commentary as { text?: unknown } | undefined;
|
||||
const text = commentary ? textOf(commentary.text) : null;
|
||||
if (!text) continue;
|
||||
const meta = x.updateMetadata as { urn?: string } | undefined;
|
||||
const activityUrn = meta?.urn ?? "";
|
||||
|
||||
// Find the matching counts (the SocialActivityCounts urn references the backing ugcPost).
|
||||
let counts = { reactions: 0, comments: 0, shares: 0 };
|
||||
for (const [urn, c] of countsByUrn) {
|
||||
const idActivity = activityUrn.match(/(\d{18,})/)?.[1];
|
||||
if (idActivity && urn.includes(idActivity)) { counts = c; break; }
|
||||
}
|
||||
|
||||
const content = x.content as Record<string, unknown> | undefined;
|
||||
const ctype = content ? String(Object.keys(content)[0] ?? "").toLowerCase() : "";
|
||||
const mediaType: ProfilePost["mediaType"] =
|
||||
ctype.includes("video") ? "video" :
|
||||
ctype.includes("image") ? "image" :
|
||||
ctype.includes("article") ? "article" : "text";
|
||||
|
||||
posts.push({
|
||||
activityUrn,
|
||||
text,
|
||||
postedAt: timeFromActivityUrn(activityUrn),
|
||||
mediaType,
|
||||
reactions: counts.reactions,
|
||||
comments: counts.comments,
|
||||
shares: counts.shares,
|
||||
});
|
||||
}
|
||||
|
||||
// Most recent first (by encoded time when available).
|
||||
posts.sort((a, b) => (b.postedAt ?? "").localeCompare(a.postedAt ?? ""));
|
||||
return posts.slice(0, 10);
|
||||
} catch {
|
||||
return [];
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
}
|
||||
|
||||
// ─── public ───────────────────────────────────────────────────────────────────
|
||||
|
||||
export async function scrapeProfile(
|
||||
ctx: BrowserContext,
|
||||
target: { sales_nav_url: string | null; linkedin_url: string | null }
|
||||
): Promise<ProfileScrape> {
|
||||
if (!target.sales_nav_url) {
|
||||
throw new Error("Contact has no sales_nav_url — cannot scrape the Sales Nav profile.");
|
||||
}
|
||||
|
||||
const career = await scrapeSalesCareer(ctx, target.sales_nav_url);
|
||||
|
||||
// Prefer the flagship /in/ URL from the career scrape; fall back to the stored one.
|
||||
const inUrl = career.linkedin_url ?? target.linkedin_url ?? null;
|
||||
const publicId = publicIdFromUrl(inUrl);
|
||||
const recent_posts = publicId ? await scrapePosts(ctx, publicId) : [];
|
||||
|
||||
return {
|
||||
full_name: career.full_name ?? null,
|
||||
first_name: career.first_name ?? null,
|
||||
last_name: career.last_name ?? null,
|
||||
headline: career.headline ?? null,
|
||||
summary: career.summary ?? null,
|
||||
location: career.location ?? null,
|
||||
degree: career.degree ?? null,
|
||||
linkedin_url: inUrl,
|
||||
num_connections: career.num_connections ?? null,
|
||||
num_shared_connections: career.num_shared_connections ?? null,
|
||||
is_premium: career.is_premium ?? false,
|
||||
is_open_profile: career.is_open_profile ?? false,
|
||||
can_inmail: career.can_inmail ?? false,
|
||||
pending_invitation: career.pending_invitation ?? false,
|
||||
websites: career.websites ?? [],
|
||||
positions: career.positions ?? [],
|
||||
educations: career.educations ?? [],
|
||||
skills: career.skills ?? [],
|
||||
languages: career.languages ?? [],
|
||||
recent_posts,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import type DatabaseType from "better-sqlite3";
|
||||
|
||||
type DB = DatabaseType.Database;
|
||||
export interface ResolvedAccount { id: string; email: string }
|
||||
|
||||
/**
|
||||
* Resolve which authenticated LinkedIn account to act through for a contact.
|
||||
* Order: explicit id → the contact's most recent run assignment → the sole
|
||||
* authenticated account. Only ever returns an `is_authenticated = 1` account
|
||||
* so callers never drive a dead session.
|
||||
*/
|
||||
export function resolveLinkedInAccount(db: DB, targetId: string, explicitId?: string): ResolvedAccount | null {
|
||||
const byId = (aid: string) =>
|
||||
db.prepare("SELECT id, email FROM accounts WHERE id = ? AND is_authenticated = 1").get(aid) as
|
||||
| ResolvedAccount
|
||||
| undefined;
|
||||
|
||||
if (explicitId) return byId(explicitId) ?? null;
|
||||
|
||||
const assigned = db.prepare(`
|
||||
SELECT r.account_id FROM run_profiles rp
|
||||
JOIN runs r ON r.id = rp.run_id
|
||||
WHERE rp.target_id = ?
|
||||
ORDER BY rp.created_at DESC LIMIT 1
|
||||
`).get(targetId) as { account_id: string } | undefined;
|
||||
if (assigned?.account_id) {
|
||||
const a = byId(assigned.account_id);
|
||||
if (a) return a;
|
||||
}
|
||||
|
||||
const all = db.prepare("SELECT id, email FROM accounts WHERE is_authenticated = 1").all() as ResolvedAccount[];
|
||||
return all.length === 1 ? all[0] : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve an authenticated LinkedIn account when there's no target/contact to
|
||||
* anchor the lookup to (e.g. Sales Navigator search, which runs before any
|
||||
* lead exists locally). Order: explicit id → the sole authenticated account.
|
||||
*/
|
||||
export function resolveAnyAuthenticatedAccount(db: DB, explicitId?: string): ResolvedAccount | null {
|
||||
const byId = (aid: string) =>
|
||||
db.prepare("SELECT id, email FROM accounts WHERE id = ? AND is_authenticated = 1").get(aid) as
|
||||
| ResolvedAccount
|
||||
| undefined;
|
||||
|
||||
if (explicitId) return byId(explicitId) ?? null;
|
||||
|
||||
const all = db.prepare("SELECT id, email FROM accounts WHERE is_authenticated = 1").all() as ResolvedAccount[];
|
||||
return all.length === 1 ? all[0] : null;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,453 @@
|
||||
/**
|
||||
* Sales Navigator list scraper using the internal Sales API.
|
||||
* Uses Playwright's browser context request (inherits browser TLS fingerprint).
|
||||
*
|
||||
* Response format discovery (from capture scripts):
|
||||
* - ctx.request returns normalized JSON: { data: { metadata: { totalDisplayCount: "106" } }, included: [...] }
|
||||
* - Profiles are in `included` filtered by entityUrn containing "salesProfile"
|
||||
* - Browser-intercepted first page returns flat: { elements: [...], paging: { total } }
|
||||
* - Query format: parentheses/commas unencoded, colons in URN encoded as %3A
|
||||
*
|
||||
* IMPORTANT — vanityName is gone from salesApiPeopleSearch (dropped by LinkedIn ~March 2026):
|
||||
* The list API no longer returns vanityName. To get the real /in/ URL we call
|
||||
* salesApiProfiles per batch of 25 after scraping, using flagshipProfileUrl.
|
||||
* See docs/linkedin-api-learnings.md for the full investigation.
|
||||
*/
|
||||
import type { BrowserContext, Page } from "playwright";
|
||||
|
||||
export interface ScrapedProfile {
|
||||
salesNavUrn: string;
|
||||
salesNavUrl: string;
|
||||
linkedinUrl: string | null; // regular /in/ URL
|
||||
fullName: string | null;
|
||||
firstName: string | null;
|
||||
lastName: string | null;
|
||||
title: string | null;
|
||||
company: string | null;
|
||||
location: string | null;
|
||||
degree: number | null;
|
||||
// Extended fields
|
||||
objectUrn: string | null; // urn:li:member:XXXX — stable LinkedIn member ID
|
||||
summary: string | null; // About / bio text
|
||||
openLink: boolean; // can message without connecting
|
||||
companyIndustry: string | null;
|
||||
companyLocation: string | null; // company HQ location
|
||||
tenureMonths: number | null; // months in current role
|
||||
spotlightBadges: string | null; // JSON array of badge displayValues
|
||||
}
|
||||
|
||||
interface SalesProfile {
|
||||
entityUrn: string;
|
||||
objectUrn?: string;
|
||||
firstName?: string;
|
||||
lastName?: string;
|
||||
fullName?: string;
|
||||
vanityName?: string;
|
||||
geoRegion?: string;
|
||||
degree?: number;
|
||||
summary?: string;
|
||||
openLink?: boolean;
|
||||
premium?: boolean;
|
||||
pendingInvitation?: boolean;
|
||||
currentPositions?: Array<{
|
||||
title?: string;
|
||||
companyName?: string;
|
||||
companyUrn?: string;
|
||||
current?: boolean;
|
||||
description?: string;
|
||||
startedOn?: { year?: number; month?: number };
|
||||
tenureAtPosition?: { numYears?: number; numMonths?: number };
|
||||
tenureAtCompany?: { numYears?: number; numMonths?: number };
|
||||
companyUrnResolutionResult?: {
|
||||
name?: string;
|
||||
industry?: string;
|
||||
location?: string;
|
||||
entityUrn?: string;
|
||||
};
|
||||
}>;
|
||||
leadAssociatedAccount?: { name?: string } | null;
|
||||
spotlightBadges?: Array<{ displayValue?: string; id?: string }>;
|
||||
}
|
||||
|
||||
// Flat response intercepted from browser
|
||||
interface FlatResponse {
|
||||
elements?: SalesProfile[];
|
||||
paging?: { total: number; count: number; start: number };
|
||||
}
|
||||
|
||||
// salesApiProfiles single-profile response
|
||||
interface ProfileDetailResponse {
|
||||
entityUrn?: string;
|
||||
flagshipProfileUrl?: string;
|
||||
fullName?: string;
|
||||
}
|
||||
|
||||
function extractListId(url: string): string | null {
|
||||
const match = url.match(/\/sales\/lists\/people\/(\d+)/);
|
||||
return match ? match[1] : null;
|
||||
}
|
||||
|
||||
function extractSavedSearchId(url: string): string | null {
|
||||
const match = url.match(/[?&]savedSearchId=(\d+)/);
|
||||
return match ? match[1] : null;
|
||||
}
|
||||
|
||||
function urnToSalesNavUrl(urn: string): string {
|
||||
const match = urn.match(/\(([^)]+)\)/);
|
||||
if (!match) return "";
|
||||
return `https://www.linkedin.com/sales/lead/${match[1]}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse profileId, authType, authToken out of a salesProfile URN.
|
||||
* urn:li:fs_salesProfile:(ACwAAEIY-4YB25mKP6R5AKfkFjhO9isSbvsVlag,NAME_SEARCH,22wq)
|
||||
*/
|
||||
function parseUrn(entityUrn: string): { profileId: string; authType: string; authToken: string } | null {
|
||||
const match = entityUrn.match(/\(([^,]+),([^,]+),([^)]+)\)/);
|
||||
if (!match) return null;
|
||||
return { profileId: match[1], authType: match[2], authToken: match[3] };
|
||||
}
|
||||
|
||||
function profileToResult(el: SalesProfile, linkedinUrl: string | null): ScrapedProfile {
|
||||
const currentPos = el.currentPositions?.find((p) => p.current) ?? el.currentPositions?.[0];
|
||||
const company = currentPos?.companyUrnResolutionResult ?? null;
|
||||
|
||||
// Tenure in months at current position
|
||||
let tenureMonths: number | null = null;
|
||||
const t = currentPos?.tenureAtPosition;
|
||||
if (t) tenureMonths = (t.numYears ?? 0) * 12 + (t.numMonths ?? 0);
|
||||
|
||||
// Spotlight badge labels as a compact JSON array e.g. ["Changed jobs", "Mentioned in news"]
|
||||
const badges = (el.spotlightBadges ?? [])
|
||||
.map((b) => b.displayValue)
|
||||
.filter(Boolean) as string[];
|
||||
|
||||
return {
|
||||
salesNavUrn: el.entityUrn,
|
||||
salesNavUrl: urnToSalesNavUrl(el.entityUrn),
|
||||
linkedinUrl,
|
||||
fullName: el.fullName ?? null,
|
||||
firstName: el.firstName ?? null,
|
||||
lastName: el.lastName ?? null,
|
||||
title: currentPos?.title ?? null,
|
||||
company: el.leadAssociatedAccount?.name ?? company?.name ?? currentPos?.companyName ?? null,
|
||||
location: el.geoRegion ?? null,
|
||||
degree: el.degree ?? null,
|
||||
objectUrn: el.objectUrn ?? null,
|
||||
summary: el.summary ?? null,
|
||||
openLink: el.openLink ?? false,
|
||||
companyIndustry: company?.industry ?? null,
|
||||
companyLocation: company?.location ?? null,
|
||||
tenureMonths,
|
||||
spotlightBadges: badges.length > 0 ? JSON.stringify(badges) : null,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Enrich profiles with their real /in/ URL via salesApiProfiles (flagshipProfileUrl).
|
||||
*
|
||||
* Uses page.evaluate fetch with JSESSIONID csrf-token. On 429 (rate limit),
|
||||
* backs off for 30s and retries. Delay between calls: 2s normally, 30s after 429.
|
||||
*
|
||||
* For 100 profiles at 2s/call = ~3.5 minutes. Happens once at import time.
|
||||
*/
|
||||
async function enrichWithFlagshipUrls(
|
||||
page: Page,
|
||||
profiles: SalesProfile[],
|
||||
onProgress?: (p: ImportProgress) => void
|
||||
): Promise<Map<string, string>> {
|
||||
const urlMap = new Map<string, string>(); // entityUrn → flagshipProfileUrl
|
||||
|
||||
const cookies = await page.context().cookies("https://www.linkedin.com");
|
||||
const jsessionid = cookies.find(c => c.name === "JSESSIONID")?.value?.replace(/"/g, "") ?? "";
|
||||
if (!jsessionid) {
|
||||
console.log("[scraper] JSESSIONID not found — skipping flagshipProfileUrl enrichment");
|
||||
return urlMap;
|
||||
}
|
||||
|
||||
let done = 0;
|
||||
for (const el of profiles) {
|
||||
const parts = parseUrn(el.entityUrn);
|
||||
if (!parts) continue;
|
||||
|
||||
const { profileId, authType, authToken } = parts;
|
||||
const apiUrl = `https://www.linkedin.com/sales-api/salesApiProfiles/(profileId:${profileId},authType:${authType},authToken:${authToken})?decoration=%28entityUrn%2CflagshipProfileUrl%29`;
|
||||
|
||||
let attempts = 0;
|
||||
while (attempts < 3) {
|
||||
const evaluatePromise = page.evaluate(async ({ url, csrfToken }: { url: string; csrfToken: string }) => {
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), 12000);
|
||||
const resp = await fetch(url, {
|
||||
credentials: "include",
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
"csrf-token": csrfToken,
|
||||
"x-restli-protocol-version": "2.0.0",
|
||||
"accept": "application/json",
|
||||
},
|
||||
});
|
||||
clearTimeout(timer);
|
||||
return { status: resp.status, body: await resp.text() };
|
||||
} catch (e: unknown) {
|
||||
return { status: -1, body: (e as Error).message };
|
||||
}
|
||||
}, { url: apiUrl, csrfToken: jsessionid });
|
||||
|
||||
const timeoutPromise = new Promise<{ status: number; body: string }>((resolve) =>
|
||||
setTimeout(() => resolve({ status: -1, body: 'timeout' }), 15000)
|
||||
);
|
||||
const result = await Promise.race([evaluatePromise, timeoutPromise]);
|
||||
|
||||
if (result.status === 200) {
|
||||
try {
|
||||
const data = JSON.parse(result.body) as ProfileDetailResponse;
|
||||
if (data.flagshipProfileUrl) {
|
||||
const normalized = data.flagshipProfileUrl.endsWith("/")
|
||||
? data.flagshipProfileUrl
|
||||
: data.flagshipProfileUrl + "/";
|
||||
urlMap.set(el.entityUrn, normalized);
|
||||
}
|
||||
} catch { /* ignore parse errors */ }
|
||||
break;
|
||||
} else if (result.status === 429) {
|
||||
attempts++;
|
||||
console.log(`[scraper] 429 rate limit — waiting 30s before retry (attempt ${attempts}/3)`);
|
||||
await page.waitForTimeout(30000);
|
||||
} else {
|
||||
const reason = result.body === 'timeout' ? 'timeout' : result.status;
|
||||
console.log(`[scraper] enrichment ${reason} for ${el.fullName ?? el.entityUrn.substring(0, 30)} — skipping`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 2s between calls — slow enough to stay under rate limits
|
||||
await page.waitForTimeout(2000);
|
||||
done++;
|
||||
console.log(`[scraper] flagship URLs: ${done}/${profiles.length} resolved`);
|
||||
onProgress?.({ phase: 'enriching', count: done, total: profiles.length });
|
||||
}
|
||||
|
||||
console.log(`[scraper] enrichment done: ${urlMap.size}/${profiles.length} profiles resolved`);
|
||||
return urlMap;
|
||||
}
|
||||
|
||||
export interface ImportProgress {
|
||||
phase: 'scraping' | 'enriching' | 'visiting';
|
||||
page?: number;
|
||||
totalPages?: number;
|
||||
count: number;
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface ScrapeOptions {
|
||||
/** 1-based page to begin this window at (for batched imports). Default 1. */
|
||||
startPage?: number;
|
||||
/** Max pages to fetch in THIS window, starting at startPage. Default 50. */
|
||||
maxPages?: number;
|
||||
onProgress?: (p: ImportProgress) => void;
|
||||
/** Polled between pages; return true to stop early (cancel / deleted list). */
|
||||
isCanceled?: () => boolean | Promise<boolean>;
|
||||
}
|
||||
|
||||
export interface WindowedScrapeResult {
|
||||
profiles: ScrapedProfile[];
|
||||
/** Last page actually fetched in this window. */
|
||||
lastPage: number;
|
||||
/** Total profiles available in the whole search. */
|
||||
knownTotal: number;
|
||||
/** True if this window reached the end of the search (nothing left to batch). */
|
||||
exhausted: boolean;
|
||||
}
|
||||
|
||||
export async function scrapeNavigatorList(
|
||||
ctx: BrowserContext,
|
||||
salesNavUrl: string,
|
||||
opts: ScrapeOptions = {}
|
||||
): Promise<WindowedScrapeResult> {
|
||||
const { startPage = 1, maxPages = 50, onProgress, isCanceled } = opts;
|
||||
const listId = extractListId(salesNavUrl);
|
||||
if (!listId) throw new Error(`Invalid Sales Navigator URL: ${salesNavUrl}`);
|
||||
|
||||
const allElements: SalesProfile[] = [];
|
||||
const seen = new Set<string>();
|
||||
const PAGE_SIZE = 25;
|
||||
const buildUrl = (n: number) =>
|
||||
`https://www.linkedin.com/sales/lists/people/${listId}?${n > 1 ? `page=${n}&` : ""}sortCriteria=CREATED_TIME&sortOrder=DESCENDING`;
|
||||
|
||||
const page = await ctx.newPage();
|
||||
let knownTotal = 0;
|
||||
let intercepted: FlatResponse | null = null;
|
||||
|
||||
const waitForIntercept = async (url: string, waitMs: number): Promise<FlatResponse | null> => {
|
||||
intercepted = null;
|
||||
page.removeAllListeners("response");
|
||||
page.on("response", async (response) => {
|
||||
if (intercepted) return;
|
||||
if (response.url().includes("salesApiPeopleSearch") && response.status() === 200) {
|
||||
try { intercepted = await response.json() as FlatResponse; } catch { /* ignore */ }
|
||||
}
|
||||
});
|
||||
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
|
||||
await page.waitForTimeout(waitMs);
|
||||
return intercepted;
|
||||
};
|
||||
|
||||
// First page of the window
|
||||
const firstData = await waitForIntercept(buildUrl(startPage), 15000);
|
||||
if (!firstData) {
|
||||
const finalUrl = page.url();
|
||||
console.error(`[scraper] no intercept after 15s. Final URL: ${finalUrl}`);
|
||||
await page.close();
|
||||
throw new Error("No data intercepted from Sales Nav — session may need re-authentication");
|
||||
}
|
||||
|
||||
knownTotal = firstData.paging?.total ?? 0;
|
||||
for (const el of firstData.elements ?? []) {
|
||||
if (!el.entityUrn || seen.has(el.entityUrn)) continue;
|
||||
seen.add(el.entityUrn);
|
||||
allElements.push(el);
|
||||
}
|
||||
console.log(`[scraper] page ${startPage}: ${allElements.length} elements, total=${knownTotal}`);
|
||||
|
||||
const totalPages = Math.ceil(knownTotal / PAGE_SIZE);
|
||||
const endPage = Math.min(totalPages, startPage + maxPages - 1);
|
||||
let lastPage = startPage;
|
||||
onProgress?.({ phase: 'scraping', page: startPage, totalPages: endPage, count: allElements.length, total: knownTotal });
|
||||
|
||||
for (let pageNum = startPage + 1; pageNum <= endPage; pageNum++) {
|
||||
if (isCanceled && (await isCanceled())) break;
|
||||
const delayMs = 60000 + Math.random() * 60000;
|
||||
console.log(`[scraper] waiting ${Math.round(delayMs / 1000)}s before page ${pageNum}...`);
|
||||
await page.waitForTimeout(delayMs);
|
||||
if (isCanceled && (await isCanceled())) break;
|
||||
|
||||
let pageData = await waitForIntercept(buildUrl(pageNum), 15000);
|
||||
if (!pageData || (pageData.elements?.length ?? 0) === 0) {
|
||||
console.log(`[scraper] page ${pageNum} empty on first try — retrying with 15s wait`);
|
||||
pageData = await waitForIntercept(buildUrl(pageNum), 15000);
|
||||
}
|
||||
if (pageData) {
|
||||
for (const el of pageData.elements ?? []) {
|
||||
if (!el.entityUrn || seen.has(el.entityUrn)) continue;
|
||||
seen.add(el.entityUrn);
|
||||
allElements.push(el);
|
||||
}
|
||||
}
|
||||
lastPage = pageNum;
|
||||
console.log(`[scraper] page ${pageNum}/${endPage}: ${allElements.length} (total ${knownTotal})`);
|
||||
onProgress?.({ phase: 'scraping', page: pageNum, totalPages: endPage, count: allElements.length, total: knownTotal });
|
||||
}
|
||||
|
||||
await page.close();
|
||||
return {
|
||||
profiles: allElements.map(el => profileToResult(el, null)),
|
||||
lastPage,
|
||||
knownTotal,
|
||||
exhausted: lastPage >= totalPages,
|
||||
};
|
||||
}
|
||||
|
||||
export async function scrapeSavedSearch(
|
||||
ctx: BrowserContext,
|
||||
savedSearchUrl: string,
|
||||
opts: ScrapeOptions = {}
|
||||
): Promise<WindowedScrapeResult> {
|
||||
const { startPage = 1, maxPages = 50, onProgress, isCanceled } = opts;
|
||||
const savedSearchId = extractSavedSearchId(savedSearchUrl);
|
||||
if (!savedSearchId) throw new Error(`Invalid Sales Navigator saved search URL: ${savedSearchUrl}`);
|
||||
|
||||
const allElements: SalesProfile[] = [];
|
||||
const seen = new Set<string>();
|
||||
const PAGE_SIZE = 25;
|
||||
const buildUrl = (n: number) =>
|
||||
`https://www.linkedin.com/sales/search/people?savedSearchId=${savedSearchId}${n > 1 ? `&page=${n}` : ""}`;
|
||||
|
||||
const page = await ctx.newPage();
|
||||
let knownTotal = 0;
|
||||
|
||||
const waitForIntercept = async (url: string, waitMs: number): Promise<FlatResponse | null> => {
|
||||
let intercepted: FlatResponse | null = null;
|
||||
page.removeAllListeners("response");
|
||||
page.on("response", async (response) => {
|
||||
if (intercepted) return;
|
||||
if (response.url().includes("salesApiLeadSearch") && response.status() === 200) {
|
||||
try { intercepted = await response.json() as FlatResponse; } catch { /* ignore */ }
|
||||
}
|
||||
});
|
||||
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
|
||||
await page.waitForTimeout(waitMs);
|
||||
return intercepted;
|
||||
};
|
||||
|
||||
// First page of the window
|
||||
const firstData = await waitForIntercept(buildUrl(startPage), 15000);
|
||||
if (!firstData) {
|
||||
await page.close();
|
||||
throw new Error("No data intercepted from saved search — session may need re-authentication");
|
||||
}
|
||||
|
||||
knownTotal = firstData.paging?.total ?? 0;
|
||||
for (const el of firstData.elements ?? []) {
|
||||
if (!el.entityUrn || seen.has(el.entityUrn)) continue;
|
||||
seen.add(el.entityUrn);
|
||||
allElements.push(el);
|
||||
}
|
||||
console.log(`[scraper:saved-search] page ${startPage}: ${allElements.length} elements, total=${knownTotal}`);
|
||||
|
||||
const totalPages = Math.ceil(knownTotal / PAGE_SIZE);
|
||||
const endPage = Math.min(totalPages, startPage + maxPages - 1);
|
||||
let lastPage = startPage;
|
||||
onProgress?.({ phase: 'scraping', page: startPage, totalPages: endPage, count: allElements.length, total: knownTotal });
|
||||
|
||||
for (let pageNum = startPage + 1; pageNum <= endPage; pageNum++) {
|
||||
if (isCanceled && (await isCanceled())) break;
|
||||
const delayMs = 60000 + Math.random() * 60000;
|
||||
console.log(`[scraper:saved-search] waiting ${Math.round(delayMs / 1000)}s before page ${pageNum}...`);
|
||||
await page.waitForTimeout(delayMs);
|
||||
if (isCanceled && (await isCanceled())) break;
|
||||
|
||||
let pageData = await waitForIntercept(buildUrl(pageNum), 15000);
|
||||
if (!pageData || (pageData.elements?.length ?? 0) === 0) {
|
||||
console.log(`[scraper:saved-search] page ${pageNum} empty on first try — retrying with 15s wait`);
|
||||
pageData = await waitForIntercept(buildUrl(pageNum), 15000);
|
||||
}
|
||||
if (pageData) {
|
||||
for (const el of pageData.elements ?? []) {
|
||||
if (!el.entityUrn || seen.has(el.entityUrn)) continue;
|
||||
seen.add(el.entityUrn);
|
||||
allElements.push(el);
|
||||
}
|
||||
}
|
||||
lastPage = pageNum;
|
||||
console.log(`[scraper:saved-search] page ${pageNum}/${endPage}: ${allElements.length} (total ${knownTotal})`);
|
||||
onProgress?.({ phase: 'scraping', page: pageNum, totalPages: endPage, count: allElements.length, total: knownTotal });
|
||||
}
|
||||
|
||||
await page.close();
|
||||
return {
|
||||
profiles: allElements.map(el => profileToResult(el, null)),
|
||||
lastPage,
|
||||
knownTotal,
|
||||
exhausted: lastPage >= totalPages,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatcher — accepts either a lead list URL or a saved search URL.
|
||||
* Callers don't need to know which type they're dealing with.
|
||||
*/
|
||||
export async function scrapeNavigatorUrl(
|
||||
ctx: BrowserContext,
|
||||
url: string,
|
||||
opts: ScrapeOptions = {}
|
||||
): Promise<WindowedScrapeResult> {
|
||||
if (extractSavedSearchId(url)) {
|
||||
return scrapeSavedSearch(ctx, url, opts);
|
||||
}
|
||||
if (extractListId(url)) {
|
||||
return scrapeNavigatorList(ctx, url, opts);
|
||||
}
|
||||
throw new Error(`Unrecognized Sales Navigator URL. Expected a list URL (/sales/lists/people/...) or saved search URL (?savedSearchId=...)`);
|
||||
}
|
||||
@@ -0,0 +1,519 @@
|
||||
import { chromium } from "playwright-extra";
|
||||
import type { Browser, BrowserContext, Page } from "playwright";
|
||||
import StealthPlugin from "puppeteer-extra-plugin-stealth";
|
||||
import { getDb } from "@/lib/db";
|
||||
import { encryptSecret, decryptSecret } from "@/lib/crypto";
|
||||
|
||||
chromium.use(StealthPlugin());
|
||||
|
||||
let browser: Browser | null = null;
|
||||
const contexts: Map<string, BrowserContext> = new Map();
|
||||
|
||||
const HEADLESS = process.env.HEADLESS !== "false";
|
||||
const CHROMIUM_PATH = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH;
|
||||
|
||||
const LAUNCH_ARGS = [
|
||||
"--no-sandbox",
|
||||
"--disable-setuid-sandbox",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-gpu",
|
||||
];
|
||||
|
||||
/**
|
||||
* Shared browser-context fingerprint. Login and runtime MUST use the identical
|
||||
* options so the LinkedIn session is BORN under the exact fingerprint it will
|
||||
* later be used with — a mismatch (or a drift) triggers a forced re-auth.
|
||||
*/
|
||||
function contextOptions(storageState?: object) {
|
||||
return {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
storageState: storageState as any,
|
||||
viewport: { width: 1920, height: 1080 },
|
||||
userAgent:
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
locale: "en-US",
|
||||
timezoneId: "America/New_York",
|
||||
permissions: ["clipboard-read", "clipboard-write"] as ("clipboard-read" | "clipboard-write")[],
|
||||
};
|
||||
}
|
||||
|
||||
async function getBrowser(headless = HEADLESS): Promise<Browser> {
|
||||
// B1: if the cached browser is disconnected, CLOSE it before relaunching.
|
||||
// Without this, a dead-but-not-reaped chromium process tree is orphaned on
|
||||
// every relaunch (the leak behind the Jun 2026 zombie pile-up).
|
||||
if (browser && !browser.isConnected()) {
|
||||
try { await browser.close(); } catch { /* already gone */ }
|
||||
browser = null;
|
||||
}
|
||||
if (!browser) {
|
||||
browser = await chromium.launch({
|
||||
headless,
|
||||
executablePath: CHROMIUM_PATH,
|
||||
args: LAUNCH_ARGS,
|
||||
});
|
||||
}
|
||||
return browser;
|
||||
}
|
||||
|
||||
async function getOrCreateContext(accountId: string): Promise<BrowserContext> {
|
||||
const db = getDb();
|
||||
const account = db.prepare("SELECT * FROM accounts WHERE id = ?").get(accountId) as
|
||||
| { cookies_json: string | null; email: string }
|
||||
| undefined;
|
||||
|
||||
if (!account) throw new Error(`Account ${accountId} not found`);
|
||||
|
||||
if (!contexts.has(accountId)) {
|
||||
const b = await getBrowser();
|
||||
|
||||
let storageState: object | undefined;
|
||||
if (account.cookies_json) {
|
||||
try {
|
||||
storageState = JSON.parse(decryptSecret(account.cookies_json)!);
|
||||
} catch {
|
||||
// Invalid storage state — will need re-auth
|
||||
}
|
||||
}
|
||||
|
||||
const ctx = await b.newContext(contextOptions(storageState));
|
||||
|
||||
// Auto-evict from map when context closes for any reason (crash, session expiry, etc.)
|
||||
ctx.on("close", () => { if (contexts.get(accountId) === ctx) contexts.delete(accountId); });
|
||||
|
||||
contexts.set(accountId, ctx);
|
||||
}
|
||||
|
||||
return contexts.get(accountId)!;
|
||||
}
|
||||
|
||||
/** Returns the BrowserContext for an account (for API calls via ctx.request) */
|
||||
export async function getSessionContext(accountId: string): Promise<BrowserContext> {
|
||||
try {
|
||||
return await getOrCreateContext(accountId);
|
||||
} catch {
|
||||
// First attempt failed — evict and retry once with a fresh context
|
||||
contexts.delete(accountId);
|
||||
return getOrCreateContext(accountId);
|
||||
}
|
||||
}
|
||||
|
||||
// B6 (Jul 2026 CPU-spike incident): closing a Playwright page doesn't mean the
|
||||
// underlying Chromium renderer OS process has actually exited — under CPU
|
||||
// contention on the 2-vCPU prod box, teardown was observed lagging 60-90s
|
||||
// behind page.close(). If the runner loop (or a concurrent MCP/API call)
|
||||
// opens its next page immediately after, two renderer processes end up alive
|
||||
// at once, which is enough to peg both cores and starve sibling containers'
|
||||
// healthchecks (NocoDB/Chatwoot flapping). Fix: serialize ALL page opens
|
||||
// app-wide through one queue, and hold the queue for a teardown buffer after
|
||||
// each page.close() before letting the next one through. PAGE_MAX_HOLD_MS is
|
||||
// a safety valve so a caller that forgets to close its page can't wedge the
|
||||
// whole app's browser access forever.
|
||||
const PAGE_TEARDOWN_GAP_MS = 3000;
|
||||
const PAGE_MAX_HOLD_MS = 120_000;
|
||||
let pageQueueTail: Promise<void> = Promise.resolve();
|
||||
|
||||
/** Returns a new Page from the account's browser context */
|
||||
export async function getSessionPage(accountId: string): Promise<Page> {
|
||||
let releaseTurn!: () => void;
|
||||
const myTurn = new Promise<void>(r => { releaseTurn = r; });
|
||||
const previousTail = pageQueueTail;
|
||||
pageQueueTail = myTurn;
|
||||
await previousTail;
|
||||
|
||||
let released = false;
|
||||
const release = () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
releaseTurn();
|
||||
};
|
||||
const safetyTimer = setTimeout(release, PAGE_MAX_HOLD_MS);
|
||||
|
||||
let page: Page;
|
||||
try {
|
||||
const ctx = await getOrCreateContext(accountId);
|
||||
try {
|
||||
page = await ctx.newPage();
|
||||
} catch {
|
||||
// B2: context was dead — CLOSE it before recreating so its underlying
|
||||
// browser process isn't left orphaned, then retry once with a fresh one.
|
||||
try { await ctx.close(); } catch { /* already gone */ }
|
||||
contexts.delete(accountId);
|
||||
const freshCtx = await getOrCreateContext(accountId);
|
||||
page = await freshCtx.newPage();
|
||||
}
|
||||
} catch (err) {
|
||||
clearTimeout(safetyTimer);
|
||||
release();
|
||||
throw err;
|
||||
}
|
||||
|
||||
const originalClose = page.close.bind(page);
|
||||
page.close = (async (options?: Parameters<Page["close"]>[0]) => {
|
||||
try {
|
||||
return await originalClose(options);
|
||||
} finally {
|
||||
clearTimeout(safetyTimer);
|
||||
setTimeout(release, PAGE_TEARDOWN_GAP_MS);
|
||||
}
|
||||
}) as Page["close"];
|
||||
|
||||
return page;
|
||||
}
|
||||
|
||||
export async function saveSessionState(accountId: string): Promise<void> {
|
||||
const ctx = contexts.get(accountId);
|
||||
if (!ctx) return;
|
||||
const db = getDb();
|
||||
const state = await ctx.storageState();
|
||||
db.prepare("UPDATE accounts SET cookies_json = ?, is_authenticated = 1 WHERE id = ?").run(
|
||||
encryptSecret(JSON.stringify(state)),
|
||||
accountId
|
||||
);
|
||||
}
|
||||
|
||||
export async function closeSession(accountId: string): Promise<void> {
|
||||
const ctx = contexts.get(accountId);
|
||||
if (ctx) {
|
||||
await ctx.close();
|
||||
contexts.delete(accountId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* B4: flag an account as logged out / needing re-auth. Clears is_authenticated
|
||||
* so the runner stops working a dead session (no more 30s-timeout fail-loop),
|
||||
* and drops the live context. The user re-authenticates from Settings.
|
||||
*/
|
||||
export async function markNeedsReauth(accountId: string): Promise<void> {
|
||||
const db = getDb();
|
||||
db.prepare("UPDATE accounts SET is_authenticated = 0 WHERE id = ?").run(accountId);
|
||||
try { await closeSession(accountId); } catch { /* ignore */ }
|
||||
console.warn(`[session] account ${accountId} flagged needs-reauth (session logged out)`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens a visible browser, navigates to LinkedIn login, and waits for the user
|
||||
* to complete login manually. Returns when the user reaches /feed.
|
||||
* Saves the full storage state to DB and marks account as authenticated.
|
||||
*/
|
||||
export async function authenticateAccount(accountId: string): Promise<void> {
|
||||
const db = getDb();
|
||||
const account = db.prepare("SELECT * FROM accounts WHERE id = ?").get(accountId) as
|
||||
| { email: string }
|
||||
| undefined;
|
||||
if (!account) throw new Error(`Account ${accountId} not found`);
|
||||
|
||||
// Close any existing context for this account — start fresh
|
||||
await closeSession(accountId);
|
||||
|
||||
// Always launch a VISIBLE browser for manual login
|
||||
const visibleBrowser = await chromium.launch({
|
||||
headless: false,
|
||||
executablePath: CHROMIUM_PATH,
|
||||
args: [
|
||||
"--no-sandbox",
|
||||
"--disable-setuid-sandbox",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-gpu",
|
||||
],
|
||||
});
|
||||
|
||||
try {
|
||||
const ctx = await visibleBrowser.newContext({
|
||||
viewport: { width: 1440, height: 900 },
|
||||
userAgent:
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
locale: "en-US",
|
||||
timezoneId: "America/New_York",
|
||||
});
|
||||
|
||||
const page = await ctx.newPage();
|
||||
await page.goto("https://www.linkedin.com/login");
|
||||
|
||||
// Pre-fill email to save the user a step
|
||||
try {
|
||||
await page.waitForSelector("input#username", { timeout: 5000 });
|
||||
await page.fill("input#username", account.email);
|
||||
} catch {
|
||||
// Input not found — page may have redirected already
|
||||
}
|
||||
|
||||
// Wait up to 3 minutes for the user to complete login and reach /feed
|
||||
await page.waitForURL("**/feed/**", { timeout: 180_000 });
|
||||
|
||||
// Save full storage state (cookies + localStorage) to DB
|
||||
const state = await ctx.storageState();
|
||||
db.prepare("UPDATE accounts SET cookies_json = ?, is_authenticated = 1 WHERE id = ?").run(
|
||||
encryptSecret(JSON.stringify(state)),
|
||||
accountId
|
||||
);
|
||||
|
||||
await ctx.close();
|
||||
} finally {
|
||||
await visibleBrowser.close();
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Server-side headless login ───────────────────────────────────────────────
|
||||
// Logs in directly on the server (no screen) so the session is born under the
|
||||
// SAME pinned Chromium fingerprint the runner uses, and so ALL cookies — incl.
|
||||
// httpOnly ones like li_ep_auth_context (the Sales Nav seat cookie that
|
||||
// document.cookie cannot read) — are captured. A login from a datacenter IP
|
||||
// almost always triggers an email/SMS PIN, so it's a two-step flow: start
|
||||
// (email+password) → maybe a challenge → verify (code).
|
||||
|
||||
export type LoginResult =
|
||||
| { status: "authenticated" }
|
||||
| { status: "challenge"; kind: "otp" | "app" | "captcha" | "unknown"; message: string }
|
||||
| { status: "error"; message: string };
|
||||
|
||||
type PendingLogin = { ctx: BrowserContext; page: Page; createdAt: number };
|
||||
const pendingLogins: Map<string, PendingLogin> = new Map();
|
||||
const PENDING_TTL_MS = 10 * 60_000;
|
||||
|
||||
// PIN/verification-code input. Named ids first, then type-based fallbacks for
|
||||
// LinkedIn's React checkpoint pages (which use dynamic ids). Safe: the login
|
||||
// page itself has no tel/one-time-code input to misfire on.
|
||||
const PIN_SELECTOR =
|
||||
"input[name='pin'], #input__email_verification_pin, input[autocomplete='one-time-code']:visible, input[type='tel']:visible";
|
||||
|
||||
async function clearPendingLogin(accountId: string): Promise<void> {
|
||||
const p = pendingLogins.get(accountId);
|
||||
if (p) {
|
||||
pendingLogins.delete(accountId);
|
||||
try { await p.ctx.close(); } catch { /* already gone */ }
|
||||
}
|
||||
}
|
||||
|
||||
function sweepPendingLogins(): void {
|
||||
const now = Date.now();
|
||||
for (const [id, p] of pendingLogins) {
|
||||
if (now - p.createdAt > PENDING_TTL_MS) void clearPendingLogin(id);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Warm the freshly-authenticated session by loading Sales Navigator once, THEN
|
||||
* persist. The bare login POST lands on /feed and only mints the ~11 core
|
||||
* LinkedIn cookies — it does NOT yet include the Sales Nav SEAT cookie
|
||||
* (li_ep_auth_context) nor the secondary auth tokens (li_a, liap) and
|
||||
* localStorage. Those are only issued once the browser actually enters Sales
|
||||
* Navigator. Without the seat cookie every Sales Nav API call returns nothing
|
||||
* ("no intercept after 15s" → import fails → account wrongly flagged
|
||||
* needs-reauth). So we navigate to /sales/ and wait for it to settle before
|
||||
* calling storageState(), capturing the FULL session the runner needs.
|
||||
* Best-effort: if the account has no Sales Nav seat the nav simply doesn't add
|
||||
* the seat cookie — the rest of the (regular-LinkedIn) session is still saved.
|
||||
*/
|
||||
async function persistLogin(accountId: string, ctx: BrowserContext, page?: Page): Promise<void> {
|
||||
if (page) {
|
||||
try {
|
||||
await page.goto("https://www.linkedin.com/sales/home", { waitUntil: "domcontentloaded", timeout: 30_000 });
|
||||
// Let Sales Nav's bootstrap requests fire so li_ep_auth_context is set.
|
||||
await page.waitForTimeout(4_000);
|
||||
} catch {
|
||||
// Non-fatal — a missing seat / slow load must not fail the whole login.
|
||||
}
|
||||
}
|
||||
const db = getDb();
|
||||
const state = await ctx.storageState();
|
||||
db.prepare("UPDATE accounts SET cookies_json = ?, is_authenticated = 1 WHERE id = ?").run(
|
||||
encryptSecret(JSON.stringify(state)),
|
||||
accountId
|
||||
);
|
||||
// Drop any stale runtime context so the runner reloads the fresh cookies.
|
||||
await closeSession(accountId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Inspect the page after a login/verify submit and classify the outcome.
|
||||
* LinkedIn varies its challenge per attempt — email/SMS code OR device (app)
|
||||
* approval — so we detect both: a visible code input = otp; a checkpoint page
|
||||
* with no code input and no captcha = device approval.
|
||||
*/
|
||||
async function classifyLoginState(page: Page): Promise<LoginResult> {
|
||||
const start = Date.now();
|
||||
const deadline = start + 20_000;
|
||||
while (Date.now() < deadline) {
|
||||
const url = page.url();
|
||||
if (/\/feed\//.test(url) || /linkedin\.com\/sales\//.test(url)) {
|
||||
return { status: "authenticated" };
|
||||
}
|
||||
|
||||
// Email/SMS PIN entry
|
||||
const pin = page.locator(PIN_SELECTOR).first();
|
||||
if ((await pin.count()) > 0 && (await pin.isVisible().catch(() => false))) {
|
||||
return {
|
||||
status: "challenge",
|
||||
kind: "otp",
|
||||
message: "LinkedIn sent you a verification code (email or SMS). Enter it below.",
|
||||
};
|
||||
}
|
||||
|
||||
if (/checkpoint\/challenge/.test(url)) {
|
||||
// CAPTCHA (Arkose / FunCaptcha) — cannot be solved headlessly
|
||||
const cap = page.locator("iframe[src*='arkoselabs'], iframe[title*='captcha'], #captcha-internal");
|
||||
if ((await cap.count().catch(() => 0)) > 0) {
|
||||
return {
|
||||
status: "challenge",
|
||||
kind: "captcha",
|
||||
message: "LinkedIn requires a CAPTCHA, which can't be solved on the server. Use cookie paste instead.",
|
||||
};
|
||||
}
|
||||
// Device/app approval: a settled checkpoint with no code input and no captcha
|
||||
if (Date.now() - start > 4_000) {
|
||||
return {
|
||||
status: "challenge",
|
||||
kind: "app",
|
||||
message: "LinkedIn sent a sign-in request to your LinkedIn mobile app. Approve it there, then click Continue.",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Wrong credentials
|
||||
const wrongPw = await page
|
||||
.getByText(/that.?s not the right password|please enter a valid|couldn.?t find a linkedin account/i)
|
||||
.count()
|
||||
.catch(() => 0);
|
||||
if (wrongPw > 0) return { status: "error", message: "Wrong email or password." };
|
||||
|
||||
await page.waitForTimeout(800);
|
||||
}
|
||||
|
||||
if (/checkpoint/.test(page.url())) {
|
||||
return {
|
||||
status: "challenge",
|
||||
kind: "unknown",
|
||||
message: "LinkedIn presented a security checkpoint. If you got a code enter it; if it's an app request, approve it and click Continue.",
|
||||
};
|
||||
}
|
||||
return { status: "error", message: `Login did not complete. Current page: ${page.url()}` };
|
||||
}
|
||||
|
||||
export async function startHeadlessLogin(
|
||||
accountId: string,
|
||||
email: string,
|
||||
password: string
|
||||
): Promise<LoginResult> {
|
||||
sweepPendingLogins();
|
||||
await clearPendingLogin(accountId);
|
||||
|
||||
const b = await getBrowser(true);
|
||||
const ctx = await b.newContext(contextOptions());
|
||||
const page = await ctx.newPage();
|
||||
try {
|
||||
await page.goto("https://www.linkedin.com/login", { waitUntil: "domcontentloaded", timeout: 30_000 });
|
||||
// LinkedIn's React login page uses dynamic ids — target by type+visibility
|
||||
// (old #username/#password kept as a fallback for the legacy layout).
|
||||
const emailInput = page.locator("input#username, input[type='email']:visible").first();
|
||||
await emailInput.waitFor({ state: "visible", timeout: 20_000 });
|
||||
await emailInput.fill(email);
|
||||
const passwordInput = page.locator("input#password, input[type='password']:visible").first();
|
||||
await passwordInput.fill(password);
|
||||
await Promise.all([
|
||||
page.waitForLoadState("domcontentloaded").catch(() => {}),
|
||||
passwordInput.press("Enter"),
|
||||
]);
|
||||
|
||||
const result = await classifyLoginState(page);
|
||||
console.log(`[login] start account=${accountId} -> ${result.status}${"kind" in result ? "/" + result.kind : ""} url=${page.url()}`);
|
||||
if (result.status === "authenticated") {
|
||||
await persistLogin(accountId, ctx, page);
|
||||
await ctx.close();
|
||||
return result;
|
||||
}
|
||||
if (result.status === "challenge" && result.kind !== "captcha") {
|
||||
pendingLogins.set(accountId, { ctx, page, createdAt: Date.now() });
|
||||
return result;
|
||||
}
|
||||
await ctx.close();
|
||||
return result;
|
||||
} catch (e) {
|
||||
console.log(`[login] start account=${accountId} ERROR ${(e as Error).message} url=${page.url()}`);
|
||||
try { await ctx.close(); } catch { /* ignore */ }
|
||||
return { status: "error", message: (e as Error).message };
|
||||
}
|
||||
}
|
||||
|
||||
export async function submitLoginChallenge(accountId: string, code: string): Promise<LoginResult> {
|
||||
const p = pendingLogins.get(accountId);
|
||||
if (!p) return { status: "error", message: "No login in progress (it may have timed out — start again)." };
|
||||
|
||||
const { ctx, page } = p;
|
||||
try {
|
||||
const pin = page.locator(PIN_SELECTOR).first();
|
||||
await pin.waitFor({ state: "visible", timeout: 15_000 });
|
||||
await pin.fill(code);
|
||||
await Promise.all([
|
||||
page.waitForLoadState("domcontentloaded").catch(() => {}),
|
||||
pin.press("Enter"),
|
||||
]);
|
||||
|
||||
const result = await classifyLoginState(page);
|
||||
console.log(`[login] verify account=${accountId} -> ${result.status}${"kind" in result ? "/" + result.kind : ""} url=${page.url()}`);
|
||||
if (result.status === "authenticated") {
|
||||
await persistLogin(accountId, ctx, page);
|
||||
await clearPendingLogin(accountId);
|
||||
return result;
|
||||
}
|
||||
if (result.status === "challenge" && result.kind !== "captcha") {
|
||||
p.createdAt = Date.now(); // keep the session alive for another step
|
||||
return result;
|
||||
}
|
||||
await clearPendingLogin(accountId);
|
||||
return result.status === "error"
|
||||
? result
|
||||
: { status: "error", message: "Code rejected or login failed." };
|
||||
} catch (e) {
|
||||
await clearPendingLogin(accountId);
|
||||
return { status: "error", message: (e as Error).message };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for a device/app-approval challenge to clear. Called after the user
|
||||
* approves the sign-in in their LinkedIn mobile app — the checkpoint page then
|
||||
* auto-advances to the feed. Also dismisses a possible "remember this browser?"
|
||||
* interstitial. If still pending, returns the challenge so the user can retry.
|
||||
*/
|
||||
export async function awaitLoginApproval(accountId: string): Promise<LoginResult> {
|
||||
const p = pendingLogins.get(accountId);
|
||||
if (!p) return { status: "error", message: "No login in progress (it may have timed out — start again)." };
|
||||
|
||||
const { ctx, page } = p;
|
||||
p.createdAt = Date.now();
|
||||
try {
|
||||
const reachedFeed = await page
|
||||
.waitForURL(/\/feed\/|linkedin\.com\/sales\//, { timeout: 50_000 })
|
||||
.then(() => true)
|
||||
.catch(() => false);
|
||||
|
||||
if (!reachedFeed) {
|
||||
// Possible post-approval interstitial (e.g. "remember this browser?")
|
||||
const btn = page
|
||||
.locator("button[type=submit]:visible, button:has-text('Yes'):visible, button:has-text('Ja'):visible")
|
||||
.first();
|
||||
if ((await btn.count().catch(() => 0)) > 0) {
|
||||
await btn.click().catch(() => {});
|
||||
await page.waitForURL(/\/feed\/|linkedin\.com\/sales\//, { timeout: 20_000 }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
const result = await classifyLoginState(page);
|
||||
console.log(`[login] await account=${accountId} -> ${result.status}${"kind" in result ? "/" + result.kind : ""} url=${page.url()}`);
|
||||
if (result.status === "authenticated") {
|
||||
await persistLogin(accountId, ctx, page);
|
||||
await clearPendingLogin(accountId);
|
||||
return result;
|
||||
}
|
||||
if (result.status === "challenge" && result.kind !== "captcha") {
|
||||
p.createdAt = Date.now();
|
||||
return result;
|
||||
}
|
||||
await clearPendingLogin(accountId);
|
||||
return result;
|
||||
} catch (e) {
|
||||
await clearPendingLogin(accountId);
|
||||
return { status: "error", message: (e as Error).message };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
import type { Page } from "playwright";
|
||||
import { getDb } from "@/lib/db";
|
||||
import { getSessionPage, saveSessionState, markNeedsReauth } from "@/lib/linkedin/session";
|
||||
|
||||
/**
|
||||
* Accepted-connection sync via the authoritative Voyager connections API.
|
||||
*
|
||||
* Replaces the old absence-inference (scroll the SENT list, treat a vanished
|
||||
* invite as "accepted"). That was measurably wrong: of 1,600 contacts it had
|
||||
* marked degree=1, only 325 were genuinely connected — 1,275 were phantoms
|
||||
* (invites that expired/withdrew/were ignored, not accepted). See docs §19.
|
||||
*
|
||||
* Sources — all proved against prod (Jul 2026), NO scrolling:
|
||||
* - DATA: GET /voyager/api/relationships/dash/connections
|
||||
* ?decorationId=…ConnectionListWithProfile-16&q=search
|
||||
* &sortType=RECENTLY_ADDED&start=N&count=100
|
||||
* → data["*elements"] (newest-first) + included[]: Connection { createdAt(ms),
|
||||
* connectedMember } and Profile { publicIdentifier (vanity), entityUrn }.
|
||||
* The API paginates to the end (30 pages for ~2947) in ~60s. It does NOT
|
||||
* expose a grand total.
|
||||
* - TOTAL (checksum): the connections PAGE header "<N> connections" <p>. Read
|
||||
* once per full pass via a single navigation (NO scroll). A full pass is
|
||||
* "verified complete" only when unique-pulled == declared total (matched ±0
|
||||
* in prod). Presence in the list = 100% proof of a 1st-degree connection.
|
||||
*
|
||||
* Behaviour:
|
||||
* - First run (boundary NULL) → FULL pass: page to the end, stamp degree=1 +
|
||||
* connected_at from createdAt for every matched contact. If the pass is
|
||||
* checksum-verified complete, ALSO un-mark phantom degree=1 contacts whose
|
||||
* vanity is absent from the authoritative list (resets degree/connected_at).
|
||||
* - Later runs → incremental: page from start=0 and STOP at the first
|
||||
* connection older than the stored boundary (minus a 24h overlap margin).
|
||||
* Incremental/incomplete passes are ADD-ONLY — they never un-mark (a partial
|
||||
* pull must not wipe real accepts). Un-marking happens ONLY on a verified
|
||||
* full pass.
|
||||
* - Reuses the runner's shared browser (getSessionPage) — never a 2nd browser.
|
||||
*/
|
||||
|
||||
const ACCEPTED_SYNC_INTERVAL_MS = 8 * 60 * 60 * 1000; // 8h — 3x per day
|
||||
const PAGE_SIZE = 100;
|
||||
const MAX_PAGES = 60; // safety cap (60 * 100 = 6000)
|
||||
const OVERLAP_MARGIN_MS = 24 * 60 * 60 * 1000; // re-check a day of overlap (idempotent)
|
||||
const DECORATION = "com.linkedin.voyager.dash.deco.web.mynetwork.ConnectionListWithProfile-16";
|
||||
|
||||
export function shouldSyncAccepted(accountId: string): boolean {
|
||||
const db = getDb();
|
||||
const row = db.prepare("SELECT accepted_sync_at FROM accounts WHERE id = ?").get(accountId) as
|
||||
| { accepted_sync_at: string | null }
|
||||
| undefined;
|
||||
if (!row?.accepted_sync_at) return true;
|
||||
return Date.now() - new Date(row.accepted_sync_at).getTime() >= ACCEPTED_SYNC_INTERVAL_MS;
|
||||
}
|
||||
|
||||
interface ApiConnection {
|
||||
vanity: string | null;
|
||||
createdAt: number; // epoch ms
|
||||
}
|
||||
|
||||
export async function syncAcceptedConnections(accountId: string): Promise<number> {
|
||||
const db = getDb();
|
||||
const page = await getSessionPage(accountId);
|
||||
let stamped = 0;
|
||||
|
||||
try {
|
||||
const boundaryRow = db.prepare("SELECT connections_synced_through_ms FROM accounts WHERE id = ?").get(accountId) as
|
||||
| { connections_synced_through_ms: number | null }
|
||||
| undefined;
|
||||
const boundary = boundaryRow?.connections_synced_through_ms ?? null;
|
||||
const isFullPass = boundary === null;
|
||||
const stopBefore = boundary === null ? null : boundary - OVERLAP_MARGIN_MS;
|
||||
|
||||
// Read the declared total from the connections PAGE header (single nav, NO
|
||||
// scroll). This is also our login-wall check and the completeness checksum.
|
||||
await page.goto("https://www.linkedin.com/mynetwork/invite-connect/connections/", {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: 35000,
|
||||
});
|
||||
await page.waitForTimeout(3500 + Math.random() * 1500);
|
||||
if (/\/login|\/authwall|\/checkpoint|\/uas\//.test(page.url())) {
|
||||
console.warn(`[sync-accepted] Session looks logged out (${page.url()}) — skipping`);
|
||||
return 0;
|
||||
}
|
||||
const declaredTotal = await page.evaluate(() => {
|
||||
const m = document.body.innerText.match(/([\d.,]+)\s+connections?/i);
|
||||
return m ? parseInt(m[1].replace(/[.,]/g, ""), 10) : null;
|
||||
});
|
||||
|
||||
const findByVanity = db.prepare(
|
||||
`SELECT id, full_name, connected_at, degree FROM targets
|
||||
WHERE linkedin_url LIKE ? AND connection_requested_at IS NOT NULL`
|
||||
);
|
||||
const stampAccepted = db.prepare(
|
||||
"UPDATE targets SET degree = 1, connected_at = COALESCE(connected_at, ?) WHERE id = ?"
|
||||
);
|
||||
|
||||
const seenVanities = new Set<string>(); // full-pass phantom check
|
||||
let uniquePulled = 0;
|
||||
let newestSeen: number | null = null;
|
||||
let reachedBoundary = false;
|
||||
|
||||
for (let pageIdx = 0; pageIdx < MAX_PAGES; pageIdx++) {
|
||||
const conns = await fetchConnectionsPage(page, pageIdx * PAGE_SIZE, PAGE_SIZE);
|
||||
if (conns === null) {
|
||||
console.warn(`[sync-accepted] connections API failed at start=${pageIdx * PAGE_SIZE} — stopping`);
|
||||
break;
|
||||
}
|
||||
if (conns.length === 0) break; // end of list
|
||||
|
||||
for (const c of conns) {
|
||||
uniquePulled++;
|
||||
if (c.vanity) seenVanities.add(c.vanity);
|
||||
if (newestSeen === null || c.createdAt > newestSeen) newestSeen = c.createdAt;
|
||||
|
||||
// Incremental early-exit (list is newest-first).
|
||||
if (stopBefore !== null && c.createdAt < stopBefore) {
|
||||
reachedBoundary = true;
|
||||
break;
|
||||
}
|
||||
if (!c.vanity) continue;
|
||||
|
||||
for (const m of findByVanity.all(`%/in/${c.vanity}/%`) as Array<{
|
||||
id: string; full_name: string | null; connected_at: string | null; degree: number | null;
|
||||
}>) {
|
||||
if (m.degree === 1 && m.connected_at) continue; // already correct
|
||||
stampAccepted.run(msToSqlite(c.createdAt), m.id);
|
||||
console.log(`[sync-accepted] Accepted: ${m.full_name ?? c.vanity}`);
|
||||
stamped++;
|
||||
}
|
||||
}
|
||||
|
||||
if (reachedBoundary) break;
|
||||
await page.waitForTimeout(900 + Math.random() * 700); // gentle, API-only
|
||||
}
|
||||
|
||||
// Completeness checksum: a full pass is trustworthy only if what we pulled
|
||||
// matches LinkedIn's own declared total (±5 slack for live churn).
|
||||
const verifiedComplete =
|
||||
isFullPass && declaredTotal !== null && Math.abs(uniquePulled - declaredTotal) <= 5;
|
||||
|
||||
// Correction: ONLY on a verified-complete full pass, un-mark phantom
|
||||
// degree=1 contacts (present nowhere in the authoritative list). Never on an
|
||||
// incremental/incomplete pass — a partial pull must not wipe real accepts.
|
||||
let unmarked = 0;
|
||||
if (verifiedComplete) {
|
||||
const deg1 = db.prepare(
|
||||
"SELECT id, full_name, linkedin_url FROM targets WHERE degree = 1 AND linkedin_url LIKE '%/in/%'"
|
||||
).all() as Array<{ id: string; full_name: string | null; linkedin_url: string }>;
|
||||
const unmark = db.prepare("UPDATE targets SET degree = NULL, connected_at = NULL WHERE id = ?");
|
||||
const tx = db.transaction((rows: typeof deg1) => {
|
||||
for (const t of rows) {
|
||||
const mm = t.linkedin_url.match(/\/in\/([^/?#]+)/);
|
||||
const v = mm ? decodeURIComponent(mm[1]).toLowerCase() : null;
|
||||
if (!v || !seenVanities.has(v)) {
|
||||
unmark.run(t.id);
|
||||
unmarked++;
|
||||
}
|
||||
}
|
||||
});
|
||||
tx(deg1);
|
||||
console.log(`[sync-accepted] Verified full pass (pulled ${uniquePulled} == declared ${declaredTotal}). Un-marked ${unmarked} phantom degree=1.`);
|
||||
} else if (isFullPass) {
|
||||
console.warn(`[sync-accepted] Full pass NOT verified complete (pulled ${uniquePulled}, declared ${declaredTotal}) — add-only, no un-marking.`);
|
||||
}
|
||||
|
||||
// Advance the boundary to the newest connection seen this run.
|
||||
if (newestSeen !== null) {
|
||||
db.prepare("UPDATE accounts SET connections_synced_through_ms = ? WHERE id = ?").run(newestSeen, accountId);
|
||||
}
|
||||
// Store the declared total for visibility (Settings shows LinkedIn's count).
|
||||
if (declaredTotal !== null) {
|
||||
db.prepare("UPDATE accounts SET li_connections = ? WHERE id = ?").run(declaredTotal, accountId);
|
||||
}
|
||||
console.log(`[sync-accepted] Stamped ${stamped} accepted, un-marked ${unmarked} phantom (boundary=${newestSeen}).`);
|
||||
} finally {
|
||||
// B5 safety: only persist the session if still on a valid page.
|
||||
let url = "";
|
||||
try { url = page.url(); } catch { /* gone */ }
|
||||
try { await page.close(); } catch { /* ignore */ }
|
||||
if (/\/login|\/authwall|\/checkpoint|\/uas\//.test(url)) {
|
||||
console.warn(`[sync-accepted] Ended on a wall (${url}) — not persisting; flagging re-auth`);
|
||||
try { await markNeedsReauth(accountId); } catch { /* ignore */ }
|
||||
} else {
|
||||
try { await saveSessionState(accountId); } catch { /* ignore */ }
|
||||
}
|
||||
db.prepare("UPDATE accounts SET accepted_sync_at = datetime('now') WHERE id = ?").run(accountId);
|
||||
}
|
||||
|
||||
return stamped;
|
||||
}
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function msToSqlite(ms: number): string {
|
||||
return new Date(ms).toISOString().replace("T", " ").slice(0, 19);
|
||||
}
|
||||
|
||||
async function fetchConnectionsPage(page: Page, start: number, count: number): Promise<ApiConnection[] | null> {
|
||||
return page.evaluate(
|
||||
async ({ start, count, decoration }): Promise<ApiConnection[] | null> => {
|
||||
const cookies = document.cookie.split("; ").reduce((a: Record<string, string>, c) => {
|
||||
const i = c.indexOf("=");
|
||||
if (i > 0) a[c.slice(0, i)] = c.slice(i + 1);
|
||||
return a;
|
||||
}, {});
|
||||
const csrf = (cookies["JSESSIONID"] || "").replace(/"/g, "");
|
||||
const url =
|
||||
`https://www.linkedin.com/voyager/api/relationships/dash/connections` +
|
||||
`?decorationId=${decoration}&count=${count}&q=search&sortType=RECENTLY_ADDED&start=${start}`;
|
||||
let json: {
|
||||
included?: Array<{
|
||||
$type?: string;
|
||||
entityUrn?: string;
|
||||
createdAt?: number;
|
||||
connectedMember?: string;
|
||||
publicIdentifier?: string;
|
||||
}>;
|
||||
};
|
||||
try {
|
||||
const r = await fetch(url, {
|
||||
headers: {
|
||||
"csrf-token": csrf,
|
||||
"accept": "application/vnd.linkedin.normalized+json+2.1",
|
||||
"x-restli-protocol-version": "2.0.0",
|
||||
"x-li-lang": "en_US",
|
||||
},
|
||||
credentials: "include",
|
||||
});
|
||||
if (!r.ok) return null;
|
||||
json = await r.json();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
const included = json.included || [];
|
||||
const vanityByUrn: Record<string, string> = {};
|
||||
for (const x of included) {
|
||||
if ((x.$type || "").includes("identity.profile.Profile") && x.entityUrn && x.publicIdentifier) {
|
||||
vanityByUrn[x.entityUrn] = x.publicIdentifier;
|
||||
}
|
||||
}
|
||||
const out: ApiConnection[] = [];
|
||||
for (const x of included) {
|
||||
if ((x.$type || "").includes("relationships.Connection") && typeof x.createdAt === "number") {
|
||||
const memberUrn = x.connectedMember || null;
|
||||
out.push({
|
||||
createdAt: x.createdAt,
|
||||
vanity: memberUrn ? vanityByUrn[memberUrn] ?? null : null,
|
||||
});
|
||||
}
|
||||
}
|
||||
out.sort((a, b) => b.createdAt - a.createdAt);
|
||||
return out;
|
||||
},
|
||||
{ start, count, decoration: DECORATION }
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import type { Page } from "playwright";
|
||||
|
||||
/**
|
||||
* Visits a LinkedIn profile page. This registers as a profile view on LinkedIn.
|
||||
* Navigates and waits, then reports whether the page shows a 1st-degree badge —
|
||||
* lets the runner backfill degree=1 for contacts that were already connected
|
||||
* before Linki ever sent them a connection request (e.g. manually added leads).
|
||||
*
|
||||
* Primary-degree signal: presence of the profile's primary "Message" link
|
||||
* (a[href*="/messaging/compose"]) — only shown to 1st-degree connections, reads
|
||||
* an href attribute rather than a CSS class or translated text, so it survives
|
||||
* both LinkedIn's periodic class-name hashing and non-English UI languages.
|
||||
*
|
||||
* MUST be scoped to the visited person's own intro/top card — a page-wide
|
||||
* search also matches "Message" links belonging to OTHER people rendered
|
||||
* elsewhere on the page (sidebar modules like "People also viewed" / suggested
|
||||
* connections). For a non-connected target, THEIR page has no such link, but
|
||||
* the sidebar still does — a page-wide `.first()` silently grabbed a random
|
||||
* unrelated 1st-degree connection's link, wrongly marked the target degree=1,
|
||||
* and stored that stranger's messaging URN, causing messages to go to the
|
||||
* wrong person entirely (incident: Jul 2026, see CLAUDE.md/memory). The top
|
||||
* card is identified structurally as the section containing the page's own
|
||||
* <h1> name heading, which is robust to LinkedIn's class-name hashing.
|
||||
*
|
||||
* Falls back to the old text-scrape (/\b1st\b/ within that same top card)
|
||||
* when the link isn't found, in case the current account's profile layout
|
||||
* doesn't render it as a plain link (e.g. buried behind a click/menu).
|
||||
*
|
||||
* The same link's href carries the messaging URN (urn:li:fsd_profile:ACoAA...)
|
||||
* needed to message this person directly later without a name-search typeahead
|
||||
* — see lib/linkedin/message.ts. Returned as messagingUrn when found.
|
||||
*/
|
||||
export async function visitProfile(page: Page, linkedinUrl: string): Promise<{ isFirstDegree: boolean; messagingUrn: string | null }> {
|
||||
await page.goto(linkedinUrl, { waitUntil: "domcontentloaded", timeout: 30000 });
|
||||
await page.waitForTimeout(3000 + Math.random() * 2000);
|
||||
|
||||
const topCard = page.locator("main section").filter({ has: page.locator("h1") }).first();
|
||||
|
||||
const messageLink = topCard.locator('a[href*="/messaging/compose"]').first();
|
||||
const messageHref = (await messageLink.count()) > 0 ? await messageLink.getAttribute("href").catch(() => null) : null;
|
||||
const urnMatch = messageHref?.match(/profileUrn=([^&]+)/);
|
||||
const messagingUrn = urnMatch ? decodeURIComponent(urnMatch[1]) : null;
|
||||
|
||||
if (messageHref) return { isFirstDegree: true, messagingUrn };
|
||||
|
||||
const pageText = await topCard.innerText().catch(() => "");
|
||||
return { isFirstDegree: /\b1st\b/.test(pageText), messagingUrn: null };
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
// lib/premium.ts — the ONE bridge from open-core code to commercial (ee/) features.
|
||||
//
|
||||
// This file lives in open-core and is present in BOTH the private and public builds.
|
||||
// It loads ee/ if the folder exists, otherwise degrades to `null`. Open-core code must
|
||||
// call premium features ONLY through the `premium` object exported here — never with a
|
||||
// direct `import ... from "@/ee/..."` anywhere else. That one rule is what makes the
|
||||
// public build (ee/ stripped) compile and run with premium cleanly absent.
|
||||
//
|
||||
// Full strategy + rules: docs/OPEN_CORE.md
|
||||
//
|
||||
// IMPORTANT: the surface type below is declared HERE, in open-core, on purpose. If we
|
||||
// imported the type from @/ee, the public build (where @/ee does not exist) would fail
|
||||
// to typecheck. ee/index.ts must SATISFY this type instead. Extend this as features land.
|
||||
|
||||
// Minimal structural shapes for what OPEN-CORE consumes from premium features. These are
|
||||
// declared here (not imported from @/ee) so the public build typechecks with ee/ stripped.
|
||||
// They are intentionally loose at the boundary — ee/ owns the rich domain types and must
|
||||
// SATISFY these. Only add a field here when open-core actually reads it. Kept deliberately
|
||||
// permissive (no index signatures) so the richer ee/ return types remain assignable.
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
|
||||
export interface AiSurface {
|
||||
// Returns at least { default_model } — ee's AgentConfig is a superset.
|
||||
getAgentConfig(): { default_model: string | null };
|
||||
// Bundle is passed straight back into the writers; open-core never inspects its innards.
|
||||
getContactWithCompany(targetId: string): { contact: any; company: any } | null;
|
||||
// Writers take a large param object (ee owns its full type); open-core builds it inline.
|
||||
writeEmail(params: any): Promise<{ subject: string; body: string }>;
|
||||
writeLinkedInMessage(params: any): Promise<{ body: string }>;
|
||||
writeSalesInMail(params: any): Promise<{ subject: string; body: string }>;
|
||||
}
|
||||
|
||||
export interface RepliesSurface {
|
||||
// AI classify + auto-followup for one stored reply (email or LinkedIn).
|
||||
classifyAndDispatch(replyId: string): Promise<void>;
|
||||
// LinkedIn inbox sync (reply detection), driven by the runner loop.
|
||||
shouldSyncInbox(accountId: string): boolean;
|
||||
syncAccountInbox(accountId: string): Promise<number>;
|
||||
}
|
||||
|
||||
export interface InMailSurface {
|
||||
// Sends a Sales Nav InMail via a live browser page (playwright Page type is open-core).
|
||||
sendInMail(page: any, salesNavUrl: string, subject: string, body: string): Promise<void>;
|
||||
}
|
||||
|
||||
export interface PremiumSurface {
|
||||
ai?: AiSurface;
|
||||
replies?: RepliesSurface;
|
||||
inmail?: InMailSurface;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
function loadPremium(): PremiumSurface | null {
|
||||
try {
|
||||
// Loaded by real path. The ee/ folder is imported IN PLACE (not copied) so its own
|
||||
// relative imports resolve normally. In the public build this require throws → null.
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
const mod = require("@/ee");
|
||||
return (mod?.default ?? mod) as PremiumSurface;
|
||||
} catch {
|
||||
return null; // public build: ee/ has been stripped
|
||||
}
|
||||
}
|
||||
|
||||
export const premium: PremiumSurface | null = loadPremium();
|
||||
|
||||
/** True in the private/commercial build, false in the public open-source build. */
|
||||
export const hasPremium: boolean = premium !== null;
|
||||
@@ -0,0 +1,53 @@
|
||||
// Minimal shape both a real NextApiRequest and NextAuth's authorize() RequestInternal
|
||||
// satisfy — socket is optional since RequestInternal doesn't expose one.
|
||||
type IpSource = {
|
||||
headers?: { [key: string]: string | string[] | undefined };
|
||||
socket?: { remoteAddress?: string };
|
||||
};
|
||||
|
||||
// In-memory sliding-window rate limiter, per IP. No Redis/external service (self-hosted,
|
||||
// single process — see CLAUDE.md). Resets on server restart; acceptable for a single-user
|
||||
// tool defending against brute force on login/signup, not a distributed-attack mitigation.
|
||||
const hits = new Map<string, number[]>();
|
||||
|
||||
// Periodically drop stale keys so this doesn't grow unbounded over a long-running process.
|
||||
const SWEEP_INTERVAL_MS = 10 * 60 * 1000;
|
||||
let lastSweep = Date.now();
|
||||
|
||||
export function isRateLimited(req: IpSource, key: string, limit: number, windowMs: number): boolean {
|
||||
const ip = clientIp(req);
|
||||
const mapKey = `${key}:${ip}`;
|
||||
const now = Date.now();
|
||||
|
||||
sweepIfDue(now, windowMs);
|
||||
|
||||
const timestamps = (hits.get(mapKey) ?? []).filter(t => now - t < windowMs);
|
||||
if (timestamps.length >= limit) {
|
||||
hits.set(mapKey, timestamps);
|
||||
return true;
|
||||
}
|
||||
|
||||
timestamps.push(now);
|
||||
hits.set(mapKey, timestamps);
|
||||
return false;
|
||||
}
|
||||
|
||||
function sweepIfDue(now: number, windowMs: number) {
|
||||
if (now - lastSweep < SWEEP_INTERVAL_MS) return;
|
||||
lastSweep = now;
|
||||
for (const [mapKey, timestamps] of hits) {
|
||||
const fresh = timestamps.filter(t => now - t < windowMs);
|
||||
if (fresh.length === 0) hits.delete(mapKey);
|
||||
else hits.set(mapKey, fresh);
|
||||
}
|
||||
}
|
||||
|
||||
function clientIp(req: IpSource): string {
|
||||
const xRealIp = req.headers?.["x-real-ip"];
|
||||
if (typeof xRealIp === "string" && xRealIp) return xRealIp;
|
||||
|
||||
const xForwardedFor = req.headers?.["x-forwarded-for"];
|
||||
if (typeof xForwardedFor === "string" && xForwardedFor) return xForwardedFor.split(",")[0].trim();
|
||||
|
||||
return req.socket?.remoteAddress || "unknown";
|
||||
}
|
||||
+302
@@ -0,0 +1,302 @@
|
||||
// lib/tour.ts — product tour engine (driver.js). Open-core: available in both builds.
|
||||
// Each page gets its own short, independent tour keyed by a "page" id. A tour is shown
|
||||
// once automatically (tracked server-side via /api/tour) and can be replayed manually
|
||||
// from Settings → General ("Restart tour"), which never touches the "seen" state.
|
||||
"use client";
|
||||
|
||||
import { driver, type DriveStep } from "driver.js";
|
||||
import "driver.js/dist/driver.css";
|
||||
|
||||
export type TourPage =
|
||||
| "dashboard"
|
||||
| "lists"
|
||||
| "contacts"
|
||||
| "companies"
|
||||
| "workflows"
|
||||
| "inbox"
|
||||
| "settings";
|
||||
|
||||
// Steps whose `element` selector isn't in the DOM are silently skipped by driver.js —
|
||||
// relied on here so a step referencing a premium-only anchor (Todos nav, InMail, MCP)
|
||||
// just no-ops cleanly in the open-core build instead of needing its own guard.
|
||||
const TOURS: Record<TourPage, DriveStep[]> = {
|
||||
dashboard: [
|
||||
{
|
||||
element: '[data-tour="nav-dashboard"]',
|
||||
popover: {
|
||||
title: "Welcome to Linki",
|
||||
description: "This is your outreach dashboard. Everything in Linki runs on one background process at a human pace — nothing fires the instant you click a button, so don't expect instant numbers here either.",
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="dashboard-filters"]',
|
||||
popover: {
|
||||
title: "LinkedIn and email, side by side",
|
||||
description: "The two channels are tracked completely separately — a campaign can run both at once, and this page reports them independently rather than blended.",
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="dashboard-funnel"]',
|
||||
popover: {
|
||||
title: "Your outreach funnel",
|
||||
description: "See exactly where leads drop off — from total targets down to replies. This is the \"is my campaign actually working\" view.",
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="dashboard-chart"]',
|
||||
popover: {
|
||||
title: "Activity over time",
|
||||
description: "Visits, connections, messages and emails per day. Click a series label to toggle it off, or change the day range to reshape the whole page.",
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="nav-settings"]',
|
||||
popover: {
|
||||
title: "First: connect a LinkedIn account",
|
||||
description: "Nothing else works until an account is authenticated in Settings → LinkedIn — that's the one true prerequisite before importing or running anything.",
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="nav-lists"]',
|
||||
popover: {
|
||||
title: "Then: import a list",
|
||||
description: "Paste a Sales Navigator search URL — that's how leads get into Linki.",
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="nav-workflows"]',
|
||||
popover: {
|
||||
title: "Then: build a campaign",
|
||||
description: "Chain steps like visit, connect, message and email into a sequence, pick the list and account, and launch.",
|
||||
},
|
||||
},
|
||||
],
|
||||
|
||||
lists: [
|
||||
{
|
||||
element: '[data-tour="lists-new"]',
|
||||
popover: {
|
||||
title: "Create a list",
|
||||
description: "Paste a LinkedIn Sales Navigator search URL to scrape leads automatically.",
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="lists-new"]',
|
||||
popover: {
|
||||
title: "Imports run in the background",
|
||||
description: "A Sales Navigator import doesn't happen instantly — it's queued and scraped in small batches with human-like delays, specifically to avoid LinkedIn flagging the account.",
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="lists-jobs"]',
|
||||
popover: {
|
||||
title: "Import jobs",
|
||||
description: "There's a shared daily import cap across every list (configurable in Settings → General) — a large list automatically spills into the next day. Track progress and cancel batches here.",
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="lists-new"]',
|
||||
popover: {
|
||||
title: "A fresh import is intentionally thin",
|
||||
description: "Newly imported leads only have basic data (name, title, company). Full profile and email enrichment happen later, automatically, right before a campaign actually needs them — you don't need to enrich everything up front.",
|
||||
},
|
||||
},
|
||||
],
|
||||
|
||||
contacts: [
|
||||
{
|
||||
element: '[data-tour="nav-contacts"]',
|
||||
popover: {
|
||||
title: "Every lead, in one place",
|
||||
description: "This is the single, deduplicated view of everyone you've ever imported — the same person can belong to several lists, and shows up here just once regardless.",
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="contacts-filters"]',
|
||||
popover: {
|
||||
title: "Search and filter",
|
||||
description: "Filter by list, title, connection status, seniority, or email verification to build a targeted sub-audience without re-importing anything.",
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="contacts-select"]',
|
||||
popover: {
|
||||
title: "Bulk-select, then add to a list",
|
||||
description: "Select multiple contacts and add them all to a list at once — an easy way to turn a filtered view (e.g. everyone who replied) into a new targeted list. Contacts already in that list are skipped.",
|
||||
},
|
||||
},
|
||||
],
|
||||
|
||||
companies: [
|
||||
{
|
||||
element: '[data-tour="nav-companies"]',
|
||||
popover: {
|
||||
title: "Companies",
|
||||
description: "Enriched organization records — industry, size, tech stack, revenue — that populate automatically as contacts get enriched. You can also add or edit one by hand.",
|
||||
},
|
||||
},
|
||||
],
|
||||
|
||||
workflows: [
|
||||
{
|
||||
element: '[data-tour="workflows-new"]',
|
||||
popover: {
|
||||
title: "Build a campaign",
|
||||
description: "A campaign chains steps — visit, connect, message, email — into a repeatable sequence, then runs it against a list of contacts through one LinkedIn account.",
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="workflows-new"]',
|
||||
popover: {
|
||||
title: "LinkedIn and email run in parallel",
|
||||
description: "LinkedIn steps and email steps are two independent tracks for the same contact. A reply on either channel stops that person's outreach on both — you don't need to manage them separately.",
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="workflows-new"]',
|
||||
popover: {
|
||||
title: "A LinkedIn account is always required",
|
||||
description: "Even an email-only campaign needs one authenticated LinkedIn account selected — set that up first in Settings if you haven't.",
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="workflows-new"]',
|
||||
popover: {
|
||||
title: "One active run at a time",
|
||||
description: "A campaign can only have one run going at once. To reach more people later, use \"Add contacts\" on the running campaign instead of starting a second run.",
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="workflows-new"]',
|
||||
popover: {
|
||||
title: "It runs at a human pace",
|
||||
description: "Launching doesn't message everyone instantly — the runner works through the list gradually in the background, respecting daily limits and working hours per account.",
|
||||
},
|
||||
},
|
||||
],
|
||||
|
||||
inbox: [
|
||||
{
|
||||
element: '[data-tour="nav-inbox"]',
|
||||
popover: {
|
||||
title: "Every reply, one place",
|
||||
description: "Replies from LinkedIn and email land here automatically — no need to check either inbox by hand. A reply also automatically stops that contact's outreach.",
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="nav-inbox"]',
|
||||
popover: {
|
||||
title: "Not every reply means interest",
|
||||
description: "Reply counts can be misleadingly high — many are auto-responders (out-of-office, \"contact my colleague\") rather than genuine interest. Open one to see the full thread and judge for yourself.",
|
||||
},
|
||||
},
|
||||
],
|
||||
|
||||
settings: [
|
||||
{
|
||||
element: '[data-tour="settings-tab-linkedin"]',
|
||||
popover: {
|
||||
title: "Connect a LinkedIn account first",
|
||||
description: "An account must be authenticated here before it can be used in any campaign — this is the single most important setup step in Linki.",
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="settings-tab-email"]',
|
||||
popover: {
|
||||
title: "Email accounts, with ramp-up",
|
||||
description: "Configure SMTP/IMAP for email steps. A new account can ramp up sending volume gradually instead of blasting at full volume from day one, which helps it avoid getting flagged as spam.",
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="settings-tab-integrations"]',
|
||||
popover: {
|
||||
title: "Apollo enrichment",
|
||||
description: "Add an Apollo API key here to enrich leads with verified emails, seniority, and company data.",
|
||||
},
|
||||
},
|
||||
{
|
||||
element: '[data-tour="settings-tab-general"]',
|
||||
popover: {
|
||||
title: "Connect an AI agent",
|
||||
description: "The General tab has a copyable MCP URL to hook up Claude Code, Cursor, or any MCP-compatible agent to manage Linki on your behalf — plus the daily import cap and this tour picker.",
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
function markSeen(page: TourPage) {
|
||||
fetch("/api/tour", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ page }),
|
||||
}).catch(() => {});
|
||||
}
|
||||
|
||||
function buildAndRun(page: TourPage, persist: boolean) {
|
||||
const steps = TOURS[page];
|
||||
if (!steps || steps.length === 0) return;
|
||||
|
||||
const d = driver({
|
||||
showProgress: true,
|
||||
overlayColor: "#000",
|
||||
overlayOpacity: 0.65,
|
||||
stagePadding: 6,
|
||||
stageRadius: 8,
|
||||
popoverOffset: 12,
|
||||
allowClose: true,
|
||||
popoverClass: "linki-tour-popover",
|
||||
nextBtnText: "Next",
|
||||
prevBtnText: "Back",
|
||||
doneBtnText: "Done",
|
||||
steps,
|
||||
onDestroyed: () => {
|
||||
if (persist) markSeen(page);
|
||||
},
|
||||
});
|
||||
|
||||
d.drive();
|
||||
}
|
||||
|
||||
export function startPageTour(page: TourPage) {
|
||||
buildAndRun(page, true);
|
||||
}
|
||||
|
||||
export function replayPageTour(page: TourPage) {
|
||||
buildAndRun(page, false);
|
||||
}
|
||||
|
||||
export async function getSeenTours(): Promise<Set<TourPage>> {
|
||||
try {
|
||||
const res = await fetch("/api/tour");
|
||||
if (!res.ok) return new Set();
|
||||
const data = await res.json();
|
||||
return new Set((data.seen ?? []) as TourPage[]);
|
||||
} catch {
|
||||
return new Set();
|
||||
}
|
||||
}
|
||||
|
||||
export const ALL_TOUR_PAGES: TourPage[] = ["dashboard", "lists", "contacts", "companies", "workflows", "inbox", "settings"];
|
||||
|
||||
export const TOUR_PAGE_LABELS: Record<TourPage, string> = {
|
||||
dashboard: "Dashboard",
|
||||
lists: "Lists",
|
||||
contacts: "Contacts",
|
||||
companies: "Companies",
|
||||
workflows: "Campaigns",
|
||||
inbox: "Inbox",
|
||||
settings: "Settings",
|
||||
};
|
||||
|
||||
// Maps a pathname to its tour page id, or null if the current page has no tour
|
||||
// (e.g. a detail route like /lists/[id] — the index page's tour covers the entry point).
|
||||
export function pathToTourPage(pathname: string): TourPage | null {
|
||||
if (pathname === "/") return "dashboard";
|
||||
if (pathname === "/lists") return "lists";
|
||||
if (pathname === "/contacts") return "contacts";
|
||||
if (pathname === "/companies") return "companies";
|
||||
if (pathname === "/workflows") return "workflows";
|
||||
if (pathname === "/inbox") return "inbox";
|
||||
if (pathname === "/settings") return "settings";
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
/**
|
||||
* Update checker — polls Docker Hub tags to detect if a newer version of the
|
||||
* image has been published. Runs once on startup, then every 12 hours.
|
||||
*
|
||||
* State is kept in memory (reset on restart, which is fine — it re-checks immediately).
|
||||
*/
|
||||
|
||||
const DOCKER_HUB_TAGS_URL =
|
||||
"https://hub.docker.com/v2/repositories/moaljumaa/linki/tags?page_size=50&ordering=last_updated";
|
||||
|
||||
const POLL_INTERVAL_MS = 12 * 60 * 60 * 1000; // 12 hours
|
||||
|
||||
export interface UpdateState {
|
||||
current: string;
|
||||
latest: string | null;
|
||||
updateAvailable: boolean;
|
||||
checkedAt: string | null;
|
||||
}
|
||||
|
||||
const state: UpdateState = {
|
||||
current: process.env.APP_VERSION ?? "dev",
|
||||
latest: null,
|
||||
updateAvailable: false,
|
||||
checkedAt: null,
|
||||
};
|
||||
|
||||
export function getUpdateState(): UpdateState {
|
||||
return { ...state };
|
||||
}
|
||||
|
||||
/** Parse semver string like "1.2.3" → [1, 2, 3]. Returns null if not semver. */
|
||||
function parseSemver(v: string): [number, number, number] | null {
|
||||
const m = v.match(/^(\d+)\.(\d+)\.(\d+)$/);
|
||||
if (!m) return null;
|
||||
return [parseInt(m[1]), parseInt(m[2]), parseInt(m[3])];
|
||||
}
|
||||
|
||||
/** Returns true if b is strictly greater than a */
|
||||
function isNewer(a: string, b: string): boolean {
|
||||
const pa = parseSemver(a);
|
||||
const pb = parseSemver(b);
|
||||
if (!pa || !pb) return false;
|
||||
if (pb[0] !== pa[0]) return pb[0] > pa[0];
|
||||
if (pb[1] !== pa[1]) return pb[1] > pa[1];
|
||||
return pb[2] > pa[2];
|
||||
}
|
||||
|
||||
async function checkForUpdate() {
|
||||
try {
|
||||
const res = await fetch(DOCKER_HUB_TAGS_URL);
|
||||
if (!res.ok) return;
|
||||
|
||||
const data = await res.json() as { results?: { name: string }[] };
|
||||
const tags: string[] = (data.results ?? [])
|
||||
.map((t) => t.name)
|
||||
.filter((n) => parseSemver(n) !== null);
|
||||
|
||||
if (tags.length === 0) return;
|
||||
|
||||
// Find highest semver tag
|
||||
const latest = tags.reduce((best, tag) => {
|
||||
return isNewer(best, tag) ? tag : best;
|
||||
}, tags[0]);
|
||||
|
||||
state.latest = latest;
|
||||
state.updateAvailable = state.current !== "dev" && isNewer(state.current, latest);
|
||||
state.checkedAt = new Date().toISOString();
|
||||
|
||||
if (state.updateAvailable) {
|
||||
console.log(`[update-check] New version available: ${latest} (running ${state.current})`);
|
||||
}
|
||||
} catch {
|
||||
// Non-fatal — silently ignore network errors
|
||||
}
|
||||
}
|
||||
|
||||
const g = global as typeof global & { __updateCheckScheduled?: boolean };
|
||||
|
||||
export function scheduleUpdateCheck() {
|
||||
if (g.__updateCheckScheduled) return;
|
||||
g.__updateCheckScheduled = true;
|
||||
|
||||
// Run immediately on startup (non-blocking)
|
||||
checkForUpdate();
|
||||
|
||||
// Then every 12 hours
|
||||
setInterval(checkForUpdate, POLL_INTERVAL_MS);
|
||||
}
|
||||
Reference in New Issue
Block a user