c0py: initial scaffolding for website registry tool

packages/c0py-core/src/crawler/crawler.ts

@@ -0,0 +1,94 @@
+export class Crawler {
+  private maxPages: number;
+  private timeoutMs: number;
+  private visited: Set<string> = new Set();
+  private pages: CrawledPage[] = [];
+  private queue: string[] = [];
+
+  constructor(options: { maxPages: number; timeoutMs: number }) {
+    this.maxPages = options.maxPages;
+    this.timeoutMs = options.timeoutMs;
+  }
+
+  async crawl(startUrl: string): Promise<CrawledPage[]> {
+    this.visited.clear();
+    this.pages = [];
+    this.queue = [startUrl];
+
+    while (this.queue.length > 0 && this.pages.length < this.maxPages) {
+      const url = this.queue.shift()!;
+      if (this.visited.has(url)) continue;
+      this.visited.add(url);
+
+      try {
+        const page = await this.fetchPage(url);
+        this.pages.push(page);
+        for (const link of page.links) {
+          if (!this.visited.has(link)) {
+            this.queue.push(link);
+          }
+        }
+      } catch {
+        // Skip unreachable pages, continue crawl
+      }
+    }
+
+    return this.pages;
+  }
+
+  private async fetchPage(url: string): Promise<CrawledPage> {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), this.timeoutMs);
+
+    try {
+      const res = await fetch(url, {
+        signal: controller.signal,
+        headers: { "User-Agent": "c0py/0.1" },
+        redirect: "follow",
+      });
+      const body = await res.text();
+
+      const links = this.extractLinks(body, url);
+
+      return {
+        url,
+        path: new URL(url).pathname,
+        title: this.extractTag(body, "title"),
+        description: this.extractMetaDescription(body),
+        statusCode: res.status,
+        contentType: res.headers.get("content-type"),
+        links,
+      };
+    } finally {
+      clearTimeout(timeout);
+    }
+  }
+
+  private extractLinks(html: string, baseUrl: string): string[] {
+    const links: string[] = [];
+    const re = /href\s*=\s*["']([^"']+)["']/gi;
+    let m;
+    while ((m = re.exec(html)) !== null) {
+      try {
+        const abs = new URL(m[1], baseUrl).href;
+        if (!links.includes(abs)) links.push(abs);
+      } catch {
+        // Skip invalid URLs
+      }
+    }
+    return links;
+  }
+
+  private extractTag(html: string, tag: string): string | null {
+    const re = new RegExp(`<${tag}[^>]*>([\\s\\S]*?)<\\/${tag}>`, "i");
+    const m = html.match(re);
+    if (!m) return null;
+    return m[1].replace(/<[^>]+>/g, "").trim().slice(0, 500) || null;
+  }
+
+  private extractMetaDescription(html: string): string | null {
+    const re = /<meta[^>]*name=["']description["'][^>]*content=["']([^"']*)["']/i;
+    const m = html.match(re);
+    return m ? m[1].slice(0, 500) : null;
+  }
+}

packages/c0py-core/src/crawler/types.ts

@@ -0,0 +1,12 @@
+import type { Page, Registry, ScanRun } from "@siax/c0py-types";
+
+export interface CrawlResult {
+  pages: Page[];
+  totalPages: number;
+  durationMs: number;
+  error: string | null;
+}
+
+export interface RegistryAnalyzer {
+  analyze(pages: Page[]): Record<string, unknown>;
+}

packages/c0py-core/src/index.ts

@@ -0,0 +1,5 @@
+export * from "./crawler/crawler";
+export * from "./registry/registry";
+export * from "./registry/analyzer";
+export * from "./crawler/types";
+export * from "./registry/types";

packages/c0py-core/src/registry/analyzer.ts

@@ -0,0 +1,39 @@
+import type { Page } from "@siax/c0py-types";
+
+export interface AnalysisResult {
+  totalPages: number;
+  uniqueDomains: number;
+  avgStatusCode: number;
+  statusCodes: Record<number, number>;
+  contentTypes: Record<string, number>;
+  hasBrokenLinks: boolean;
+  brokenLinks: number;
+}
+
+export function analyzePages(pages: Page[]): AnalysisResult {
+  const statusCodes: Record<number, number> = {};
+  const contentTypes: Record<string, number> = {};
+  let brokenLinks = 0;
+
+  for (const page of pages) {
+    statusCodes[page.statusCode] = (statusCodes[page.statusCode] ?? 0) + 1;
+    const ct = page.contentType?.split(";")[0].trim() ?? "unknown";
+    contentTypes[ct] = (contentTypes[ct] ?? 0) + 1;
+    if (page.statusCode >= 400) brokenLinks++;
+  }
+
+  const avgStatusCode =
+    pages.length > 0
+      ? pages.reduce((sum, p) => sum + p.statusCode, 0) / pages.length
+      : 0;
+
+  return {
+    totalPages: pages.length,
+    uniqueDomains: new Set(pages.map((p) => new URL(p.url).host)).size,
+    avgStatusCode: Math.round(avgStatusCode * 100) / 100,
+    statusCodes,
+    contentTypes,
+    hasBrokenLinks: brokenLinks > 0,
+    brokenLinks,
+  };
+}

packages/c0py-core/src/registry/registry.ts

@@ -0,0 +1,31 @@
+import type { Page, Registry, ScanRun } from "@siax/c0py-types";
+import type { CrawledPage } from "./types";
+
+export function assembleRegistry(
+  url: string,
+  name: string,
+  crawled: CrawledPage[]
+): { registry: Registry; pages: Page[] } {
+  const registry: Registry = {
+    id: crypto.randomUUID(),
+    url,
+    name,
+    createdAt: new Date(),
+    updatedAt: new Date(),
+  };
+
+  const pages: Page[] = crawled.map((c) => ({
+    id: crypto.randomUUID(),
+    registryId: registry.id,
+    url: c.url,
+    path: c.path,
+    title: c.title,
+    description: c.description,
+    statusCode: c.statusCode,
+    contentType: c.contentType,
+    discoveredAt: new Date(),
+    analysis: null,
+  }));
+
+  return { registry, pages };
+}

packages/c0py-core/src/registry/types.ts

@@ -0,0 +1,15 @@
+export interface CrawlOptions {
+  url: string;
+  maxPages: number;
+  timeoutMs: number;
+}
+
+export interface CrawledPage {
+  url: string;
+  path: string;
+  title: string | null;
+  description: string | null;
+  statusCode: number;
+  contentType: string | null;
+  links: string[];
+}