c0py: initial scaffolding for website registry tool

packages/c0py-core/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "@siax/c0py-core",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "main": "./src/index.ts",
+  "types": "./src/index.ts",
+  "scripts": {
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run"
+  },
+  "dependencies": {
+    "@siax/c0py-types": "workspace:*"
+  },
+  "devDependencies": {
+    "typescript": "^5.7.2",
+    "vitest": "^3.0.0"
+  }
+}

packages/c0py-core/src/crawler/crawler.ts

@@ -0,0 +1,94 @@
+export class Crawler {
+  private maxPages: number;
+  private timeoutMs: number;
+  private visited: Set<string> = new Set();
+  private pages: CrawledPage[] = [];
+  private queue: string[] = [];
+
+  constructor(options: { maxPages: number; timeoutMs: number }) {
+    this.maxPages = options.maxPages;
+    this.timeoutMs = options.timeoutMs;
+  }
+
+  async crawl(startUrl: string): Promise<CrawledPage[]> {
+    this.visited.clear();
+    this.pages = [];
+    this.queue = [startUrl];
+
+    while (this.queue.length > 0 && this.pages.length < this.maxPages) {
+      const url = this.queue.shift()!;
+      if (this.visited.has(url)) continue;
+      this.visited.add(url);
+
+      try {
+        const page = await this.fetchPage(url);
+        this.pages.push(page);
+        for (const link of page.links) {
+          if (!this.visited.has(link)) {
+            this.queue.push(link);
+          }
+        }
+      } catch {
+        // Skip unreachable pages, continue crawl
+      }
+    }
+
+    return this.pages;
+  }
+
+  private async fetchPage(url: string): Promise<CrawledPage> {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), this.timeoutMs);
+
+    try {
+      const res = await fetch(url, {
+        signal: controller.signal,
+        headers: { "User-Agent": "c0py/0.1" },
+        redirect: "follow",
+      });
+      const body = await res.text();
+
+      const links = this.extractLinks(body, url);
+
+      return {
+        url,
+        path: new URL(url).pathname,
+        title: this.extractTag(body, "title"),
+        description: this.extractMetaDescription(body),
+        statusCode: res.status,
+        contentType: res.headers.get("content-type"),
+        links,
+      };
+    } finally {
+      clearTimeout(timeout);
+    }
+  }
+
+  private extractLinks(html: string, baseUrl: string): string[] {
+    const links: string[] = [];
+    const re = /href\s*=\s*["']([^"']+)["']/gi;
+    let m;
+    while ((m = re.exec(html)) !== null) {
+      try {
+        const abs = new URL(m[1], baseUrl).href;
+        if (!links.includes(abs)) links.push(abs);
+      } catch {
+        // Skip invalid URLs
+      }
+    }
+    return links;
+  }
+
+  private extractTag(html: string, tag: string): string | null {
+    const re = new RegExp(`<${tag}[^>]*>([\\s\\S]*?)<\\/${tag}>`, "i");
+    const m = html.match(re);
+    if (!m) return null;
+    return m[1].replace(/<[^>]+>/g, "").trim().slice(0, 500) || null;
+  }
+
+  private extractMetaDescription(html: string): string | null {
+    const re = /<meta[^>]*name=["']description["'][^>]*content=["']([^"']*)["']/i;
+    const m = html.match(re);
+    return m ? m[1].slice(0, 500) : null;
+  }
+}

packages/c0py-core/src/crawler/types.ts

@@ -0,0 +1,12 @@
+import type { Page, Registry, ScanRun } from "@siax/c0py-types";
+
+export interface CrawlResult {
+  pages: Page[];
+  totalPages: number;
+  durationMs: number;
+  error: string | null;
+}
+
+export interface RegistryAnalyzer {
+  analyze(pages: Page[]): Record<string, unknown>;
+}

packages/c0py-core/src/index.ts

@@ -0,0 +1,5 @@
+export * from "./crawler/crawler";
+export * from "./registry/registry";
+export * from "./registry/analyzer";
+export * from "./crawler/types";
+export * from "./registry/types";

packages/c0py-core/src/registry/analyzer.ts

@@ -0,0 +1,39 @@
+import type { Page } from "@siax/c0py-types";
+
+export interface AnalysisResult {
+  totalPages: number;
+  uniqueDomains: number;
+  avgStatusCode: number;
+  statusCodes: Record<number, number>;
+  contentTypes: Record<string, number>;
+  hasBrokenLinks: boolean;
+  brokenLinks: number;
+}
+
+export function analyzePages(pages: Page[]): AnalysisResult {
+  const statusCodes: Record<number, number> = {};
+  const contentTypes: Record<string, number> = {};
+  let brokenLinks = 0;
+
+  for (const page of pages) {
+    statusCodes[page.statusCode] = (statusCodes[page.statusCode] ?? 0) + 1;
+    const ct = page.contentType?.split(";")[0].trim() ?? "unknown";
+    contentTypes[ct] = (contentTypes[ct] ?? 0) + 1;
+    if (page.statusCode >= 400) brokenLinks++;
+  }
+
+  const avgStatusCode =
+    pages.length > 0
+      ? pages.reduce((sum, p) => sum + p.statusCode, 0) / pages.length
+      : 0;
+
+  return {
+    totalPages: pages.length,
+    uniqueDomains: new Set(pages.map((p) => new URL(p.url).host)).size,
+    avgStatusCode: Math.round(avgStatusCode * 100) / 100,
+    statusCodes,
+    contentTypes,
+    hasBrokenLinks: brokenLinks > 0,
+    brokenLinks,
+  };
+}

packages/c0py-core/src/registry/registry.ts

@@ -0,0 +1,31 @@
+import type { Page, Registry, ScanRun } from "@siax/c0py-types";
+import type { CrawledPage } from "./types";
+
+export function assembleRegistry(
+  url: string,
+  name: string,
+  crawled: CrawledPage[]
+): { registry: Registry; pages: Page[] } {
+  const registry: Registry = {
+    id: crypto.randomUUID(),
+    url,
+    name,
+    createdAt: new Date(),
+    updatedAt: new Date(),
+  };
+
+  const pages: Page[] = crawled.map((c) => ({
+    id: crypto.randomUUID(),
+    registryId: registry.id,
+    url: c.url,
+    path: c.path,
+    title: c.title,
+    description: c.description,
+    statusCode: c.statusCode,
+    contentType: c.contentType,
+    discoveredAt: new Date(),
+    analysis: null,
+  }));
+
+  return { registry, pages };
+}

packages/c0py-core/src/registry/types.ts

@@ -0,0 +1,15 @@
+export interface CrawlOptions {
+  url: string;
+  maxPages: number;
+  timeoutMs: number;
+}
+
+export interface CrawledPage {
+  url: string;
+  path: string;
+  title: string | null;
+  description: string | null;
+  statusCode: number;
+  contentType: string | null;
+  links: string[];
+}

packages/c0py-core/tests/analyzer.test.ts

@@ -0,0 +1,65 @@
+import { describe, it, expect } from "vitest";
+import { analyzePages } from "../src/registry/analyzer";
+import type { Page } from "@siax/c0py-types";
+
+const pages: Page[] = [
+  {
+    id: "1",
+    registryId: "r1",
+    url: "https://example.com/",
+    path: "/",
+    title: "Home",
+    description: null,
+    statusCode: 200,
+    contentType: "text/html",
+    discoveredAt: new Date(),
+    analysis: null,
+  },
+  {
+    id: "2",
+    registryId: "r1",
+    url: "https://example.com/about",
+    path: "/about",
+    title: "About",
+    description: null,
+    statusCode: 200,
+    contentType: "text/html",
+    discoveredAt: new Date(),
+    analysis: null,
+  },
+  {
+    id: "3",
+    registryId: "r1",
+    url: "https://example.com/missing",
+    path: "/missing",
+    title: null,
+    description: null,
+    statusCode: 404,
+    contentType: "text/html",
+    discoveredAt: new Date(),
+    analysis: null,
+  },
+];
+
+describe("analyzePages", () => {
+  it("counts pages correctly", () => {
+    const result = analyzePages(pages);
+    expect(result.totalPages).toBe(3);
+    expect(result.brokenLinks).toBe(1);
+    expect(result.hasBrokenLinks).toBe(true);
+    expect(result.uniqueDomains).toBe(1);
+  });
+
+  it("aggregates status codes", () => {
+    const result = analyzePages(pages);
+    expect(result.statusCodes[200]).toBe(2);
+    expect(result.statusCodes[404]).toBe(1);
+  });
+
+  it("handles empty input", () => {
+    const result = analyzePages([]);
+    expect(result.totalPages).toBe(0);
+    expect(result.avgStatusCode).toBe(0);
+    expect(result.hasBrokenLinks).toBe(false);
+  });
+});

packages/c0py-core/tests/crawler.test.ts

@@ -0,0 +1,16 @@
+import { describe, it, expect } from "vitest";
+import { Crawler } from "../src/crawler/crawler";
+
+describe("Crawler", () => {
+  it("instantiates with options", () => {
+    const c = new Crawler({ maxPages: 10, timeoutMs: 5000 });
+    expect(c).toBeDefined();
+  });
+
+  it("crawls a real page", async () => {
+    const c = new Crawler({ maxPages: 1, timeoutMs: 10000 });
+    const pages = await c.crawl("https://example.com");
+    expect(pages.length).toBeGreaterThanOrEqual(1);
+    expect(pages[0].statusCode).toBe(200);
+  }, 15000);
+});

packages/c0py-core/tests/registry.test.ts

@@ -0,0 +1,24 @@
+import { describe, it, expect } from "vitest";
+import { assembleRegistry } from "../src/registry/registry";
+
+describe("assembleRegistry", () => {
+  it("creates a registry with pages", () => {
+    const result = assembleRegistry("https://example.com", "Example", [
+      {
+        url: "https://example.com/",
+        path: "/",
+        title: "Home",
+        description: null,
+        statusCode: 200,
+        contentType: "text/html",
+        links: [],
+      },
+    ]);
+
+    expect(result.registry.url).toBe("https://example.com");
+    expect(result.registry.name).toBe("Example");
+    expect(result.pages).toHaveLength(1);
+    expect(result.pages[0].id).toBeDefined();
+    expect(result.pages[0].registryId).toBe(result.registry.id);
+  });
+});

packages/c0py-core/tests/setup.ts

@@ -0,0 +1 @@
+testTimeout: 10000,

packages/c0py-core/tsconfig.json

@@ -0,0 +1,22 @@
+# C0PY Core Package -- tsconfig.json
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "declaration": true,
+    "outDir": "dist",
+    "baseUrl": ".",
+    "paths": {
+      "@siax/c0py-types": ["../types/src/index.ts"]
+    }
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

packages/c0py-core/vitest.config.ts

@@ -0,0 +1 @@
+{"type": "vitest"}

packages/config/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "@siax/c0py-config",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "main": "./src/index.ts",
+  "types": "./src/index.ts",
+  "scripts": {
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run"
+  },
+  "dependencies": {
+    "zod": "^3.24.0"
+  },
+  "devDependencies": {
+    "typescript": "^5.7.2",
+    "vitest": "^3.0.0"
+  }
+}

packages/config/src/index.ts

@@ -0,0 +1,4 @@
+import { envSchema } from "@siax/c0py-config";
+
+export type Env = typeof envSchema;
+export { envSchema };

packages/config/tsconfig.json

@@ -0,0 +1,19 @@
+# C0PY Config Package -- tsconfig.json
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "declaration": true,
+    "outDir": "dist",
+    "baseUrl": "."
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

packages/config/vitest.config.ts

@@ -0,0 +1 @@
+{"type": "vitest"}

packages/types/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "@siax/c0py-types",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "main": "./src/index.ts",
+  "types": "./src/index.ts",
+  "scripts": {
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run"
+  },
+  "devDependencies": {
+    "typescript": "^5.7.2",
+    "vitest": "^3.0.0"
+  }
+}

packages/types/src/index.ts

@@ -0,0 +1,30 @@
+export interface Page {
+  id: string;
+  registryId: string;
+  url: string;
+  path: string;
+  title: string | null;
+  description: string | null;
+  statusCode: number;
+  contentType: string | null;
+  discoveredAt: Date;
+  analysis: Record<string, unknown> | null;
+}
+
+export interface ScanRun {
+  id: string;
+  registryId: string;
+  url: string;
+  status: "pending" | "running" | "completed" | "failed";
+  startedAt: Date;
+  completedAt: Date | null;
+  error: string | null;
+}
+
+export interface Registry {
+  id: string;
+  url: string;
+  name: string;
+  createdAt: Date;
+  updatedAt: Date;
+}

packages/types/tsconfig.json

@@ -0,0 +1,19 @@
+# C0PY Types Package -- tsconfig.json
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "declaration": true,
+    "outDir": "dist",
+    "baseUrl": "."
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

packages/types/vitest.config.ts

@@ -0,0 +1 @@
+{"type": "vitest"}