From f991f469cfcd808d2d1573d16d0a7c84370472ef Mon Sep 17 00:00:00 2001 From: Samraaj Bath Date: Sat, 4 Jul 2026 14:11:24 -0700 Subject: [PATCH] Pattern catalog: pinned library fingerprints as hint-only recipe evidence Ported from PR #15: a SHA256-pinned catalog of 49 generic library and platform fingerprints (carousel/consent/chat/commerce vendors) with an O(1) signature index. Catalog hits strengthen recipe recognition (small confidence deltas, marquee->logo-cloud, carousel false-grid skip, lightbox->gallery, commerce prior) and are recorded in the manifest as pattern evidence with node references - they never override captured geometry, which remains gated by the existing plan-agreement checks. The integrity pin is unconditional: a catalog/lock mismatch always throws (the fork's env-var soft-warning mode is removed). 515 tests pass (10 new), typecheck clean. Co-Authored-By: Claude Fable 5 --- compiler/data/pattern-catalog.json | 55 +++++++ compiler/data/pattern-catalog.lock | 1 + compiler/src/generate/manifest.ts | 21 ++- compiler/src/generate/pipeline.ts | 13 +- compiler/src/index.ts | 4 + compiler/src/infer/recipes.ts | 65 +++++++- compiler/src/knowledge/patternIndex.ts | 206 +++++++++++++++++++++++++ compiler/test/patternCatalog.test.ts | 150 ++++++++++++++++++ 8 files changed, 504 insertions(+), 11 deletions(-) create mode 100644 compiler/data/pattern-catalog.json create mode 100644 compiler/data/pattern-catalog.lock create mode 100644 compiler/src/knowledge/patternIndex.ts create mode 100644 compiler/test/patternCatalog.test.ts diff --git a/compiler/data/pattern-catalog.json b/compiler/data/pattern-catalog.json new file mode 100644 index 0000000..1ea3180 --- /dev/null +++ b/compiler/data/pattern-catalog.json @@ -0,0 +1,55 @@ +{ + "version": 1, + "description": "Frozen pattern catalog: signature-indexed widget/platform/animation detectors. Matched against IR srcClass/tag/attrs (frozen capture evidence) by src/knowledge/patternIndex.ts. Deterministic: same catalog + same IR => same hints. Regenerate the lock after edits: node --import tsx src/knowledge/patternIndex.ts --write-lock", + "patterns": [ + { "id": "carousel_slick", "kind": "carousel", "flags": ["deferred_interactive", "gallery"], "match": { "classTokens": ["slick-slider", "slick-track", "slick-slide", "slick-dots", "slick-list"] } }, + { "id": "carousel_swiper", "kind": "carousel", "flags": ["deferred_interactive", "gallery"], "match": { "classTokens": ["swiper", "swiper-wrapper", "swiper-slide", "swiper-container", "swiper-pagination"] } }, + { "id": "carousel_splide", "kind": "carousel", "flags": ["deferred_interactive", "gallery"], "match": { "classTokens": ["splide", "splide__track", "splide__slide", "splide__list"] } }, + { "id": "carousel_owl", "kind": "carousel", "flags": ["deferred_interactive", "gallery"], "match": { "classTokens": ["owl-carousel", "owl-stage", "owl-item"] } }, + { "id": "carousel_flickity", "kind": "carousel", "flags": ["deferred_interactive", "gallery"], "match": { "classTokens": ["flickity-enabled", "flickity-viewport", "flickity-slider"] } }, + { "id": "carousel_glide", "kind": "carousel", "flags": ["deferred_interactive", "gallery"], "match": { "classTokens": ["glide", "glide__track", "glide__slides", "glide__slide"] } }, + { "id": "carousel_embla", "kind": "carousel", "flags": ["deferred_interactive", "gallery"], "match": { "classTokens": ["embla", "embla__container", "embla__slide"] } }, + { "id": "carousel_keen", "kind": "carousel", "flags": ["deferred_interactive", "gallery"], "match": { "classTokens": ["keen-slider", "keen-slider__slide"] } }, + { "id": "carousel_tiny", "kind": "carousel", "flags": ["deferred_interactive", "gallery"], "match": { "classTokens": ["tns-outer", "tns-inner", "tns-slider", "tns-item"] } }, + { "id": "carousel_bootstrap", "kind": "carousel", "flags": ["deferred_interactive", "gallery"], "match": { "classTokens": ["carousel-inner", "carousel-item", "carousel-indicators"] } }, + { "id": "marquee_rfm", "kind": "marquee", "flags": ["deferred_interactive"], "match": { "classPrefixes": ["rfm-"] } }, + { "id": "marquee_generic", "kind": "marquee", "flags": ["deferred_interactive"], "match": { "classTokens": ["marquee", "ticker", "scroller"], "tags": ["marquee"] } }, + { "id": "counter_odometer", "kind": "counter", "flags": ["counter", "deferred_interactive"], "match": { "classTokens": ["odometer", "odometer-inside", "odometer-digit"] } }, + { "id": "counter_countup", "kind": "counter", "flags": ["counter", "deferred_interactive"], "match": { "classTokens": ["countup", "counter-up", "purecounter"] } }, + { "id": "typed_text", "kind": "text_effect", "flags": ["deferred_interactive"], "match": { "classTokens": ["typed-cursor", "typewrite", "typewriter"] } }, + { "id": "anim_aos", "kind": "scroll_animation", "flags": ["motion_lib"], "match": { "classTokens": ["aos-init", "aos-animate"], "attrNames": ["data-aos"] } }, + { "id": "anim_wow", "kind": "scroll_animation", "flags": ["motion_lib"], "match": { "classTokens": ["wow"], "attrNames": ["data-wow-delay", "data-wow-duration"] } }, + { "id": "anim_animate_css", "kind": "css_animation", "flags": ["motion_lib"], "match": { "classPrefixes": ["animate__"], "classTokens": ["animated"] } }, + { "id": "anim_locomotive", "kind": "scroll_animation", "flags": ["motion_lib"], "match": { "attrNames": ["data-scroll", "data-scroll-container", "data-scroll-section"] } }, + { "id": "anim_rellax", "kind": "parallax", "flags": ["motion_lib"], "match": { "classTokens": ["rellax"], "attrNames": ["data-rellax-speed"] } }, + { "id": "anim_gsap_scrolltrigger", "kind": "scroll_animation", "flags": ["motion_lib"], "match": { "classPrefixes": ["gsap-"], "attrNames": ["data-gsap"] } }, + { "id": "anim_sal", "kind": "scroll_animation", "flags": ["motion_lib"], "match": { "attrNames": ["data-sal", "data-sal-delay"] } }, + { "id": "lottie_widget", "kind": "lottie", "flags": ["deferred_interactive", "motion_lib"], "match": { "classTokens": ["lottie", "lottie-animation"], "tags": ["lottie-player", "dotlottie-player"], "attrNames": ["data-lottie", "data-animation-path"] } }, + { "id": "particles_bg", "kind": "background_effect", "flags": ["deferred_interactive"], "match": { "classTokens": ["particles-js", "tsparticles"], "idPrefixes": ["particles-js"] } }, + { "id": "video_player_lib", "kind": "video_player", "flags": ["deferred_interactive"], "match": { "classTokens": ["video-js", "plyr", "jwplayer", "vjs-tech"] } }, + { "id": "lightbox_gallery", "kind": "lightbox", "flags": ["deferred_interactive", "gallery"], "match": { "classPrefixes": ["lg-", "pswp"], "classTokens": ["fancybox", "glightbox", "lightbox"] } }, + { "id": "platform_elementor", "kind": "platform", "flags": ["platform_elementor"], "match": { "classPrefixes": ["elementor-"], "classTokens": ["elementor"] } }, + { "id": "platform_wp_blocks", "kind": "platform", "flags": ["platform_wordpress"], "match": { "classPrefixes": ["wp-block-", "wp-container-"] } }, + { "id": "platform_wpbakery", "kind": "platform", "flags": ["platform_wordpress"], "match": { "classPrefixes": ["vc_"], "classTokens": ["wpb_wrapper", "wpb_row"] } }, + { "id": "platform_divi", "kind": "platform", "flags": ["platform_wordpress"], "match": { "classPrefixes": ["et_pb_"] } }, + { "id": "platform_webflow", "kind": "platform", "flags": ["platform_webflow"], "match": { "classTokens": ["w-nav", "w-slider", "w-dropdown", "w-container", "w-richtext"], "attrNames": ["data-wf-page", "data-wf-site"] } }, + { "id": "platform_shopify", "kind": "platform", "flags": ["platform_shopify"], "match": { "classTokens": ["shopify-section"], "idPrefixes": ["shopify-section"] } }, + { "id": "platform_squarespace", "kind": "platform", "flags": ["platform_squarespace"], "match": { "classPrefixes": ["sqs-"], "attrNames": ["data-block-type"] } }, + { "id": "platform_framer", "kind": "platform", "flags": ["platform_framer"], "match": { "classPrefixes": ["framer-"], "attrNames": ["data-framer-name"] } }, + { "id": "platform_woocommerce", "kind": "platform", "flags": ["platform_wordpress", "ecommerce"], "match": { "classTokens": ["woocommerce", "woocommerce-page"] } }, + { "id": "nav_hamburger", "kind": "nav_toggle", "flags": ["nav_toggle"], "match": { "classTokens": ["hamburger", "navbar-toggler", "menu-toggle", "nav-toggle", "burger", "menu-btn"] } }, + { "id": "nav_mega_menu", "kind": "nav_menu", "flags": ["nav_toggle"], "match": { "classTokens": ["mega-menu", "megamenu", "dropdown-menu", "sub-menu"] } }, + { "id": "consent_onetrust", "kind": "consent", "flags": ["consent_overlay", "third_party_widget"], "match": { "classPrefixes": ["ot-sdk-", "onetrust-"], "idPrefixes": ["onetrust-"] } }, + { "id": "consent_cookiebot", "kind": "consent", "flags": ["consent_overlay", "third_party_widget"], "match": { "idPrefixes": ["CybotCookiebot"] } }, + { "id": "consent_osano", "kind": "consent", "flags": ["consent_overlay", "third_party_widget"], "match": { "classPrefixes": ["osano-"] } }, + { "id": "consent_cookieyes", "kind": "consent", "flags": ["consent_overlay", "third_party_widget"], "match": { "classPrefixes": ["cky-"] } }, + { "id": "consent_complianz", "kind": "consent", "flags": ["consent_overlay", "third_party_widget"], "match": { "classPrefixes": ["cmplz-"] } }, + { "id": "chat_intercom", "kind": "chat_widget", "flags": ["third_party_widget"], "match": { "classPrefixes": ["intercom-"], "idPrefixes": ["intercom-"] } }, + { "id": "chat_crisp", "kind": "chat_widget", "flags": ["third_party_widget"], "match": { "classPrefixes": ["crisp-"], "idPrefixes": ["crisp-"] } }, + { "id": "chat_drift", "kind": "chat_widget", "flags": ["third_party_widget"], "match": { "classPrefixes": ["drift-"], "idPrefixes": ["drift-"] } }, + { "id": "chat_hubspot", "kind": "chat_widget", "flags": ["third_party_widget"], "match": { "classPrefixes": ["hs-chat-", "hubspot-"], "idPrefixes": ["hubspot-"] } }, + { "id": "chat_tawk", "kind": "chat_widget", "flags": ["third_party_widget"], "match": { "idPrefixes": ["tawk-"] } }, + { "id": "accordion_generic", "kind": "accordion", "flags": ["nav_toggle"], "match": { "classTokens": ["accordion", "accordion-item", "accordion-header", "collapsible"] } }, + { "id": "tabs_generic", "kind": "tabs", "flags": ["nav_toggle"], "match": { "classTokens": ["tabs", "tab-content", "tab-pane", "tablist"] } } + ] +} diff --git a/compiler/data/pattern-catalog.lock b/compiler/data/pattern-catalog.lock new file mode 100644 index 0000000..2cd702a --- /dev/null +++ b/compiler/data/pattern-catalog.lock @@ -0,0 +1 @@ +3f46aca40bb69d5c114e957c28b3a4924c58902cecb35332d34f3065114d0e75 diff --git a/compiler/src/generate/manifest.ts b/compiler/src/generate/manifest.ts index 5c65b48..57ee6b2 100644 --- a/compiler/src/generate/manifest.ts +++ b/compiler/src/generate/manifest.ts @@ -4,6 +4,7 @@ import type { Tokens } from "../infer/tokens.js"; import type { AssetGraph } from "../infer/assets.js"; import type { FontGraph } from "../infer/fonts.js"; import type { CaptureResult } from "../capture/capture.js"; +import type { PatternHints } from "../knowledge/patternIndex.js"; export const COMPILER_VERSION = "0.1.0"; export const SCHEMA_VERSION = 1; @@ -16,8 +17,9 @@ export function buildManifest(args: { fontGraph: FontGraph; capture: CaptureResult; componentCount: number; + patternHints?: PatternHints; }): Record { - const { ir, sections, tokens, assetGraph, fontGraph, capture, componentCount } = args; + const { ir, sections, tokens, assetGraph, fontGraph, capture, componentCount, patternHints } = args; const byType: Record = {}; let downloaded = 0, skipped = 0; @@ -56,6 +58,23 @@ export function buildManifest(args: { fallback: fontGraph.entries.filter((f) => f.status === "fallback").length, }, components: { count: componentCount }, + // Frozen-catalog pattern evidence (hint-only, additive): library/platform fingerprints + // detected in the IR, with the node cids that carried each signature (bounded pre-order + // sample). Deterministic — matches are id-sorted and cids are pre-order, so the same + // catalog + IR yields byte-identical evidence. Generated docs read this to say e.g. + // "Swiper carousel detected". Omitted when no hints were computed. + ...(patternHints + ? { + patterns: { + catalogVersion: patternHints.catalogVersion, + catalogHash: patternHints.catalogHash, + flags: patternHints.flags, + platforms: patternHints.platforms, + simpleStatic: patternHints.simpleStatic, + matches: patternHints.matches.map((m) => ({ id: m.id, kind: m.kind, count: m.count, cids: m.cids })), + }, + } + : {}), // Fidelity note: containers whose children were a DIFFERENT SET at some band viewport(s) // (content-identity drift — the source deterministically served other content there). The // clone shows the canonical-viewport children at those widths (faithful-at-canonical) instead diff --git a/compiler/src/generate/pipeline.ts b/compiler/src/generate/pipeline.ts index ee52d67..8411a45 100644 --- a/compiler/src/generate/pipeline.ts +++ b/compiler/src/generate/pipeline.ts @@ -13,6 +13,7 @@ import { buildCodeQualityReport, codeQualityReportToMarkdown, type CodeQualityRe import { interactionRejectedSet } from "./interactive.js"; import { buildManifest } from "./manifest.js"; import { buildSeoInventory, seoInventoryToMarkdown, type SeoInventory } from "./seo.js"; +import { resolvePatternHints, type PatternHints } from "../knowledge/patternIndex.js"; import { writeJSON, writeText, readJSON, fileExists } from "../util/fsx.js"; import type { CaptureResult } from "../capture/capture.js"; @@ -24,6 +25,7 @@ export type GenerateAllResult = { fontGraph: FontGraph; recipeReport: RecipeReport; interactionRecipeReport: InteractionRecipeReport; + patternHints: PatternHints; seoInventory: SeoInventory; codeQuality: CodeQualityReport; manifest: Record; @@ -59,6 +61,10 @@ export function generateAll(opts: { const sections = detectSections(ir); const tokens = extractTokens(ir); + // Pattern hints: frozen-catalog signature scan over the IR (deterministic; the + // pin is asserted here and throws on catalog/lock drift). Fed as ADDITIVE evidence + // into recipe recognition — never overrides captured geometry. + const patternHints = resolvePatternHints(ir); const assetGraph = buildAssetGraph(capture); const fontGraph = buildFontGraph(capture.fontFaces, assetGraph, url); const seoInventory = buildSeoInventory(ir, assetGraph, capture); @@ -69,7 +75,7 @@ export function generateAll(opts: { const tokensCss = (palette.css ? palette.css + "\n" : "") + tokensToCss(tokens, true); const tokenResolver = buildTokenResolver(tokens); const primitives = recognizePrimitives(ir); - const recipeReport = buildRecipeReport(ir, sections, primitives); + const recipeReport = buildRecipeReport(ir, sections, primitives, patternHints); const interactionRecipeReport = buildInteractionRecipeReport(ir, sections, capture.interaction); // Patterns the interaction gate previously rejected (don't reproduce) → left static. const rejPath = join(sourceDir, "interaction-rejected.json"); @@ -95,6 +101,7 @@ export function generateAll(opts: { writeJSON(join(outDir, "fonts.json"), fontGraph.entries); const inventory = inventoryOf(ir, primitives); writeJSON(join(outDir, "components.json"), inventory); + writeJSON(join(outDir, "patterns.json"), patternHints); writeJSON(join(outDir, "recipes.json"), recipeReport); writeText(join(outDir, "recipes.md"), recipeReportToMarkdown(recipeReport)); writeJSON(join(outDir, "interaction-recipes.json"), interactionRecipeReport); @@ -104,8 +111,8 @@ export function generateAll(opts: { const codeQuality = buildCodeQualityReport(appDir, recipeReport); writeJSON(join(outDir, "code-quality.json"), codeQuality); writeText(join(outDir, "code-quality.md"), codeQualityReportToMarkdown(codeQuality)); - const manifest = buildManifest({ ir, sections, tokens, assetGraph, fontGraph, capture, componentCount: inventory.count }); + const manifest = buildManifest({ ir, sections, tokens, assetGraph, fontGraph, capture, componentCount: inventory.count, patternHints }); writeJSON(join(outDir, "manifest.json"), manifest); - return { ir, sections, tokens, assetGraph, fontGraph, recipeReport, interactionRecipeReport, seoInventory, codeQuality, manifest, assetsCopied: mat.copied, assetsMissing: mat.missing }; + return { ir, sections, tokens, assetGraph, fontGraph, recipeReport, interactionRecipeReport, patternHints, seoInventory, codeQuality, manifest, assetsCopied: mat.copied, assetsMissing: mat.missing }; } diff --git a/compiler/src/index.ts b/compiler/src/index.ts index 277a51e..bd325b2 100644 --- a/compiler/src/index.ts +++ b/compiler/src/index.ts @@ -32,6 +32,10 @@ export type { GateResult } from "./validate/gates.js"; export { buildIR } from "./normalize/ir.js"; export type { IR } from "./normalize/ir.js"; +// ---- Pattern knowledge (frozen catalog; deterministic hints; pin asserted on load) ---- +export { resolvePatternHints, loadPatternIndex, assertPinnedCatalog, matchCatalogNode } from "./knowledge/patternIndex.js"; +export type { PatternHints, PatternMatch, PatternDef, PatternCatalog } from "./knowledge/patternIndex.js"; + // ---- Capture surface + version ---- export { captureSite, REQUIRED_VIEWPORTS } from "./capture/capture.js"; export type { CaptureResult } from "./capture/capture.js"; diff --git a/compiler/src/infer/recipes.ts b/compiler/src/infer/recipes.ts index 2d4813e..52d47e0 100644 --- a/compiler/src/infer/recipes.ts +++ b/compiler/src/infer/recipes.ts @@ -3,6 +3,7 @@ import { isTextChild } from "../normalize/ir.js"; import type { PrimitiveType } from "./primitives.js"; import type { Section } from "./sections.js"; import { round } from "../util/canonical.js"; +import { matchCatalogNode, type PatternDef, type PatternHints } from "../knowledge/patternIndex.js"; export type RecipeKind = "logo-cloud" | "feature-grid" | "card-grid" | "product-grid" | "gallery-showcase" | "cta-band"; @@ -60,6 +61,8 @@ export type RecipeReport = { highConfidence: number; byKind: Record; templateReadyKinds: RecipeKind[]; + /** page-level frozen-catalog pattern ids (from resolvePatternHints), when provided */ + catalogPatterns?: string[]; }; candidates: RecipeCandidate[]; }; @@ -78,6 +81,10 @@ type RecipeContext = { parentById: ParentMap; sectionByNodeId: SectionMap; primitives: Map; + /** per-node frozen-catalog matches, memoized (pattern catalog as a 4th evidence source) */ + catalogByCid: Map; + /** page-level catalog flags from resolvePatternHints (e.g. platform_shopify, ecommerce) */ + pageFlags: Set; }; type ItemStats = { @@ -233,7 +240,7 @@ function nodeStats(ctx: RecipeContext, n: IRNode): ItemStats { }; } -function buildContext(ir: IR, sections: Section[], primitives: Map): RecipeContext { +function buildContext(ir: IR, sections: Section[], primitives: Map, patternHints?: PatternHints): RecipeContext { const nodes: IRNode[] = []; const byId: NodeMap = new Map(); const parentById: ParentMap = new Map(); @@ -254,9 +261,32 @@ function buildContext(ir: IR, sections: Section[], primitives: Map [s.nodeId, s])), primitives, + catalogByCid: new Map(), + pageFlags: new Set(patternHints?.flags ?? []), }; } +function catalogDefs(ctx: RecipeContext, node: IRNode): PatternDef[] { + let defs = ctx.catalogByCid.get(node.id); + if (!defs) { + defs = matchCatalogNode(node); + ctx.catalogByCid.set(node.id, defs); + } + return defs; +} + +/** First catalog pattern in `root`'s subtree matching `pred` (pre-order; memoized + * per node). The frozen catalog is deterministic, so recipe evidence stays + * byte-stable across regenerations. */ +function subtreeCatalogMatch(ctx: RecipeContext, root: IRNode, pred: (d: PatternDef) => boolean): PatternDef | null { + for (const d of catalogDefs(ctx, root)) if (pred(d)) return d; + for (const c of elementChildren(root)) { + const hit = subtreeCatalogMatch(ctx, c, pred); + if (hit) return hit; + } + return null; +} + function nearestSection(ctx: RecipeContext, n: IRNode): Section | undefined { let cur: IRNode | undefined = n; while (cur) { @@ -545,18 +575,23 @@ function detectLogoClouds(ctx: RecipeContext): RecipeDraft[] { const rootText = textContent(root, 1200); const layout = isLayoutish(ctx, parent, logos.map((s) => s.node)); const avgHeight = logos.reduce((sum, s) => sum + (s.bbox?.height ?? 0), 0) / logos.length; + // Logo strips often ride marquee libraries (react-fast-marquee, ticker) — + // a catalog marquee hit is strong logo-cloud evidence. + const catalogMarquee = subtreeCatalogMatch(ctx, parent, (d) => d.kind === "marquee"); const signals = [ `${logos.length} repeated media-light children`, layout ? "item parent behaves like grid/flex/wrapped row" : "item parent has repeated logo geometry", TRUSTED_COPY.test(rootText) ? "nearby copy matches trusted-by/brand language" : "", avgHeight <= 96 ? "logo item boxes stay small" : "", + catalogMarquee ? `pattern catalog identifies a ${catalogMarquee.id} strip` : "", ].filter(Boolean); const confidence = 0.58 + Math.min(0.18, logos.length * 0.025) + (logos.length / kids.length) * 0.12 + (layout ? 0.08 : 0) + (TRUSTED_COPY.test(rootText) ? 0.10 : 0) - + (avgHeight <= 96 ? 0.04 : 0); + + (avgHeight <= 96 ? 0.04 : 0) + + (catalogMarquee ? 0.04 : 0); out.push(baseDraft(ctx, "logo-cloud", parent, logos, confidence, signals)); } return out; @@ -568,6 +603,10 @@ function detectGrids(ctx: RecipeContext): RecipeDraft[] { for (const parent of ctx.nodes) { if (parent.id === ctx.ir.root.id) continue; if (subtreeSourceClassMatches(parent, DEFERRED_INTERACTIVE_COLLECTION)) continue; + // Catalog-known carousel/marquee libraries the regex misses (embla, flickity, + // keen-slider, glide, tns, …) are the same deferred-interactive class: their + // children are slides, not a static grid. + if (subtreeCatalogMatch(ctx, parent, (d) => d.kind === "carousel" || d.kind === "marquee")) continue; const kids = visibleElementChildren(parent, ctx.cw); if (kids.length < 2 || kids.length > 36) continue; const stats = kids.map((k) => nodeStats(ctx, k)); @@ -581,7 +620,10 @@ function detectGrids(ctx: RecipeContext): RecipeDraft[] { const rootTooBroad = rootInfo.root.id === ctx.ir.root.id || (!!pageH && !!rootBox && rootBox.height > pageH * 0.55); const localText = textContent(parent, 1200); const parentText = rootTooBroad ? localText : textContent(rootInfo.root, 1600); - const galleryContext = subtreeAttrOrSourceMatches(parent, GALLERY_SOURCE) || (!rootTooBroad && subtreeAttrOrSourceMatches(rootInfo.root, GALLERY_SOURCE)); + // Lightbox libraries (fancybox, photoswipe, lightGallery, …) mark a thumbnail + // grid as a gallery even when no gallery-ish class names are present. + const catalogLightbox = subtreeCatalogMatch(ctx, parent, (d) => d.kind === "lightbox"); + const galleryContext = subtreeAttrOrSourceMatches(parent, GALLERY_SOURCE) || (!rootTooBroad && subtreeAttrOrSourceMatches(rootInfo.root, GALLERY_SOURCE)) || !!catalogLightbox; const testimonialContext = subtreeAttrOrSourceMatches(parent, TESTIMONIAL_SOURCE) || TESTIMONIAL_COPY.test(localText); const featureItems = stats.filter(isLikelyFeatureItem); const cardItems = stats.filter(isLikelyCardItem); @@ -598,6 +640,7 @@ function detectGrids(ctx: RecipeContext): RecipeDraft[] { const signals = [ `${cardLikeItems.length} repeated ${testimonialContext ? "testimonial/gallery" : "media/gallery"} items`, galleryContext ? "source attributes/classes identify a gallery or carousel track" : "", + catalogLightbox ? `pattern catalog identifies a ${catalogLightbox.id} lightbox gallery` : "", testimonialContext ? "source text/classes identify a testimonial or horizontal story strip" : "", oneRowRegimes >= 2 ? "items remain in a horizontal gallery row across sampled widths" : "", mediaRatio >= 0.6 ? "most gallery items include media" : "", @@ -619,9 +662,13 @@ function detectGrids(ctx: RecipeContext): RecipeDraft[] { const productItems = stats.filter(isLikelyProductItem); const forbiddenProductContext = subtreeAttrOrSourceMatches(parent, /\b(?:footer|directory|nav|menu)\b/i) || (!rootTooBroad && subtreeAttrOrSourceMatches(rootInfo.root, /\b(?:footer|directory|nav|menu)\b/i)); + // Page-level catalog prior: a Shopify/WooCommerce page makes repeated card + // grids product grids even when per-node class names carry no commerce words. + const catalogCommerce = ctx.pageFlags.has("ecommerce") || ctx.pageFlags.has("platform_shopify"); const productContext = !forbiddenProductContext && (PRODUCT_COPY.test(localText) || subtreeAttrOrSourceMatches(parent, PRODUCT_SOURCE) - || (!rootTooBroad && (PRODUCT_COPY.test(parentText) || subtreeAttrOrSourceMatches(rootInfo.root, PRODUCT_SOURCE)))); + || (!rootTooBroad && (PRODUCT_COPY.test(parentText) || subtreeAttrOrSourceMatches(rootInfo.root, PRODUCT_SOURCE))) + || catalogCommerce); const productStats = productItems.length >= 2 ? productItems : (productContext ? cardLikeItems : []); if (productStats.length >= 2 && productStats.length / kids.length >= 0.45 && productContext) { const key = itemSetKey("product-grid", productStats); @@ -635,6 +682,7 @@ function detectGrids(ctx: RecipeContext): RecipeDraft[] { productCopyRatio >= 0.5 ? "most items contain commerce/product copy" : "", ctaRatio >= 0.5 ? "most products include link/button affordances" : "", (subtreeAttrOrSourceMatches(parent, PRODUCT_SOURCE) || (!rootTooBroad && subtreeAttrOrSourceMatches(rootInfo.root, PRODUCT_SOURCE))) ? "source context is product/promo/commerce-like" : "", + catalogCommerce ? "pattern catalog identifies an e-commerce platform page" : "", ].filter(Boolean); const confidence = 0.58 + Math.min(0.14, productStats.length * 0.025) @@ -642,7 +690,8 @@ function detectGrids(ctx: RecipeContext): RecipeDraft[] { + (layout ? 0.10 : 0) + (productCopyRatio >= 0.5 ? 0.07 : 0) + (ctaRatio >= 0.5 ? 0.05 : 0) - + ((subtreeAttrOrSourceMatches(parent, PRODUCT_SOURCE) || (!rootTooBroad && subtreeAttrOrSourceMatches(rootInfo.root, PRODUCT_SOURCE))) ? 0.06 : 0); + + ((subtreeAttrOrSourceMatches(parent, PRODUCT_SOURCE) || (!rootTooBroad && subtreeAttrOrSourceMatches(rootInfo.root, PRODUCT_SOURCE))) ? 0.06 : 0) + + (catalogCommerce ? 0.05 : 0); out.push(baseDraft(ctx, "product-grid", parent, productStats, confidence, signals)); } continue; @@ -832,8 +881,8 @@ function candidateSort(ctx: RecipeContext, a: RecipeDraft, b: RecipeDraft): numb || b.confidence - a.confidence; } -export function buildRecipeReport(ir: IR, sections: Section[], primitives: Map): RecipeReport { - const ctx = buildContext(ir, sections, primitives); +export function buildRecipeReport(ir: IR, sections: Section[], primitives: Map, patternHints?: PatternHints): RecipeReport { + const ctx = buildContext(ir, sections, primitives, patternHints); const drafts = suppressDuplicates(ctx, [ ...detectLogoClouds(ctx), ...detectGrids(ctx), @@ -853,6 +902,7 @@ export function buildRecipeReport(ir: IR, sections: Section[], primitives: Map c.confidence >= 0.82).length, byKind, templateReadyKinds: TEMPLATE_READY, + ...(patternHints ? { catalogPatterns: patternHints.matches.map((m) => m.id) } : {}), }, candidates, }; @@ -880,6 +930,7 @@ export function recipeReportToMarkdown(report: RecipeReport): string { lines.push(""); lines.push(`- Candidates: ${report.summary.totalCandidates}`); lines.push(`- High confidence: ${report.summary.highConfidence}`); + if (report.summary.catalogPatterns?.length) lines.push(`- Catalog patterns on page: ${report.summary.catalogPatterns.join(", ")}`); lines.push(`- By kind: ${Object.entries(report.summary.byKind).map(([k, v]) => `${k} ${v}`).join(", ") || "none"}`); lines.push(""); lines.push("## Candidates"); diff --git a/compiler/src/knowledge/patternIndex.ts b/compiler/src/knowledge/patternIndex.ts new file mode 100644 index 0000000..e46e475 --- /dev/null +++ b/compiler/src/knowledge/patternIndex.ts @@ -0,0 +1,206 @@ +/** + * Pattern index: O(1) signature lookup over the frozen pattern catalog + * (data/pattern-catalog.json). Matches known widget/platform/animation + * signatures against the IR's diagnostic evidence (srcClass, tags, attrs) + * and produces deterministic PatternHints consumed as a generation artifact + * (patterns.json) and as an ADDITIVE evidence source for recipe recognition. + * + * Determinism contract: the catalog is FROZEN data pinned by sha256 + * (data/pattern-catalog.lock). Same catalog + same IR => byte-identical hints. + * There is no learning layer here — hints are catalog-only by construction. + * The pin is UNCONDITIONAL: a catalog/lock mismatch (or a missing lock) always + * throws, so pinned evidence can never drift silently. + */ +import { createHash } from "node:crypto"; +import { readFileSync, writeFileSync, existsSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import type { IR, IRNode, IRChild } from "../normalize/ir.js"; + +export type PatternMatchSpec = { + /** exact class-token equality (lowercase) */ + classTokens?: string[]; + /** class-token prefix, e.g. "elementor-", "wp-block-" */ + classPrefixes?: string[]; + /** element tag names, e.g. custom elements like "lottie-player" */ + tags?: string[]; + /** attribute presence, e.g. "data-aos" */ + attrNames?: string[]; + /** id-attribute prefix, e.g. "shopify-section" */ + idPrefixes?: string[]; +}; + +export type PatternDef = { + id: string; + kind: string; + flags: string[]; + match: PatternMatchSpec; +}; + +export type PatternCatalog = { + version: number; + description?: string; + patterns: PatternDef[]; +}; + +export type PatternMatch = { + id: string; + kind: string; + flags: string[]; + /** how many IR nodes matched at least one signature of this pattern */ + count: number; + /** first few matching node ids, pre-order (bounded sample for diagnostics) */ + cids: string[]; +}; + +export type PatternHints = { + catalogVersion: number; + catalogHash: string; + /** matched patterns, sorted by id for determinism */ + matches: PatternMatch[]; + /** union of flags across matches, sorted */ + flags: string[]; + /** platform_* flags with the prefix stripped, sorted (e.g. ["elementor","wordpress"]) */ + platforms: string[]; + /** no deferred-interactive / motion-lib / counter / nav-toggle signatures and a small tree — + * callers may skip optional capture/inference stages for such pages */ + simpleStatic: boolean; +}; + +const CATALOG_PATH = fileURLToPath(new URL("../../data/pattern-catalog.json", import.meta.url)); +const LOCK_PATH = fileURLToPath(new URL("../../data/pattern-catalog.lock", import.meta.url)); +const SAMPLE_CIDS = 5; +/** simpleStatic requires the whole tree under this node count (matches "small landing page"). */ +const SIMPLE_STATIC_MAX_NODES = 1500; +const SIMPLE_STATIC_BLOCKERS = ["deferred_interactive", "motion_lib", "counter", "nav_toggle"]; + +type CompiledIndex = { + catalog: PatternCatalog; + hash: string; + byClassToken: Map; + classPrefixes: Array<[string, PatternDef]>; + byTag: Map; + byAttrName: Map; + idPrefixes: Array<[string, PatternDef]>; +}; + +let cached: CompiledIndex | null = null; + +function addTo(map: Map, key: K, def: PatternDef): void { + const list = map.get(key); + if (list) list.push(def); + else map.set(key, [def]); +} + +export function loadPatternIndex(): CompiledIndex { + if (cached) return cached; + const raw = readFileSync(CATALOG_PATH, "utf8"); + const hash = createHash("sha256").update(raw).digest("hex"); + const catalog = JSON.parse(raw) as PatternCatalog; + if (!Array.isArray(catalog.patterns)) throw new Error("pattern catalog: missing patterns[]"); + const idx: CompiledIndex = { + catalog, + hash, + byClassToken: new Map(), + classPrefixes: [], + byTag: new Map(), + byAttrName: new Map(), + idPrefixes: [], + }; + for (const p of catalog.patterns) { + for (const t of p.match.classTokens ?? []) addTo(idx.byClassToken, t.toLowerCase(), p); + for (const pre of p.match.classPrefixes ?? []) idx.classPrefixes.push([pre.toLowerCase(), p]); + for (const t of p.match.tags ?? []) addTo(idx.byTag, t.toLowerCase(), p); + for (const a of p.match.attrNames ?? []) addTo(idx.byAttrName, a.toLowerCase(), p); + for (const pre of p.match.idPrefixes ?? []) idx.idPrefixes.push([pre, p]); + } + cached = idx; + return idx; +} + +/** Enforce the frozen-catalog pin. UNCONDITIONAL: a missing lock or a hash + * mismatch always throws — pinned evidence must never drift silently, so the + * generator refuses to run against an unpinned/edited catalog. Regenerate the + * lock deliberately after a catalog edit with `--write-lock`. */ +export function assertPinnedCatalog(): void { + const idx = loadPatternIndex(); + if (!existsSync(LOCK_PATH)) { + throw new Error("pattern catalog lock missing (data/pattern-catalog.lock)"); + } + const pinned = readFileSync(LOCK_PATH, "utf8").trim(); + if (pinned !== idx.hash) { + throw new Error( + `pattern catalog hash mismatch: pinned ${pinned.slice(0, 12)}… actual ${idx.hash.slice(0, 12)}…`, + ); + } +} + +function isElement(c: IRChild): c is IRNode { + return (c as IRNode).id !== undefined; +} + +/** Match ONE node's own evidence (srcClass tokens, tag, attr names, id prefix) + * against the catalog. Shared by the page-level hint scan and per-candidate + * recipe scoring. Returns matched defs, deduped, in catalog order. */ +export function matchCatalogNode(node: Pick): PatternDef[] { + const idx = loadPatternIndex(); + const hits = new Set(); + if (node.srcClass) { + for (const tok of node.srcClass.toLowerCase().split(/\s+/)) { + if (!tok) continue; + for (const def of idx.byClassToken.get(tok) ?? []) hits.add(def); + for (const [pre, def] of idx.classPrefixes) if (tok.startsWith(pre)) hits.add(def); + } + } + for (const def of idx.byTag.get(node.tag.toLowerCase()) ?? []) hits.add(def); + for (const attr of Object.keys(node.attrs)) { + for (const def of idx.byAttrName.get(attr.toLowerCase()) ?? []) hits.add(def); + } + const id = node.attrs.id; + if (id) for (const [pre, def] of idx.idPrefixes) if (id.startsWith(pre)) hits.add(def); + return idx.catalog.patterns.filter((p) => hits.has(p)); +} + +/** Single deterministic pre-order walk of the IR: tokenize srcClass, check tag / + * attr / id signatures, accumulate per-pattern counts + a bounded cid sample. + * Asserts the frozen-catalog pin first (throws on mismatch). */ +export function resolvePatternHints(ir: IR): PatternHints { + const idx = loadPatternIndex(); + assertPinnedCatalog(); + + const counts = new Map(); + const hit = (def: PatternDef, cid: string, seen: Set) => { + if (seen.has(def.id)) return; // one hit per node per pattern + seen.add(def.id); + let e = counts.get(def.id); + if (!e) { e = { def, count: 0, cids: [] }; counts.set(def.id, e); } + e.count++; + if (e.cids.length < SAMPLE_CIDS) e.cids.push(cid); + }; + + const visit = (node: IRNode) => { + const seen = new Set(); + for (const def of matchCatalogNode(node)) hit(def, node.id, seen); + for (const c of node.children) if (isElement(c)) visit(c); + }; + visit(ir.root); + + const matches: PatternMatch[] = [...counts.values()] + .map((e) => ({ id: e.def.id, kind: e.def.kind, flags: e.def.flags, count: e.count, cids: e.cids })) + .sort((a, b) => a.id.localeCompare(b.id)); + const flags = [...new Set(matches.flatMap((m) => m.flags))].sort(); + const platforms = flags + .filter((f) => f.startsWith("platform_")) + .map((f) => f.slice("platform_".length)) + .sort(); + const simpleStatic = + ir.doc.nodeCount < SIMPLE_STATIC_MAX_NODES && !flags.some((f) => SIMPLE_STATIC_BLOCKERS.includes(f)); + + return { catalogVersion: idx.catalog.version, catalogHash: idx.hash, matches, flags, platforms, simpleStatic }; +} + +// `--write-lock` maintenance entry: refresh the pin after a deliberate catalog edit. +if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href && process.argv.includes("--write-lock")) { + const idx = loadPatternIndex(); + writeFileSync(LOCK_PATH, idx.hash + "\n"); + console.log("pinned pattern catalog:", idx.hash); +} diff --git a/compiler/test/patternCatalog.test.ts b/compiler/test/patternCatalog.test.ts new file mode 100644 index 0000000..8f353e5 --- /dev/null +++ b/compiler/test/patternCatalog.test.ts @@ -0,0 +1,150 @@ +import { describe, it, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync, writeFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { + loadPatternIndex, + assertPinnedCatalog, + resolvePatternHints, + matchCatalogNode, +} from "../src/knowledge/patternIndex.js"; +import { planForFrameUrl } from "../src/capture/graft.js"; +import type { IR, IRNode, IRChild } from "../src/normalize/ir.js"; + +const CW = 1280; +const LOCK_PATH = fileURLToPath(new URL("../data/pattern-catalog.lock", import.meta.url)); + +/** Element node at the canonical viewport (ids assigned in pre-order by `page`). */ +function el( + tag: string, + opts: { srcClass?: string; attrs?: Record; children?: IRChild[] } = {}, +): IRNode { + return { + id: "", + tag, + attrs: opts.attrs ?? {}, + srcClass: opts.srcClass, + visibleByVp: { [CW]: true }, + bboxByVp: { [CW]: { x: 0, y: 0, width: CW, height: 100 } }, + computedByVp: { [CW]: { display: "block" } }, + children: opts.children ?? [], + }; +} + +/** Wrap children in a root and assign stable pre-order ids (n0, n1, …). */ +function page(children: IRNode[]): IR { + const root = el("body", { children }); + let i = 0; + const assign = (n: IRNode): void => { + n.id = `n${i++}`; + for (const c of n.children) if ((c as IRNode).tag) assign(c as IRNode); + }; + assign(root); + return { + doc: { + sourceUrl: "https://example.test/", title: "Fixture", lang: "en", charset: "UTF-8", + metaViewport: "width=device-width, initial-scale=1", + viewports: [CW], sampleViewports: [CW], canonicalViewport: CW, + perViewport: { [CW]: { scrollHeight: 100, scrollWidth: CW, htmlBg: "", bodyBg: "", bodyColor: "", bodyFont: "" } }, + nodeCount: i, keyframes: [], + }, + root, + }; +} + +describe("pattern catalog: load + pin", () => { + it("loads the frozen catalog and pins by sha256 (lock verifies)", () => { + const idx = loadPatternIndex(); + assert.equal(idx.catalog.version, 1); + assert.ok(idx.catalog.patterns.length > 0, "catalog has patterns"); + // The on-disk lock must equal the catalog's live hash — no throw. + assert.doesNotThrow(() => assertPinnedCatalog()); + const pinned = readFileSync(LOCK_PATH, "utf8").trim(); + assert.equal(pinned, idx.hash, "lock matches catalog hash"); + }); +}); + +describe("pattern catalog: integrity check throws unconditionally", () => { + const original = readFileSync(LOCK_PATH, "utf8"); + afterEach(() => { + // Always restore the real lock so later tests / runs see the pinned hash. + writeFileSync(LOCK_PATH, original); + }); + + it("throws on a lock/catalog hash mismatch (no env-var escape hatch)", () => { + writeFileSync(LOCK_PATH, "deadbeef".repeat(8) + "\n"); + assert.throws(() => assertPinnedCatalog(), /hash mismatch/); + // Restoring must make it pass again. + writeFileSync(LOCK_PATH, original); + assert.doesNotThrow(() => assertPinnedCatalog()); + }); +}); + +describe("pattern catalog: fingerprint fixtures hit", () => { + it("matchCatalogNode identifies a generic carousel fingerprint by class token", () => { + const hits = matchCatalogNode({ tag: "div", attrs: {}, srcClass: "swiper swiper-wrapper" }); + assert.deepEqual(hits.map((h) => h.id), ["carousel_swiper"]); + assert.equal(hits[0].kind, "carousel"); + }); + + it("matchCatalogNode identifies a platform fingerprint by class prefix", () => { + const hits = matchCatalogNode({ tag: "div", attrs: {}, srcClass: "elementor-widget-container" }); + assert.ok(hits.some((h) => h.id === "platform_elementor"), "elementor prefix hit"); + }); + + it("matchCatalogNode identifies a scroll-animation fingerprint by attribute presence", () => { + const hits = matchCatalogNode({ tag: "div", attrs: { "data-aos": "fade-up" }, srcClass: undefined }); + assert.ok(hits.some((h) => h.id === "anim_aos"), "data-aos attr hit"); + }); + + it("resolvePatternHints walks the IR deterministically and records id + cid evidence", () => { + const ir = page([ + el("div", { srcClass: "swiper", children: [el("div", { srcClass: "swiper-slide" })] }), + el("div", { srcClass: "swiper-slide" }), + ]); + const hints = resolvePatternHints(ir); + const swiper = hints.matches.find((m) => m.id === "carousel_swiper"); + assert.ok(swiper, "swiper detected"); + // Three nodes carried a swiper token → count 3, cids are pre-order node ids. + assert.equal(swiper!.count, 3); + assert.deepEqual(swiper!.cids, ["n1", "n2", "n3"]); + // matches are sorted by id → deterministic evidence ordering. + const ids = hints.matches.map((m) => m.id); + assert.deepEqual(ids, [...ids].sort((a, b) => a.localeCompare(b))); + // Same catalog + same IR ⇒ byte-identical hints. + assert.deepEqual(resolvePatternHints(page([ + el("div", { srcClass: "swiper", children: [el("div", { srcClass: "swiper-slide" })] }), + el("div", { srcClass: "swiper-slide" }), + ])), hints); + }); + + it("resolvePatternHints exposes platform flags (stripped) and simpleStatic", () => { + const ir = page([el("section", { srcClass: "shopify-section" })]); + const hints = resolvePatternHints(ir); + assert.ok(hints.platforms.includes("shopify"), "shopify platform surfaced"); + assert.ok(hints.flags.includes("platform_shopify")); + }); +}); + +describe("popup-vendor skip list: FRAME_SKIP semantics preserved (single source of truth)", () => { + // The catalog's consent/chat entries are DOM class/id fingerprints; the capture + // pipeline's popup-vendor skip list (graft.ts FRAME_SKIP_RE) matches iframe host + // URLs. They are orthogonal axes — the catalog carries no URL hosts, so there is + // nothing to consolidate; capture's list stays the single source of truth for + // frame-graft decisions. Assert its semantics are unchanged. + it("skips promo/consent creative iframe hosts", () => { + assert.equal(planForFrameUrl("https://widget.privy.com/assets/popup.html"), "skip"); + assert.equal(planForFrameUrl("https://app.recart.com/creative"), "skip"); + assert.equal(planForFrameUrl("https://www.googletagmanager.com/ns.html"), "skip"); + }); + + it("still grafts inline form embeds and same-origin blank frames", () => { + assert.equal(planForFrameUrl("https://static-forms.klaviyo.com/form"), "graft"); + assert.equal(planForFrameUrl("about:blank"), "graft"); + assert.equal(planForFrameUrl(""), "graft"); + }); + + it("still renders media players as stills", () => { + assert.equal(planForFrameUrl("https://www.youtube.com/embed/abc"), "still"); + }); +});