webpack.config.js

const path = require("path");
const { CleanWebpackPlugin } = require("clean-webpack-plugin");
const CopyPlugin = require("copy-webpack-plugin");
const BundleAnalyzerPlugin = require("webpack-bundle-analyzer").BundleAnalyzerPlugin;
const Handlebars = require("handlebars");
const fs = require("fs");
const ImageMinimizerPlugin = require("image-minimizer-webpack-plugin");
const HtmlMinimizerPlugin = require("html-minimizer-webpack-plugin");

const FRAGMENTS_PATH = "src/fragments";

// Load the fragments from the fragments directory and caches it
const loadFragmentsMap = (() => {
    let cachedFragments = null;
    return async () => {
        if (cachedFragments === null) {
            cachedFragments = {};
            const walkDir = async (dir, basePath = '') => {
                const files = fs.readdirSync(dir);
                await Promise.all(files.map(async file => {
                    const filePath = path.join(dir, file);
                    const relativePath = path.join(basePath, file);
                    if (fs.statSync(filePath).isDirectory()) {
                        await walkDir(filePath, relativePath);
                    } else {
                        // Remove the .html extension before creating the dotted path
                        const nameWithoutExt = relativePath.replace(/\.html$/, '');
                        const dottedPath = 'fragment-' + nameWithoutExt.replace(/\\/g, '-').replace(/\//g, '-').replace(/\./g, '-');
                        const content = fs.readFileSync(filePath, "utf8");
                        // Minify the HTML content using swcMinifyFragment
                        const minifiedRes = await HtmlMinimizerPlugin.swcMinifyFragment({"tmp.html": content})
                        if (minifiedRes.errors) {
                            console.error(minifiedRes.errors)
                        }
                        const minifiedContent = minifiedRes.code;
                        cachedFragments[dottedPath] = minifiedContent;
                    }
                }));
            };
            await walkDir(FRAGMENTS_PATH);
        }
        return cachedFragments;
    };
})();

const transformMarkdownWithFragments = async (data, filepath) => {
    const fragments = await loadFragmentsMap();
    console.log(`Available fragments: ${Object.keys(fragments).join(', ')}`);
    
    // Read the markdown file
    const markdown = require('markdown-it')({
        html: true,
        linkify: true,
        typographer: true
    });
    
    const markdownContent = data.toString('utf8');
    const htmlContent = markdown.render(markdownContent);
    
    // Process with Handlebars for fragment insertion
    const template = Handlebars.compile(htmlContent);
    return template(fragments);
};

module.exports = {
    entry: {
        distill: "./src/distill.js",
        main: "./src/index.js",
    },
    output: {
        filename: "[name].bundle.js",
        path: path.resolve(__dirname, "dist"),
    },
    module: {
        rules: [
            { test: /\.css$/, use: ["style-loader", "css-loader"] },
            {
                test: /\.(js|mjs)$/,
                exclude: /node_modules/,
                use: {
                    loader: "babel-loader",
                    options: {
                        presets: ["@babel/preset-env"],
                    },
                },
            },
            {}
        ],
    },
    plugins: [
        new CleanWebpackPlugin(),
        new CopyPlugin({
            patterns: [
                { from: "src/fragments/*", to: "fragments/[name].html" },
                { from: "src/style.css", to: "style.css" },
                { from: "content/*.png", to: "static/[name][ext]" },
                { from: "content/*.svg", to: "static/[name][ext]" },
                { from: "content/*.html", to: "static/[name][ext]" },
                { 
                    from: "content/article.md", 
                    to: "index.html",
                    transform: async (content, path) => {
                        const fragments = await loadFragmentsMap();
                        
                        // Convert markdown to HTML
                        const markdown = require('markdown-it')({
                            html: true,
                            linkify: true,
                            typographer: true
                        });
                        
                        const markdownContent = content.toString('utf8');
                        const htmlContent = markdown.render(markdownContent);
                        
                        // Extract headings for TOC generation
                        const tocScript = `
                        <script>
                        function initializeTOC() {
                            const article = document.querySelector('d-article');
                            const toc = document.querySelector('d-contents');
                            if (toc) {
                                const headings = article.querySelectorAll('h1, h2, h3, h4');
                                let ToC = '<nav role="navigation" class="l-text figcaption">';
                                ToC += '<div class="toc-header"><span class="toc-title">Table of Contents</span></div>';
                                ToC += '<div class="toc-content">';
                                
                                headings.forEach((heading, index) => {
                                    const id = heading.id || 'heading-' + index;
                                    if (!heading.id) heading.id = id;
                                    const level = parseInt(heading.tagName.charAt(1));
                                    const indent = level === 1 ? '' : 'style="margin-left: ' + ((level - 1) * 1.2) + 'em;"';
                                    ToC += '<div ' + indent + '><a href="#' + id + '">' + heading.textContent + '</a></div>';
                                });
                                
                                ToC += '</div></nav>';
                                toc.innerHTML = ToC;
                                toc.setAttribute('prerendered', 'true');
                                
                                // Extract tenet text for tooltips
                                const tenetTooltips = {
                                    'source-of-truth': 'We should be a source of truth for all model definitions. Model implementations should be reliable, reproducible, and faithful to the original performances.',
                                    'one-model-one-file': 'All inference (and most of training, loss is separate, not a part of model) logic visible, top‑to‑bottom.',
                                    'code-is-product': 'Optimize for reading, diffing, and tweaking, our users are power users. Variables can be explicit, full words, even several words, readability is primordial.',
                                    'standardize-dont-abstract': 'If it\\'s model behavior, keep it in the file; abstractions only for generic infra.',
                                    'do-repeat-yourself': 'Copy when it helps users; keep successors in sync without centralizing behavior.',
                                    'minimal-user-api': 'Config, model, preprocessing; from_pretrained, save_pretrained, push_to_hub. We want the least amount of codepaths.',
                                    'backwards-compatibility': 'Evolve by additive standardization, never break public APIs.',
                                    'consistent-public-surface': 'Same argument names, same outputs, hidden states and attentions exposed.',
                                };

                                // Add smooth scrolling and active state
                                const tocLinks = document.querySelectorAll('d-contents a');
                                tocLinks.forEach(link => {
                                    const href = link.getAttribute('href');
                                    const anchor = href ? href.substring(1) : '';
                                    
                                    // Add tooltip if this is a tenet link
                                    if (tenetTooltips[anchor]) {
                                        link.setAttribute('title', tenetTooltips[anchor]);
                                        link.style.position = 'relative';
                                    }
                                    
                                    link.addEventListener('click', function(e) {
                                        e.preventDefault();
                                        const target = document.querySelector(this.getAttribute('href'));
                                        if (target) {
                                            target.scrollIntoView({ behavior: 'smooth' });
                                        }
                                    });
                                });
                                
                                // Update active state on scroll
                                window.addEventListener('scroll', function() {
                                    const scrollPos = window.scrollY + 100;
                                    headings.forEach((heading) => {
                                        const link = document.querySelector('d-contents a[href="#' + heading.id + '"]');
                                        if (link) {
                                            if (heading.offsetTop <= scrollPos && 
                                                heading.offsetTop + heading.offsetHeight > scrollPos) {
                                                link.classList.add('active');
                                            } else {
                                                link.classList.remove('active');
                                            }
                                        }
                                    });
                                });
                            }
                        }
                        
                        // Initialize Prism syntax highlighting
                        function initializeSyntaxHighlighting() {
                            if (typeof Prism !== 'undefined') {
                                Prism.highlightAll();
                            }
                        }
                        
                        // Try multiple times to ensure it runs after distill.js
                        document.addEventListener('DOMContentLoaded', function() {
                            initializeTOC();
                            initializeSyntaxHighlighting();
                        });
                        setTimeout(function() {
                            initializeTOC();
                            initializeSyntaxHighlighting();
                        }, 100);
                        setTimeout(function() {
                            initializeTOC();
                            initializeSyntaxHighlighting();
                        }, 500);
                        setTimeout(function() {
                            initializeTOC();
                            initializeSyntaxHighlighting();
                        }, 1000);
                        </script>`;
                        
                        // Create full HTML document with distill template
                        const template = `<!DOCTYPE html>
<html>
<head>
    <script src="distill.bundle.js" type="module" fetchpriority="high" blocking></script>
    <script src="main.bundle.js" type="module" fetchpriority="low" defer></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/components/prism-core.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/plugins/autoloader/prism-autoloader.min.js"></script>
    <script src="https://d3js.org/d3.v7.min.js"></script>
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <meta charset="utf8">
    <title>Transformers Feature Showcase</title>
    <link rel="stylesheet" href="style.css">
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/themes/prism.min.css">
</head>
<body>
    <d-front-matter>
        <script id='distill-front-matter' type="text/json">{
    "title": "Transformers Feature Showcase",
    "description": "An interactive demonstration of transformers library features and design philosophy.",
    "published": "Aug 21, 2025",
    "authors": [{"author": "Pablo Pernot", "authorURL": "https://huggingface.co/pablo"}]
  }</script>
    </d-front-matter>
    <d-title>
        <h1>Transformers Feature Showcase</h1>
        <p>An interactive demonstration of transformers library features and design philosophy.</p>
    </d-title>
    <d-byline></d-byline>
    <d-article>
        <d-contents>
            <nav role="navigation" class="l-text figcaption">
                <div class="toc-header"><span class="toc-title">Table of Contents</span></div>
                <div class="toc-content">
                    <div><a href="#introduction">Introduction</a></div>
                    <div style="margin-left: 1.2em;"><a href="#what-you-will-learn">What you will learn</a></div>
                    <div><a href="#source-of-truth">0. Source of truth</a></div>
                    <div><a href="#one-model-one-file">1. One model, one file</a></div>
                    <div><a href="#code-is-product">2. Code is product</a></div>
                    <div><a href="#standardize-dont-abstract">3. Standardize, don't abstract</a></div>
                    <div><a href="#do-repeat-yourself">4. DRY* (DO Repeat Yourself)</a></div>
                    <div><a href="#minimal-user-api">5. Minimal user API</a></div>
                    <div><a href="#backwards-compatibility">6. Backwards compatibility</a></div>
                    <div><a href="#consistent-public-surface">7. Consistent public surface</a></div>
                    <div><a href="#modular">Going modular</a></div>
                    <div><a href="#attention-classes">External Attention classes</a></div>
                    <div><a href="#encoders-ftw">Encoders win!</a></div>
                </div>
            </nav>
        </d-contents>
        ${htmlContent}
    </d-article>
    ${tocScript}
</body>
</html>`;
                        
                        // Process with Handlebars for fragment insertion
                        const handlebars = Handlebars.compile(template);
                        return handlebars(fragments);
                    }
                },
            ],
        }),
    ],
    devtool: process.env.NODE_ENV === 'production' ? 'source-map' : 'eval-source-map',
    devServer: {
        static: {
            directory: path.join(__dirname, 'dist'),
        },
        hot: true,
        watchFiles: ['src/**/*'],
        client: {
            overlay: true,
        },
    },
    mode: process.env.NODE_ENV === 'production' ? 'production' : 'development',
    optimization: {
        minimizer: [
            new ImageMinimizerPlugin({
                minimizer: [{
                    implementation: ImageMinimizerPlugin.sharpMinify,
                    options: {
                        encodeOptions: {
                            jpeg: {
                                quality: 80
                            },
                            png: {
                                quality: 80
                            },
                            webp: {
                                quality: 80
                            }
                        }
                    }
                },
                {
                    implementation: ImageMinimizerPlugin.svgoMinify,
                    options: {
                        encodeOptions: {
                            multipass: true,
                            plugins: [
                                'preset-default',
                            ]
                        }
                    }
                }
            ]
            }),
            new HtmlMinimizerPlugin({
                test: /fragments\/.*\.html$/i,
                minify: HtmlMinimizerPlugin.swcMinifyFragment,
            })
        ]
    },
};

console.log(process.env.NODE_ENV)