update

app.py

@@ -13,7 +13,7 @@ from pathlib import Path
 import gradio as gr
 
 # —— refactored helpers ——
-from modular_graph_and_candidates import build_graph_json, generate_html
+from modular_graph_and_candidates import build_graph_json, generate_html, build_timeline_json, generate_timeline_html
 
 HF_MAIN_REPO = "https://github.com/huggingface/transformers"
 
@@ -51,8 +51,8 @@ def _escape_srcdoc(text: str) -> str:
     )
 
 
-def run(repo_url: str, threshold: float, multimodal: bool, sim_method: str):
-    # Always download repo for now - let build_graph_json decide if it needs it
+def run_graph(repo_url: str, threshold: float, multimodal: bool, sim_method: str):
+    """Generate the dependency graph visualization."""
     repo_path = clone_or_cache(repo_url)
 
     graph = build_graph_json(
@@ -73,26 +73,63 @@ def run(repo_url: str, threshold: float, multimodal: bool, sim_method: str):
     tmp_json.write_text(json.dumps(graph), encoding="utf-8")
     return iframe_html, str(tmp_json)
 
+def run_timeline(repo_url: str, threshold: float, multimodal: bool, sim_method: str):
+    """Generate the chronological timeline visualization."""
+    repo_path = clone_or_cache(repo_url)
+
+    timeline = build_timeline_json(
+        transformers_dir=repo_path,
+        threshold=threshold,
+        multimodal=multimodal,
+        sim_method=sim_method,
+    )
+
+    raw_html = generate_timeline_html(timeline)
+
+    iframe_html = (
+        f'<iframe style="width:100%;height:85vh;border:none;" '
+        f'srcdoc="{_escape_srcdoc(raw_html)}"></iframe>'
+    )
+
+    tmp_json = Path(tempfile.mktemp(suffix="_timeline.json"))
+    tmp_json.write_text(json.dumps(timeline), encoding="utf-8")
+    return iframe_html, str(tmp_json)
+
 # ───────────────────────────── UI ────────────────────────────────────────────────
 
 CUSTOM_CSS = """
-#graph_html iframe {height:85vh !important; width:100% !important; border:none;}
+#graph_html iframe, #timeline_html iframe {height:85vh !important; width:100% !important; border:none;}
 """
 
 with gr.Blocks(css=CUSTOM_CSS) as demo:
     gr.Markdown("## 🔍 Modular‑candidate explorer for 🤗 Transformers")
 
-    with gr.Row():
-        repo_in   = gr.Text(value=HF_MAIN_REPO, label="Repo / fork URL")
-        thresh    = gr.Slider(0.50, 0.95, value=0.5, step=0.01, label="Similarity ≥")
-        multi_cb  = gr.Checkbox(label="Only multimodal models")
-        sim_radio = gr.Radio(["jaccard", "embedding"], value="jaccard", label="Similarity metric")
-        go_btn    = gr.Button("Build graph")
+    with gr.Tabs():
+        with gr.Tab("Dependency Graph"):
+            with gr.Row():
+                repo_in   = gr.Text(value=HF_MAIN_REPO, label="Repo / fork URL")
+                thresh    = gr.Slider(0.50, 0.95, value=0.5, step=0.01, label="Similarity ≥")
+                multi_cb  = gr.Checkbox(label="Only multimodal models")
+                sim_radio = gr.Radio(["jaccard", "embedding"], value="jaccard", label="Similarity metric")
+                go_btn    = gr.Button("Build graph")
+
+            graph_html_out  = gr.HTML(elem_id="graph_html", show_label=False)
+            graph_json_out  = gr.File(label="Download graph.json")
+
+            go_btn.click(run_graph, [repo_in, thresh, multi_cb, sim_radio], [graph_html_out, graph_json_out])
+
+        with gr.Tab("Chronological Timeline"):
+            with gr.Row():
+                timeline_repo_in = gr.Text(value=HF_MAIN_REPO, label="Repo / fork URL")
+                timeline_thresh = gr.Slider(0.50, 0.95, value=0.5, step=0.01, label="Similarity ≥")
+                timeline_multi_cb = gr.Checkbox(label="Only multimodal models")
+                timeline_sim_radio = gr.Radio(["jaccard", "embedding"], value="jaccard", label="Similarity metric")
+                timeline_btn = gr.Button("Build timeline")
 
-    html_out  = gr.HTML(elem_id="graph_html", show_label=False)
-    json_out  = gr.File(label="Download graph.json")
+            timeline_html_out = gr.HTML(elem_id="timeline_html", show_label=False)
+            timeline_json_out = gr.File(label="Download timeline.json")
 
-    go_btn.click(run, [repo_in, thresh, multi_cb, sim_radio], [html_out, json_out])
+            timeline_btn.click(run_timeline, [timeline_repo_in, timeline_thresh, timeline_multi_cb, timeline_sim_radio], [timeline_html_out, timeline_json_out])
 
 if __name__ == "__main__":
     demo.launch(allowed_paths=["static"])

modular_graph_and_candidates.py

@@ -32,6 +32,7 @@ import argparse
 import ast
 import json
 import re
+import subprocess
 import tokenize
 from collections import Counter, defaultdict
 from itertools import combinations
@@ -42,6 +43,7 @@ from tqdm import tqdm
 import numpy as np
 import spaces
 import torch
+from datetime import datetime
 
 # ────────────────────────────────────────────────────────────────────────────────
 # CONFIG
@@ -454,11 +456,163 @@ def build_graph_json(
     return graph
 
 
+# ────────────────────────────────────────────────────────────────────────────────
+# Timeline functions for chronological visualization
+# ────────────────────────────────────────────────────────────────────────────────
+
+
+def get_model_creation_dates(transformers_dir: Path) -> Dict[str, datetime]:
+    """Get creation dates for all model directories by finding the earliest add of the directory path."""
+    models_root = transformers_dir / "src/transformers/models"
+    creation_dates: Dict[str, datetime] = {}
+
+    if not models_root.exists():
+        return creation_dates
+
+    def run_git(args: list[str]) -> subprocess.CompletedProcess:
+        return subprocess.run(
+            ["git"] + args,
+            cwd=transformers_dir,
+            capture_output=True,
+            text=True,
+            timeout=120,
+        )
+
+    # Ensure full history; shallow clones make every path look newly added "today".
+    shallow = run_git(["rev-parse", "--is-shallow-repository"])
+    if shallow.returncode == 0 and shallow.stdout.strip() == "true":
+        # Try best-effort unshallow; if it fails, we still proceed.
+        run_git(["fetch", "--unshallow", "--tags", "--prune"])  # ignore return code
+        # Fallback if server forbids --unshallow
+        run_git(["fetch", "--depth=100000", "--tags", "--prune"])
+
+    for model_dir in models_root.iterdir():
+        if not model_dir.is_dir():
+            continue
+
+        rel = f"src/transformers/models/{model_dir.name}/"
+
+        # Earliest commit that ADDED something under this directory.
+        # Use a stable delimiter to avoid locale/spacing issues.
+        proc = run_git([
+            "log",
+            "--reverse",               # oldest → newest
+            "--diff-filter=A",         # additions only
+            "--date=short",            # YYYY-MM-DD
+            '--format=%H|%ad',         # hash|date
+            "--",
+            rel,
+        ])
+
+        if proc.returncode != 0 or not proc.stdout.strip():
+            # As a fallback, look at the earliest commit touching any tracked file under the dir.
+            # This can catch cases where files were moved (rename) rather than added.
+            ls = run_git(["ls-files", rel])
+            files = [ln for ln in ls.stdout.splitlines() if ln.strip()]
+            best_date: datetime | None = None
+            if files:
+                for fp in files:
+                    proc_file = run_git([
+                        "log",
+                        "--reverse",
+                        "--diff-filter=A",
+                        "--date=short",
+                        "--format=%H|%ad",
+                        "--",
+                        fp,
+                    ])
+                    line = proc_file.stdout.splitlines()[0].strip() if proc_file.stdout else ""
+                    if line and "|" in line:
+                        _, d = line.split("|", 1)
+                        try:
+                            dt = datetime.strptime(d.strip(), "%Y-%m-%d")
+                            if best_date is None or dt < best_date:
+                                best_date = dt
+                        except ValueError:
+                            pass
+            if best_date is not None:
+                creation_dates[model_dir.name] = best_date
+                print(f"✅ {model_dir.name}: {best_date.strftime('%Y-%m-%d')}")
+            else:
+                print(f"❌ {model_dir.name}: no add commit found")
+            continue
+
+        first_line = proc.stdout.splitlines()[0].strip()  # oldest add
+        if "|" in first_line:
+            _, date_str = first_line.split("|", 1)
+            try:
+                creation_dates[model_dir.name] = datetime.strptime(date_str.strip(), "%Y-%m-%d")
+                print(f"✅ {model_dir.name}: {date_str.strip()}")
+            except ValueError:
+                print(f"❌ {model_dir.name}: bad date format: {date_str!r}")
+        else:
+            print(f"❌ {model_dir.name}: unexpected log format: {first_line!r}")
+
+    return creation_dates
+
+
+def build_timeline_json(
+    transformers_dir: Path,
+    threshold: float = SIM_DEFAULT,
+    multimodal: bool = False,
+    sim_method: str = "jaccard",
+) -> dict:
+    """Build chronological timeline with modular connections."""
+    # Get the standard dependency graph for connections
+    graph = build_graph_json(transformers_dir, threshold, multimodal, sim_method)
+
+    # Get creation dates for chronological positioning
+    creation_dates = get_model_creation_dates(transformers_dir)
+
+    # Enhance nodes with chronological data
+    for node in graph["nodes"]:
+        model_name = node["id"]
+        if model_name in creation_dates:
+            creation_date = creation_dates[model_name]
+            node.update({
+                "date": creation_date.isoformat(),
+                "year": creation_date.year,
+                "timestamp": creation_date.timestamp()
+            })
+        else:
+            # Fallback for models without date info
+            node.update({
+                "date": "2020-01-01T00:00:00",  # Default date
+                "year": 2020,
+                "timestamp": datetime(2020, 1, 1).timestamp()
+            })
+
+    # Add timeline metadata
+    valid_dates = [n for n in graph["nodes"] if n["timestamp"] > 0]
+    if valid_dates:
+        min_year = min(n["year"] for n in valid_dates)
+        max_year = max(n["year"] for n in valid_dates)
+        graph["timeline_meta"] = {
+            "min_year": min_year,
+            "max_year": max_year,
+            "total_models": len(graph["nodes"]),
+            "dated_models": len(valid_dates)
+        }
+    else:
+        graph["timeline_meta"] = {
+            "min_year": 2018,
+            "max_year": 2024,
+            "total_models": len(graph["nodes"]),
+            "dated_models": 0
+        }
+
+    return graph
+
 def generate_html(graph: dict) -> str:
     """Return the full HTML string with inlined CSS/JS + graph JSON."""
     js = JS.replace("__GRAPH_DATA__", json.dumps(graph, separators=(",", ":")))
     return HTML.replace("__CSS__", CSS).replace("__JS__", js)
 
+def generate_timeline_html(timeline: dict) -> str:
+    """Return the full HTML string for chronological timeline visualization."""
+    js = TIMELINE_JS.replace("__TIMELINE_DATA__", json.dumps(timeline, separators=(",", ":")))
+    return TIMELINE_HTML.replace("__TIMELINE_CSS__", TIMELINE_CSS).replace("__TIMELINE_JS__", js)
+
 
 
 # ────────────────────────────────────────────────────────────────────────────────
@@ -614,6 +768,381 @@ HTML = """
 <script>__JS__</script></body></html>
 """
 
+# ────────────────────────────────────────────────────────────────────────────────
+# Timeline HTML Templates
+# ────────────────────────────────────────────────────────────────────────────────
+
+TIMELINE_CSS = """
+@import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;600&display=swap');
+
+:root{
+  --bg:#ffffff;
+  --text:#222222;
+  --muted:#555555;
+  --outline:#ffffff;
+  --timeline-line:#dee2e6;
+  --base-color:#ffbe0b;
+  --derived-color:#1f77b4;
+  --candidate-color:#e63946;
+}
+@media (prefers-color-scheme: dark){
+  :root{
+    --bg:#0b0d10;
+    --text:#e8e8e8;
+    --muted:#c8c8c8;
+    --outline:#000000;
+    --timeline-line:#343a40;
+  }
+}
+
+body{
+  margin:0;
+  font-family:'Inter',Arial,sans-serif;
+  background:var(--bg);
+  overflow:hidden;
+}
+svg{ width:100vw; height:100vh; }
+
+/* Enhanced link styles for chronological flow */
+.link{
+  stroke:#4a90e2;
+  stroke-opacity:0.6;
+  stroke-width:1.5;
+  fill:none;
+  transition: stroke-opacity 0.3s ease;
+}
+.link.cand{
+  stroke:var(--candidate-color);
+  stroke-width:2.5;
+  stroke-opacity:0.8;
+  stroke-dasharray: 4,4;
+}
+.link:hover{
+  stroke-opacity:1;
+  stroke-width:3;
+}
+
+/* Improved node label styling */
+.node-label{
+  fill:var(--text);
+  pointer-events:none;
+  text-anchor:middle;
+  font-weight:600;
+  font-size:13px;
+  paint-order:stroke fill;
+  stroke:var(--outline);
+  stroke-width:3px;
+  cursor:default;
+}
+
+/* Enhanced node styling with better visual hierarchy */
+.node.base circle{
+  fill:var(--base-color);
+  stroke:#d4a000;
+  stroke-width:2;
+}
+.node.derived circle{
+  fill:var(--derived-color);
+  stroke:#1565c0;
+  stroke-width:2;
+}
+.node.cand circle{
+  fill:var(--candidate-color);
+  stroke:#c62828;
+  stroke-width:2;
+}
+
+.node circle{
+  transition: r 0.3s ease, stroke-width 0.3s ease;
+  cursor:grab;
+}
+.node:hover circle{
+  r:22;
+  stroke-width:3;
+}
+.node:active{
+  cursor:grabbing;
+}
+
+/* Timeline axis styling */
+.timeline-axis {
+  stroke: var(--timeline-line);
+  stroke-width: 3px;
+  stroke-opacity: 0.8;
+}
+
+.timeline-tick {
+  stroke: var(--timeline-line);
+  stroke-width: 2px;
+  stroke-opacity: 0.6;
+}
+
+.timeline-label {
+  fill: var(--muted);
+  font-size: 14px;
+  font-weight: 600;
+  text-anchor: middle;
+}
+
+/* Enhanced controls panel */
+#controls{
+  position:fixed; top:20px; left:20px;
+  background:rgba(255,255,255,.95);
+  padding:20px 26px; border-radius:12px; border:1.5px solid #e0e0e0;
+  font-size:14px; box-shadow:0 4px 16px rgba(0,0,0,.12);
+  z-index: 100;
+  backdrop-filter: blur(8px);
+  max-width: 280px;
+}
+@media (prefers-color-scheme: dark){
+  #controls{
+    background:rgba(20,22,25,.95);
+    color:#e8e8e8;
+    border-color:#404040;
+  }
+}
+
+#controls label{
+  display:flex;
+  align-items:center;
+  margin-top:10px;
+  cursor:pointer;
+}
+#controls input[type="checkbox"]{
+  margin-right:8px;
+  cursor:pointer;
+}
+"""
+
+TIMELINE_JS = """
+function updateVisibility() {
+  const show = document.getElementById('toggleRed').checked;
+  svg.selectAll('.link.cand').style('display', show ? null : 'none');
+  svg.selectAll('.node.cand').style('display', show ? null : 'none');
+}
+document.getElementById('toggleRed').addEventListener('change', updateVisibility);
+
+const HF_LOGO_URI = "./static/hf-logo.png";
+const timeline = __TIMELINE_DATA__;
+const W = innerWidth, H = innerHeight;
+
+// Enhanced timeline configuration for maximum horizontal spread
+const MARGIN = { top: 60, right: 200, bottom: 120, left: 200 };
+const CONTENT_HEIGHT = H - MARGIN.top - MARGIN.bottom;
+const VERTICAL_LANES = 4; // Number of horizontal lanes for better organization
+
+// Create SVG with zoom behavior
+const svg = d3.select('#timeline-svg');
+const zoomBehavior = d3.zoom()
+  .scaleExtent([0.1, 8])
+  .on('zoom', handleZoom);
+svg.call(zoomBehavior);
+
+const g = svg.append('g');
+
+// Time scale for chronological positioning with much wider spread
+const timeExtent = d3.extent(timeline.nodes.filter(d => d.timestamp > 0), d => d.timestamp);
+let timeScale;
+
+if (timeExtent[0] && timeExtent[1]) {
+  // Much wider timeline for maximum horizontal spread
+  const timeWidth = Math.max(W * 8, 8000);
+  timeScale = d3.scaleTime()
+    .domain(timeExtent.map(t => new Date(t * 1000)))
+    .range([MARGIN.left, timeWidth - MARGIN.right]);
+
+  // Timeline axis at the bottom
+  const timelineG = g.append('g').attr('class', 'timeline');
+  const timelineY = H - 80;
+
+  timelineG.append('line')
+    .attr('class', 'timeline-axis')
+    .attr('x1', MARGIN.left)
+    .attr('y1', timelineY)
+    .attr('x2', timeWidth - MARGIN.right)
+    .attr('y2', timelineY);
+
+  // Enhanced year markers with better spacing
+  const years = d3.timeYear.range(new Date(timeExtent[0] * 1000), new Date(timeExtent[1] * 1000 + 365*24*60*60*1000));
+  timelineG.selectAll('.timeline-tick')
+    .data(years)
+    .join('line')
+    .attr('class', 'timeline-tick')
+    .attr('x1', d => timeScale(d))
+    .attr('y1', timelineY - 15)
+    .attr('x2', d => timeScale(d))
+    .attr('y2', timelineY + 15);
+
+  timelineG.selectAll('.timeline-label')
+    .data(years)
+    .join('text')
+    .attr('class', 'timeline-label')
+    .attr('x', d => timeScale(d))
+    .attr('y', timelineY + 30)
+    .text(d => d.getFullYear());
+}
+
+function handleZoom(event) {
+  const { transform } = event;
+  g.attr('transform', transform);
+}
+
+// Enhanced curved links for better chronological flow visualization
+const link = g.selectAll('path.link')
+  .data(timeline.links)
+  .join('path')
+  .attr('class', d => d.cand ? 'link cand' : 'link')
+  .attr('fill', 'none')
+  .attr('stroke-width', d => d.cand ? 2.5 : 1.5);
+
+// Nodes with improved positioning strategy
+const node = g.selectAll('g.node')
+  .data(timeline.nodes)
+  .join('g')
+  .attr('class', d => `node ${d.cls}`)
+  .call(d3.drag().on('start', dragStart).on('drag', dragged).on('end', dragEnd));
+
+const baseSel = node.filter(d => d.cls === 'base');
+if (HF_LOGO_URI) {
+  baseSel.append('image')
+    .attr('href', HF_LOGO_URI)
+    .attr('width', 35)
+    .attr('height', 35)
+    .attr('x', -17.5)
+    .attr('y', -17.5)
+    .on('error', function() {
+      d3.select(this.parentNode).append('circle')
+        .attr('r', 20).attr('fill', '#ffbe0b');
+    });
+} else {
+  baseSel.append('circle').attr('r', 20).attr('fill', '#ffbe0b');
+}
+node.filter(d => d.cls !== 'base').append('circle').attr('r', 18);
+
+node.append('text')
+  .attr('class', 'node-label')
+  .attr('dy', '-2.2em')
+  .text(d => d.id);
+
+// Organize nodes by chronological lanes for better vertical distribution
+timeline.nodes.forEach((d, i) => {
+  if (d.timestamp > 0) {
+    // Assign lane based on chronological order within similar timeframes
+    const yearNodes = timeline.nodes.filter(n =>
+      n.timestamp > 0 &&
+      Math.abs(n.timestamp - d.timestamp) < 365*24*60*60
+    );
+    d.lane = yearNodes.indexOf(d) % VERTICAL_LANES;
+  } else {
+    d.lane = i % VERTICAL_LANES;
+  }
+});
+
+// Enhanced force simulation for optimal horizontal chronological layout
+const sim = d3.forceSimulation(timeline.nodes)
+  .force('link', d3.forceLink(timeline.links).id(d => d.id)
+    .distance(d => d.cand ? 100 : 200)
+    .strength(d => d.cand ? 0.1 : 0.3))
+  .force('charge', d3.forceManyBody().strength(-300))
+  .force('collide', d3.forceCollide(d => 45).strength(0.8));
+
+// Very strong chronological X positioning for proper horizontal spread
+if (timeScale) {
+  sim.force('chronological', d3.forceX(d => {
+    if (d.timestamp > 0) {
+      return timeScale(new Date(d.timestamp * 1000));
+    }
+    // Place undated models at the end
+    return timeScale.range()[1] + 100;
+  }).strength(0.95));
+}
+
+// Organized Y positioning using lanes instead of random spread
+sim.force('lanes', d3.forceY(d => {
+  const centerY = H / 2 - 100; // Position above timeline
+  const laneHeight = (H - 200) / (VERTICAL_LANES + 1); // Account for timeline space
+  const targetY = centerY - ((H - 200) / 2) + (d.lane + 1) * laneHeight;
+  return targetY;
+}).strength(0.7));
+
+// Add center force to prevent rightward drift
+sim.force('center', d3.forceCenter(timeScale ? (timeScale.range()[0] + timeScale.range()[1]) / 2 : W / 2, H / 2 - 100).strength(0.1));
+
+// Custom path generator for curved links that follow chronological flow
+function linkPath(d) {
+  const sourceX = d.source.x || 0;
+  const sourceY = d.source.y || 0;
+  const targetX = d.target.x || 0;
+  const targetY = d.target.y || 0;
+
+  // Create curved paths for better visual flow
+  const dx = targetX - sourceX;
+  const dy = targetY - sourceY;
+  const dr = Math.sqrt(dx * dx + dy * dy) * 0.3;
+
+  // Curve direction based on chronological order
+  const curve = dx > 0 ? dr : -dr;
+
+  return `M${sourceX},${sourceY}A${dr},${dr} 0 0,1 ${targetX},${targetY}`;
+}
+
+sim.on('tick', () => {
+  link.attr('d', linkPath);
+  node.attr('transform', d => `translate(${d.x},${d.y})`);
+});
+
+function dragStart(e, d) {
+  if (!e.active) sim.alphaTarget(.3).restart();
+  d.fx = d.x;
+  d.fy = d.y;
+}
+function dragged(e, d) {
+  d.fx = e.x;
+  d.fy = e.y;
+}
+function dragEnd(e, d) {
+  if (!e.active) sim.alphaTarget(0);
+  d.fx = d.fy = null;
+}
+
+// Initialize
+updateVisibility();
+
+// Auto-fit timeline view with better zoom for horizontal spread
+setTimeout(() => {
+  if (timeScale && timeExtent[0] && timeExtent[1]) {
+    const timeWidth = timeScale.range()[1] - timeScale.range()[0];
+    const scale = Math.min((W * 0.9) / timeWidth, 1);
+    const translateX = (W - timeWidth * scale) / 2;
+    const translateY = 0;
+
+    svg.transition()
+      .duration(2000)
+      .call(zoomBehavior.transform,
+        d3.zoomIdentity.translate(translateX, translateY).scale(scale));
+  }
+}, 1500);
+"""
+
+TIMELINE_HTML = """
+<!DOCTYPE html>
+<html lang='en'><head><meta charset='UTF-8'>
+<title>Transformers Chronological Timeline</title>
+<style>__TIMELINE_CSS__</style></head><body>
+<div id='controls'>
+  <div style='font-weight:600; margin-bottom:8px;'>Chronological Timeline</div>
+  🟡 base<br>🔵 modular<br>🔴 candidate<br>
+  <label><input type="checkbox" id="toggleRed" checked> Show candidates</label>
+  <div style='margin-top:10px; font-size:11px; color:var(--muted);'>
+    Models positioned by creation date<br>
+    Scroll & zoom to explore timeline
+  </div>
+</div>
+<svg id='timeline-svg'></svg>
+<script src='https://d3js.org/d3.v7.min.js'></script>
+<script>__TIMELINE_JS__</script></body></html>
+"""
+
 # ────────────────────────────────────────────────────────────────────────────────
 # HTML writer
 # ────────────────────────────────────────────────────────────────────────────────