Merge pull request #224 from biggraph/darabos-merge-public

.github/workflows/test.yaml

@@ -6,19 +6,20 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
-    env:
-      UV_SYSTEM_PYTHON: 1
     steps:
       - uses: actions/checkout@v4
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
-        with:
-            enable-cache: true
+        uses: astral-sh/setup-uv@v6
 
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.12"
+      - name: Set up Python
+        run: uv python install
+
+      # - name: Debug ty issue
+      #   run: |
+      #     uv pip install ty
+      #     uv pip list
+      #     uv run python -m ty check
 
       - name: Install dependencies
         run: |
@@ -37,32 +38,16 @@ jobs:
       - name: Run pre-commits
         run: |
           uv pip install pre-commit
-          pre-commit run --all-files
-
-      - name: Run core tests
-        run: |
-          cd lynxkite-core
-          pytest
+          uv run pre-commit run --all-files
 
-      - name: Run app tests
+      - name: Run Python unittests
         run: |
-          cd lynxkite-app
-          pytest
+          uv run pytest --asyncio-mode=auto
 
-      - name: Run graph analytics tests
-        run: |
-          cd lynxkite-graph-analytics
-          pytest
-
-      - name: Run LynxScribe tests
-        run: |
-          cd lynxkite-lynxscribe
-          pytest
-
-      - name: Try building the documentation
+      - name: Build the documentation
         run: |
           uv pip install mkdocs-material mkdocstrings[python]
-          mkdocs build
+          uv run mkdocs build
 
       - uses: actions/setup-node@v4
         with:
@@ -76,9 +61,7 @@ jobs:
 
       - name: Run Playwright tests
         run: |
-          cd lynxkite-app/web
-          npm run build
-          npm run test
+          uv run bash -c 'cd lynxkite-app/web; npm run build; npm run test'
 
       - uses: actions/upload-artifact@v4
         name: Upload playwright report

.pre-commit-config.yaml

@@ -15,3 +15,28 @@ repos:
   rev: v1.9.4
   hooks:
     - id: biome-check
+# https://github.com/astral-sh/ty/issues/269
+- repo: local
+  hooks:
+    - id: ty-check
+      name: ty-check
+      language: python
+      entry: uv run ty check
+      pass_filenames: false
+      args: [--python=.venv/]
+      additional_dependencies: [ty]
+- repo: https://github.com/fpgmaas/deptry.git
+  rev: "0.23.0"
+  hooks:
+    - id: deptry
+      name: deptry for lynxkite-app
+      entry: bash -c 'cd lynxkite-app && deptry .'
+    - id: deptry
+      name: deptry for lynxkite-core
+      entry: bash -c 'cd lynxkite-core && deptry .'
+    - id: deptry
+      name: deptry for lynxkite-graph-analytics
+      entry: bash -c 'cd lynxkite-graph-analytics && deptry .'
+    - id: deptry
+      name: deptry for lynxkite-pillow-example
+      entry: bash -c 'cd lynxkite-pillow-example && deptry .'

.python-version

@@ -1 +1 @@
-3.11
+3.12

docs/contributing.md

@@ -22,7 +22,7 @@ Install everything like this:
 uv venv
 source .venv/bin/activate
 uvx pre-commit install
-uv pip install -e 'lynxkite-core/[dev]' -e 'lynxkite-app/[dev]' -e 'lynxkite-graph-analytics/[dev]' -e lynxkite-pillow-example/ -e lynxkite-bio -e lynxkite-lynxscribe/
+uv sync
 ```
 
 This also builds the frontend, hopefully very quickly. To run it:
@@ -41,10 +41,10 @@ npm run dev
 
 ## Executing tests
 
-Run all tests with a single command, or look inside to see how to run them individually:
-
 ```bash
-./test.sh
+pytest                                # Runs all backend unit tests.
+pytest lynxkite-core                  # Runs tests for one package.
+cd lynxkite-app/web && npm run test   # Runs frontend tests.
 ```
 
 ## Documentation
@@ -52,6 +52,5 @@ Run all tests with a single command, or look inside to see how to run them indiv
 To work on the documentation:
 
 ```bash
-uv pip install mkdocs-material mkdocstrings[python]
 mkdocs serve
 ```

docs/guides/plugins.md

@@ -57,7 +57,12 @@ Let's review the changes we made.
 
 The [`@op`](../reference/lynxkite-core/ops.md#lynxkite.core.ops.op) decorator registers a
 function as a LynxKite operation. The first argument is the name of the environment,
-the second argument is the name of the operation.
+the last argument is the name of the operation. Between the two, you can list the hierarchy of
+categories the operation belongs to. For example:
+
+```python
+@op("LynxKite Graph Analytics", "Machine learning", "Preprocessing", "Split train/test set")
+```
 
 When defining multiple operations, you can use
 [`ops.op_registration`](../reference/lynxkite-core/ops.md#lynxkite.core.ops.op_registration)

examples/Model definition.lynxkite.json

@@ -81,7 +81,10 @@
         "error": null,
         "input_metadata": null,
         "meta": {
+          "categories": [],
           "color": "green",
+          "doc": null,
+          "id": "Optimizer",
           "inputs": [
             {
               "name": "loss",
@@ -110,7 +113,7 @@
               }
             },
             {
-              "default": 0.001,
+              "default": 0.0001,
               "name": "lr",
               "type": {
                 "type": "<class 'float'>"
@@ -119,6 +122,7 @@
           ],
           "type": "basic"
         },
+        "op_id": "Optimizer",
         "params": {
           "lr": "0.1",
           "type": "SGD"
@@ -126,7 +130,7 @@
         "status": "done",
         "title": "Optimizer"
       },
-      "dragHandle": ".bg-primary",
+      "dragHandle": ".drag-handle",
       "height": 250.0,
       "id": "Optimizer 2",
       "position": {
@@ -144,7 +148,10 @@
         "error": null,
         "input_metadata": null,
         "meta": {
+          "categories": [],
           "color": "orange",
+          "doc": null,
+          "id": "Activation",
           "inputs": [
             {
               "name": "x",
@@ -170,23 +177,30 @@
               "name": "type",
               "type": {
                 "enum": [
+                  "ELU",
+                  "GELU",
+                  "LeakyReLU",
+                  "Mish",
+                  "PReLU",
                   "ReLU",
-                  "Leaky_ReLU",
-                  "Tanh",
-                  "Mish"
+                  "Sigmoid",
+                  "SiLU",
+                  "Softplus",
+                  "Tanh"
                 ]
               }
             }
           ],
           "type": "basic"
         },
+        "op_id": "Activation",
         "params": {
-          "type": "Leaky_ReLU"
+          "type": "LeakyReLU"
         },
         "status": "done",
         "title": "Activation"
       },
-      "dragHandle": ".bg-primary",
+      "dragHandle": ".drag-handle",
       "height": 200.0,
       "id": "Activation 1",
       "position": {
@@ -204,7 +218,10 @@
         "error": null,
         "input_metadata": null,
         "meta": {
-          "color": "orange",
+          "categories": [],
+          "color": "gray",
+          "doc": null,
+          "id": "Input: tensor",
           "inputs": [],
           "name": "Input: tensor",
           "outputs": [
@@ -227,13 +244,14 @@
           ],
           "type": "basic"
         },
+        "op_id": "Input: tensor",
         "params": {
           "name": "Y"
         },
         "status": "done",
         "title": "Input: tensor"
       },
-      "dragHandle": ".bg-primary",
+      "dragHandle": ".drag-handle",
       "height": 200.0,
       "id": "Input: tensor 3",
       "position": {
@@ -251,7 +269,10 @@
         "error": null,
         "input_metadata": null,
         "meta": {
+          "categories": [],
           "color": "orange",
+          "doc": null,
+          "id": "MSE loss",
           "inputs": [
             {
               "name": "x",
@@ -281,11 +302,12 @@
           "params": [],
           "type": "basic"
         },
+        "op_id": "MSE loss",
         "params": {},
         "status": "done",
         "title": "MSE loss"
       },
-      "dragHandle": ".bg-primary",
+      "dragHandle": ".drag-handle",
       "height": 200.0,
       "id": "MSE loss 2",
       "position": {
@@ -303,7 +325,10 @@
         "error": null,
         "input_metadata": null,
         "meta": {
+          "categories": [],
           "color": "orange",
+          "doc": null,
+          "id": "Repeat",
           "inputs": [
             {
               "name": "input",
@@ -341,6 +366,7 @@
           ],
           "type": "basic"
         },
+        "op_id": "Repeat",
         "params": {
           "same_weights": false,
           "times": "2"
@@ -348,7 +374,7 @@
         "status": "done",
         "title": "Repeat"
       },
-      "dragHandle": ".bg-primary",
+      "dragHandle": ".drag-handle",
       "height": 200.0,
       "id": "Repeat 1",
       "position": {
@@ -366,7 +392,10 @@
         "error": null,
         "input_metadata": null,
         "meta": {
+          "categories": [],
           "color": "blue",
+          "doc": null,
+          "id": "Linear",
           "inputs": [
             {
               "name": "x",
@@ -397,13 +426,14 @@
           ],
           "type": "basic"
         },
+        "op_id": "Linear",
         "params": {
           "output_dim": "4"
         },
         "status": "done",
         "title": "Linear"
       },
-      "dragHandle": ".bg-primary",
+      "dragHandle": ".drag-handle",
       "height": 189.0,
       "id": "Linear 1",
       "position": {
@@ -421,7 +451,10 @@
         "error": null,
         "input_metadata": null,
         "meta": {
-          "color": "orange",
+          "categories": [],
+          "color": "gray",
+          "doc": null,
+          "id": "Input: tensor",
           "inputs": [],
           "name": "Input: tensor",
           "outputs": [
@@ -444,13 +477,14 @@
           ],
           "type": "basic"
         },
+        "op_id": "Input: tensor",
         "params": {
           "name": "X"
         },
         "status": "done",
         "title": "Input: tensor"
       },
-      "dragHandle": ".bg-primary",
+      "dragHandle": ".drag-handle",
       "height": 200.0,
       "id": "Input: tensor 1",
       "position": {
@@ -468,7 +502,10 @@
         "error": null,
         "input_metadata": null,
         "meta": {
+          "categories": [],
           "color": "orange",
+          "doc": null,
+          "id": "Constant vector",
           "inputs": [],
           "name": "Constant vector",
           "outputs": [
@@ -498,6 +535,7 @@
           ],
           "type": "basic"
         },
+        "op_id": "Constant vector",
         "params": {
           "size": "1",
           "value": "1"
@@ -505,7 +543,7 @@
         "status": "done",
         "title": "Constant vector"
       },
-      "dragHandle": ".bg-primary",
+      "dragHandle": ".drag-handle",
       "height": 258.0,
       "id": "Constant vector 1",
       "position": {
@@ -523,7 +561,10 @@
         "error": null,
         "input_metadata": null,
         "meta": {
+          "categories": [],
           "color": "orange",
+          "doc": null,
+          "id": "Add",
           "inputs": [
             {
               "name": "a",
@@ -553,11 +594,12 @@
           "params": [],
           "type": "basic"
         },
+        "op_id": "Add",
         "params": {},
         "status": "done",
         "title": "Add"
       },
-      "dragHandle": ".bg-primary",
+      "dragHandle": ".drag-handle",
       "height": 200.0,
       "id": "Add 1",
       "position": {
@@ -575,7 +617,10 @@
         "error": null,
         "input_metadata": null,
         "meta": {
-          "color": "orange",
+          "categories": [],
+          "color": "gray",
+          "doc": null,
+          "id": "Output",
           "inputs": [
             {
               "name": "x",
@@ -606,11 +651,12 @@
           ],
           "type": "basic"
         },
+        "op_id": "Output",
         "params": {},
         "status": "done",
         "title": "Output"
       },
-      "dragHandle": ".bg-primary",
+      "dragHandle": ".drag-handle",
       "height": 200.0,
       "id": "Output 1",
       "position": {
@@ -620,5 +666,6 @@
       "type": "basic",
       "width": 200.0
     }
-  ]
+  ],
+  "paused": false
 }

examples/Multi-output demo.lynxkite.json

@@ -25,6 +25,9 @@
         "error": null,
         "input_metadata": [],
         "meta": {
+          "categories": [
+            "Examples"
+          ],
           "color": "orange",
           "doc": [
             {
@@ -57,6 +60,7 @@
               ]
             }
           ],
+          "id": "Examples > Multi-output example",
           "inputs": [],
           "name": "Multi-output example",
           "outputs": [
@@ -93,6 +97,7 @@
           ],
           "type": "basic"
         },
+        "op_id": "Examples > Multi-output example",
         "params": {
           "a_limit": "2",
           "b_limit": "10"
@@ -100,7 +105,7 @@
         "status": "done",
         "title": "Multi-output example"
       },
-      "dragHandle": ".bg-primary",
+      "dragHandle": ".drag-handle",
       "height": 275.0,
       "id": "Multi-output example 1",
       "position": {
@@ -146,8 +151,10 @@
           }
         ],
         "meta": {
+          "categories": [],
           "color": "orange",
           "doc": null,
+          "id": "View tables",
           "inputs": [
             {
               "name": "bundle",
@@ -170,13 +177,14 @@
           ],
           "type": "table_view"
         },
+        "op_id": "View tables",
         "params": {
           "limit": 100.0
         },
         "status": "done",
         "title": "View tables"
       },
-      "dragHandle": ".bg-primary",
+      "dragHandle": ".drag-handle",
       "height": 200.0,
       "id": "View tables 1",
       "position": {
@@ -246,8 +254,10 @@
           }
         ],
         "meta": {
+          "categories": [],
           "color": "orange",
           "doc": null,
+          "id": "View tables",
           "inputs": [
             {
               "name": "bundle",
@@ -270,13 +280,14 @@
           ],
           "type": "table_view"
         },
+        "op_id": "View tables",
         "params": {
           "limit": 100.0
         },
         "status": "done",
         "title": "View tables"
       },
-      "dragHandle": ".bg-primary",
+      "dragHandle": ".drag-handle",
       "height": 215.0,
       "id": "View tables 2",
       "position": {

examples/fake_data.py

@@ -1,5 +1,5 @@
 from lynxkite.core.ops import op
-from faker import Faker
+from faker import Faker  # ty: ignore[unresolved-import]
 import pandas as pd
 
 faker = Faker()

examples/multi_output_demo.py

@@ -2,7 +2,7 @@ from lynxkite.core.ops import op
 import pandas as pd
 
 
-@op("LynxKite Graph Analytics", "Multi-output example", outputs=["one", "two"])
+@op("LynxKite Graph Analytics", "Examples", "Multi-output example", outputs=["one", "two"])
 def multi_output(*, a_limit=4, b_limit=10):
     """
     Returns two outputs. Also demonstrates Numpy-style docstrings.

examples/ode_lstm.py

@@ -0,0 +1,54 @@
+from lynxkite.core.ops import op_registration, LongStr
+from lynxkite_graph_analytics.core import Bundle
+from matplotlib import pyplot as plt
+import numpy as np
+import pandas as pd
+import json
+
+op = op_registration("LynxKite Graph Analytics")
+
+
+@op("Drop NA")
+def drop_na(df: pd.DataFrame):
+    return df.replace("", np.nan).dropna()
+
+
+@op("Sort by")
+def sort_by(df: pd.DataFrame, *, key_columns: str):
+    df = df.copy()
+    df.sort_values(
+        by=[k.strip() for k in key_columns.split(",")],
+        inplace=True,
+        ignore_index=True,
+    )
+    return df
+
+
+@op("Group by")
+def group_by(df: pd.DataFrame, *, key_columns: str, aggregation: LongStr):
+    key_columns = [k.strip() for k in key_columns.split(",")]
+    j = json.loads(aggregation)
+    for k, vs in j.items():
+        j[k] = [list if v == "list" else v for v in vs]
+    res = df.groupby(key_columns).agg(j).reset_index()
+    res.columns = ["_".join(col) for col in res.columns]
+    return res
+
+
+@op("Take first element of list")
+def take_first_element(df: pd.DataFrame, *, column: str):
+    df = df.copy()
+    df[f"{column}_first_element"] = df[column].apply(lambda x: x[0])
+    return df
+
+
+@op("Plot time series", view="matplotlib")
+def plot_time_series(bundle: Bundle, *, table_name: str, index: int, x_column: str, y_columns: str):
+    df = bundle.dfs[table_name]
+    y_columns = [y.strip() for y in y_columns.split(",")]
+    x = df[x_column].iloc[index]
+    for y_column in y_columns:
+        y = df[y_column].iloc[index]
+        plt.plot(x, y, "o-", label=y_column)
+    plt.xlabel(x_column)
+    plt.legend()

examples/word2vec.py

@@ -6,7 +6,7 @@ ENV = "LynxKite Graph Analytics"
 
 @op(ENV, "Word2vec for the top 1000 words", slow=True)
 def word2vec_1000():
-    import staticvectors
+    import staticvectors  # ty: ignore[unresolved-import]
 
     model = staticvectors.StaticVectors("neuml/word2vec-quantized")
     df = pd.read_csv(

lynxkite-app/pyproject.toml

@@ -6,25 +6,28 @@ readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
     "fastapi[standard]>=0.115.6",
+    "griffe>=1.7.3",
+    "joblib>=1.5.1",
     "lynxkite-core",
-    "orjson>=3.10.13",
     "pycrdt-websocket>=0.16",
+    "pycrdt>=0.12.26",
+    "pydantic>=2.11.7",
     "sse-starlette>=2.2.1",
-    "griffe>=1.7.3",
+    "uvicorn>=0.35.0",
 ]
 classifiers = ["Private :: Do Not Upload"]
 
 [project.urls]
 Homepage = "https://github.com/lynxkite/lynxkite-2000/"
 
-[project.optional-dependencies]
+[dependency-groups]
 dev = [
     "pydantic-to-typescript>=2.0.0",
-    "pytest>=8.3.4",
+    "setuptools>=80.9.0",
 ]
 
 [tool.uv.sources]
-lynxkite-core = { path = "../lynxkite-core" }
+lynxkite-core = { workspace = true }
 
 [build-system]
 requires = ["setuptools", "wheel", "setuptools-scm"]
@@ -47,3 +50,11 @@ build_py = "build_frontend.build_py"
 
 [project.scripts]
 lynxkite = "lynxkite_app.__main__:main"
+
+[tool.deptry.package_module_name_map]
+lynxkite-core = "lynxkite"
+sse-starlette = "starlette"
+
+[tool.deptry.per_rule_ignores]
+DEP002 = ["pycrdt-websocket", "griffe"]
+DEP004 = ["setuptools"]

lynxkite-app/src/lynxkite_app/crdt.py

@@ -7,8 +7,8 @@ import pathlib
 import fastapi
 import os.path
 import pycrdt.websocket
-import pycrdt.store
-import uvicorn
+import pycrdt.store.file
+import uvicorn.protocols.utils
 import builtins
 from lynxkite.core import workspace, ops
 
@@ -59,7 +59,8 @@ class WorkspaceWebsocketServer(pycrdt.websocket.WebsocketServer):
         room = pycrdt.websocket.YRoom(
             ystore=ystore, ydoc=ydoc, exception_handler=ws_exception_handler
         )
-        room.ws = ws
+        # We hang the YDoc pointer on the room, so it only gets garbage collected when the room does.
+        room.ws = ws  # ty: ignore[unresolved-attribute]
 
         def on_change(changes):
             task = asyncio.create_task(workspace_changed(name, changes, ws))
@@ -106,7 +107,8 @@ class CodeWebsocketServer(WorkspaceWebsocketServer):
         room = pycrdt.websocket.YRoom(
             ystore=ystore, ydoc=ydoc, exception_handler=ws_exception_handler
         )
-        room.text = text
+        # We hang the YDoc pointer on the room, so it only gets garbage collected when the room does.
+        room.text = text  # ty: ignore[unresolved-attribute]
 
         def on_change(changes):
             asyncio.create_task(code_changed(name, changes, text))
@@ -128,10 +130,12 @@ def clean_input(ws_pyd):
         for p in list(node.data.params):
             if p.startswith("_"):
                 del node.data.params[p]
-        if node.data.title == "Comment":
+        if node.data.op_id == "Comment":
             node.data.params = {}
         node.position.x = 0
         node.position.y = 0
+        node.width = 0
+        node.height = 0
         if node.model_extra:
             for key in list(node.model_extra.keys()):
                 delattr(node, key)
@@ -161,6 +165,7 @@ def crdt_update(
         ValueError: If the Python object provided is not a dict or list.
     """
     if isinstance(python_obj, dict):
+        assert isinstance(crdt_obj, pycrdt.Map), "CRDT object must be a Map for a dict input"
         for key, value in python_obj.items():
             if key in non_collaborative_fields:
                 crdt_obj[key] = value
@@ -177,6 +182,7 @@ def crdt_update(
             else:
                 crdt_obj[key] = value
     elif isinstance(python_obj, list):
+        assert isinstance(crdt_obj, pycrdt.Array), "CRDT object must be an Array for a list input"
         for i, value in enumerate(python_obj):
             if isinstance(value, dict):
                 if i >= len(crdt_obj):
@@ -218,7 +224,7 @@ last_known_versions = {}
 delayed_executions = {}
 
 
-async def workspace_changed(name: str, changes: pycrdt.MapEvent, ws_crdt: pycrdt.Map):
+async def workspace_changed(name: str, changes: list[pycrdt.MapEvent], ws_crdt: pycrdt.Map):
     """Callback to react to changes in the workspace.
 
     Args:
@@ -242,6 +248,10 @@ async def workspace_changed(name: str, changes: pycrdt.MapEvent, ws_crdt: pycrdt
         getattr(change, "keys", {}).get("__execution_delay", {}).get("newValue", 0)
         for change in changes
     )
+    # Check if workspace is paused - if so, skip automatic execution
+    if getattr(ws_pyd, "paused", False):
+        print(f"Skipping automatic execution for {name} in {ws_pyd.env} - workspace is paused")
+        return
     if delay:
         task = asyncio.create_task(execute(name, ws_crdt, ws_pyd, delay))
         delayed_executions[name] = task
@@ -289,6 +299,14 @@ async def code_changed(name: str, changes: pycrdt.TextEvent, text: pycrdt.Text):
         f.write(contents)
 
 
+ws_websocket_server: WorkspaceWebsocketServer
+code_websocket_server: CodeWebsocketServer
+
+
+def get_room(name):
+    return ws_websocket_server.get_room(name)
+
+
 @contextlib.asynccontextmanager
 async def lifespan(app):
     global ws_websocket_server

lynxkite-app/src/lynxkite_app/main.py

@@ -4,20 +4,24 @@ import shutil
 import pydantic
 import fastapi
 import importlib
+import joblib
 import pathlib
 import pkgutil
 from fastapi.staticfiles import StaticFiles
 from fastapi.middleware.gzip import GZipMiddleware
-import starlette
+import starlette.exceptions
 from lynxkite.core import ops
 from lynxkite.core import workspace
 from . import crdt
 
+mem = joblib.Memory(".joblib-cache")
+ops.CACHE_WRAPPER = mem.cache
+
 
 def detect_plugins():
     plugins = {}
     for _, name, _ in pkgutil.iter_modules():
-        if name.startswith("lynxkite_"):
+        if name.startswith("lynxkite_") and name != "lynxkite_app":
             print(f"Importing {name}")
             plugins[name] = importlib.import_module(name)
     if not plugins:
@@ -136,7 +140,7 @@ async def upload(req: fastapi.Request):
 @app.post("/api/execute_workspace")
 async def execute_workspace(name: str):
     """Trigger and await the execution of a workspace."""
-    room = await crdt.ws_websocket_server.get_room(name)
+    room = await crdt.get_room(name)
     ws_pyd = workspace.Workspace.model_validate(room.ws.to_py())
     await crdt.execute(name, room.ws, ws_pyd)
 

lynxkite-app/web/package-lock.json

@@ -23,6 +23,7 @@
         "daisyui": "^4.12.20",
         "echarts": "^5.5.1",
         "fuse.js": "^7.0.0",
+        "jmespath": "^0.16.0",
         "json-schema-to-typescript": "^15.0.3",
         "monaco-editor": "^0.52.2",
         "react": "^18.3.1",
@@ -40,6 +41,7 @@
       "devDependencies": {
         "@playwright/test": "^1.50.1",
         "@tailwindcss/typography": "^0.5.16",
+        "@types/jmespath": "^0.15.2",
         "@types/node": "^22.13.1",
         "@types/react": "^18.3.14",
         "@types/react-dom": "^18.3.2",
@@ -1894,6 +1896,13 @@
         "@types/unist": "*"
       }
     },
+    "node_modules/@types/jmespath": {
+      "version": "0.15.2",
+      "resolved": "https://registry.npmjs.org/@types/jmespath/-/jmespath-0.15.2.tgz",
+      "integrity": "sha512-pegh49FtNsC389Flyo9y8AfkVIZn9MMPE9yJrO9svhq6Fks2MwymULWjZqySuxmctd3ZH4/n7Mr98D+1Qo5vGA==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/@types/json-schema": {
       "version": "7.0.15",
       "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz",
@@ -2357,9 +2366,9 @@
       }
     },
     "node_modules/caniuse-lite": {
-      "version": "1.0.30001687",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001687.tgz",
-      "integrity": "sha512-0S/FDhf4ZiqrTUiQ39dKeUjYRjkv7lOZU1Dgif2rIqrTzX/1wV2hfKu9TOm1IHkdSijfLswxTFzl/cvir+SLSQ==",
+      "version": "1.0.30001723",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001723.tgz",
+      "integrity": "sha512-1R/elMjtehrFejxwmexeXAtae5UO9iSyFn6G/I806CYC/BLyyBk1EPhrKBkWhy6wM6Xnm47dSJQec+tLJ39WHw==",
       "funding": [
         {
           "type": "opencollective",
@@ -3667,6 +3676,15 @@
         "jiti": "bin/jiti.js"
       }
     },
+    "node_modules/jmespath": {
+      "version": "0.16.0",
+      "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz",
+      "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">= 0.6.0"
+      }
+    },
     "node_modules/js-tokens": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",

lynxkite-app/web/package.json

@@ -25,6 +25,7 @@
     "daisyui": "^4.12.20",
     "echarts": "^5.5.1",
     "fuse.js": "^7.0.0",
+    "jmespath": "^0.16.0",
     "json-schema-to-typescript": "^15.0.3",
     "monaco-editor": "^0.52.2",
     "react": "^18.3.1",
@@ -42,6 +43,7 @@
   "devDependencies": {
     "@playwright/test": "^1.50.1",
     "@tailwindcss/typography": "^0.5.16",
+    "@types/jmespath": "^0.15.2",
     "@types/node": "^22.13.1",
     "@types/react": "^18.3.14",
     "@types/react-dom": "^18.3.2",

lynxkite-app/web/src/Code.tsx

@@ -50,9 +50,13 @@ export default function Code() {
     yDocRef.current = new Y.Doc();
     const text = yDocRef.current.getText("text");
     const proto = location.protocol === "https:" ? "wss:" : "ws:";
+    const encodedPath = path!
+      .split("/")
+      .map((segment) => encodeURIComponent(segment))
+      .join("/");
     wsProviderRef.current = new WebsocketProvider(
       `${proto}//${location.host}/ws/code/crdt`,
-      path!,
+      encodedPath!,
       yDocRef.current,
     );
     editorRef.current.getModel()!.setEOL(0); // https://github.com/yjs/y-monaco/issues/6
@@ -73,10 +77,11 @@ export default function Code() {
   return (
     <div className="workspace">
       <div className="top-bar bg-neutral">
-        <Link className="logo" to="">
+        <Link className="logo" to="/">
           <img alt="" src={favicon} />
         </Link>
         <div className="ws-name">{path}</div>
+        <title>{path}</title>
         <div className="tools text-secondary">
           <button className="btn btn-link">
             <Atom />
@@ -84,7 +89,13 @@ export default function Code() {
           <button className="btn btn-link">
             <Backspace />
           </button>
-          <Link to={`/dir/${parentDir}`} className="btn btn-link">
+          <Link
+            to={`/dir/${parentDir
+              .split("/")
+              .map((segment) => encodeURIComponent(segment))
+              .join("/")}`}
+            className="btn btn-link"
+          >
             <Close />
           </Link>
         </div>

lynxkite-app/web/src/Directory.tsx

@@ -29,23 +29,55 @@ function EntryCreator(props: {
   onCreate: (name: string) => void;
 }) {
   const [isCreating, setIsCreating] = useState(false);
+  const [nameValidationError, setNameValidationError] = useState("");
+
+  function validateName(name: string): boolean {
+    if (name.includes("/")) {
+      setNameValidationError("Name cannot contain '/' characters");
+      return false;
+    }
+    if (name.trim() === "") {
+      setNameValidationError("Name cannot be empty");
+      return false;
+    }
+    setNameValidationError("");
+    return true;
+  }
+
   return (
     <>
       {isCreating ? (
         <form
           onSubmit={(e) => {
             e.preventDefault();
-            props.onCreate((e.target as HTMLFormElement).entryName.value.trim());
+            const name = (e.target as HTMLFormElement).entryName.value.trim();
+            if (validateName(name)) {
+              props.onCreate(name);
+              setIsCreating(false);
+            }
           }}
         >
           <input
-            className="input input-ghost w-full"
+            className={`input input-ghost w-full ${nameValidationError ? "input-error" : ""}`}
             autoFocus
             type="text"
             name="entryName"
             onBlur={() => setIsCreating(false)}
+            onChange={(e) => validateName(e.target.value)}
             placeholder={`${props.label} name`}
           />
+          {nameValidationError && (
+            <div
+              className="error-message"
+              role="alert"
+              style={{ position: "absolute", zIndex: 10 }}
+            >
+              <span className="error-icon" aria-hidden="true">
+                ⚠️
+              </span>
+              <span className="error-text">{nameValidationError}</span>
+            </div>
+          )}
         </form>
       ) : (
         <button type="button" onClick={() => setIsCreating(true)}>
@@ -67,13 +99,14 @@ export default function Directory() {
   const navigate = useNavigate();
 
   function link(item: DirectoryEntry) {
+    const encodedName = encodePathSegments(item.name);
     if (item.type === "directory") {
-      return `/dir/${item.name}`;
+      return `/dir/${encodedName}`;
     }
     if (item.type === "workspace") {
-      return `/edit/${item.name}`;
+      return `/edit/${encodedName}`;
     }
-    return `/code/${item.name}`;
+    return `/code/${encodedName}`;
   }
 
   function shortName(item: DirectoryEntry) {
@@ -83,13 +116,20 @@ export default function Directory() {
       ?.replace(/[.]lynxkite[.]json$/, "");
   }
 
+  function encodePathSegments(path: string): string {
+    const segments = path.split("/");
+    return segments.map((segment) => encodeURIComponent(segment)).join("/");
+  }
+
   function newWorkspaceIn(path: string, workspaceName: string) {
-    const pathSlash = path ? `${path}/` : "";
-    navigate(`/edit/${pathSlash}${workspaceName}.lynxkite.json`, { replace: true });
+    const pathSlash = path ? `${encodePathSegments(path)}/` : "";
+    navigate(`/edit/${pathSlash}${encodeURIComponent(workspaceName)}.lynxkite.json`, {
+      replace: true,
+    });
   }
   function newCodeFile(path: string, name: string) {
-    const pathSlash = path ? `${path}/` : "";
-    navigate(`/code/${pathSlash}${name}`, { replace: true });
+    const pathSlash = path ? `${encodePathSegments(path)}/` : "";
+    navigate(`/code/${pathSlash}${encodeURIComponent(name)}`, { replace: true });
   }
   async function newFolderIn(path: string, folderName: string) {
     const pathSlash = path ? `${path}/` : "";
@@ -99,7 +139,8 @@ export default function Directory() {
       body: JSON.stringify({ path: pathSlash + folderName }),
     });
     if (res.ok) {
-      navigate(`/dir/${pathSlash}${folderName}`);
+      const pathSlash = path ? `${encodePathSegments(path)}/` : "";
+      navigate(`/dir/${pathSlash}${encodeURIComponent(folderName)}`);
     } else {
       alert("Failed to create folder.");
     }

lynxkite-app/web/src/apiTypes.ts

@@ -25,6 +25,7 @@ export interface SaveRequest {
  */
 export interface Workspace {
   env?: string;
+  paused?: boolean;
   nodes?: WorkspaceNode[];
   edges?: WorkspaceEdge[];
   [k: string]: unknown;

lynxkite-app/web/src/index.css

@@ -81,7 +81,7 @@ body {
     display: flex;
     flex-direction: column;
 
-    > :not(.title) {
+    > :not(.title, .react-flow__handle) {
       user-select: text;
       cursor: default;
     }
@@ -277,11 +277,6 @@ body {
       padding: 2px 8px;
       border-radius: 4px 4px 0 0;
     }
-
-    .collapsed-param {
-      min-height: 20px;
-      line-height: 10px;
-    }
   }
 
   .node-search {
@@ -704,3 +699,29 @@ body {
   left: -4px;
   top: -5px;
 }
+
+.error-message {
+  display: flex;
+  align-items: center;
+  gap: 0.5rem;
+  margin-top: 0.25rem;
+  padding: 0.5rem;
+  background-color: #fee2e2;
+  border: 1px solid #fecaca;
+  border-radius: 0.375rem;
+  color: #dc2626;
+  font-size: 0.875rem;
+}
+
+.error-icon {
+  flex-shrink: 0;
+}
+
+.error-text {
+  line-height: 1.4;
+}
+
+.input-error {
+  border-color: #dc2626;
+  box-shadow: 0 0 0 1px #dc2626;
+}

lynxkite-app/web/src/workspace/NodeSearch.tsx

@@ -3,6 +3,8 @@ import { useEffect, useMemo, useRef, useState } from "react";
 
 export type OpsOp = {
   name: string;
+  id: string;
+  categories: string[];
   type: string;
   position: { x: number; y: number };
   params: { name: string; default: any }[];
@@ -81,6 +83,7 @@ export default function NodeSearch(props: {
             onClick={addSelected}
             className={`search-result ${index === selectedIndex ? "selected" : ""}`}
           >
+            {box.item.categories.map((category) => `${category}\u00A0›\u00A0`)}
             {box.item.name}
           </div>
         ))}

lynxkite-app/web/src/workspace/Workspace.tsx

@@ -29,6 +29,10 @@ import UngroupIcon from "~icons/tabler/library-minus.jsx";
 // @ts-ignore
 import GroupIcon from "~icons/tabler/library-plus.jsx";
 // @ts-ignore
+import Pause from "~icons/tabler/player-pause.jsx";
+// @ts-ignore
+import Play from "~icons/tabler/player-play.jsx";
+// @ts-ignore
 import Restart from "~icons/tabler/rotate-clockwise.jsx";
 // @ts-ignore
 import Close from "~icons/tabler/x.jsx";
@@ -71,12 +75,24 @@ function LynxKiteFlow() {
     .replace(/[.]lynxkite[.]json$/, "");
   const [state, setState] = useState({ workspace: {} as WorkspaceType });
   const [message, setMessage] = useState(null as string | null);
+  const [pausedUIState, setPausedUIState] = useState(false);
   useEffect(() => {
     const state = syncedStore({ workspace: {} as WorkspaceType });
     setState(state);
     const doc = getYjsDoc(state);
     const proto = location.protocol === "https:" ? "wss:" : "ws:";
-    const wsProvider = new WebsocketProvider(`${proto}//${location.host}/ws/crdt`, path!, doc);
+    const encodedPath = path!
+      .split("/")
+      .map((segment) => encodeURIComponent(segment))
+      .join("/");
+    const wsProvider = new WebsocketProvider(
+      `${proto}//${location.host}/ws/crdt`,
+      encodedPath,
+      doc,
+    );
+    if (state.workspace && typeof state.workspace.paused === "undefined") {
+      state.workspace.paused = false;
+    }
     const onChange = (_update: any, origin: any, _doc: any, _tr: any) => {
       if (origin === wsProvider) {
         // An update from the CRDT. Apply it to the local state.
@@ -100,6 +116,7 @@ function LynxKiteFlow() {
           // Make sure the internal copies are updated.
           updateNodeInternals(node.id);
         }
+        setPausedUIState(state.workspace.paused || false);
       }
     };
     doc.on("update", onChange);
@@ -128,30 +145,44 @@ function LynxKiteFlow() {
           !Number.isNaN(ch.position.y)
         ) {
           getYjsDoc(state).transact(() => {
-            Object.assign(node.position, ch.position);
+            node.position.x = ch.position.x;
+            node.position.y = ch.position.y;
           });
+          // Update edge positions.
+          updateNodeInternals(ch.id);
         } else if (ch.type === "select") {
         } else if (ch.type === "dimensions") {
-          getYjsDoc(state).transact(() => Object.assign(node, ch.dimensions));
+          getYjsDoc(state).transact(() => {
+            node.width = ch.dimensions.width;
+            node.height = ch.dimensions.height;
+          });
+          // Update edge positions when node size changes.
+          updateNodeInternals(ch.id);
         } else if (ch.type === "remove") {
           wnodes.splice(nodeIndex, 1);
         } else if (ch.type === "replace") {
           // Ideally we would only update the parameter that changed. But ReactFlow does not give us that detail.
-          const u = {
-            collapsed: ch.item.data.collapsed,
-            // The "..." expansion on a Y.map returns an empty object. Copying with fromEntries/entries instead.
-            params: {
-              ...Object.fromEntries(Object.entries(ch.item.data.params)),
-            },
-            __execution_delay: ch.item.data.__execution_delay,
-          };
-          getYjsDoc(state).transact(() => Object.assign(node.data, u));
+          getYjsDoc(state).transact(() => {
+            if (node.data.collapsed !== ch.item.data.collapsed) {
+              node.data.collapsed = ch.item.data.collapsed;
+              // Update edge positions when node collapses/expands.
+              setTimeout(() => updateNodeInternals(ch.id), 0);
+            }
+            if (node.data.__execution_delay !== ch.item.data.__execution_delay) {
+              node.data.__execution_delay = ch.item.data.__execution_delay;
+            }
+            for (const [key, value] of Object.entries(ch.item.data.params)) {
+              if (node.data.params[key] !== value) {
+                node.data.params[key] = value;
+              }
+            }
+          });
         } else {
           console.log("Unknown node change", ch);
         }
       }
     },
-    [state],
+    [state, updateNodeInternals],
   );
   const onEdgesChange = useCallback(
     (changes: any[]) => {
@@ -173,7 +204,11 @@ function LynxKiteFlow() {
 
   const fetcher: Fetcher<Catalogs> = (resource: string, init?: RequestInit) =>
     fetch(resource, init).then((res) => res.json());
-  const catalog = useSWR(`/api/catalog?workspace=${path}`, fetcher);
+  const encodedPathForAPI = path!
+    .split("/")
+    .map((segment) => encodeURIComponent(segment))
+    .join("/");
+  const catalog = useSWR(`/api/catalog?workspace=${encodedPathForAPI}`, fetcher);
   const [suppressSearchUntil, setSuppressSearchUntil] = useState(0);
   const [nodeSearchSettings, setNodeSearchSettings] = useState(
     undefined as
@@ -308,6 +343,7 @@ function LynxKiteFlow() {
       data: {
         meta: { value: meta },
         title: meta.name,
+        op_id: meta.id,
         params: Object.fromEntries(meta.params.map((p) => [p.name, p.default])),
       },
     };
@@ -386,11 +422,15 @@ function LynxKiteFlow() {
     }
   }
   async function executeWorkspace() {
-    const response = await axios.post(`/api/execute_workspace?name=${path}`);
+    const response = await axios.post(`/api/execute_workspace?name=${encodeURIComponent(path)}`);
     if (response.status !== 200) {
       setMessage("Workspace execution failed.");
     }
   }
+  function togglePause() {
+    state.workspace.paused = !state.workspace.paused;
+    setPausedUIState(state.workspace.paused);
+  }
   function deleteSelection() {
     const selectedNodes = nodes.filter((n) => n.selected);
     const selectedEdges = edges.filter((e) => e.selected);
@@ -526,13 +566,25 @@ function LynxKiteFlow() {
               <Backspace />
             </button>
           </Tooltip>
+          <Tooltip doc={pausedUIState ? "Resume automatic execution" : "Pause automatic execution"}>
+            <button className="btn btn-link" onClick={togglePause}>
+              {pausedUIState ? <Play /> : <Pause />}
+            </button>
+          </Tooltip>
           <Tooltip doc="Re-run the workspace">
             <button className="btn btn-link" onClick={executeWorkspace}>
               <Restart />
             </button>
           </Tooltip>
           <Tooltip doc="Close workspace">
-            <Link className="btn btn-link" to={`/dir/${parentDir}`} aria-label="close">
+            <Link
+              className="btn btn-link"
+              to={`/dir/${parentDir
+                .split("/")
+                .map((segment) => encodeURIComponent(segment))
+                .join("/")}`}
+              aria-label="close"
+            >
               <Close />
             </Link>
           </Tooltip>

lynxkite-app/web/src/workspace/nodes/LynxKiteNode.tsx

@@ -55,7 +55,45 @@ function getHandles(inputs: any[], outputs: any[]) {
   return handles;
 }
 
-const stopPropagation = (e: Event) => e.stopPropagation();
+function canScrollX(element: HTMLElement) {
+  const style = getComputedStyle(element);
+  return style.overflowX === "auto" || style.overflow === "auto";
+}
+function canScrollY(element: HTMLElement) {
+  const style = getComputedStyle(element);
+  return style.overflowY === "auto" || style.overflow === "auto";
+}
+function canScrollUp(e: HTMLElement) {
+  return canScrollY(e) && e.scrollTop > 0;
+}
+function canScrollDown(e: HTMLElement) {
+  return canScrollY(e) && e.scrollTop < e.scrollHeight - e.clientHeight - 1;
+}
+function canScrollLeft(e: HTMLElement) {
+  return canScrollX(e) && e.scrollLeft > 0;
+}
+function canScrollRight(e: HTMLElement) {
+  return canScrollX(e) && e.scrollLeft < e.scrollWidth - e.clientWidth - 1;
+}
+
+function onWheel(e: WheelEvent) {
+  if (e.ctrlKey) return; // Zoom, not scroll.
+  let t = e.target as HTMLElement;
+  // If we find an element inside the node container that can apply this scroll event, we stop propagation.
+  // Otherwise ReactFlow can have it and pan the workspace.
+  while (t && !t.classList.contains("node-container")) {
+    if (
+      (e.deltaX < 0 && canScrollLeft(t)) ||
+      (e.deltaX > 0 && canScrollRight(t)) ||
+      (e.deltaY < 0 && canScrollUp(t)) ||
+      (e.deltaY > 0 && canScrollDown(t))
+    ) {
+      e.stopPropagation();
+      return;
+    }
+    t = t.parentElement as HTMLElement;
+  }
+}
 
 function LynxKiteNodeComponent(props: LynxKiteNodeProps) {
   const reactFlow = useReactFlow();
@@ -67,9 +105,11 @@ function LynxKiteNodeComponent(props: LynxKiteNodeProps) {
     // ReactFlow handles wheel events to zoom/pan and this would prevent scrolling inside the node.
     // To stop the event from reaching ReactFlow, we stop propagation on the wheel event.
     // This must be done with a "passive: false" listener, which we can only register like this.
-    containerRef.current?.addEventListener("wheel", stopPropagation, { passive: false });
+    containerRef.current?.addEventListener("wheel", onWheel, {
+      passive: false,
+    });
     return () => {
-      containerRef.current?.removeEventListener("wheel", stopPropagation);
+      containerRef.current?.removeEventListener("wheel", onWheel);
     };
   }, [containerRef]);
   function titleClicked() {

lynxkite-app/web/src/workspace/nodes/ModelMappingParameter.tsx

@@ -0,0 +1,169 @@
+import { useRef } from "react";
+// @ts-ignore
+import ArrowsHorizontal from "~icons/tabler/arrows-horizontal.jsx";
+// @ts-ignore
+import Help from "~icons/tabler/question-mark.jsx";
+import ParameterInput from "./ParameterInput";
+
+type Bindings = {
+  [key: string]: {
+    df: string;
+    column: string;
+  };
+};
+
+type NamedId = {
+  name: string;
+  id: string;
+};
+
+function getModelBindings(
+  data: any,
+  variant: "training input" | "inference input" | "output",
+): NamedId[] {
+  function bindingsOfModel(m: any): string[] {
+    switch (variant) {
+      case "training input":
+        return [
+          ...m.model_inputs,
+          ...m.loss_inputs.filter((i: string) => !m.model_outputs.includes(i)),
+        ];
+      case "inference input":
+        return m.model_inputs;
+      case "output":
+        return m.model_outputs;
+    }
+  }
+  const bindings = new Set<NamedId>();
+  const inputs = data?.input_metadata?.value ?? data?.input_metadata ?? [];
+  for (const input of inputs) {
+    const other = input.other ?? {};
+    for (const e of Object.values(other) as any[]) {
+      if (e.type === "model") {
+        for (const id of bindingsOfModel(e.model)) {
+          bindings.add({ id, name: e.model.input_output_names[id] ?? id });
+        }
+      }
+    }
+  }
+  const list = [...bindings];
+  list.sort((a, b) => {
+    if (a.name < b.name) return -1;
+    if (a.name > b.name) return 1;
+    return 0;
+  });
+  return list;
+}
+
+function parseJsonOrEmpty(json: string): object {
+  try {
+    const j = JSON.parse(json);
+    if (j !== null && typeof j === "object") {
+      return j;
+    }
+  } catch (e) {}
+  return {};
+}
+
+export default function ModelMapping({ value, onChange, data, variant }: any) {
+  const dfsRef = useRef({} as { [binding: string]: HTMLSelectElement | null });
+  const columnsRef = useRef(
+    {} as { [binding: string]: HTMLSelectElement | HTMLInputElement | null },
+  );
+  const v: any = parseJsonOrEmpty(value);
+  v.map ??= {};
+  const dfs: { [df: string]: string[] } = {};
+  const inputs = data?.input_metadata?.value ?? data?.input_metadata ?? [];
+  for (const input of inputs) {
+    if (!input.dataframes) continue;
+    const dataframes = input.dataframes as {
+      [df: string]: { columns: string[] };
+    };
+    for (const [df, { columns }] of Object.entries(dataframes)) {
+      dfs[df] = columns;
+    }
+  }
+  const bindings = getModelBindings(data, variant);
+  function getMap() {
+    const map: Bindings = {};
+    for (const binding of bindings) {
+      const df = dfsRef.current[binding.id]?.value ?? "";
+      const column = columnsRef.current[binding.id]?.value ?? "";
+      if (df.length || column.length) {
+        map[binding.id] = { df, column };
+      }
+    }
+    return map;
+  }
+  return (
+    <table className="model-mapping-param">
+      <tbody>
+        {bindings.length > 0 ? (
+          bindings.map((binding: NamedId) => (
+            <tr key={binding.id}>
+              <td>{binding.name}</td>
+              <td>
+                <ArrowsHorizontal />
+              </td>
+              <td>
+                <select
+                  className="select select-ghost"
+                  value={v.map?.[binding.id]?.df}
+                  ref={(el) => {
+                    dfsRef.current[binding.id] = el;
+                  }}
+                  onChange={() => onChange(JSON.stringify({ map: getMap() }))}
+                >
+                  <option key="" value="" />
+                  {Object.keys(dfs).map((df: string) => (
+                    <option key={df} value={df}>
+                      {df}
+                    </option>
+                  ))}
+                </select>
+              </td>
+              <td>
+                {variant === "output" ? (
+                  <ParameterInput
+                    inputRef={(el) => {
+                      columnsRef.current[binding.id] = el;
+                    }}
+                    value={v.map?.[binding.id]?.column}
+                    onChange={(column, options) => {
+                      const map = getMap();
+                      // At this point the <input> has not been updated yet. We use the value from the event.
+                      const df = dfsRef.current[binding.id]?.value ?? "";
+                      map[binding.id] ??= { df, column };
+                      map[binding.id].column = column;
+                      onChange(JSON.stringify({ map }), options);
+                    }}
+                  />
+                ) : (
+                  <select
+                    className="select select-ghost"
+                    value={v.map?.[binding.id]?.column}
+                    ref={(el) => {
+                      columnsRef.current[binding.id] = el;
+                    }}
+                    onChange={() => onChange(JSON.stringify({ map: getMap() }))}
+                  >
+                    <option key="" value="" />
+                    {dfs[v.map?.[binding.id]?.df]?.map((col: string) => (
+                      <option key={col} value={col}>
+                        {col}
+                      </option>
+                    ))}
+                  </select>
+                )}
+              </td>
+            </tr>
+          ))
+        ) : (
+          <tr>
+            <td>no bindings</td>
+          </tr>
+        )}
+      </tbody>
+    </table>
+  );
+}

lynxkite-app/web/src/workspace/nodes/NodeParameter.tsx

@@ -1,10 +1,12 @@
-import { useRef } from "react";
+import jmespath from "jmespath";
 // @ts-ignore
 import ArrowsHorizontal from "~icons/tabler/arrows-horizontal.jsx";
 // @ts-ignore
 import Help from "~icons/tabler/question-mark.jsx";
 import Tooltip from "../../Tooltip";
+import ModelMapping from "./ModelMappingParameter";
 import NodeGroupParameter from "./NodeGroupParameter";
+import ParameterInput from "./ParameterInput";
 
 const BOOLEAN = "<class 'bool'>";
 const MODEL_TRAINING_INPUT_MAPPING =
@@ -27,178 +29,6 @@ function ParamName({ name, doc }: { name: string; doc: string }) {
   );
 }
 
-function Input({
-  value,
-  onChange,
-  inputRef,
-}: {
-  value: string;
-  onChange: (value: string, options?: { delay: number }) => void;
-  inputRef?: React.Ref<HTMLInputElement>;
-}) {
-  return (
-    <input
-      className="input input-bordered w-full"
-      ref={inputRef}
-      value={value ?? ""}
-      onChange={(evt) => onChange(evt.currentTarget.value, { delay: 2 })}
-      onBlur={(evt) => onChange(evt.currentTarget.value, { delay: 0 })}
-      onKeyDown={(evt) => evt.code === "Enter" && onChange(evt.currentTarget.value, { delay: 0 })}
-    />
-  );
-}
-
-type Bindings = {
-  [key: string]: {
-    df: string;
-    column: string;
-  };
-};
-
-function getModelBindings(
-  data: any,
-  variant: "training input" | "inference input" | "output",
-): string[] {
-  function bindingsOfModel(m: any): string[] {
-    switch (variant) {
-      case "training input":
-        return [...m.inputs, ...m.loss_inputs.filter((i: string) => !m.outputs.includes(i))];
-      case "inference input":
-        return m.inputs;
-      case "output":
-        return m.outputs;
-    }
-  }
-  const bindings = new Set<string>();
-  const inputs = data?.input_metadata?.value ?? data?.input_metadata ?? [];
-  for (const input of inputs) {
-    const other = input.other ?? {};
-    for (const e of Object.values(other) as any[]) {
-      if (e.type === "model") {
-        for (const b of bindingsOfModel(e.model)) {
-          bindings.add(b);
-        }
-      }
-    }
-  }
-  const list = [...bindings];
-  list.sort();
-  return list;
-}
-
-function parseJsonOrEmpty(json: string): object {
-  try {
-    const j = JSON.parse(json);
-    if (j !== null && typeof j === "object") {
-      return j;
-    }
-  } catch (e) {}
-  return {};
-}
-
-function ModelMapping({ value, onChange, data, variant }: any) {
-  const dfsRef = useRef({} as { [binding: string]: HTMLSelectElement | null });
-  const columnsRef = useRef(
-    {} as { [binding: string]: HTMLSelectElement | HTMLInputElement | null },
-  );
-  const v: any = parseJsonOrEmpty(value);
-  v.map ??= {};
-  const dfs: { [df: string]: string[] } = {};
-  const inputs = data?.input_metadata?.value ?? data?.input_metadata ?? [];
-  for (const input of inputs) {
-    if (!input.dataframes) continue;
-    const dataframes = input.dataframes as {
-      [df: string]: { columns: string[] };
-    };
-    for (const [df, { columns }] of Object.entries(dataframes)) {
-      dfs[df] = columns;
-    }
-  }
-  const bindings = getModelBindings(data, variant);
-  function getMap() {
-    const map: Bindings = {};
-    for (const binding of bindings) {
-      const df = dfsRef.current[binding]?.value ?? "";
-      const column = columnsRef.current[binding]?.value ?? "";
-      if (df.length || column.length) {
-        map[binding] = { df, column };
-      }
-    }
-    return map;
-  }
-  return (
-    <table className="model-mapping-param">
-      <tbody>
-        {bindings.length > 0 ? (
-          bindings.map((binding: string) => (
-            <tr key={binding}>
-              <td>{binding}</td>
-              <td>
-                <ArrowsHorizontal />
-              </td>
-              <td>
-                <select
-                  className="select select-ghost"
-                  value={v.map?.[binding]?.df}
-                  ref={(el) => {
-                    dfsRef.current[binding] = el;
-                  }}
-                  onChange={() => onChange(JSON.stringify({ map: getMap() }))}
-                >
-                  <option key="" value="" />
-                  {Object.keys(dfs).map((df: string) => (
-                    <option key={df} value={df}>
-                      {df}
-                    </option>
-                  ))}
-                </select>
-              </td>
-              <td>
-                {variant === "output" ? (
-                  <Input
-                    inputRef={(el) => {
-                      columnsRef.current[binding] = el;
-                    }}
-                    value={v.map?.[binding]?.column}
-                    onChange={(column, options) => {
-                      const map = getMap();
-                      // At this point the <input> has not been updated yet. We use the value from the event.
-                      const df = dfsRef.current[binding]?.value ?? "";
-                      map[binding] ??= { df, column };
-                      map[binding].column = column;
-                      onChange(JSON.stringify({ map }), options);
-                    }}
-                  />
-                ) : (
-                  <select
-                    className="select select-ghost"
-                    value={v.map?.[binding]?.column}
-                    ref={(el) => {
-                      columnsRef.current[binding] = el;
-                    }}
-                    onChange={() => onChange(JSON.stringify({ map: getMap() }))}
-                  >
-                    <option key="" value="" />
-                    {dfs[v.map?.[binding]?.df]?.map((col: string) => (
-                      <option key={col} value={col}>
-                        {col}
-                      </option>
-                    ))}
-                  </select>
-                )}
-              </td>
-            </tr>
-          ))
-        ) : (
-          <tr>
-            <td>no bindings</td>
-          </tr>
-        )}
-      </tbody>
-    </table>
-  );
-}
-
 interface NodeParameterProps {
   name: string;
   value: any;
@@ -226,22 +56,32 @@ export default function NodeParameter({ name, value, meta, data, setParam }: Nod
   function onChange(value: any, opts?: UpdateOptions) {
     setParam(meta.name, value, opts || {});
   }
-  return meta?.type?.format === "collapsed" ? (
-    <label className="param">
-      <ParamName name={name} doc={doc} />
-      <button className="collapsed-param">⋯</button>
-    </label>
-  ) : meta?.type?.format === "textarea" ? (
+  return meta?.type?.format === "textarea" ? (
     <label className="param">
       <ParamName name={name} doc={doc} />
       <textarea
         className="textarea textarea-bordered w-full"
-        rows={6}
-        value={value || ""}
+        rows={(value ?? "").split("\n").length}
+        value={value ?? ""}
         onChange={(evt) => onChange(evt.currentTarget.value, { delay: 2 })}
         onBlur={(evt) => onChange(evt.currentTarget.value, { delay: 0 })}
       />
     </label>
+  ) : meta?.type?.format === "dropdown" ? (
+    <label className="param">
+      <ParamName name={name} doc={doc} />
+      <select
+        className="select select-bordered w-full"
+        value={value ?? ""}
+        onChange={(evt) => onChange(evt.currentTarget.value)}
+      >
+        {getDropDownValues(data, meta).map((option: string) => (
+          <option key={option} value={option}>
+            {option}
+          </option>
+        ))}
+      </select>
+    </label>
   ) : meta?.type === "group" ? (
     <NodeGroupParameter meta={meta} data={data} setParam={setParam} />
   ) : meta?.type?.enum ? (
@@ -289,7 +129,19 @@ export default function NodeParameter({ name, value, meta, data, setParam }: Nod
   ) : (
     <label className="param">
       <ParamName name={name} doc={doc} />
-      <Input value={value} onChange={onChange} />
+      <ParameterInput value={value} onChange={onChange} />
     </label>
   );
 }
+
+function getDropDownValues(data: any, meta: any): string[] {
+  const metadata = data.input_metadata.value;
+  let query = meta.type.metadata_query;
+  // Substitute parameters in the query.
+  for (const p in data.params) {
+    query = query.replace(`<${p}>`, data.params[p]);
+  }
+  const res = ["", ...jmespath.search(metadata, query)];
+  res.sort();
+  return res;
+}

lynxkite-app/web/src/workspace/nodes/NodeWithTableView.tsx

@@ -54,8 +54,8 @@ function NodeWithTableView(props: any) {
                 <dl key={`${name}-dl`}>
                   {df.columns.map((c: string, i: number) => (
                     <React.Fragment key={`${name}-${c}`}>
-                      <dt>{c}</dt>
-                      <dd>
+                      {df.columns.length > 1 && <dt>{c}</dt>}
+                      <dd className="prose">
                         <Markdown>{toMD(df.data[0][i])}</Markdown>
                       </dd>
                     </React.Fragment>

lynxkite-app/web/src/workspace/nodes/ParameterInput.tsx

@@ -0,0 +1,20 @@
+export default function ParameterInput({
+  value,
+  onChange,
+  inputRef,
+}: {
+  value: string;
+  onChange: (value: string, options?: { delay: number }) => void;
+  inputRef?: React.Ref<HTMLInputElement>;
+}) {
+  return (
+    <input
+      className="input input-bordered w-full"
+      ref={inputRef}
+      value={value ?? ""}
+      onChange={(evt) => onChange(evt.currentTarget.value, { delay: 2 })}
+      onBlur={(evt) => onChange(evt.currentTarget.value, { delay: 0 })}
+      onKeyDown={(evt) => evt.code === "Enter" && onChange(evt.currentTarget.value, { delay: 0 })}
+    />
+  );
+}

lynxkite-app/web/tests/basic.spec.ts

@@ -21,7 +21,10 @@ test("Box creation & deletion per env", async () => {
   const envs = await workspace.getEnvs();
   for (const env of envs) {
     await workspace.setEnv(env);
-    const catalog = (await workspace.getCatalog()).filter((box) => box !== "Comment");
+    // Op categories don't have a finished UI yet. I just skip NetworkX ops for now.
+    const catalog = (await workspace.getCatalog()).filter(
+      (box) => box !== "Comment" && !box.includes("NetworkX"),
+    );
     expect(catalog).not.toHaveLength(0);
     const op = catalog[0];
     await workspace.addBox(op);
@@ -32,9 +35,9 @@ test("Box creation & deletion per env", async () => {
 });
 
 test("Delete multi-handle boxes", async () => {
-  await workspace.addBox("NX › PageRank");
-  await workspace.deleteBoxes(["NX › PageRank 1"]);
-  await expect(workspace.getBox("NX › PageRank 1")).not.toBeVisible();
+  await workspace.addBox("NetworkX › Algorithms › Link analysis › PageRank alg › PageRank");
+  await workspace.deleteBoxes(["PageRank 1"]);
+  await expect(workspace.getBox("PageRank 1")).not.toBeVisible();
 });
 
 test("Drag box", async () => {

lynxkite-app/web/tests/errors.spec.ts

@@ -20,8 +20,8 @@ test.afterEach(async () => {
 test("missing parameter", async () => {
   // Test the correct error message is displayed when a required parameter is missing,
   // and that the error message is removed when the parameter is filled.
-  await workspace.addBox("NX › Scale-Free Graph");
-  const graphBox = workspace.getBox("NX › Scale-Free Graph 1");
+  await workspace.addBox("NetworkX › Generators › Directed › Scale-free graph");
+  const graphBox = workspace.getBox("Scale-free graph 1");
   await expect(graphBox.locator(".error")).toHaveText("n is unset.");
   await graphBox.getByLabel("n", { exact: true }).fill("10");
   await expect(graphBox.locator(".error")).not.toBeVisible();
@@ -30,11 +30,11 @@ test("missing parameter", async () => {
 test("unknown operation", async () => {
   // Test that the correct error is displayed when the operation does not belong to
   // the current environment.
-  await workspace.addBox("NX › Scale-Free Graph");
-  const graphBox = workspace.getBox("NX › Scale-Free Graph 1");
+  await workspace.addBox("NetworkX › Generators › Directed › Scale-free graph");
+  const graphBox = workspace.getBox("Scale-free graph 1");
   await graphBox.getByLabel("n", { exact: true }).fill("10");
   await workspace.setEnv("Pillow");
-  const csvBox = workspace.getBox("NX › Scale-Free Graph 1");
+  const csvBox = workspace.getBox("Scale-free graph 1");
   await expect(csvBox.locator(".error")).toHaveText("Unknown operation.");
   await workspace.setEnv("LynxKite Graph Analytics");
   await expect(csvBox.locator(".error")).not.toBeVisible();

lynxkite-app/web/tests/examples.spec.ts

@@ -2,13 +2,7 @@
 import { expect, test } from "@playwright/test";
 import { Workspace } from "./lynxkite";
 
-const WORKSPACES = [
-  "Airlines demo",
-  "Bio Cypher demo",
-  "Image processing",
-  "NetworkX demo",
-  "Model use",
-];
+const WORKSPACES = ["Airlines demo", "Bio Cypher demo", "Image processing", "NetworkX demo"];
 
 for (const name of WORKSPACES) {
   test(name, async ({ page }) => {
@@ -17,3 +11,18 @@ for (const name of WORKSPACES) {
     await ws.expectErrorFree();
   });
 }
+
+test("Model use", async ({ page }) => {
+  const ws = await Workspace.open(page, "Model use");
+  await ws.execute({ timeout: 30000 }); // Actually trains the model.
+  await ws.expectErrorFree();
+  let b = ws.boxByTitle("Train/test split");
+  await b.expectParameterOptions("table name", ["", "df"]);
+  b = ws.boxByTitle("Train model");
+  await b.expectParameterOptions("model name", ["", "model"]);
+  b = ws.boxByTitle("View vectors");
+  await b.locator.locator(".params-expander").click();
+  await b.expectParameterOptions("table name", ["", "df", "df_test", "df_train", "training"]);
+  await b.expectParameterOptions("vector column", ["", "index", "pred", "x", "y"]);
+  await b.expectParameterOptions("label column", ["", "index", "pred", "x", "y"]);
+});

lynxkite-app/web/tests/graph_creation.spec.ts

@@ -6,10 +6,10 @@ let workspace: Workspace;
 
 test.beforeEach(async ({ browser }) => {
   workspace = await Workspace.empty(await browser.newPage(), "graph_creation_spec_test");
-  await workspace.addBox("NX › Scale-Free Graph");
-  await workspace.getBox("NX › Scale-Free Graph 1").getByLabel("n", { exact: true }).fill("10");
-  await workspace.addBox("Create graph");
-  await workspace.connectBoxes("NX › Scale-Free Graph 1", "Create graph 1");
+  await workspace.addBox("NetworkX › Generators › Directed › Scale-free graph");
+  await workspace.getBox("Scale-free graph 1").getByLabel("n", { exact: true }).fill("10");
+  await workspace.addBox("Organize");
+  await workspace.connectBoxes("Scale-free graph 1", "Organize 1");
 });
 
 test.afterEach(async () => {
@@ -22,7 +22,7 @@ test.afterEach(async () => {
 });
 
 test("Tables are displayed in the Graph creation box", async () => {
-  const graphBox = await workspace.getBox("Create graph 1");
+  const graphBox = await workspace.getBox("Organize 1");
   const nodesTableHeader = graphBox.locator(".graph-tables .df-head", {
     hasText: "nodes",
   });
@@ -42,7 +42,7 @@ test("Tables are displayed in the Graph creation box", async () => {
 });
 
 test("Adding and removing relationships", async () => {
-  const graphBox = await workspace.getBox("Create graph 1");
+  const graphBox = await workspace.getBox("Organize 1");
   const addRelationshipButton = await graphBox.locator(".add-relationship-button");
   await addRelationshipButton.click();
   const formData: Record<string, string> = {
@@ -64,7 +64,7 @@ test("Adding and removing relationships", async () => {
   await graphBox.locator(".submit-relationship-button").click();
   // check that the relationship has been saved in the backend
   await workspace.page.reload();
-  const graphBoxAfterReload = await workspace.getBox("Create graph 1");
+  const graphBoxAfterReload = await workspace.getBox("Organize 1");
   const relationHeader = await graphBoxAfterReload.locator(".graph-relations .df-head", {
     hasText: "relation_1",
   });
@@ -76,7 +76,7 @@ test("Adding and removing relationships", async () => {
 test("Output of the box is a bundle", async () => {
   await workspace.addBox("View tables");
   const tableView = await workspace.getBox("View tables 1");
-  await workspace.connectBoxes("Create graph 1", "View tables 1");
+  await workspace.connectBoxes("Organize 1", "View tables 1");
   const nodesTableHeader = await tableView.locator(".df-head", {
     hasText: "nodes",
   });

lynxkite-app/web/tests/lynxkite.ts

@@ -91,6 +91,10 @@ export class Workspace {
   getBox(boxId: string) {
     return this.page.locator(`[data-id="${boxId}"]`);
   }
+  boxByTitle(title: string): Box {
+    const titleLocator = this.page.getByText(title, { exact: true });
+    return new Box(this.page, titleLocator.locator("../.."));
+  }
 
   getBoxes() {
     return this.page.locator(".react-flow__node");
@@ -150,8 +154,8 @@ export class Workspace {
     }
   }
 
-  async execute() {
-    const request = this.page.waitForResponse(/api[/]execute_workspace/);
+  async execute(opts?) {
+    const request = this.page.waitForResponse(/api[/]execute_workspace/, opts);
     await this.page.keyboard.press("r");
     await request;
   }
@@ -165,6 +169,21 @@ export class Workspace {
   }
 }
 
+export class Box {
+  constructor(
+    readonly page: Page,
+    readonly locator: Locator,
+  ) {}
+  getParameter(name: string) {
+    return this.locator.getByLabel(name);
+  }
+  async expectParameterOptions(parameter: string, options: string[]) {
+    const param = this.getParameter(parameter);
+    const optionsLocator = param.locator("option");
+    await expect(optionsLocator).toHaveText(options);
+  }
+}
+
 export class Splash {
   page: Page;
   root: Locator;

lynxkite-core/pyproject.toml

@@ -5,16 +5,21 @@ description = "A lightweight dependency for authoring LynxKite operations and ex
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
+    "pydantic>=2.11.7",
 ]
 classifiers = ["Private :: Do Not Upload"]
 
 [project.urls]
 Homepage = "https://github.com/lynxkite/lynxkite-2000/"
 
-[project.optional-dependencies]
-dev = [
-    "pytest",
-]
+[tool.deptry.per_rule_ignores]
+DEP001 = ["matplotlib", "griffe", "pycrdt"]
+DEP003 = ["matplotlib", "griffe", "pycrdt"]
+
+[build-system]
+requires = ["setuptools", "wheel", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
 
-[tool.pytest.ini_options]
-asyncio_mode = "auto"
+[tool.setuptools.packages.find]
+namespaces = true
+where = ["src"]

lynxkite-core/src/lynxkite/core/executors/one_by_one.py

@@ -4,9 +4,6 @@ A LynxKite executor that assumes most operations operate on their input one by o
 
 from .. import ops
 from .. import workspace
-import orjson
-import pandas as pd
-import pydantic
 import traceback
 import inspect
 import typing
@@ -35,9 +32,6 @@ def _has_ctx(op):
     return "_ctx" in sig.parameters
 
 
-CACHES = {}
-
-
 def register(env: str, cache: bool = True):
     """Registers the one-by-one executor.
 
@@ -46,12 +40,7 @@ def register(env: str, cache: bool = True):
         from lynxkite.core.executors import one_by_one
         one_by_one.register("My Environment")
     """
-    if cache:
-        CACHES[env] = {}
-        cache = CACHES[env]
-    else:
-        cache = None
-    ops.EXECUTORS[env] = lambda ws: _execute(ws, ops.CATALOGS[env], cache=cache)
+    ops.EXECUTORS[env] = lambda ws: _execute(ws, ops.CATALOGS[env])
 
 
 def _get_stages(ws, catalog: ops.Catalog):
@@ -64,7 +53,7 @@ def _get_stages(ws, catalog: ops.Catalog):
     for edge in ws.edges:
         inputs.setdefault(edge.target, []).append(edge.source)
         node = nodes[edge.target]
-        op = catalog[node.data.title]
+        op = catalog[node.data.op_id]
         if op.get_input(edge.targetHandle).position.is_vertical():
             batch_inputs.setdefault(edge.target, []).append(edge.source)
     stages = []
@@ -83,26 +72,13 @@ def _get_stages(ws, catalog: ops.Catalog):
     return stages
 
 
-def _default_serializer(obj):
-    if isinstance(obj, pydantic.BaseModel):
-        return obj.dict()
-    return {"__nonserializable__": id(obj)}
-
-
-def _make_cache_key(obj):
-    return orjson.dumps(obj, default=_default_serializer)
-
-
-EXECUTOR_OUTPUT_CACHE = {}
-
-
 async def _await_if_needed(obj):
     if inspect.isawaitable(obj):
         return await obj
     return obj
 
 
-async def _execute(ws: workspace.Workspace, catalog: ops.Catalog, cache=None):
+async def _execute(ws: workspace.Workspace, catalog: ops.Catalog):
     nodes = {n.id: n for n in ws.nodes}
     contexts = {n.id: Context(node=n) for n in ws.nodes}
     edges = {n.id: [] for n in ws.nodes}
@@ -111,9 +87,9 @@ async def _execute(ws: workspace.Workspace, catalog: ops.Catalog, cache=None):
     tasks = {}
     NO_INPUT = object()  # Marker for initial tasks.
     for node in ws.nodes:
-        op = catalog.get(node.data.title)
+        op = catalog.get(node.data.op_id)
         if op is None:
-            node.publish_error(f'Operation "{node.data.title}" not found.')
+            node.publish_error(f'Operation "{node.data.op_id}" not found.')
             continue
         node.publish_error(None)
         # Start tasks for nodes that have no non-batch inputs.
@@ -130,7 +106,7 @@ async def _execute(ws: workspace.Workspace, catalog: ops.Catalog, cache=None):
                 next_stage.setdefault(n, []).extend(ts)
                 continue
             node = nodes[n]
-            op = catalog[node.data.title]
+            op = catalog[node.data.op_id]
             params = {**node.data.params}
             if _has_ctx(op):
                 params["_ctx"] = contexts[node.id]
@@ -155,15 +131,7 @@ async def _execute(ws: workspace.Workspace, catalog: ops.Catalog, cache=None):
                     if missing:
                         node.publish_error(f"Missing input: {', '.join(missing)}")
                         break
-                    if cache is not None:
-                        key = _make_cache_key((inputs, params))
-                        if key not in cache:
-                            result: ops.Result = op(*inputs, **params)
-                            result.output = await _await_if_needed(result.output)
-                            cache[key] = result
-                        result = cache[key]
-                    else:
-                        result = op(*inputs, **params)
+                    result = op(*inputs, **params)
                     output = await _await_if_needed(result.output)
                 except Exception as e:
                     traceback.print_exc()
@@ -171,7 +139,7 @@ async def _execute(ws: workspace.Workspace, catalog: ops.Catalog, cache=None):
                     break
                 contexts[node.id].last_result = output
                 # Returned lists and DataFrames are considered multiple tasks.
-                if isinstance(output, pd.DataFrame):
+                if hasattr(output, "to_dict"):
                     output = _df_to_list(output)
                 elif not isinstance(output, list):
                     output = [output]
@@ -181,7 +149,7 @@ async def _execute(ws: workspace.Workspace, catalog: ops.Catalog, cache=None):
                     result.display = await _await_if_needed(result.display)
                 for edge in edges[node.id]:
                     t = nodes[edge.target]
-                    op = catalog[t.data.title]
+                    op = catalog[t.data.op_id]
                     if op.get_input(edge.targetHandle).position.is_vertical():
                         batch_inputs.setdefault((edge.target, edge.targetHandle), []).extend(
                             results

lynxkite-core/src/lynxkite/core/executors/simple.py

@@ -37,7 +37,7 @@ async def execute(ws: workspace.Workspace, catalog: ops.Catalog):
     ts = graphlib.TopologicalSorter(dependencies)
     for node_id in ts.static_order():
         node = nodes[node_id]
-        op = catalog[node.data.title]
+        op = catalog[node.data.op_id]
         params = {**node.data.params}
         node.publish_started()
         try:

lynxkite-core/src/lynxkite/core/ops.py

@@ -6,7 +6,7 @@ import asyncio
 import enum
 import functools
 import json
-import importlib
+import importlib.util
 import inspect
 import pathlib
 import subprocess
@@ -15,9 +15,7 @@ import types
 import typing
 from dataclasses import dataclass
 
-import joblib
 import pydantic
-from typing_extensions import Annotated
 
 if typing.TYPE_CHECKING:
     from . import workspace
@@ -26,10 +24,17 @@ Catalog = dict[str, "Op"]
 Catalogs = dict[str, Catalog]
 CATALOGS: Catalogs = {}
 EXECUTORS = {}
-mem = joblib.Memory(".joblib-cache")
 
 typeof = type  # We have some arguments called "type".
 
+CACHE_WRAPPER = None  # Overwrite this to configure a caching mechanism.
+
+
+def _cache_wrap(func):
+    if CACHE_WRAPPER is None:
+        return func
+    return CACHE_WRAPPER(func)
+
 
 def type_to_json(t):
     if isinstance(t, type) and issubclass(t, enum.Enum):
@@ -39,13 +44,10 @@ def type_to_json(t):
     return {"type": str(t)}
 
 
-Type = Annotated[typing.Any, pydantic.PlainSerializer(type_to_json, return_type=dict)]
-LongStr = Annotated[str, {"format": "textarea"}]
+Type = typing.Annotated[typing.Any, pydantic.PlainSerializer(type_to_json, return_type=dict)]
+LongStr = typing.Annotated[str, {"format": "textarea"}]
 """LongStr is a string type for parameters that will be displayed as a multiline text area in the UI."""
-PathStr = Annotated[str, {"format": "path"}]
-CollapsedStr = Annotated[str, {"format": "collapsed"}]
-NodeAttribute = Annotated[str, {"format": "node attribute"}]
-EdgeAttribute = Annotated[str, {"format": "edge attribute"}]
+PathStr = typing.Annotated[str, {"format": "path"}]
 # https://github.com/python/typing/issues/182#issuecomment-1320974824
 ReadOnlyJSON: typing.TypeAlias = (
     typing.Mapping[str, "ReadOnlyJSON"]
@@ -76,10 +78,6 @@ class Parameter(BaseConfig):
         e = enum.Enum(f"OptionsFor_{name}", options)
         return Parameter.basic(name, default or options[0], e)
 
-    @staticmethod
-    def collapsed(name, default, type=None):
-        return Parameter.basic(name, default, CollapsedStr)
-
     @staticmethod
     def basic(name, default=None, type=None):
         if default is inspect._empty:
@@ -110,17 +108,31 @@ class Position(str, enum.Enum):
     def is_vertical(self):
         return self in (self.TOP, self.BOTTOM)
 
+    @staticmethod
+    def from_dir(dir: str) -> tuple[Position, Position]:
+        """Returns the input and output positions based on the direction."""
+        if dir == "left-to-right":
+            return Position.LEFT, Position.RIGHT
+        elif dir == "right-to-left":
+            return Position.RIGHT, Position.LEFT
+        elif dir == "top-to-bottom":
+            return Position.TOP, Position.BOTTOM
+        elif dir == "bottom-to-top":
+            return Position.BOTTOM, Position.TOP
+        else:
+            raise ValueError(f"Invalid direction: {dir}")
+
 
 class Input(BaseConfig):
     name: str
     type: Type
-    position: Position = Position.LEFT
+    position: Position
 
 
 class Output(BaseConfig):
     name: str
     type: Type
-    position: Position = Position.RIGHT
+    position: Position
 
 
 @dataclass
@@ -139,17 +151,6 @@ class Result:
     input_metadata: ReadOnlyJSON | None = None
 
 
-MULTI_INPUT = Input(name="multi", type="*")
-
-
-def basic_inputs(*names):
-    return {name: Input(name=name, type=None) for name in names}
-
-
-def basic_outputs(*names):
-    return {name: Output(name=name, type=None) for name in names}
-
-
 def get_optional_type(type):
     """For a type like `int | None`, returns `int`. Returns `None` otherwise."""
     if isinstance(type, types.UnionType):
@@ -184,6 +185,7 @@ def _param_to_type(name, value, type):
 
 class Op(BaseConfig):
     func: typing.Callable = pydantic.Field(exclude=True)
+    categories: list[str]
     name: str
     params: list[Parameter | ParameterGroup]
     inputs: list[Input]
@@ -192,6 +194,8 @@ class Op(BaseConfig):
     type: str = "basic"  # The UI to use for this operation.
     color: str = "orange"  # The color of the operation in the UI.
     doc: object = None
+    # ID is automatically set from the name and categories.
+    id: str = pydantic.Field(default=None)
 
     def __call__(self, *inputs, **params):
         # Convert parameters.
@@ -234,18 +238,44 @@ class Op(BaseConfig):
                 res[p.name] = _param_to_type(p.name, params[p.name], p.type)
         return res
 
+    @pydantic.model_validator(mode="after")
+    def compute_id(self):
+        self.id = " > ".join(self.categories + [self.name])
+        return self
+
 
 def op(
     env: str,
-    name: str,
-    *,
-    view="basic",
-    outputs=None,
-    params=None,
-    slow=False,
-    color=None,
+    *names: str,
+    view: str = "basic",
+    outputs: list[str] | None = None,
+    params: list[Parameter] | None = None,
+    slow: bool = False,
+    color: str | None = None,
+    cache: bool | None = None,
+    dir: str = "left-to-right",
 ):
-    """Decorator for defining an operation."""
+    """
+    Decorator for defining an operation.
+
+    Parameters:
+        env: The environment (workspace type) to which the operation belongs.
+        names: The list of categories this operation belongs to, followed by the name of the operation.
+        view: How the operation will be displayed in the UI. One of "basic", "visualization",
+              "table_view", "graph_creation_view", "image", "molecule", "matplotlib".
+        outputs: A list of output names. If not provided, defaults to ["output"] for "basic" view.
+        params: Normally the parameters are taken from the function signature.
+                Use "params" to override this.
+        slow: If True, the operation results will be cached.
+              If the function is not async, it will be run in a separate thread.
+        color: The color of the operation in the UI. Defaults to "orange".
+        cache: Set to False to disable caching for a slow operation.
+               You may need this for slow operations with parameters/outputs that can't be serialized.
+        dir: Sets the default input and output positions. The default is "left-to-right", meaning
+             inputs are on the left and outputs are on the right. Other options are "right-to-left",
+             "top-to-bottom", and "bottom-to-top".
+    """
+    [*categories, name] = names
 
     def decorator(func):
         doc = parse_doc(func)
@@ -256,10 +286,12 @@ def op(
             func = matplotlib_to_image(func)
         if slow:
             func = make_async(func)
-            func = mem.cache(func)
+            if cache is not False:
+                func = _cache_wrap(func)
         # Positional arguments are inputs.
+        ipos, opos = Position.from_dir(dir)
         inputs = [
-            Input(name=name, type=param.annotation)
+            Input(name=name, type=param.annotation, position=ipos)
             for name, param in sig.parameters.items()
             if param.kind not in (param.KEYWORD_ONLY, param.VAR_KEYWORD)
         ]
@@ -270,13 +302,14 @@ def op(
         if params:
             _params.extend(params)
         if outputs is not None:
-            _outputs = [Output(name=name, type=None) for name in outputs]
+            _outputs = [Output(name=name, type=None, position=opos) for name in outputs]
         else:
-            _outputs = [Output(name="output", type=None)] if view == "basic" else []
+            _outputs = [Output(name="output", type=None, position=opos)] if view == "basic" else []
         op = Op(
             func=func,
             doc=doc,
             name=name,
+            categories=categories,
             params=_params,
             inputs=inputs,
             outputs=_outputs,
@@ -284,7 +317,7 @@ def op(
             color=color or "orange",
         )
         CATALOGS.setdefault(env, {})
-        CATALOGS[env][name] = op
+        CATALOGS[env][op.id] = op
         func.__op__ = op
         return func
 
@@ -362,24 +395,34 @@ def no_op(*args, **kwargs):
     return None
 
 
-def register_passive_op(env: str, name: str, inputs=[], outputs=["output"], params=[], **kwargs):
+def register_passive_op(
+    env: str, *names: str, inputs=[], outputs=["output"], params=[], dir="left-to-right", **kwargs
+):
     """A passive operation has no associated code."""
+    ipos, opos = Position.from_dir(dir)
+    [*categories, name] = names
     op = Op(
         func=no_op,
         name=name,
+        categories=categories,
         params=params,
-        inputs=[Input(name=i, type=None) if isinstance(i, str) else i for i in inputs],
-        outputs=[Output(name=o, type=None) if isinstance(o, str) else o for o in outputs],
+        inputs=[
+            Input(name=i, type=None, position=ipos) if isinstance(i, str) else i for i in inputs
+        ],
+        outputs=[
+            Output(name=o, type=None, position=opos) if isinstance(o, str) else o for o in outputs
+        ],
         **kwargs,
     )
     CATALOGS.setdefault(env, {})
-    CATALOGS[env][name] = op
+    CATALOGS[env][op.id] = op
     return op
 
 
 COMMENT_OP = Op(
     func=no_op,
     name="Comment",
+    categories=[],
     params=[Parameter.basic("text", "", LongStr)],
     inputs=[],
     outputs=[],
@@ -403,14 +446,14 @@ def register_executor(env: str):
     return decorator
 
 
-def op_registration(env: str):
+def op_registration(env: str, *categories: str, **kwargs):
     """Returns a decorator that can be used for registering functions as operations."""
-    return functools.partial(op, env)
+    return functools.partial(op, env, *categories, **kwargs)
 
 
-def passive_op_registration(env: str):
+def passive_op_registration(env: str, *categories: str, **kwargs):
     """Returns a function that can be used to register operations without associated code."""
-    return functools.partial(register_passive_op, env)
+    return functools.partial(register_passive_op, env, *categories, **kwargs)
 
 
 def make_async(func):
@@ -475,6 +518,7 @@ def install_requirements(req: pathlib.Path):
 
 def run_user_script(script_path: pathlib.Path):
     spec = importlib.util.spec_from_file_location(script_path.stem, str(script_path))
+    assert spec
     module = importlib.util.module_from_spec(spec)
     spec.loader.exec_module(module)
 
@@ -508,7 +552,7 @@ def _get_griffe_function(func):
         if param.annotation is inspect.Parameter.empty:
             annotation = None
         else:
-            annotation = param.annotation.__name__
+            annotation = getattr(param.annotation, "__name__", str(param.annotation))
         parameters.append(
             griffe.Parameter(
                 name,

lynxkite-core/src/lynxkite/core/workspace.py

@@ -1,15 +1,18 @@
 """For working with LynxKite workspaces."""
 
 import json
-from typing import Optional
+from typing import Optional, TYPE_CHECKING
 import dataclasses
 import enum
 import os
-import pycrdt
 import pydantic
 import tempfile
 from . import ops
 
+if TYPE_CHECKING:
+    import pycrdt
+    from lynxkite.core import ops
+
 
 class BaseConfig(pydantic.BaseModel):
     model_config = pydantic.ConfigDict(
@@ -30,13 +33,28 @@ class NodeStatus(str, enum.Enum):
 
 class WorkspaceNodeData(BaseConfig):
     title: str
+    op_id: str
     params: dict
     display: Optional[object] = None
     input_metadata: Optional[object] = None
     error: Optional[str] = None
     status: NodeStatus = NodeStatus.done
-    # Also contains a "meta" field when going out.
-    # This is ignored when coming back from the frontend.
+    meta: Optional["ops.Op"] = None
+
+    @pydantic.model_validator(mode="before")
+    @classmethod
+    def fill_op_id_if_missing(cls, data: dict) -> dict:
+        """Compatibility with old workspaces that don't have op_id."""
+        if "op_id" not in data:
+            data["op_id"] = data["title"]
+        return data
+
+    @pydantic.model_validator(mode="before")
+    @classmethod
+    def ignore_meta(cls, data: dict) -> dict:
+        """Metadata is never loaded. We will use fresh metadata."""
+        data["meta"] = None
+        return data
 
 
 class WorkspaceNode(BaseConfig):
@@ -47,13 +65,13 @@ class WorkspaceNode(BaseConfig):
     position: Position
     width: Optional[float] = None
     height: Optional[float] = None
-    _crdt: pycrdt.Map
+    _crdt: Optional["pycrdt.Map"] = None
 
     def publish_started(self):
         """Notifies the frontend that work has started on this node."""
         self.data.error = None
         self.data.status = NodeStatus.active
-        if hasattr(self, "_crdt") and "data" in self._crdt:
+        if self._crdt and "data" in self._crdt:
             with self._crdt.doc.transaction():
                 self._crdt["data"]["error"] = None
                 self._crdt["data"]["status"] = NodeStatus.active
@@ -64,7 +82,7 @@ class WorkspaceNode(BaseConfig):
         self.data.input_metadata = result.input_metadata
         self.data.error = result.error
         self.data.status = NodeStatus.done
-        if hasattr(self, "_crdt") and "data" in self._crdt:
+        if self._crdt and "data" in self._crdt:
             with self._crdt.doc.transaction():
                 try:
                     self._crdt["data"]["status"] = NodeStatus.done
@@ -100,13 +118,13 @@ class Workspace(BaseConfig):
     env: str = ""
     nodes: list[WorkspaceNode] = dataclasses.field(default_factory=list)
     edges: list[WorkspaceEdge] = dataclasses.field(default_factory=list)
-    _crdt: pycrdt.Map
+    _crdt: Optional["pycrdt.Map"] = None
 
     def normalize(self):
         if self.env not in ops.CATALOGS:
             return self
         catalog = ops.CATALOGS[self.env]
-        _ops = {n.id: catalog[n.data.title] for n in self.nodes if n.data.title in catalog}
+        _ops = {n.id: catalog[n.data.op_id] for n in self.nodes if n.data.op_id in catalog}
         valid_targets = set()
         valid_sources = set()
         for n in self.nodes:
@@ -128,10 +146,16 @@ class Workspace(BaseConfig):
     async def execute(self):
         return await ops.EXECUTORS[self.env](self)
 
-    def save(self, path: str):
-        """Persist the workspace to a local file in JSON format."""
+    def model_dump_json(self) -> str:
+        """Returns the workspace as JSON."""
+        # Pydantic can't sort the keys. TODO: Keep an eye on https://github.com/pydantic/pydantic-core/pull/1637.
         j = self.model_dump()
         j = json.dumps(j, indent=2, sort_keys=True) + "\n"
+        return j
+
+    def save(self, path: str):
+        """Persist the workspace to a local file in JSON format."""
+        j = self.model_dump_json()
         dirname, basename = os.path.split(path)
         if dirname:
             os.makedirs(dirname, exist_ok=True)
@@ -174,28 +198,30 @@ class Workspace(BaseConfig):
         catalog = ops.CATALOGS[self.env]
         for node in self.nodes:
             data = node.data
-            op = catalog.get(data.title)
+            op = catalog.get(data.op_id)
             if op:
                 if getattr(data, "meta", None) != op:
                     data.meta = op
                     # If the node is connected to a CRDT, update that too.
-                    if hasattr(node, "_crdt"):
+                    if node._crdt:
                         node._crdt["data"]["meta"] = op.model_dump()
                 if node.type != op.type:
                     node.type = op.type
-                    if hasattr(node, "_crdt"):
+                    if node._crdt:
                         node._crdt["type"] = op.type
                 if data.error == "Unknown operation.":
                     data.error = None
-                    if hasattr(node, "_crdt"):
+                    if node._crdt:
                         node._crdt["data"]["error"] = None
             else:
                 data.error = "Unknown operation."
-                if hasattr(node, "_crdt"):
+                if node._crdt:
                     node._crdt["data"]["meta"] = {}
                     node._crdt["data"]["error"] = "Unknown operation."
 
-    def connect_crdt(self, ws_crdt: pycrdt.Map):
+    def connect_crdt(self, ws_crdt: "pycrdt.Map"):
+        import pycrdt
+
         self._crdt = ws_crdt
         with ws_crdt.doc.transaction():
             for nc, np in zip(ws_crdt["nodes"], self.nodes):
@@ -208,13 +234,19 @@ class Workspace(BaseConfig):
         random_string = os.urandom(4).hex()
         if func:
             kwargs["type"] = func.__op__.type
-            kwargs["data"] = WorkspaceNodeData(title=func.__op__.name, params={})
+            kwargs["data"] = WorkspaceNodeData(
+                title=func.__op__.name, op_id=func.__op__.id, params={}
+            )
+        elif "title" in kwargs:
+            kwargs["data"] = WorkspaceNodeData(
+                title=kwargs["title"], op_id=kwargs["title"], params=kwargs.get("params", {})
+            )
         kwargs.setdefault("type", "basic")
         kwargs.setdefault("id", f"{kwargs['data'].title} {random_string}")
         kwargs.setdefault("position", Position(x=0, y=0))
         kwargs.setdefault("width", 100)
         kwargs.setdefault("height", 100)
-        node = WorkspaceNode(**kwargs)
+        node = WorkspaceNode(**kwargs)  # ty: ignore[missing-argument]
         self.nodes.append(node)
         return node
 

lynxkite-core/tests/test_one_by_one.py

@@ -13,8 +13,8 @@ async def test_optional_inputs():
         return [a + b for a, b in zip(a, b)] if b else a
 
     assert maybe_add.__op__.inputs == [
-        ops.Input(name="a", type=list[int], position="bottom"),
-        ops.Input(name="b", type=list[int] | None, position="bottom"),
+        ops.Input(name="a", type=list[int], position=ops.Position.BOTTOM),
+        ops.Input(name="b", type=list[int] | None, position=ops.Position.BOTTOM),
     ]
     one_by_one.register("test")
     ws = workspace.Workspace(env="test", nodes=[], edges=[])

lynxkite-core/tests/test_ops.py

@@ -4,17 +4,17 @@ import enum
 
 
 def test_op_decorator_no_params_no_types_default_positions():
-    @ops.op(env="test", name="add", view="basic", outputs=["result"])
+    @ops.op("test", "add", view="basic", outputs=["result"])
     def add(a, b):
         return a + b
 
     assert add.__op__.name == "add"
     assert add.__op__.params == []
     assert add.__op__.inputs == [
-        ops.Input(name="a", type=inspect._empty, position="left"),
-        ops.Input(name="b", type=inspect._empty, position="left"),
+        ops.Input(name="a", type=inspect._empty, position=ops.Position.LEFT),
+        ops.Input(name="b", type=inspect._empty, position=ops.Position.LEFT),
     ]
-    assert add.__op__.outputs == [ops.Output(name="result", type=None, position="right")]
+    assert add.__op__.outputs == [ops.Output(name="result", type=None, position=ops.Position.RIGHT)]
     assert add.__op__.type == "basic"
     assert ops.CATALOGS["test"]["add"] == add.__op__
 
@@ -22,33 +22,37 @@ def test_op_decorator_no_params_no_types_default_positions():
 def test_op_decorator_custom_positions():
     @ops.input_position(a="right", b="top")
     @ops.output_position(result="bottom")
-    @ops.op(env="test", name="add", view="basic", outputs=["result"])
+    @ops.op("test", "add", view="basic", outputs=["result"])
     def add(a, b):
         return a + b
 
     assert add.__op__.name == "add"
     assert add.__op__.params == []
     assert add.__op__.inputs == [
-        ops.Input(name="a", type=inspect._empty, position="right"),
-        ops.Input(name="b", type=inspect._empty, position="top"),
+        ops.Input(name="a", type=inspect._empty, position=ops.Position.RIGHT),
+        ops.Input(name="b", type=inspect._empty, position=ops.Position.TOP),
+    ]
+    assert add.__op__.outputs == [
+        ops.Output(name="result", type=None, position=ops.Position.BOTTOM)
     ]
-    assert add.__op__.outputs == [ops.Output(name="result", type=None, position="bottom")]
     assert add.__op__.type == "basic"
     assert ops.CATALOGS["test"]["add"] == add.__op__
 
 
 def test_op_decorator_with_params_and_types_():
-    @ops.op(env="test", name="multiply", view="basic", outputs=["result"])
+    @ops.op("test", "multiply", view="basic", outputs=["result"])
     def multiply(a: int, b: float = 2.0, *, param: str = "param"):
         return a * b
 
     assert multiply.__op__.name == "multiply"
     assert multiply.__op__.params == [ops.Parameter(name="param", default="param", type=str)]
     assert multiply.__op__.inputs == [
-        ops.Input(name="a", type=int, position="left"),
-        ops.Input(name="b", type=float, position="left"),
+        ops.Input(name="a", type=int, position=ops.Position.LEFT),
+        ops.Input(name="b", type=float, position=ops.Position.LEFT),
+    ]
+    assert multiply.__op__.outputs == [
+        ops.Output(name="result", type=None, position=ops.Position.RIGHT)
     ]
-    assert multiply.__op__.outputs == [ops.Output(name="result", type=None, position="right")]
     assert multiply.__op__.type == "basic"
     assert ops.CATALOGS["test"]["multiply"] == multiply.__op__
 
@@ -59,24 +63,26 @@ def test_op_decorator_with_complex_types():
         GREEN = 2
         BLUE = 3
 
-    @ops.op(env="test", name="color_op", view="basic", outputs=["result"])
+    @ops.op("test", "color_op", view="basic", outputs=["result"])
     def complex_op(color: Color, color_list: list[Color], color_dict: dict[str, Color]):
         return color.name
 
     assert complex_op.__op__.name == "color_op"
     assert complex_op.__op__.params == []
     assert complex_op.__op__.inputs == [
-        ops.Input(name="color", type=Color, position="left"),
-        ops.Input(name="color_list", type=list[Color], position="left"),
-        ops.Input(name="color_dict", type=dict[str, Color], position="left"),
+        ops.Input(name="color", type=Color, position=ops.Position.LEFT),
+        ops.Input(name="color_list", type=list[Color], position=ops.Position.LEFT),
+        ops.Input(name="color_dict", type=dict[str, Color], position=ops.Position.LEFT),
     ]
     assert complex_op.__op__.type == "basic"
-    assert complex_op.__op__.outputs == [ops.Output(name="result", type=None, position="right")]
+    assert complex_op.__op__.outputs == [
+        ops.Output(name="result", type=None, position=ops.Position.RIGHT)
+    ]
     assert ops.CATALOGS["test"]["color_op"] == complex_op.__op__
 
 
 def test_operation_can_return_non_result_instance():
-    @ops.op(env="test", name="subtract", view="basic", outputs=["result"])
+    @ops.op("test", "subtract", view="basic", outputs=["result"])
     def subtract(a, b):
         return a - b
 
@@ -87,7 +93,7 @@ def test_operation_can_return_non_result_instance():
 
 
 def test_operation_can_return_result_instance():
-    @ops.op(env="test", name="subtract", view="basic", outputs=["result"])
+    @ops.op("test", "subtract", view="basic", outputs=["result"])
     def subtract(a, b):
         return ops.Result(output=a - b, display=None)
 
@@ -98,7 +104,7 @@ def test_operation_can_return_result_instance():
 
 
 def test_visualization_operations_display_is_populated_automatically():
-    @ops.op(env="test", name="display_op", view="visualization", outputs=["result"])
+    @ops.op("test", "display_op", view="visualization", outputs=["result"])
     def display_op():
         return {"display_value": 1}
 

lynxkite-core/tests/test_simple.py

@@ -9,11 +9,12 @@ async def test_optional_inputs():
 
     @ops.op("test", "maybe add")
     def maybe_add(a: int, b: int | None = None):
+        """b is optional"""
         return a + (b or 0)
 
     assert maybe_add.__op__.inputs == [
-        ops.Input(name="a", type=int, position="left"),
-        ops.Input(name="b", type=int | None, position="left"),
+        ops.Input(name="a", type=int, position=ops.Position.LEFT),
+        ops.Input(name="b", type=int | None, position=ops.Position.LEFT),
     ]
     simple.register("test")
     ws = workspace.Workspace(env="test", nodes=[], edges=[])

lynxkite-core/tests/test_workspace.py

@@ -9,12 +9,12 @@ def test_save_load():
     ws.add_node(
         id="1",
         type="node_type",
-        data=workspace.WorkspaceNodeData(title="Node 1", params={}),
+        title="Node 1",
     )
     ws.add_node(
         id="2",
         type="node_type",
-        data=workspace.WorkspaceNodeData(title="Node 2", params={}),
+        title="Node 2",
     )
     ws.edges.append(
         workspace.WorkspaceEdge(
@@ -42,6 +42,7 @@ def test_save_load():
             assert node.id == loaded_node.id
             assert node.type == loaded_node.type
             assert node.data.title == loaded_node.data.title
+            assert node.data.op_id == loaded_node.data.op_id
             assert node.data.params == loaded_node.data.params
             assert node.position.x == loaded_node.position.x
             assert node.position.y == loaded_node.position.y
@@ -61,7 +62,7 @@ def test_save_load():
 def populate_ops_catalog():
     from lynxkite.core import ops
 
-    ops.register_passive_op("test", "Test Operation", [])
+    ops.register_passive_op("test", "Test Operation", inputs=[])
 
 
 def test_update_metadata():
@@ -69,17 +70,18 @@ def test_update_metadata():
     ws.add_node(
         id="1",
         type="basic",
-        data=workspace.WorkspaceNodeData(title="Test Operation", params={}),
+        title="Test Operation",
     )
     ws.add_node(
         id="2",
         type="basic",
-        data=workspace.WorkspaceNodeData(title="Unknown Operation", params={}),
+        title="Unknown Operation",
     )
     ws.update_metadata()
+    assert ws.nodes[0].data.meta
     assert ws.nodes[0].data.meta.name == "Test Operation"
     assert ws.nodes[0].data.error is None
-    assert not hasattr(ws.nodes[1].data, "meta")
+    assert ws.nodes[1].data.meta is None
     assert ws.nodes[1].data.error == "Unknown operation."
 
 

lynxkite-graph-analytics/pyproject.toml

@@ -5,19 +5,24 @@ description = "The graph analytics executor and boxes for LynxKite"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
+    "cudf-cu12>=25.6.0",
     "fsspec>=2025.3.2",
     "grand-cypher>=0.13.0",
-    "joblib>=1.4.2",
     "lynxkite-core",
     "matplotlib>=3.10.1",
     "networkx[default]>=3.4.2",
+    "numba>=0.61.2",
+    "numpy>=2.2.6",
     "osmnx>=2.0.2",
     "pandas>=2.2.3",
     "polars>=1.25.2",
     "pyarrow>=19.0.1",
+    "pydantic>=2.11.7",
     "torch>=2.7.0",
     "torch-geometric>=2.6.1",
-    "umap-learn>=0.5.7",
+    "torchdiffeq>=0.2.5",
+    "tqdm>=4.67.1",
+    "umap-learn>=0.5.9.post2",
 ]
 classifiers = ["License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)"]
 
@@ -25,10 +30,6 @@ classifiers = ["License :: OSI Approved :: GNU Affero General Public License v3
 Homepage = "https://github.com/lynxkite/lynxkite-2000/"
 
 [project.optional-dependencies]
-dev = [
-    "pytest>=8.3.5",
-    "pytest-asyncio>=0.26.0",
-]
 gpu = [
     "cuml-cu12>=25.2.1",
     "nx-cugraph-cu12>=25.4.0",
@@ -37,7 +38,7 @@ gpu = [
 ]
 
 [tool.uv.sources]
-lynxkite-core = { path = "../lynxkite-core" }
+lynxkite-core = { workspace = true }
 pylibcugraph-cu12 = { index = "nvidia" }
 
 [tool.pytest.ini_options]
@@ -46,3 +47,21 @@ asyncio_mode = "auto"
 [[tool.uv.index]]
 name = "nvidia"
 url = "https://pypi.nvidia.com"
+
+[tool.deptry.per_rule_ignores]
+DEP002 = ["numba", "pyarrow", "nx-cugraph-cu12", "pylibcugraph-cu12"]
+
+[tool.deptry.package_module_name_map]
+grand-cypher = "grandcypher"
+lynxkite-core = "lynxkite"
+umap-learn = "umap"
+cuml-cu12 = "cuml"
+cudf-cu12 = "cudf"
+
+[build-system]
+requires = ["setuptools", "wheel", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+namespaces = true
+where = ["src"]

lynxkite-graph-analytics/src/lynxkite_graph_analytics/__init__.py

@@ -4,7 +4,7 @@ import os
 import pandas as pd
 
 if os.environ.get("NX_CUGRAPH_AUTOCONFIG", "").strip().lower() == "true":
-    import cudf.pandas
+    import cudf.pandas  # ty: ignore[unresolved-import]
 
     cudf.pandas.install()
 

lynxkite-graph-analytics/src/lynxkite_graph_analytics/core.py

@@ -14,6 +14,56 @@ import typing
 
 ENV = "LynxKite Graph Analytics"
 
+# Annotated types with format "dropdown" let you specify the available options
+# as a query on the input_metadata. These query expressions are JMESPath expressions.
+TableName = typing.Annotated[
+    str, {"format": "dropdown", "metadata_query": "[].dataframes[].keys(@)[]"}
+]
+"""A type annotation to be used for parameters of an operation. TableName is
+rendered as a dropdown in the frontend, listing all DataFrames in the Bundle.
+The table name is passed to the operation as a string."""
+
+NodePropertyName = typing.Annotated[
+    str, {"format": "dropdown", "metadata_query": "[].dataframes[].nodes[].columns[]"}
+]
+"""A type annotation to be used for parameters of an operation. NodePropertyName is
+rendered as a dropdown in the frontend, listing the columns of the "nodes" DataFrame.
+The column name is passed to the operation as a string."""
+
+EdgePropertyName = typing.Annotated[
+    str, {"format": "dropdown", "metadata_query": "[].dataframes[].edges[].columns[]"}
+]
+"""A type annotation to be used for parameters of an operation. EdgePropertyName is
+rendered as a dropdown in the frontend, listing the columns of the "edges" DataFrame.
+The column name is passed to the operation as a string."""
+
+OtherName = typing.Annotated[str, {"format": "dropdown", "metadata_query": "[].other.keys(@)[]"}]
+"""A type annotation to be used for parameters of an operation. OtherName is
+rendered as a dropdown in the frontend, listing the keys on the "other" part of the Bundle.
+The key is passed to the operation as a string."""
+
+ModelName = typing.Annotated[
+    str,
+    {
+        "format": "dropdown",
+        "metadata_query": "[].other.*[] | [?type == 'model'].key",
+    },
+]
+"""A type annotation to be used for parameters of an operation. ModelName is
+rendered as a dropdown in the frontend, listing the models in the Bundle.
+The model name is passed to the operation as a string."""
+
+# Parameter names in angle brackets, like <table_name>, will be replaced with the parameter
+# values. (This is not part of JMESPath.)
+# ColumnNameByTableName will list the columns of the DataFrame with the name
+# specified by the `table_name` parameter.
+ColumnNameByTableName = typing.Annotated[
+    str, {"format": "dropdown", "metadata_query": "[].dataframes[].<table_name>.columns[]"}
+]
+"""A type annotation to be used for parameters of an operation. ColumnNameByTableName is
+rendered as a dropdown in the frontend, listing the columns of the DataFrame
+named by the "table_name" parameter. The column name is passed to the operation as a string."""
+
 
 @dataclasses.dataclass
 class RelationDefinition:
@@ -141,12 +191,15 @@ class Bundle:
         return {
             "dataframes": {
                 name: {
+                    "key": name,
                     "columns": sorted(str(c) for c in df.columns),
                 }
                 for name, df in self.dfs.items()
             },
             "relations": [dataclasses.asdict(relation) for relation in self.relations],
-            "other": {k: getattr(v, "metadata", lambda: {})() for k, v in self.other.items()},
+            "other": {
+                k: {"key": k, **getattr(v, "metadata", lambda: {})()} for k, v in self.other.items()
+            },
         }
 
 
@@ -173,14 +226,17 @@ def disambiguate_edges(ws: workspace.Workspace):
     seen = set()
     for edge in reversed(ws.edges):
         dst_node = nodes[edge.target]
-        op = catalog.get(dst_node.data.title)
-        if op.get_input(edge.targetHandle).type == list[Bundle]:
+        op = catalog.get(dst_node.data.op_id)
+        if not op:
+            continue
+        t = op.get_input(edge.targetHandle).type
+        if t is list or typing.get_origin(t) is list:
             # Takes multiple bundles as an input. No need to disambiguate.
             continue
         if (edge.target, edge.targetHandle) in seen:
             i = ws.edges.index(edge)
             del ws.edges[i]
-            if hasattr(ws, "_crdt"):
+            if ws._crdt:
                 del ws._crdt["edges"][i]
         seen.add((edge.target, edge.targetHandle))
 
@@ -220,44 +276,65 @@ async def await_if_needed(obj):
     return obj
 
 
+def _to_bundle(x):
+    if isinstance(x, nx.Graph):
+        x = Bundle.from_nx(x)
+    elif isinstance(x, pd.DataFrame):
+        x = Bundle.from_df(x)
+    assert isinstance(x, Bundle), f"Input must be a graph or dataframe. Got: {x}"
+    return x
+
+
 async def _execute_node(
     node: workspace.WorkspaceNode, ws: workspace.Workspace, catalog: ops.Catalog, outputs: Outputs
 ):
     params = {**node.data.params}
-    op = catalog.get(node.data.title)
+    op = catalog.get(node.data.op_id)
     if not op:
         node.publish_error("Operation not found in catalog")
         return
     node.publish_started()
-    # TODO: Handle multi-inputs.
-    input_map = {
-        edge.targetHandle: outputs[edge.source, edge.sourceHandle]
-        for edge in ws.edges
-        if edge.target == node.id
-    }
+    input_map = {}
+    for edge in ws.edges:
+        if edge.target == node.id:
+            input_map.setdefault(edge.targetHandle, []).append(
+                outputs[edge.source, edge.sourceHandle]
+            )
     # Convert inputs types to match operation signature.
     try:
         inputs = []
         missing = []
         for p in op.inputs:
+            is_list = typing.get_origin(p.type) is list
             if p.name not in input_map:
                 opt_type = ops.get_optional_type(p.type)
                 if opt_type is not None:
                     inputs.append(None)
+                elif is_list:
+                    inputs.append([])
                 else:
                     missing.append(p.name)
                 continue
             x = input_map[p.name]
+            if p.type == list[Bundle]:
+                x = [_to_bundle(i) for i in x]
+            elif is_list:
+                pass
+            else:
+                [x] = x  # There should never be multiple inputs.
             if p.type == nx.Graph:
                 if isinstance(x, Bundle):
                     x = x.to_nx()
                 assert isinstance(x, nx.Graph), f"Input must be a graph. Got: {x}"
             elif p.type == Bundle:
+                x = _to_bundle(x)
+            if p.type == pd.DataFrame:
                 if isinstance(x, nx.Graph):
                     x = Bundle.from_nx(x)
-                elif isinstance(x, pd.DataFrame):
-                    x = Bundle.from_df(x)
-                assert isinstance(x, Bundle), f"Input must be a graph or dataframe. Got: {x}"
+                if isinstance(x, Bundle):
+                    assert len(x.dfs) == 1, "Bundle must contain a single DataFrame."
+                    [x] = list(x.dfs.values())
+                assert isinstance(x, pd.DataFrame), f"Input must be a DataFrame. Got: {x}"
             inputs.append(x)
     except Exception as e:
         if not os.environ.get("LYNXKITE_SUPPRESS_OP_ERRORS"):
@@ -277,12 +354,17 @@ async def _execute_node(
             traceback.print_exc()
         result = ops.Result(error=str(e))
     result.input_metadata = [_get_metadata(i) for i in inputs]
-    if isinstance(result.output, dict):
-        for k, v in result.output.items():
-            outputs[node.id, k] = v
-    elif result.output is not None:
-        [k] = op.outputs
-        outputs[node.id, k.name] = result.output
+    try:
+        if isinstance(result.output, dict):
+            for k, v in result.output.items():
+                outputs[node.id, k] = v
+        elif result.output is not None:
+            [k] = op.outputs
+            outputs[node.id, k.name] = result.output
+    except Exception as e:
+        if not os.environ.get("LYNXKITE_SUPPRESS_OP_ERRORS"):
+            traceback.print_exc()
+        result = ops.Result(error=str(e))
     node.publish_result(result)
 
 

lynxkite-graph-analytics/src/lynxkite_graph_analytics/lynxkite_ops.py

@@ -8,7 +8,8 @@ from collections import deque
 
 from . import core
 import grandcypher
-import matplotlib
+import matplotlib.cm
+import matplotlib.colors
 import networkx as nx
 import pandas as pd
 import polars as pl
@@ -43,9 +44,10 @@ class FileFormat(enum.StrEnum):
             default=FileFormat.csv,
         ),
     ],
+    slow=True,
 )
 def import_file(
-    *, file_path: str, table_name: str, file_format: FileFormat, **kwargs
+    *, file_path: str, table_name: str, file_format: FileFormat = FileFormat.csv, **kwargs
 ) -> core.Bundle:
     """Read the contents of the a file into a `Bundle`.
 
@@ -74,6 +76,36 @@ def import_file(
     return core.Bundle(dfs={table_name: df})
 
 
+@op("Export to file")
+def export_to_file(
+    bundle: core.Bundle,
+    *,
+    table_name: str,
+    filename: str,
+    file_format: FileFormat = FileFormat.csv,
+):
+    """Exports a DataFrame to a file.
+
+    Args:
+        bundle: The bundle containing the DataFrame to export.
+        table_name: The name of the DataFrame in the bundle to export.
+        filename: The name of the file to export to.
+        file_format: The format of the file to export to. Defaults to CSV.
+    """
+
+    df = bundle.dfs[table_name]
+    if file_format == FileFormat.csv:
+        df.to_csv(filename, index=False)
+    elif file_format == FileFormat.json:
+        df.to_json(filename, orient="records", lines=True)
+    elif file_format == FileFormat.parquet:
+        df.to_parquet(filename, index=False)
+    elif file_format == FileFormat.excel:
+        df.to_excel(filename, index=False)
+    else:
+        raise ValueError(f"Unsupported file format: {file_format}")
+
+
 @op("Import Parquet")
 def import_parquet(*, filename: str):
     """Imports a Parquet file."""
@@ -146,17 +178,6 @@ def cypher(bundle: core.Bundle, *, query: ops.LongStr, save_as: str = "result"):
     return bundle
 
 
-@op("Organize")
-def organize(bundle: list[core.Bundle], *, code: ops.LongStr) -> core.Bundle:
-    """Lets you rename/copy/delete DataFrames, and modify relations.
-
-    TODO: Merge this with "Create graph".
-    """
-    bundle = bundle.copy()
-    exec(code, globals(), {"bundle": bundle})
-    return bundle
-
-
 @op("Sample graph")
 def sample_graph(graph: nx.Graph, *, nodes: int = 100):
     """Takes a (preferably connected) subgraph."""
@@ -185,6 +206,7 @@ def _map_color(value):
     else:
         cmap = matplotlib.cm.get_cmap("Paired")
         categories = pd.Index(value.unique())
+        assert isinstance(cmap, matplotlib.colors.ListedColormap)
         colors = cmap.colors[: len(categories)]
         return [
             "#{:02x}{:02x}{:02x}".format(int(r * 255), int(g * 255), int(b * 255))
@@ -196,9 +218,9 @@ def _map_color(value):
 def visualize_graph(
     graph: core.Bundle,
     *,
-    color_nodes_by: ops.NodeAttribute = None,
-    label_by: ops.NodeAttribute = None,
-    color_edges_by: ops.EdgeAttribute = None,
+    color_nodes_by: core.NodePropertyName = None,
+    label_by: core.NodePropertyName = None,
+    color_edges_by: core.EdgePropertyName = None,
 ):
     nodes = core.df_for_frontend(graph.dfs["nodes"], 10_000)
     if color_nodes_by:
@@ -288,29 +310,25 @@ def view_tables(bundle: core.Bundle, *, _tables_open: str = "", limit: int = 100
 
 
 @op(
-    "Create graph",
+    "Organize",
     view="graph_creation_view",
     outputs=["output"],
 )
-def create_graph(bundle: core.Bundle, *, relations: str = None) -> core.Bundle:
-    """Replace relations of the given bundle
-
-    relations is a stringified JSON, instead of a dict, because complex Yjs types (arrays, maps)
-    are not currently supported in the UI.
+def organize(bundles: list[core.Bundle], *, relations: str = ""):
+    """Merge multiple inputs and construct graphs from the tables.
 
-    Args:
-        bundle: Bundle to modify
-        relations (str, optional): Set of relations to set for the bundle. The parameter
-            should be a JSON object where the keys are relation names and the values are
-            a dictionary representation of a `RelationDefinition`.
-            Defaults to None.
-
-    Returns:
-        Bundle: The input bundle with the new relations set.
+    To create a graph, import tables for edges and nodes, and combine them in this operation.
     """
-    bundle = bundle.copy()
-    if not (relations is None or relations.strip() == ""):
-        bundle.relations = [core.RelationDefinition(**r) for r in json.loads(relations).values()]
+    bundle = core.Bundle()
+    for b in bundles:
+        bundle.dfs.update(b.dfs)
+        bundle.relations.extend(b.relations)
+        bundle.other.update(b.other)
+    if relations.strip():
+        bundle.relations = [
+            core.RelationDefinition(**r)  # ty: ignore[missing-argument]
+            for r in json.loads(relations).values()
+        ]
     return ops.Result(output=bundle, display=bundle.to_dict(limit=100))
 
 

lynxkite-graph-analytics/src/lynxkite_graph_analytics/ml_ops.py

@@ -55,23 +55,53 @@ class ModelOutputMapping(pytorch_core.ModelMapping):
     pass
 
 
+def _get_num_samples(bundle: core.Bundle, input_mapping: pytorch_core.ModelMapping):
+    """Returns the number of samples in the input mapping."""
+    num_samples = None
+    for k, v in input_mapping.map.items():
+        if v.df in bundle.dfs and v.column in bundle.dfs[v.df]:
+            if num_samples is None:
+                num_samples = len(bundle.dfs[v.df][v.column])
+            else:
+                assert num_samples == len(bundle.dfs[v.df][v.column]), (
+                    f"Input '{k}' has different number of samples ({len(bundle.dfs[v.df][v.column])}) "
+                    f"than other inputs ({num_samples})."
+                )
+    return num_samples
+
+
 @op("Train model", slow=True)
 def train_model(
     bundle: core.Bundle,
     *,
-    model_name: str = "model",
+    model_name: core.ModelName = "model",
     input_mapping: ModelTrainingInputMapping,
     epochs: int = 1,
+    batch_size: int = 1,
 ):
-    """Trains the selected model on the selected dataset. Most training parameters are set in the model definition."""
+    """
+    Trains the selected model on the selected dataset.
+    Training parameters specific to the model are set in the model definition,
+    while parameters specific to the hardware environment and dataset are set here.
+    """
+    if input_mapping is None:
+        return ops.Result(bundle, error="No inputs are selected.")
     m = bundle.other[model_name].copy()
-    inputs = pytorch_core.to_tensors(bundle, input_mapping)
-    t = tqdm(range(epochs), desc="Training model")
+    num_samples = _get_num_samples(bundle, input_mapping)
+    if num_samples is None:
+        return ops.Result(bundle, error="No inputs are selected.")
+    num_batches = num_samples // batch_size
+    tepochs = tqdm(range(epochs), desc="Training model")
     losses = []
-    for _ in t:
-        loss = m.train(inputs)
-        t.set_postfix({"loss": loss})
-        losses.append(loss)
+    for _ in tepochs:
+        total_loss = 0
+        for i in tqdm(range(num_batches)):
+            inputs = pytorch_core.to_batch_tensors(bundle, batch_size, i, input_mapping)
+            loss = m.train(inputs)
+            total_loss += loss
+        mean_loss = total_loss / len(inputs)
+        tepochs.set_postfix({"loss": mean_loss})
+        losses.append(mean_loss)
     m.trained = True
     bundle = bundle.copy()
     bundle.dfs["training"] = pd.DataFrame({"training_loss": losses})
@@ -83,17 +113,27 @@ def train_model(
 def model_inference(
     bundle: core.Bundle,
     *,
-    model_name: str = "model",
+    model_name: core.ModelName = "model",
     input_mapping: ModelInferenceInputMapping,
     output_mapping: ModelOutputMapping,
+    batch_size: int = 1,
 ):
     """Executes a trained model."""
     if input_mapping is None or output_mapping is None:
         return ops.Result(bundle, error="Mapping is unset.")
     m = bundle.other[model_name]
     assert m.trained, "The model is not trained."
-    inputs = pytorch_core.to_tensors(bundle, input_mapping)
-    outputs = m.inference(inputs)
+    num_samples = _get_num_samples(bundle, input_mapping)
+    if num_samples is None:
+        return ops.Result(bundle, error="No inputs are selected.")
+    num_batches = num_samples // batch_size
+    outputs = {}
+    for i in tqdm(range(num_batches)):
+        inputs = pytorch_core.to_batch_tensors(bundle, batch_size, i, input_mapping)
+        batch_outputs = m.inference(inputs)
+        for k, v in batch_outputs.items():
+            v = v.detach().numpy().reshape(batch_size, -1)
+            outputs.setdefault(k, []).extend(v.tolist())
     bundle = bundle.copy()
     copied = set()
     for k, v in output_mapping.map.items():
@@ -102,15 +142,17 @@ def model_inference(
         if v.df not in copied:
             bundle.dfs[v.df] = bundle.dfs[v.df].copy()
             copied.add(v.df)
-        bundle.dfs[v.df][v.column] = outputs[k].detach().numpy().tolist()
+        bundle.dfs[v.df][v.column] = outputs[k]
     return bundle
 
 
 @op("Train/test split")
-def train_test_split(bundle: core.Bundle, *, table_name: str, test_ratio: float = 0.1):
+def train_test_split(
+    bundle: core.Bundle, *, table_name: core.TableName, test_ratio: float = 0.1, seed=1234
+):
     """Splits a dataframe in the bundle into separate "_train" and "_test" dataframes."""
     df = bundle.dfs[table_name]
-    test = df.sample(frac=test_ratio).reset_index()
+    test = df.sample(frac=test_ratio, random_state=seed).reset_index()
     train = df.drop(test.index).reset_index()
     bundle = bundle.copy()
     bundle.dfs[f"{table_name}_train"] = train
@@ -166,15 +208,15 @@ class UMAPMetric(str, enum.Enum):
 def view_vectors(
     bundle: core.Bundle,
     *,
-    table_name: str = "nodes",
-    vector_column: str = "",
-    label_column: str = "",
+    table_name: core.TableName = "nodes",
+    vector_column: core.ColumnNameByTableName = "",
+    label_column: core.ColumnNameByTableName = "",
     n_neighbors: int = 15,
     min_dist: float = 0.1,
     metric: UMAPMetric = UMAPMetric.euclidean,
 ):
     try:
-        from cuml.manifold.umap import UMAP
+        from cuml.manifold.umap import UMAP  # ty: ignore[unresolved-import]
     except ImportError:
         from umap import UMAP
     vec = np.stack(bundle.dfs[table_name][vector_column].to_numpy())

lynxkite-graph-analytics/src/lynxkite_graph_analytics/networkx_ops.py

@@ -1,14 +1,14 @@
 """Automatically wraps all NetworkX functions as LynxKite operations."""
 
-import collections
-import types
 from lynxkite.core import ops
+import collections.abc
+import enum
 import functools
 import inspect
 import networkx as nx
-import re
-
 import pandas as pd
+import re
+import types
 
 ENV = "LynxKite Graph Analytics"
 
@@ -17,20 +17,22 @@ class UnsupportedParameterType(Exception):
     pass
 
 
-_UNSUPPORTED = object()
-_SKIP = object()
+class Failure(str, enum.Enum):
+    UNSUPPORTED = "unsupported"  # This parameter will be hidden.
+    SKIP = "skip"  # We have to skip the whole function.
 
 
-def doc_to_type(name: str, type_hint: str) -> type:
+def doc_to_type(name: str, type_hint: str) -> type | types.UnionType | Failure:
     type_hint = type_hint.lower()
     type_hint = re.sub("[(][^)]+[)]", "", type_hint).strip().strip(".")
     if " " in name or "http" in name:
-        return _UNSUPPORTED  # Not a parameter type.
+        return Failure.UNSUPPORTED  # Not a parameter type.
     if type_hint.endswith(", optional"):
         w = doc_to_type(name, type_hint.removesuffix(", optional").strip())
-        if w is _UNSUPPORTED:
-            return _SKIP
-        return w if w is _SKIP else w | None
+        if w is Failure.UNSUPPORTED or w is Failure.SKIP:
+            return Failure.SKIP
+        assert not isinstance(w, Failure)
+        return w | None
     if type_hint in [
         "a digraph or multidigraph",
         "a graph g",
@@ -54,15 +56,15 @@ def doc_to_type(name: str, type_hint: str) -> type:
     ]:
         return nx.DiGraph
     elif type_hint == "node":
-        return _UNSUPPORTED
+        return Failure.UNSUPPORTED
     elif type_hint == '"node (optional)"':
-        return _SKIP
+        return Failure.SKIP
     elif type_hint == '"edge"':
-        return _UNSUPPORTED
+        return Failure.UNSUPPORTED
     elif type_hint == '"edge (optional)"':
-        return _SKIP
+        return Failure.SKIP
     elif type_hint in ["class", "data type"]:
-        return _UNSUPPORTED
+        return Failure.UNSUPPORTED
     elif type_hint in ["string", "str", "node label"]:
         return str
     elif type_hint in ["string or none", "none or string", "string, or none"]:
@@ -72,27 +74,27 @@ def doc_to_type(name: str, type_hint: str) -> type:
     elif type_hint in ["bool", "boolean"]:
         return bool
     elif type_hint == "tuple":
-        return _UNSUPPORTED
+        return Failure.UNSUPPORTED
     elif type_hint == "set":
-        return _UNSUPPORTED
+        return Failure.UNSUPPORTED
     elif type_hint == "list of floats":
-        return _UNSUPPORTED
+        return Failure.UNSUPPORTED
     elif type_hint == "list of floats or float":
         return float
     elif type_hint in ["dict", "dictionary"]:
-        return _UNSUPPORTED
+        return Failure.UNSUPPORTED
     elif type_hint == "scalar or dictionary":
         return float
     elif type_hint == "none or dict":
-        return _SKIP
+        return Failure.SKIP
     elif type_hint in ["function", "callable"]:
-        return _UNSUPPORTED
+        return Failure.UNSUPPORTED
     elif type_hint in [
         "collection",
         "container of nodes",
         "list of nodes",
     ]:
-        return _UNSUPPORTED
+        return Failure.UNSUPPORTED
     elif type_hint in [
         "container",
         "generator",
@@ -104,13 +106,13 @@ def doc_to_type(name: str, type_hint: str) -> type:
         "list or tuple",
         "list",
     ]:
-        return _UNSUPPORTED
+        return Failure.UNSUPPORTED
     elif type_hint == "generator of sets":
-        return _UNSUPPORTED
+        return Failure.UNSUPPORTED
     elif type_hint == "dict or a set of 2 or 3 tuples":
-        return _UNSUPPORTED
+        return Failure.UNSUPPORTED
     elif type_hint == "set of 2 or 3 tuples":
-        return _UNSUPPORTED
+        return Failure.UNSUPPORTED
     elif type_hint == "none, string or function":
         return str | None
     elif type_hint == "string or function" and name == "weight":
@@ -135,8 +137,8 @@ def doc_to_type(name: str, type_hint: str) -> type:
     elif name == "weight":
         return str
     elif type_hint == "object":
-        return _UNSUPPORTED
-    return _SKIP
+        return Failure.UNSUPPORTED
+    return Failure.SKIP
 
 
 def types_from_doc(doc: str) -> dict[str, type]:
@@ -186,13 +188,13 @@ def wrapped(name: str, func):
     return wrapper
 
 
-def _get_params(func) -> dict | None:
+def _get_params(func) -> list[ops.Parameter | ops.ParameterGroup]:
     sig = inspect.signature(func)
     # Get types from docstring.
     types = types_from_doc(func.__doc__)
     # Always hide these.
     for k in ["backend", "backend_kwargs", "create_using"]:
-        types[k] = _SKIP
+        types[k] = Failure.SKIP
     # Add in types based on signature.
     for k, param in sig.parameters.items():
         if k in types:
@@ -203,10 +205,10 @@ def _get_params(func) -> dict | None:
             types[k] = int
     params = []
     for name, param in sig.parameters.items():
-        _type = types.get(name, _UNSUPPORTED)
-        if _type is _UNSUPPORTED:
+        _type = types.get(name, Failure.UNSUPPORTED)
+        if _type is Failure.UNSUPPORTED:
             raise UnsupportedParameterType(name)
-        if _type is _SKIP or _type in [nx.Graph, nx.DiGraph]:
+        if _type is Failure.SKIP or _type in [nx.Graph, nx.DiGraph]:
             continue
         p = ops.Parameter.basic(
             name=name,
@@ -218,32 +220,104 @@ def _get_params(func) -> dict | None:
 
 
 _REPLACEMENTS = [
-    ("Barabasi Albert", "Barabasi–Albert"),
-    ("Bellman Ford", "Bellman–Ford"),
-    ("Bethe Hessian", "Bethe–Hessian"),
-    ("Bfs", "BFS"),
-    ("Dag ", "DAG "),
-    ("Dfs", "DFS"),
-    ("Dorogovtsev Goltsev Mendes", "Dorogovtsev–Goltsev–Mendes"),
-    ("Erdos Renyi", "Erdos–Renyi"),
-    ("Floyd Warshall", "Floyd–Warshall"),
-    ("Gnc", "G(n,c)"),
-    ("Gnm", "G(n,m)"),
-    ("Gnp", "G(n,p)"),
-    ("Gnr", "G(n,r)"),
-    ("Havel Hakimi", "Havel–Hakimi"),
-    ("Hkn", "H(k,n)"),
-    ("Hnm", "H(n,m)"),
-    ("Kl ", "KL "),
-    ("Moebius Kantor", "Moebius–Kantor"),
-    ("Pagerank", "PageRank"),
-    ("Scale Free", "Scale-Free"),
-    ("Vf2Pp", "VF2++"),
-    ("Watts Strogatz", "Watts–Strogatz"),
-    ("Weisfeiler Lehman", "Weisfeiler–Lehman"),
+    (" at free", " AT-free"),
+    (" dag", " DAG"),
+    (" k out ", " k-out "),
+    (" rary", " r-ary"),
+    ("2d ", "2D "),
+    ("3d ", "3D "),
+    ("adamic adar", "Adamic–Adar"),
+    ("barabasi albert", "Barabasi–Albert"),
+    ("bellman ford", "Bellman–Ford"),
+    ("bethe hessian", "Bethe–Hessian"),
+    ("bfs", "BFS"),
+    ("d separator", "d-separator"),
+    ("dag ", "DAG "),
+    ("dfs", "DFS"),
+    ("dijkstra", "Dijkstra"),
+    ("dorogovtsev goltsev mendes", "Dorogovtsev–Goltsev–Mendes"),
+    ("erdos renyi", "Erdos–Renyi"),
+    ("euler", "Euler"),
+    ("floyd warshall", "Floyd–Warshall"),
+    ("forceatlas2", "ForceAtlas2"),
+    ("gexf ", "GEXF "),
+    ("gml", "GML"),
+    ("gnc", "G(n,c)"),
+    ("gnm", "G(n,m)"),
+    ("gnp", "G(n,p)"),
+    ("gnr", "G(n,r)"),
+    ("graphml", "GraphML"),
+    ("harary", "Harary"),
+    ("havel hakimi", "Havel–Hakimi"),
+    ("hkn", "H(k,n)"),
+    ("hnm", "H(n,m)"),
+    ("internet", "Internet"),
+    ("k core", "k-core"),
+    ("k corona", "k-corona"),
+    ("k crust", "k-crust"),
+    ("k shell", "k-shell"),
+    ("k truss", "k-truss"),
+    ("kl ", "KL "),
+    ("laplacian", "Laplacian"),
+    ("lfr ", "LFR "),
+    ("margulis gabber galil", "Margulis–Gabber–Galil"),
+    ("moebius kantor", "Moebius–Kantor"),
+    ("newman watts strogatz", "Newman–Watts–Strogatz"),
+    ("numpy", "NumPy"),
+    ("pagerank", "PageRank"),
+    ("pajek", "Pajek"),
+    ("pandas", "Pandas"),
+    ("parse leda", "Parse LEDA"),
+    ("powerlaw", "power-law"),
+    ("prufer", "Prüfer"),
+    ("radzik", "Radzik"),
+    ("s metric", "s-metric"),
+    ("scale free", "Scale-free"),
+    ("scipy", "SciPy"),
+    ("small world", "small-world"),
+    ("soundarajan hopcroft", "Soundarajan–Hopcroft"),
+    ("southern women", "Southern women"),
+    ("vf2pp", "VF2++"),
+    ("watts strogatz", "Watts–Strogatz"),
+    ("weisfeiler lehman", "Weisfeiler–Lehman"),
+]
+_CATEGORY_REPLACEMENTS = [
+    ("Networkx", "NetworkX"),
+    ("D separation", "D-separation"),
+    ("Dag", "DAG"),
+    ("Pagerank alg", "PageRank alg"),
+    ("Richclub", "Rich-club"),
+    ("Smallworld", "Small-world"),
+    ("Smetric", "S-metric"),
+    ("Structuralholes", "Structural holes"),
+    ("Edgedfs", "Edge DFS"),
+    ("Edgebfs", "Edge BFS"),
+    ("Edge_kcomponents", "Edge k-components"),
+    ("Mincost", "Min cost"),
+    ("Networksimplex", "Network simplex"),
+    ("Vf2pp", "VF2++"),
+    ("Mst", "MST"),
+    ("Attrmatrix", "Attr matrix"),
+    ("Graphmatrix", "Graph matrix"),
+    ("Laplacianmatrix", "Laplacian matrix"),
+    ("Algebraicconnectivity", "Algebraic connectivity"),
+    ("Modularitymatrix", "Modularity matrix"),
+    ("Bethehessianmatrix", "Bethe–Hessian matrix"),
 ]
 
 
+def _categories(func) -> list[str]:
+    """Extract categories from the function's docstring."""
+    path = func.__module__.split(".")
+    cats = []
+    for p in path:
+        p = p.replace("_", " ").capitalize()
+        for a, b in _CATEGORY_REPLACEMENTS:
+            p = p.replace(a, b)
+        cats.append(p)
+    return cats
+
+
 def register_networkx(env: str):
     cat = ops.CATALOGS.setdefault(env, {})
     counter = 0
@@ -253,19 +327,24 @@ def register_networkx(env: str):
                 params = _get_params(func)
             except UnsupportedParameterType:
                 continue
-            inputs = [ops.Input(name=k, type=nx.Graph) for k in func.graphs]
-            nicename = "NX › " + name.replace("_", " ").title()
+            inputs = [
+                ops.Input(name=k, type=nx.Graph, position=ops.Position.LEFT) for k in func.graphs
+            ]
+            nicename = name.replace("_", " ")
             for a, b in _REPLACEMENTS:
                 nicename = nicename.replace(a, b)
+            if nicename[1] != "-":
+                nicename = nicename[0].upper() + nicename[1:]
             op = ops.Op(
                 func=wrapped(name, func),
                 name=nicename,
+                categories=_categories(func),
                 params=params,
                 inputs=inputs,
-                outputs=[ops.Output(name="output", type=nx.Graph)],
+                outputs=[ops.Output(name="output", type=nx.Graph, position=ops.Position.RIGHT)],
                 type="basic",
             )
-            cat[nicename] = op
+            cat[op.id] = op
             counter += 1
     print(f"Registered {counter} NetworkX operations.")
 

lynxkite-graph-analytics/src/lynxkite_graph_analytics/pytorch/__init__.py

@@ -1,2 +1,9 @@
+"""
+This module provides the "PyTorch model" LynxKite environment. This is a passive
+environment: you can build PyTorch models here from neural network layers,
+but the workspace can't be executed. Instead, it can be loaded as a model
+definition in a "LynxKite Graph Analytics" workspace.
+"""
+
 from . import pytorch_core  # noqa
 from . import pytorch_ops  # noqa