Working demo

app.py

@@ -0,0 +1,62 @@
+import gradio as gr
+
+_remove_color = "rgb(103,6,12)"
+_addition_color = "rgb(6,103,12)"
+
+def mark_text(text, add=True):
+    if add:
+        color = _addition_color
+    else:
+        color = _remove_color
+    return f'<mark style="background-color:{color}!important;color:white!important">{text}</mark>'
+
+def highlight(option):
+    filename = option.lower().replace(' ', '_')
+    with open(f"code_samples/{filename}") as f:
+        output = f.read()
+    lines = output.split("\n")
+    for i,line in enumerate(lines):
+        if line.startswith("-"):
+            lines[i] = "- " + line[1:]
+            lines[i] = mark_text(lines[i], False)
+        elif line.startswith("+"):
+            lines[i] = "+ " + line[1:]
+            lines[i] = mark_text(lines[i], True)
+        else:
+            lines[i] = "  " + line
+    return "\n".join(lines).rstrip()
+
+with open("code_samples/initial") as f:
+    template = f.read()
+
+with open("code_samples/accelerate") as f:
+    accelerated_template = f.read()
+
+with open("code_samples/initial_with_metrics") as f:
+    metrics_template = f.read()
+
+def change(inp):
+    if inp == "Basic":
+        return (template, highlight(inp), "## Accelerate Code (Base Integration)")
+    elif inp == "Calculating Metrics":
+        return (metrics_template, highlight(inp), f"## Accelerate Code ({inp})")
+    else:
+        return (accelerated_template, highlight(inp), f"## Accelerate Code ({inp})")
+
+with gr.Blocks() as demo:
+    gr.Markdown(f'''# Accelerate Template Generator
+Here is a very basic Python training loop.
+Select how you would like to introduce an Accelerate capability to add to it.''')
+    inp = gr.Radio(
+        ["Basic", "Calculating Metrics", "Checkpointing", "Gradient Accumulation", ], 
+        label="Select a feature"
+    )
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown("## Initial Code")
+            code = gr.Markdown(template)
+        with gr.Column():
+            feature = gr.Markdown("## Accelerate Code")
+            out = gr.Markdown()
+    inp.change(fn=change, inputs=inp, outputs=[code, out, feature])  
+demo.launch()

code_samples/accelerate

@@ -0,0 +1,17 @@
+```
+from accelerate import Accelerator
+accelerator = Accelerator()
+train_dataloader, model, optimizer scheduler = accelerator.prepare(
+        dataloader, model, optimizer, scheduler
+)
+
+model.train()
+for batch in train_dataloader:
+    optimizer.zero_grad()
+    inputs, targets = batch
+    outputs = model(inputs)
+    loss = loss_function(outputs, targets)
+    accelerator.backward(loss)
+    optimizer.step()
+    scheduler.step()
+```

code_samples/basic

@@ -0,0 +1,19 @@
+<pre>
++from accelerate import Accelerator
++accelerator = Accelerator()
++dataloader, model, optimizer scheduler = accelerator.prepare(
++        dataloader, model, optimizer, scheduler
++)
+
+for batch in dataloader:
+    optimizer.zero_grad()
+    inputs, targets = batch
+-    inputs = inputs.to(device)
+-    targets = targets.to(device)
+    outputs = model(inputs)
+    loss = loss_function(outputs, targets)
+-    loss.backward()
++    accelerator.backward(loss)
+    optimizer.step()
+    scheduler.step()
+</pre>

code_samples/calculating_metrics

@@ -0,0 +1,36 @@
+<pre>
+import evaluate
++from accelerate import Accelerator
++accelerator = Accelerator()
++dataloader, model, optimizer scheduler = accelerator.prepare(
++        dataloader, model, optimizer, scheduler
++)
+metric = evaluate.load("accuracy")
+for batch in train_dataloader:
+    optimizer.zero_grad()
+    inputs, targets = batch
+-    inputs = inputs.to(device)
+-    targets = targets.to(device)
+    outputs = model(inputs)
+    loss = loss_function(outputs, targets)
+    loss.backward()
+    optimizer.step()
+    scheduler.step()
+
+model.eval()
+for batch in eval_dataloader:
+    inputs, targets = batch
+-    inputs = inputs.to(device)
+-    targets = targets.to(device)
+    with torch.no_grad():
+        outputs = model(inputs)
+    predictions = outputs.argmax(dim=-1)
++    predictions, references = accelerator.gather_for_metrics(
++        (predictions, references)
++    )
+    metric.add_batch(
+        predictions = predictions,
+        references = references
+    )
+print(metric.compute())
+<pre>

code_samples/checkpointing

@@ -0,0 +1,17 @@
+<pre>
+from accelerate import Accelerator
+accelerator = Accelerator()
+dataloader, model, optimizer scheduler = accelerator.prepare(
+        dataloader, model, optimizer, scheduler
+)
+
+for batch in dataloader:
+    optimizer.zero_grad()
+    inputs, targets = batch
+    outputs = model(inputs)
+    loss = loss_function(outputs, targets)
+    accelerator.backward(loss)
+    optimizer.step()
+    scheduler.step()
++accelerator.save_state("checkpoint_dir")
+</pre>

code_samples/gradient_accumulation

@@ -0,0 +1,19 @@
+<pre>
+from accelerate import Accelerator
+accelerator = Accelerator(
++    gradient_accumulation_steps=2,
+)
+dataloader, model, optimizer scheduler = accelerator.prepare(
+        dataloader, model, optimizer, scheduler
+)
+
+for batch in dataloader:
++  with accelerator.accumulate(model):
+      optimizer.zero_grad()
+      inputs, targets = batch
+      outputs = model(inputs)
+      loss = loss_function(outputs, targets)
+      accelerator.backward(loss)
+      optimizer.step()
+      scheduler.step()
+</pre>

code_samples/initial

@@ -0,0 +1,12 @@
+```
+for batch in dataloader:
+    optimizer.zero_grad()
+    inputs, targets = batch
+    inputs = inputs.to(device)
+    targets = targets.to(device)
+    outputs = model(inputs)
+    loss = loss_function(outputs, targets)
+    loss.backward()
+    optimizer.step()
+    scheduler.step()
+```

code_samples/initial_with_metrics

@@ -0,0 +1,28 @@
+```
+import evaluate
+metric = evaluate.load("accuracy")
+for batch in train_dataloader:
+    optimizer.zero_grad()
+    inputs, targets = batch
+    inputs = inputs.to(device)
+    targets = targets.to(device)
+    outputs = model(inputs)
+    loss = loss_function(outputs, targets)
+    loss.backward()
+    optimizer.step()
+    scheduler.step()
+
+model.eval()
+for batch in eval_dataloader:
+    inputs, targets = batch
+    inputs = inputs.to(device)
+    targets = targets.to(device)
+    with torch.no_grad():
+        outputs = model(inputs)
+    predictions = outputs.argmax(dim=-1)
+    metric.add_batch(
+        predictions = predictions,
+        references = references
+    )
+print(metric.compute())
+```