Hugging Face Profiling

This guide demonstrates how to profile Transformers inference and Accelerate workloads on Intel GPUs. It includes scheduled trace collection and explains the instrumentation needed for Trainer-based runs.

Transformers and Accelerate

The Transformers Trainer has no built-in torch.profiler integration or control surface. Profiling a Trainer-based run requires manual instrumentation of the training/eval step with torch.profiler, or use of an external harness.

Profiling Transformers inference

This example wraps a plain Transformers inference call directly with torch.profiler, independent of Trainer. It downloads the model from Hugging Face on first run unless it is already cached locally.

from pathlib import Path

import torch
from transformers import AutoTokenizer, AutoModelForSequenceClassification
from torch.profiler import profile, ProfilerActivity, record_function, schedule

model_name = "distilbert-base-uncased-finetuned-sst-2-english"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSequenceClassification.from_pretrained(model_name)
model = model.to("xpu")
model.eval()

inputs = tokenizer("This profiler guide is helpful.", return_tensors="pt")
inputs = {k: v.to("xpu") for k, v in inputs.items()}
trace_dir = Path("transformers_traces")
trace_dir.mkdir(parents=True, exist_ok=True)

def trace_handler(prof):
    print(
        prof.key_averages().table(
            sort_by="self_xpu_time_total", row_limit=10
        )
    )
    prof.export_chrome_trace(
        str(trace_dir / f"transformers_trace_{prof.step_num}.json")
    )

with profile(
    activities=[ProfilerActivity.CPU, ProfilerActivity.XPU],
    schedule=schedule(wait=1, warmup=1, active=2, repeat=1),
    on_trace_ready=trace_handler,
) as prof:
    with torch.inference_mode():
        for _ in range(4):
            with record_function("transformers_inference"):
                model(**inputs)
            torch.accelerator.synchronize()
            prof.step()

Profiling with Accelerate

Hugging Face Accelerate is a separate library from Transformers. It provides a supported profiler wrapper through ProfileKwargs. Its implementation accepts xpu as an activity name and maps it to ProfilerActivity.XPU when PyTorch and XPU support are available.

from pathlib import Path

import torch
import torch.nn as nn
from accelerate import Accelerator
from accelerate.utils import ProfileKwargs

trace_dir = Path("accelerate_traces")
trace_dir.mkdir(parents=True, exist_ok=True)

def trace_handler(prof):
    print(prof.key_averages().table(sort_by="self_xpu_time_total"))
    prof.export_chrome_trace(
        str(trace_dir / f"accelerate_trace_{prof.step_num}.json")
    )

profile_kwargs = ProfileKwargs(
    activities=["cpu", "xpu"],
    schedule_option={"wait": 1, "warmup": 1, "active": 2, "repeat": 1},
    on_trace_ready=trace_handler,
    record_shapes=True,
    profile_memory=True,
)

accelerator = Accelerator(kwargs_handlers=[profile_kwargs])

model = nn.Sequential(
    nn.Linear(1024, 1024),
    nn.ReLU(),
    nn.Linear(1024, 1024),
)
model = accelerator.prepare(model)

x = torch.randn(64, 1024, device=accelerator.device)

with accelerator.profile() as prof:
    for _ in range(4):
        model(x)
        torch.accelerator.synchronize()
        prof.step()

For general guidance on scheduling, trace output, and overhead, see the PyTorch profiling guide.