Code behind the measured architecture

Line-numbered excerpts from the actual GPU executions on 15 September 2026. Every complete source file was checked against its recorded SHA-256 before these excerpts were generated. These are archived references, not the current development files.

The model implementation is identical in both executions. Adapter and reporting fixes between runs are recorded separately. Optional graph, dilation, ensemble and attention code does not mean those paths were enabled: the published clip suite uses LegibleTCN, no dilation, one joint stream, no attention and the first actor. The saved recipe and evaluation record identify the active configuration.

HMDB51 and UCF101 execution: skeletonBenchCommon.py

SHA-256 of complete original file: e4655a8691afef33713e9b0e63ce7fe8ff3cee4b893947dcd0aa74022dde65cd

43class TimestepGroupNorm(nn.Module):
44    """GroupNorm with per-timestep statistics: each frame is normalized over
45    its channel group only. Standard GroupNorm on (N, C, T) pools stats over
46    the WHOLE clip, so a causal conv stack followed by it is not actually
47    causal — this is."""
48
49    def __init__(self, num_groups, num_channels):
50        super().__init__()
51        self.num_groups = num_groups
52        self.weight = nn.Parameter(torch.ones(num_channels))
53        self.bias = nn.Parameter(torch.zeros(num_channels))
54
55    def forward(self, x):  # (N, C, T)
56        n, c, t = x.shape
57        z = x.permute(0, 2, 1).reshape(n * t, c)
58        z = F.group_norm(z, self.num_groups, self.weight, self.bias)
59        return z.reshape(n, t, c).permute(0, 2, 1)
60
61
62class LegibleTCN(nn.Module):
63    """Same architecture as the BoxingVI model, with V/C_IN/classes/K as args.
64
65    causal=True: left-pad only — every conv sees ONLY past frames, so the model
66    can emit a verdict at every frame with zero lookahead. The head is linear,
67    so applying it before or after temporal mean-pooling is identical.
68    """
69
70    def __init__(self, *, v, c_in, num_classes, filters=8, hidden=64, k=9,
71                 causal=False, attn=False, per_timestep_norm=None,
72                 temporal_dilations=None):
73        super().__init__()
74        self.v = v
75        self.c_in = c_in
76        self.filters = filters
77        self.hidden = hidden
78        self.k = k
79        self.causal = causal
80        self.attn = attn
81        self.last_a = None
82        # temporal_dilations: WaveNet-style dilated depthwise stack replacing the
83        # single l3, to see WHOLE actions. Receptive field (causal) =
84        #   (k-1)   [l1]  +  (k-1)*Σ dilations  [stack]  + 1.
85        # PKU-MMD actions median 98 frames; [1,2,4,8] k9 → ~129-frame (4.3s)
86        # field. Each depthwise layer stays a NAMED per-unit temporal template
87        # at its dilation, so legibility survives. None → original single l3.
88        self.temporal_dilations = temporal_dilations
89        pad = 0 if causal else k // 2
90        # (N, V*C_IN, T) grouped by joint: each group reads one joint's channels.
91        self.l1 = nn.Conv1d(v * c_in, v * filters, k, padding=pad, groups=v)
92        # GroupNorm, not BatchNorm: BN's fold-dependent batch statistics caused
93        # a deterministic training collapse on the BoxingVI V4 holdout.
94        # Per-timestep GroupNorm is REQUIRED whenever activations are consumed
95        # per-frame at a different sequence length than training:
96        #   - causal streaming (else the norm leaks the future — 1.14 logit delta)
97        #   - framewise DETECTION trained on short windows but run on long
98        #     sequences (plain GroupNorm pools over T, so its stats depend on
99        #     sequence length → train/test mismatch → mAP collapses to ~0.01).
100        # Clip classification (fixed T, mean-pooled) is fine with plain GN.
101        # Default: tie to causal; detectors pass per_timestep_norm=True explicitly.
102        use_ts = causal if per_timestep_norm is None else per_timestep_norm
103        norm = TimestepGroupNorm if use_ts else nn.GroupNorm
104        self.bn1 = norm(v, v * filters)
105        self.mix = nn.Conv1d(v * filters, hidden, 1)
106        self.bn2 = norm(8, hidden)
107        if temporal_dilations:
108            self._norm_cls = norm
109            self.l3_stack = nn.ModuleList([
110                nn.Conv1d(hidden, hidden, k, padding=(0 if causal else (k // 2) * d),
111                          dilation=d, groups=hidden)
112                for d in temporal_dilations
113            ])
114            self.l3_norms = nn.ModuleList([norm(8, hidden) for _ in temporal_dilations])
115        else:
116            self.l3 = nn.Conv1d(hidden, hidden, k, padding=pad, groups=hidden)
117            self.bn3 = norm(8, hidden)
118        self.head = nn.Linear(hidden, num_classes)
119        self.att = nn.Conv1d(hidden, 1, 1) if attn else None
120
121    def _trunk(self, x):  # x: (N, C_IN, T, V)
122        n, c, t, v = x.shape
123        z = x.permute(0, 3, 1, 2).reshape(n, v * c, t)  # joint-major
124        if self.causal:
125            z = F.pad(z, (self.k - 1, 0))
126        z = F.relu(self.bn1(self.l1(z)))
127        z = F.relu(self.bn2(self.mix(z)))
128        if self.temporal_dilations:
129            # Dilated depthwise TCN stack with residual connections; each layer
130            # left-pads (k-1)*d in causal mode so it stays zero-lookahead.
131            for conv, bn, d in zip(self.l3_stack, self.l3_norms, self.temporal_dilations):
132                res = z
133                if self.causal:
134                    z = F.pad(z, ((self.k - 1) * d, 0))
135                z = F.relu(bn(conv(z)) + res)
136            return z
137        if self.causal:
138            z = F.pad(z, (self.k - 1, 0))
139        return F.relu(self.bn3(self.l3(z)))  # (N, H, T)
140
141    def forward(self, x):
142        z = self._trunk(x)
143        if self.att is not None:
144            a = torch.sigmoid(self.att(z))
145            self.last_a = a.squeeze(1)
146            fw = self.head(z.transpose(1, 2))
147            return (a.transpose(1, 2) * fw).sum(1) / (1 + a.sum(-1))
148        return self.head(z.mean(-1))
149
150    def framewise_logits(self, x):  # (N, T, classes), zero lookahead when causal
151        return self.head(self._trunk(x).transpose(1, 2))

Unrelated source lines omitted.

551def train_classifier(*, model, X, Y, epochs, batch, device, lr=2e-3,
552                     log=print, log_every=1, val=None, optimizer="adamw",
553                     weight_decay=1e-4, warmup_epochs=0, augment=False):
554    """X: (N, C, T, V) float32 CPU tensor; Y: (N,) int64 CPU tensor. Batches
555    are moved to `device` per step so NTU-scale data stays in host RAM.
556
557    Full-recipe knobs (default off → original BoxingVI behavior): optimizer
558    'sgd' (momentum 0.9) or 'adamw'; weight_decay; linear warmup_epochs then
559    cosine; augment=True enables on-the-fly skeleton augmentation."""
560    num_classes = model.head.out_features
561    counts = np.bincount(Y.numpy(), minlength=num_classes).astype(np.float32)
562    counts[counts == 0] = 1
563    w = torch.from_numpy((1.0 / counts)[Y.numpy()])
564    if optimizer == "sgd":
565        opt = torch.optim.SGD(model.parameters(), lr=lr, momentum=0.9,
566                              weight_decay=weight_decay, nesterov=True)
567    else:
568        opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=weight_decay)
569    # linear warmup for warmup_epochs, then cosine to 0 over the remainder
570    def lr_at(ep):
571        if warmup_epochs and ep < warmup_epochs:
572            return (ep + 1) / warmup_epochs
573        prog = (ep - warmup_epochs) / max(1, epochs - warmup_epochs)
574        return 0.5 * (1 + np.cos(np.pi * prog))
575    sched = torch.optim.lr_scheduler.LambdaLR(opt, lr_lambda=lr_at)
576    n = len(Y)
577    import time as _time
578    for ep in range(epochs):
579        t0 = _time.time()
580        model.train()
581        idx = torch.multinomial(w, n, replacement=True)
582        tot = 0.0
583        for b0 in range(0, n, batch):
584            bidx = idx[b0: b0 + batch]
585            # NO non_blocking=True: async copies from unpinned host memory race
586            # on MPS — the labels arrive as garbage (observed: loss pinned at
587            # the label-smoothing floor while accuracy stays at chance).
588            # .float(): host X may be float16 (PYSKL benchmarks keep it half to
589            # fit Kinetics-scale data in 16GB RAM); the model runs in float32.
590            xb = X[bidx].to(device).float()
591            if augment:
592                xb = augment_batch(xb)
593            yb = Y[bidx].to(device)
594            loss = F.cross_entropy(model(xb), yb, label_smoothing=0.1)
595            opt.zero_grad()
596            loss.backward()
597            opt.step()
598            tot += loss.item() * len(bidx)
599        sched.step()
600        if (ep + 1) % log_every == 0 or ep == 0:
601            msg = f"epoch {ep+1}/{epochs} loss={tot/n:.4f} [{_time.time()-t0:.0f}s]"
602            if val is not None and ((ep + 1) % (log_every * 5) == 0 or ep + 1 == epochs):
603                msg += f" val={eval_top1(model=model, X=val[0], Y=val[1], device=device)*100:.1f}%"
604            log(msg)
605    return model
606
607
608def eval_top1(*, model, X, Y, device, batch=256, two_body_max=None):
609    """Top-1 accuracy. two_body_max: (N, C, T, V) second-body tensor — logits
610    are elementwise-maxed over the two single-body passes, but ONLY for samples
611    whose second body is actually present. An all-zero second body (the common
612    single-actor case) still produces finite logits that would otherwise win
613    the max and wreck accuracy, so those samples keep body-1 logits alone."""
614    model.eval()
615    correct = 0
616    with torch.no_grad():
617        for b0 in range(0, len(Y), batch):
618            xb = X[b0: b0 + batch].to(device).float()
619            logits = model(xb)
620            if two_body_max is not None:
621                x2 = two_body_max[b0: b0 + batch].to(device).float()
622                present = (x2.abs().flatten(1).sum(1) > 0)  # (B,)
623                l2 = model(x2)
624                logits = torch.where(present[:, None], torch.maximum(logits, l2), logits)
625            correct += (logits.argmax(1).cpu() == Y[b0: b0 + batch]).sum().item()
626    return correct / len(Y)

HMDB51 and UCF101 execution: pysklBench.py

SHA-256 of complete original file: bd735aff62734666b09d145b6b6b1f17a2797d711c418a6aabe870805b095f43

34
35# COCO-17 keypoint order (HRNet / DETRPose).
36COCO17_JOINTS = [
37    "nose", "L-eye", "R-eye", "L-ear", "R-ear",
38    "L-shoulder", "R-shoulder", "L-elbow", "R-elbow", "L-wrist", "R-wrist",
39    "L-hip", "R-hip", "L-knee", "R-knee", "L-ankle", "R-ankle",
40]
41COCO17_EDGES = [
42    (0, 1), (0, 2), (1, 3), (2, 4),          # head
43    (0, 5), (0, 6),                           # nose->shoulders: LINKS head to body

Unrelated source lines omitted.

105
106
107def annotation_to_bodies(ann, t_fix):
108    """One PYSKL annotation -> two (C_IN, t_fix, 17) float32 bodies in their
109    ORIGINAL tracked order (absent 2nd body = zeros).
110
111    Normalization EXACTLY matches PYSKL's PreNormalize2D (mode 'fix') — the
112    reference recipe: center on the IMAGE center and divide by half-image-size,
113    then zero the x,y of any joint whose HRNet confidence <= 0.01. Crucially it
114    does NOT hip-center — keeping absolute in-frame position preserves the spatial
115    relationship between the two actors, which NTU-120's many two-person actions
116    depend on. Bones/velocities derive from these image-centered coords.
117    """
118    kp = ann["keypoint"].astype(np.float32)        # (M, T, 17, 2) pixels
119    sc = ann["keypoint_score"].astype(np.float32)  # (M, T, 17)
120    h, w = ann["img_shape"]
121    kp = kp.copy()
122    kp[..., 0] = (kp[..., 0] - w / 2.0) / (w / 2.0)   # image-center, -> ~[-1,1]
123    kp[..., 1] = (kp[..., 1] - h / 2.0) / (h / 2.0)
124    low = sc <= 0.01                                  # zero out low-confidence joints
125    kp[..., 0][low] = 0.0
126    kp[..., 1][low] = 0.0
127
128    outs = []
129    for m in range(kp.shape[0]):
130        coords = _uniform_sample(kp[m], t_fix)    # (t_fix, 17, 2)
131        conf = _uniform_sample(sc[m], t_fix)      # (t_fix, 17)
132        vel = np.zeros_like(coords); vel[1:] = coords[1:] - coords[:-1]
133        bone = coords - coords[:, BONE_PARENT, :]
134        bvel = np.zeros_like(bone); bvel[1:] = bone[1:] - bone[:-1]
135        feat = np.concatenate(
136            [coords, conf[..., None], vel, bone, bvel], axis=2
137        ).transpose(2, 0, 1).astype(np.float32)   # (9, t_fix, 17)
138        outs.append(feat)
139
140    bodies = outs[:2]
141    while len(bodies) < 2:
142        bodies.append(np.zeros((C_IN, t_fix, V), dtype=np.float32))
143    return bodies[0], bodies[1]
144
145
146def _uniform_sample(arr, t_fix):
147    """(T, ...) -> (t_fix, ...) uniform index sampling; short clips repeat."""
148    t = arr.shape[0]
149    idx = np.linspace(0, t - 1, t_fix).round().astype(int)
150    return arr[idx]

HMDB51 and UCF101 execution: pysklBenchmark.py

SHA-256 of complete original file: 657a1e9aee900bfa903cc0498a95a41cc0c10bf2cc796ff4318a1fb0d37a57c9

118CH = {"x": 0, "y": 1, "conf": 2, "dx": 3, "dy": 4, "bx": 5, "by": 6, "dbx": 7, "dby": 8}
119STREAM_CHANNELS = {
120    "joint":  ["x", "y", "conf", "dx", "dy"],           # positions + motion (the original)
121    "bone":   ["bx", "by", "conf", "dbx", "dby"],        # local limb geometry + its motion
122    "motion": ["dx", "dy", "dbx", "dby"],                # pure velocity
123    "all":    ["x", "y", "conf", "dx", "dy", "bx", "by", "dbx", "dby"],  # input-level fusion
124    # The canonical 4-stream fusion (each 3ch: spatial pair + conf), the literature
125    # standard. augment_batch rotates channels [0,1], correct for all four.
126    "j":  ["x", "y", "conf"],       # joint positions
127    "b":  ["bx", "by", "conf"],     # bone vectors
128    "jm": ["dx", "dy", "conf"],     # joint motion
129    "bm": ["dbx", "dby", "conf"],   # bone motion
130}
131FOUR_STREAM = "j,b,jm,bm"
132
133
134def resolve_channels(spec):
135    names = STREAM_CHANNELS[spec] if spec in STREAM_CHANNELS else spec.split(",")
136    return [CH[n] for n in names], names
137
138
139def parse_dilations(spec):
140    return [int(x) for x in spec.split(",")] if spec else None
141
142
143def predict_probs(model, X, device, batch=256):
144    model.eval()
145    out = []
146    with torch.no_grad():
147        for b0 in range(0, len(X), batch):
148            xb = X[b0:b0 + batch].to(device).float()
149            out.append(torch.softmax(model(xb), 1).cpu())
150    return torch.cat(out)
151
152
153def train_stream(*, seed, d, cfg, args, device, ch_idx, dilations, hidden, log, tag=""):
154    """Train one LegibleTCN on a channel subset; return (probs_test, nparam, secs)."""
155    torch.manual_seed(seed); np.random.seed(seed)
156    model = build_model(arch=args.arch, c_in=len(ch_idx),
157                        num_classes=cfg["num_classes"], args=args,

Unrelated source lines omitted.

300    data = load_dataset(pkl_path=pkl_path, dataset=dataset, t_fix=t_fix, test_len=tl,
301                        limit=limit, log=log)
302    log(f"[bench] adapted in {time.time()-t0:.0f}s; caching -> {cache}")
303    np.savez_compressed(cache, **data)
304    return data
305
306
307def run_one_seed(*, seed, d, cfg, args, device, log):
308    torch.manual_seed(seed)
309    np.random.seed(seed)
310    ch_idx, _ = resolve_channels(args.channels)
311    model = LegibleTCN(v=V, c_in=len(ch_idx), num_classes=cfg["num_classes"],
312                       filters=args.filters, hidden=args.hidden,
313                       temporal_dilations=parse_dilations(args.dilations)).to(device)
314    nparam = sum(p.numel() for p in model.parameters())
315    X = torch.from_numpy(d["x1_train"][:, ch_idx])
316    Y = torch.from_numpy(d["y_train"])
317    Xte = torch.from_numpy(d["x1_test"][:, ch_idx])
318    Yte = torch.from_numpy(d["y_test"])
319    log(f"[bench] seed={seed} train={len(Y)} test={len(Yte)} "
320        f"classes={cfg['num_classes']} params={nparam/1e3:.1f}k "
321        f"filters={args.filters} hidden={args.hidden} lr={args.lr} epochs={args.epochs}")
322    t0 = time.time()
323    train_classifier(model=model, X=X, Y=Y, epochs=args.epochs, batch=args.batch,
324                     device=device, lr=args.lr, log=lambda m: log(f"[bench] s{seed} {m}"),
325                     log_every=max(1, args.epochs // 8), val=(Xte, Yte))
326    acc1 = eval_top1(model=model, X=Xte, Y=Yte, device=device)
327    row = {"seed": seed, "acc1": acc1, "params": nparam, "secs": time.time() - t0}
328    predicted = predict_probs(model, Xte, device).argmax(1)
329    class_counts = torch.bincount(Yte, minlength=cfg["num_classes"])
330    assert torch.all(class_counts > 0), "Every class must be represented in the test split"
331    class_correct = torch.bincount(Yte[predicted == Yte], minlength=cfg["num_classes"])
332    row["mean_class_accuracy"] = (class_correct / class_counts).mean().item()
333    if args.results_json:
334        checkpoint = os.path.splitext(args.results_json)[0] + f"-seed{seed}.pt"
335        torch.save(model.state_dict(), checkpoint)
336        row["checkpoint_bytes"] = os.path.getsize(checkpoint)
337    if cfg["two_body"]:
338        Xte2 = torch.from_numpy(d["x2_test"][:, ch_idx])
339        row["acc2"] = eval_top1(model=model, X=Xte, Y=Yte, device=device, two_body_max=Xte2)
340    log(f"[bench] seed={seed} top-1 body1={100*acc1:.2f}%"
341        + (f" max2body={100*row['acc2']:.2f}%" if 'acc2' in row else "")
342        + f" [{row['secs']:.0f}s]")
343    return row
344
345
346def dump_errors(*, dataset, d, cfg, args, device, out_path, log):

Unrelated source lines omitted.

440    ap.add_argument("--data-dir", default="~/Datasets/pyskl")
441    ap.add_argument("--cache-dir", default="~/Datasets/pyskl/cache")
442    ap.add_argument("--epochs", type=int, default=80)
443    ap.add_argument("--batch", type=int, default=64)
444    ap.add_argument("--lr", type=float, default=2e-3)
445    ap.add_argument("--filters", type=int, default=8)
446    ap.add_argument("--hidden", type=int, default=96)
447    ap.add_argument("--t-fix", type=int, default=64)
448    ap.add_argument("--seeds", default="42,43,44")
449    ap.add_argument("--limit", type=int, default=0, help="pilot: random subset of N annotations")
450    ap.add_argument("--adapt-only", action="store_true", help="build the cache and exit (no training)")
451    ap.add_argument("--dump-errors", default=None, help="train 1 seed, write per-class/confusion JSON to this path")
452    # levers 1-3
453    ap.add_argument("--channels", default="joint", help="joint|bone|motion|all or comma channel names (single-stream)")
454    ap.add_argument("--streams", default=None, help="comma stream names for multi-stream ensemble, e.g. joint,bone,motion")
455    ap.add_argument("--dilations", default=None, help="temporal dilations, e.g. 1,2,4 (bigger receptive field)")
456    ap.add_argument("--experiment", action="store_true", help="run the channels/streams/capacity experiment path")
457    ap.add_argument("--arch", default="legible", choices=["legible", "graph"], help="legible per-joint TCN or compact ST-GCN")
458    ap.add_argument("--gchannels", default="64,64,128,128", help="GraphTCN block channels")
459    ap.add_argument("--partitioned", action="store_true", help="ST-GCN 3-partition adjacency (self/centripetal/centrifugal)")
460    # full-recipe knobs
461    ap.add_argument("--optimizer", default="adamw", choices=["adamw", "sgd"])
462    ap.add_argument("--weight-decay", type=float, default=1e-4)
463    ap.add_argument("--warmup", type=int, default=0, help="linear warmup epochs before cosine")
464    ap.add_argument("--augment", action="store_true", help="on-the-fly skeleton augmentation (rotation/scale/jitter)")
465    ap.add_argument("--four-stream", action="store_true", help="shortcut for --streams j,b,jm,bm (canonical 4-stream)")
466    ap.add_argument("--reference", action="store_true", help="PYSKL reference recipe: image-norm, no aug")
467    ap.add_argument("--num-person", type=int, default=2, help="1 = single actor (single-person case); 2 = both actors")
468    ap.add_argument("--test-len", type=int, default=None, help="frames stored per test seq (>= t_fix) so test clips can differ")
469    ap.add_argument("--test-clips", type=int, default=1, help="N-clip test-time averaging (PYSKL uses 10)")
470    ap.add_argument("--stgcnpp", action="store_true", help="SOTA grafts: learnable adaptive graph + multi-scale temporal conv (ST-GCN++)")
471    ap.add_argument("--ctrgc", action="store_true", help="CTR-GCN channel-wise topology refinement (+ MS-TCN)")
472    ap.add_argument("--dct", action="store_true", help="NOVEL: DCT frequency-domain temporal residual with learnable band gains")
473    ap.add_argument("--antisym", action="store_true", help="NOVEL: time-reversal antisymmetry feature (motion direction) for reversal-pair classes")
474    args = ap.parse_args()
475    if args.four_stream:
476        args.streams = FOUR_STREAM
477
478    data_dir = os.path.expanduser(args.data_dir)

Diving48, Gym99, NTU60 and NTU120 execution: skeletonBenchCommon.py

SHA-256 of complete original file: e4655a8691afef33713e9b0e63ce7fe8ff3cee4b893947dcd0aa74022dde65cd

43class TimestepGroupNorm(nn.Module):
44    """GroupNorm with per-timestep statistics: each frame is normalized over
45    its channel group only. Standard GroupNorm on (N, C, T) pools stats over
46    the WHOLE clip, so a causal conv stack followed by it is not actually
47    causal — this is."""
48
49    def __init__(self, num_groups, num_channels):
50        super().__init__()
51        self.num_groups = num_groups
52        self.weight = nn.Parameter(torch.ones(num_channels))
53        self.bias = nn.Parameter(torch.zeros(num_channels))
54
55    def forward(self, x):  # (N, C, T)
56        n, c, t = x.shape
57        z = x.permute(0, 2, 1).reshape(n * t, c)
58        z = F.group_norm(z, self.num_groups, self.weight, self.bias)
59        return z.reshape(n, t, c).permute(0, 2, 1)
60
61
62class LegibleTCN(nn.Module):
63    """Same architecture as the BoxingVI model, with V/C_IN/classes/K as args.
64
65    causal=True: left-pad only — every conv sees ONLY past frames, so the model
66    can emit a verdict at every frame with zero lookahead. The head is linear,
67    so applying it before or after temporal mean-pooling is identical.
68    """
69
70    def __init__(self, *, v, c_in, num_classes, filters=8, hidden=64, k=9,
71                 causal=False, attn=False, per_timestep_norm=None,
72                 temporal_dilations=None):
73        super().__init__()
74        self.v = v
75        self.c_in = c_in
76        self.filters = filters
77        self.hidden = hidden
78        self.k = k
79        self.causal = causal
80        self.attn = attn
81        self.last_a = None
82        # temporal_dilations: WaveNet-style dilated depthwise stack replacing the
83        # single l3, to see WHOLE actions. Receptive field (causal) =
84        #   (k-1)   [l1]  +  (k-1)*Σ dilations  [stack]  + 1.
85        # PKU-MMD actions median 98 frames; [1,2,4,8] k9 → ~129-frame (4.3s)
86        # field. Each depthwise layer stays a NAMED per-unit temporal template
87        # at its dilation, so legibility survives. None → original single l3.
88        self.temporal_dilations = temporal_dilations
89        pad = 0 if causal else k // 2
90        # (N, V*C_IN, T) grouped by joint: each group reads one joint's channels.
91        self.l1 = nn.Conv1d(v * c_in, v * filters, k, padding=pad, groups=v)
92        # GroupNorm, not BatchNorm: BN's fold-dependent batch statistics caused
93        # a deterministic training collapse on the BoxingVI V4 holdout.
94        # Per-timestep GroupNorm is REQUIRED whenever activations are consumed
95        # per-frame at a different sequence length than training:
96        #   - causal streaming (else the norm leaks the future — 1.14 logit delta)
97        #   - framewise DETECTION trained on short windows but run on long
98        #     sequences (plain GroupNorm pools over T, so its stats depend on
99        #     sequence length → train/test mismatch → mAP collapses to ~0.01).
100        # Clip classification (fixed T, mean-pooled) is fine with plain GN.
101        # Default: tie to causal; detectors pass per_timestep_norm=True explicitly.
102        use_ts = causal if per_timestep_norm is None else per_timestep_norm
103        norm = TimestepGroupNorm if use_ts else nn.GroupNorm
104        self.bn1 = norm(v, v * filters)
105        self.mix = nn.Conv1d(v * filters, hidden, 1)
106        self.bn2 = norm(8, hidden)
107        if temporal_dilations:
108            self._norm_cls = norm
109            self.l3_stack = nn.ModuleList([
110                nn.Conv1d(hidden, hidden, k, padding=(0 if causal else (k // 2) * d),
111                          dilation=d, groups=hidden)
112                for d in temporal_dilations
113            ])
114            self.l3_norms = nn.ModuleList([norm(8, hidden) for _ in temporal_dilations])
115        else:
116            self.l3 = nn.Conv1d(hidden, hidden, k, padding=pad, groups=hidden)
117            self.bn3 = norm(8, hidden)
118        self.head = nn.Linear(hidden, num_classes)
119        self.att = nn.Conv1d(hidden, 1, 1) if attn else None
120
121    def _trunk(self, x):  # x: (N, C_IN, T, V)
122        n, c, t, v = x.shape
123        z = x.permute(0, 3, 1, 2).reshape(n, v * c, t)  # joint-major
124        if self.causal:
125            z = F.pad(z, (self.k - 1, 0))
126        z = F.relu(self.bn1(self.l1(z)))
127        z = F.relu(self.bn2(self.mix(z)))
128        if self.temporal_dilations:
129            # Dilated depthwise TCN stack with residual connections; each layer
130            # left-pads (k-1)*d in causal mode so it stays zero-lookahead.
131            for conv, bn, d in zip(self.l3_stack, self.l3_norms, self.temporal_dilations):
132                res = z
133                if self.causal:
134                    z = F.pad(z, ((self.k - 1) * d, 0))
135                z = F.relu(bn(conv(z)) + res)
136            return z
137        if self.causal:
138            z = F.pad(z, (self.k - 1, 0))
139        return F.relu(self.bn3(self.l3(z)))  # (N, H, T)
140
141    def forward(self, x):
142        z = self._trunk(x)
143        if self.att is not None:
144            a = torch.sigmoid(self.att(z))
145            self.last_a = a.squeeze(1)
146            fw = self.head(z.transpose(1, 2))
147            return (a.transpose(1, 2) * fw).sum(1) / (1 + a.sum(-1))
148        return self.head(z.mean(-1))
149
150    def framewise_logits(self, x):  # (N, T, classes), zero lookahead when causal
151        return self.head(self._trunk(x).transpose(1, 2))

Unrelated source lines omitted.

551def train_classifier(*, model, X, Y, epochs, batch, device, lr=2e-3,
552                     log=print, log_every=1, val=None, optimizer="adamw",
553                     weight_decay=1e-4, warmup_epochs=0, augment=False):
554    """X: (N, C, T, V) float32 CPU tensor; Y: (N,) int64 CPU tensor. Batches
555    are moved to `device` per step so NTU-scale data stays in host RAM.
556
557    Full-recipe knobs (default off → original BoxingVI behavior): optimizer
558    'sgd' (momentum 0.9) or 'adamw'; weight_decay; linear warmup_epochs then
559    cosine; augment=True enables on-the-fly skeleton augmentation."""
560    num_classes = model.head.out_features
561    counts = np.bincount(Y.numpy(), minlength=num_classes).astype(np.float32)
562    counts[counts == 0] = 1
563    w = torch.from_numpy((1.0 / counts)[Y.numpy()])
564    if optimizer == "sgd":
565        opt = torch.optim.SGD(model.parameters(), lr=lr, momentum=0.9,
566                              weight_decay=weight_decay, nesterov=True)
567    else:
568        opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=weight_decay)
569    # linear warmup for warmup_epochs, then cosine to 0 over the remainder
570    def lr_at(ep):
571        if warmup_epochs and ep < warmup_epochs:
572            return (ep + 1) / warmup_epochs
573        prog = (ep - warmup_epochs) / max(1, epochs - warmup_epochs)
574        return 0.5 * (1 + np.cos(np.pi * prog))
575    sched = torch.optim.lr_scheduler.LambdaLR(opt, lr_lambda=lr_at)
576    n = len(Y)
577    import time as _time
578    for ep in range(epochs):
579        t0 = _time.time()
580        model.train()
581        idx = torch.multinomial(w, n, replacement=True)
582        tot = 0.0
583        for b0 in range(0, n, batch):
584            bidx = idx[b0: b0 + batch]
585            # NO non_blocking=True: async copies from unpinned host memory race
586            # on MPS — the labels arrive as garbage (observed: loss pinned at
587            # the label-smoothing floor while accuracy stays at chance).
588            # .float(): host X may be float16 (PYSKL benchmarks keep it half to
589            # fit Kinetics-scale data in 16GB RAM); the model runs in float32.
590            xb = X[bidx].to(device).float()
591            if augment:
592                xb = augment_batch(xb)
593            yb = Y[bidx].to(device)
594            loss = F.cross_entropy(model(xb), yb, label_smoothing=0.1)
595            opt.zero_grad()
596            loss.backward()
597            opt.step()
598            tot += loss.item() * len(bidx)
599        sched.step()
600        if (ep + 1) % log_every == 0 or ep == 0:
601            msg = f"epoch {ep+1}/{epochs} loss={tot/n:.4f} [{_time.time()-t0:.0f}s]"
602            if val is not None and ((ep + 1) % (log_every * 5) == 0 or ep + 1 == epochs):
603                msg += f" val={eval_top1(model=model, X=val[0], Y=val[1], device=device)*100:.1f}%"
604            log(msg)
605    return model
606
607
608def eval_top1(*, model, X, Y, device, batch=256, two_body_max=None):
609    """Top-1 accuracy. two_body_max: (N, C, T, V) second-body tensor — logits
610    are elementwise-maxed over the two single-body passes, but ONLY for samples
611    whose second body is actually present. An all-zero second body (the common
612    single-actor case) still produces finite logits that would otherwise win
613    the max and wreck accuracy, so those samples keep body-1 logits alone."""
614    model.eval()
615    correct = 0
616    with torch.no_grad():
617        for b0 in range(0, len(Y), batch):
618            xb = X[b0: b0 + batch].to(device).float()
619            logits = model(xb)
620            if two_body_max is not None:
621                x2 = two_body_max[b0: b0 + batch].to(device).float()
622                present = (x2.abs().flatten(1).sum(1) > 0)  # (B,)
623                l2 = model(x2)
624                logits = torch.where(present[:, None], torch.maximum(logits, l2), logits)
625            correct += (logits.argmax(1).cpu() == Y[b0: b0 + batch]).sum().item()
626    return correct / len(Y)

Diving48, Gym99, NTU60 and NTU120 execution: pysklBench.py

SHA-256 of complete original file: 83ab7c94a16349d5fe369113244f76fbb708fd77c8fbe619943e0155083c311e

34
35# COCO-17 keypoint order (HRNet / DETRPose).
36COCO17_JOINTS = [
37    "nose", "L-eye", "R-eye", "L-ear", "R-ear",
38    "L-shoulder", "R-shoulder", "L-elbow", "R-elbow", "L-wrist", "R-wrist",
39    "L-hip", "R-hip", "L-knee", "R-knee", "L-ankle", "R-ankle",
40]
41COCO17_EDGES = [
42    (0, 1), (0, 2), (1, 3), (2, 4),          # head
43    (0, 5), (0, 6),                           # nose->shoulders: LINKS head to body

Unrelated source lines omitted.

105
106
107def annotation_to_bodies(ann, t_fix):
108    """One PYSKL annotation -> two (C_IN, t_fix, 17) float32 bodies in their
109    ORIGINAL tracked order (absent 2nd body = zeros).
110
111    Normalization EXACTLY matches PYSKL's PreNormalize2D (mode 'fix') — the
112    reference recipe: center on the IMAGE center and divide by half-image-size,
113    then zero the x,y of any joint whose HRNet confidence <= 0.01. Crucially it
114    does NOT hip-center — keeping absolute in-frame position preserves the spatial
115    relationship between the two actors, which NTU-120's many two-person actions
116    depend on. Bones/velocities derive from these image-centered coords.
117    """
118    kp = ann["keypoint"].astype(np.float32)        # (M, T, 17, 2) pixels
119    sc = ann["keypoint_score"].astype(np.float32)  # (M, T, 17)
120    h, w = ann["img_shape"]
121    kp = kp.copy()
122    kp[..., 0] = (kp[..., 0] - w / 2.0) / (w / 2.0)   # image-center, -> ~[-1,1]
123    kp[..., 1] = (kp[..., 1] - h / 2.0) / (h / 2.0)
124    low = sc <= 0.01                                  # zero out low-confidence joints
125    kp[..., 0][low] = 0.0
126    kp[..., 1][low] = 0.0
127
128    outs = []
129    for m in range(kp.shape[0]):
130        coords = _uniform_sample(kp[m], t_fix)    # (t_fix, 17, 2)
131        conf = _uniform_sample(sc[m], t_fix)      # (t_fix, 17)
132        vel = np.zeros_like(coords); vel[1:] = coords[1:] - coords[:-1]
133        bone = coords - coords[:, BONE_PARENT, :]
134        bvel = np.zeros_like(bone); bvel[1:] = bone[1:] - bone[:-1]
135        feat = np.concatenate(
136            [coords, conf[..., None], vel, bone, bvel], axis=2
137        ).transpose(2, 0, 1).astype(np.float32)   # (9, t_fix, 17)
138        outs.append(feat)
139
140    bodies = outs[:2]
141    while len(bodies) < 2:
142        bodies.append(np.zeros((C_IN, t_fix, V), dtype=np.float32))
143    return bodies[0], bodies[1]
144
145
146def _uniform_sample(arr, t_fix):
147    """(T, ...) -> (t_fix, ...) uniform index sampling; short clips repeat."""
148    t = arr.shape[0]
149    idx = np.linspace(0, t - 1, t_fix).round().astype(int)
150    return arr[idx]

Diving48, Gym99, NTU60 and NTU120 execution: pysklBenchmark.py

SHA-256 of complete original file: 98f3b41e8bf3e7614ebfc5d52e8dae79ac9a2e4a4b2ce985e16d52f0e258f156

118CH = {"x": 0, "y": 1, "conf": 2, "dx": 3, "dy": 4, "bx": 5, "by": 6, "dbx": 7, "dby": 8}
119STREAM_CHANNELS = {
120    "joint":  ["x", "y", "conf", "dx", "dy"],           # positions + motion (the original)
121    "bone":   ["bx", "by", "conf", "dbx", "dby"],        # local limb geometry + its motion
122    "motion": ["dx", "dy", "dbx", "dby"],                # pure velocity
123    "all":    ["x", "y", "conf", "dx", "dy", "bx", "by", "dbx", "dby"],  # input-level fusion
124    # The canonical 4-stream fusion (each 3ch: spatial pair + conf), the literature
125    # standard. augment_batch rotates channels [0,1], correct for all four.
126    "j":  ["x", "y", "conf"],       # joint positions
127    "b":  ["bx", "by", "conf"],     # bone vectors
128    "jm": ["dx", "dy", "conf"],     # joint motion
129    "bm": ["dbx", "dby", "conf"],   # bone motion
130}
131FOUR_STREAM = "j,b,jm,bm"
132
133
134def resolve_channels(spec):
135    names = STREAM_CHANNELS[spec] if spec in STREAM_CHANNELS else spec.split(",")
136    return [CH[n] for n in names], names
137
138
139def parse_dilations(spec):
140    return [int(x) for x in spec.split(",")] if spec else None
141
142
143def predict_probs(model, X, device, batch=256):
144    model.eval()
145    out = []
146    with torch.no_grad():
147        for b0 in range(0, len(X), batch):
148            xb = X[b0:b0 + batch].to(device).float()
149            out.append(torch.softmax(model(xb), 1).cpu())
150    return torch.cat(out)
151
152
153def train_stream(*, seed, d, cfg, args, device, ch_idx, dilations, hidden, log, tag=""):
154    """Train one LegibleTCN on a channel subset; return (probs_test, nparam, secs)."""
155    torch.manual_seed(seed); np.random.seed(seed)
156    model = build_model(arch=args.arch, c_in=len(ch_idx),
157                        num_classes=cfg["num_classes"], args=args,

Unrelated source lines omitted.

300    data = load_dataset(pkl_path=pkl_path, dataset=dataset, t_fix=t_fix, test_len=tl,
301                        limit=limit, log=log)
302    log(f"[bench] adapted in {time.time()-t0:.0f}s; caching -> {cache}")
303    np.savez_compressed(cache, **data)
304    return data
305
306
307def run_one_seed(*, seed, d, cfg, args, device, log):
308    torch.manual_seed(seed)
309    np.random.seed(seed)
310    ch_idx, _ = resolve_channels(args.channels)
311    model = LegibleTCN(v=V, c_in=len(ch_idx), num_classes=cfg["num_classes"],
312                       filters=args.filters, hidden=args.hidden,
313                       temporal_dilations=parse_dilations(args.dilations)).to(device)
314    nparam = sum(p.numel() for p in model.parameters())
315    X = torch.from_numpy(d["x1_train"][:, ch_idx])
316    Y = torch.from_numpy(d["y_train"])
317    Xte = torch.from_numpy(d["x1_test"][:, ch_idx])
318    Yte = torch.from_numpy(d["y_test"])
319    log(f"[bench] seed={seed} train={len(Y)} test={len(Yte)} "
320        f"classes={cfg['num_classes']} params={nparam/1e3:.1f}k "
321        f"filters={args.filters} hidden={args.hidden} lr={args.lr} epochs={args.epochs}")
322    t0 = time.time()
323    train_classifier(model=model, X=X, Y=Y, epochs=args.epochs, batch=args.batch,
324                     device=device, lr=args.lr, log=lambda m: log(f"[bench] s{seed} {m}"),
325                     log_every=max(1, args.epochs // 8), val=(Xte, Yte))
326    acc1 = eval_top1(model=model, X=Xte, Y=Yte, device=device)
327    row = {"seed": seed, "acc1": acc1, "params": nparam, "secs": time.time() - t0}
328    predicted = predict_probs(model, Xte, device).argmax(1)
329    class_counts = torch.bincount(Yte, minlength=cfg["num_classes"])
330    class_correct = torch.bincount(Yte[predicted == Yte], minlength=cfg["num_classes"])
331    supported = class_counts > 0
332    row["mean_class_accuracy"] = (class_correct[supported] / class_counts[supported]).mean().item()
333    row["test_class_counts"] = class_counts.tolist()
334    row["test_class_correct"] = class_correct.tolist()
335    row["test_classes_present"] = int(supported.sum())
336    if args.results_json:
337        checkpoint = os.path.splitext(args.results_json)[0] + f"-seed{seed}.pt"
338        torch.save(model.state_dict(), checkpoint)
339        row["checkpoint_bytes"] = os.path.getsize(checkpoint)
340    if cfg["two_body"]:
341        Xte2 = torch.from_numpy(d["x2_test"][:, ch_idx])
342        row["acc2"] = eval_top1(model=model, X=Xte, Y=Yte, device=device, two_body_max=Xte2)
343    log(f"[bench] seed={seed} top-1 body1={100*acc1:.2f}%"
344        + (f" max2body={100*row['acc2']:.2f}%" if 'acc2' in row else "")
345        + f" [{row['secs']:.0f}s]")
346    return row

Unrelated source lines omitted.

440    ap = argparse.ArgumentParser()
441    ap.add_argument("--dataset", required=True, choices=list(SPLITS.keys()))
442    ap.add_argument("--results-json", help="Save final-epoch metrics, recipe, sizes and per-seed checkpoints")
443    ap.add_argument("--inspect-data", action="store_true", help="Print annotation schema and split counts without adapting or training")
444    ap.add_argument("--data-dir", default="~/Datasets/pyskl")
445    ap.add_argument("--cache-dir", default="~/Datasets/pyskl/cache")
446    ap.add_argument("--epochs", type=int, default=80)
447    ap.add_argument("--batch", type=int, default=64)
448    ap.add_argument("--lr", type=float, default=2e-3)
449    ap.add_argument("--filters", type=int, default=8)
450    ap.add_argument("--hidden", type=int, default=96)
451    ap.add_argument("--t-fix", type=int, default=64)
452    ap.add_argument("--seeds", default="42,43,44")
453    ap.add_argument("--limit", type=int, default=0, help="pilot: random subset of N annotations")
454    ap.add_argument("--adapt-only", action="store_true", help="build the cache and exit (no training)")
455    ap.add_argument("--dump-errors", default=None, help="train 1 seed, write per-class/confusion JSON to this path")
456    # levers 1-3
457    ap.add_argument("--channels", default="joint", help="joint|bone|motion|all or comma channel names (single-stream)")
458    ap.add_argument("--streams", default=None, help="comma stream names for multi-stream ensemble, e.g. joint,bone,motion")
459    ap.add_argument("--dilations", default=None, help="temporal dilations, e.g. 1,2,4 (bigger receptive field)")
460    ap.add_argument("--experiment", action="store_true", help="run the channels/streams/capacity experiment path")
461    ap.add_argument("--arch", default="legible", choices=["legible", "graph"], help="legible per-joint TCN or compact ST-GCN")
462    ap.add_argument("--gchannels", default="64,64,128,128", help="GraphTCN block channels")
463    ap.add_argument("--partitioned", action="store_true", help="ST-GCN 3-partition adjacency (self/centripetal/centrifugal)")
464    # full-recipe knobs
465    ap.add_argument("--optimizer", default="adamw", choices=["adamw", "sgd"])
466    ap.add_argument("--weight-decay", type=float, default=1e-4)
467    ap.add_argument("--warmup", type=int, default=0, help="linear warmup epochs before cosine")
468    ap.add_argument("--augment", action="store_true", help="on-the-fly skeleton augmentation (rotation/scale/jitter)")
469    ap.add_argument("--four-stream", action="store_true", help="shortcut for --streams j,b,jm,bm (canonical 4-stream)")
470    ap.add_argument("--reference", action="store_true", help="PYSKL reference recipe: image-norm, no aug")
471    ap.add_argument("--num-person", type=int, default=2, help="1 = single actor (single-person case); 2 = both actors")
472    ap.add_argument("--test-len", type=int, default=None, help="frames stored per test seq (>= t_fix) so test clips can differ")
473    ap.add_argument("--test-clips", type=int, default=1, help="N-clip test-time averaging (PYSKL uses 10)")
474    ap.add_argument("--stgcnpp", action="store_true", help="SOTA grafts: learnable adaptive graph + multi-scale temporal conv (ST-GCN++)")
475    ap.add_argument("--ctrgc", action="store_true", help="CTR-GCN channel-wise topology refinement (+ MS-TCN)")
476    ap.add_argument("--dct", action="store_true", help="NOVEL: DCT frequency-domain temporal residual with learnable band gains")
477    ap.add_argument("--antisym", action="store_true", help="NOVEL: time-reversal antisymmetry feature (motion direction) for reversal-pair classes")
478    args = ap.parse_args()