Line-numbered excerpts from the actual GPU executions on 15 September 2026. Every complete source file was checked against its recorded SHA-256 before these excerpts were generated. These are archived references, not the current development files.
The model implementation is identical in both executions. Adapter and reporting fixes between runs are recorded separately. Optional graph, dilation, ensemble and attention code does not mean those paths were enabled: the published clip suite uses LegibleTCN, no dilation, one joint stream, no attention and the first actor. The saved recipe and evaluation record identify the active configuration.
SHA-256 of complete original file: e4655a8691afef33713e9b0e63ce7fe8ff3cee4b893947dcd0aa74022dde65cd
43class TimestepGroupNorm(nn.Module):44"""GroupNorm with per-timestep statistics: each frame is normalized over45its channel group only. Standard GroupNorm on (N, C, T) pools stats over46the WHOLE clip, so a causal conv stack followed by it is not actually47causal — this is."""4849def __init__(self, num_groups, num_channels):50super().__init__()51self.num_groups = num_groups52self.weight = nn.Parameter(torch.ones(num_channels))53self.bias = nn.Parameter(torch.zeros(num_channels))5455def forward(self, x): # (N, C, T)56n, c, t = x.shape57z = x.permute(0, 2, 1).reshape(n * t, c)58z = F.group_norm(z, self.num_groups, self.weight, self.bias)59return z.reshape(n, t, c).permute(0, 2, 1)606162class LegibleTCN(nn.Module):63"""Same architecture as the BoxingVI model, with V/C_IN/classes/K as args.6465causal=True: left-pad only — every conv sees ONLY past frames, so the model66can emit a verdict at every frame with zero lookahead. The head is linear,67so applying it before or after temporal mean-pooling is identical.68"""6970def __init__(self, *, v, c_in, num_classes, filters=8, hidden=64, k=9,71causal=False, attn=False, per_timestep_norm=None,72temporal_dilations=None):73super().__init__()74self.v = v75self.c_in = c_in76self.filters = filters77self.hidden = hidden78self.k = k79self.causal = causal80self.attn = attn81self.last_a = None82# temporal_dilations: WaveNet-style dilated depthwise stack replacing the83# single l3, to see WHOLE actions. Receptive field (causal) =84# (k-1) [l1] + (k-1)*Σ dilations [stack] + 1.85# PKU-MMD actions median 98 frames; [1,2,4,8] k9 → ~129-frame (4.3s)86# field. Each depthwise layer stays a NAMED per-unit temporal template87# at its dilation, so legibility survives. None → original single l3.88self.temporal_dilations = temporal_dilations89pad = 0 if causal else k // 290# (N, V*C_IN, T) grouped by joint: each group reads one joint's channels.91self.l1 = nn.Conv1d(v * c_in, v * filters, k, padding=pad, groups=v)92# GroupNorm, not BatchNorm: BN's fold-dependent batch statistics caused93# a deterministic training collapse on the BoxingVI V4 holdout.94# Per-timestep GroupNorm is REQUIRED whenever activations are consumed95# per-frame at a different sequence length than training:96# - causal streaming (else the norm leaks the future — 1.14 logit delta)97# - framewise DETECTION trained on short windows but run on long98# sequences (plain GroupNorm pools over T, so its stats depend on99# sequence length → train/test mismatch → mAP collapses to ~0.01).100# Clip classification (fixed T, mean-pooled) is fine with plain GN.101# Default: tie to causal; detectors pass per_timestep_norm=True explicitly.102use_ts = causal if per_timestep_norm is None else per_timestep_norm103norm = TimestepGroupNorm if use_ts else nn.GroupNorm104self.bn1 = norm(v, v * filters)105self.mix = nn.Conv1d(v * filters, hidden, 1)106self.bn2 = norm(8, hidden)107if temporal_dilations:108self._norm_cls = norm109self.l3_stack = nn.ModuleList([110nn.Conv1d(hidden, hidden, k, padding=(0 if causal else (k // 2) * d),111dilation=d, groups=hidden)112for d in temporal_dilations113])114self.l3_norms = nn.ModuleList([norm(8, hidden) for _ in temporal_dilations])115else:116self.l3 = nn.Conv1d(hidden, hidden, k, padding=pad, groups=hidden)117self.bn3 = norm(8, hidden)118self.head = nn.Linear(hidden, num_classes)119self.att = nn.Conv1d(hidden, 1, 1) if attn else None120121def _trunk(self, x): # x: (N, C_IN, T, V)122n, c, t, v = x.shape123z = x.permute(0, 3, 1, 2).reshape(n, v * c, t) # joint-major124if self.causal:125z = F.pad(z, (self.k - 1, 0))126z = F.relu(self.bn1(self.l1(z)))127z = F.relu(self.bn2(self.mix(z)))128if self.temporal_dilations:129# Dilated depthwise TCN stack with residual connections; each layer130# left-pads (k-1)*d in causal mode so it stays zero-lookahead.131for conv, bn, d in zip(self.l3_stack, self.l3_norms, self.temporal_dilations):132res = z133if self.causal:134z = F.pad(z, ((self.k - 1) * d, 0))135z = F.relu(bn(conv(z)) + res)136return z137if self.causal:138z = F.pad(z, (self.k - 1, 0))139return F.relu(self.bn3(self.l3(z))) # (N, H, T)140141def forward(self, x):142z = self._trunk(x)143if self.att is not None:144a = torch.sigmoid(self.att(z))145self.last_a = a.squeeze(1)146fw = self.head(z.transpose(1, 2))147return (a.transpose(1, 2) * fw).sum(1) / (1 + a.sum(-1))148return self.head(z.mean(-1))149150def framewise_logits(self, x): # (N, T, classes), zero lookahead when causal151return self.head(self._trunk(x).transpose(1, 2))
Unrelated source lines omitted.
551def train_classifier(*, model, X, Y, epochs, batch, device, lr=2e-3,552log=print, log_every=1, val=None, optimizer="adamw",553weight_decay=1e-4, warmup_epochs=0, augment=False):554"""X: (N, C, T, V) float32 CPU tensor; Y: (N,) int64 CPU tensor. Batches555are moved to `device` per step so NTU-scale data stays in host RAM.556557Full-recipe knobs (default off → original BoxingVI behavior): optimizer558'sgd' (momentum 0.9) or 'adamw'; weight_decay; linear warmup_epochs then559cosine; augment=True enables on-the-fly skeleton augmentation."""560num_classes = model.head.out_features561counts = np.bincount(Y.numpy(), minlength=num_classes).astype(np.float32)562counts[counts == 0] = 1563w = torch.from_numpy((1.0 / counts)[Y.numpy()])564if optimizer == "sgd":565opt = torch.optim.SGD(model.parameters(), lr=lr, momentum=0.9,566weight_decay=weight_decay, nesterov=True)567else:568opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=weight_decay)569# linear warmup for warmup_epochs, then cosine to 0 over the remainder570def lr_at(ep):571if warmup_epochs and ep < warmup_epochs:572return (ep + 1) / warmup_epochs573prog = (ep - warmup_epochs) / max(1, epochs - warmup_epochs)574return 0.5 * (1 + np.cos(np.pi * prog))575sched = torch.optim.lr_scheduler.LambdaLR(opt, lr_lambda=lr_at)576n = len(Y)577import time as _time578for ep in range(epochs):579t0 = _time.time()580model.train()581idx = torch.multinomial(w, n, replacement=True)582tot = 0.0583for b0 in range(0, n, batch):584bidx = idx[b0: b0 + batch]585# NO non_blocking=True: async copies from unpinned host memory race586# on MPS — the labels arrive as garbage (observed: loss pinned at587# the label-smoothing floor while accuracy stays at chance).588# .float(): host X may be float16 (PYSKL benchmarks keep it half to589# fit Kinetics-scale data in 16GB RAM); the model runs in float32.590xb = X[bidx].to(device).float()591if augment:592xb = augment_batch(xb)593yb = Y[bidx].to(device)594loss = F.cross_entropy(model(xb), yb, label_smoothing=0.1)595opt.zero_grad()596loss.backward()597opt.step()598tot += loss.item() * len(bidx)599sched.step()600if (ep + 1) % log_every == 0 or ep == 0:601msg = f"epoch {ep+1}/{epochs} loss={tot/n:.4f} [{_time.time()-t0:.0f}s]"602if val is not None and ((ep + 1) % (log_every * 5) == 0 or ep + 1 == epochs):603msg += f" val={eval_top1(model=model, X=val[0], Y=val[1], device=device)*100:.1f}%"604log(msg)605return model606607608def eval_top1(*, model, X, Y, device, batch=256, two_body_max=None):609"""Top-1 accuracy. two_body_max: (N, C, T, V) second-body tensor — logits610are elementwise-maxed over the two single-body passes, but ONLY for samples611whose second body is actually present. An all-zero second body (the common612single-actor case) still produces finite logits that would otherwise win613the max and wreck accuracy, so those samples keep body-1 logits alone."""614model.eval()615correct = 0616with torch.no_grad():617for b0 in range(0, len(Y), batch):618xb = X[b0: b0 + batch].to(device).float()619logits = model(xb)620if two_body_max is not None:621x2 = two_body_max[b0: b0 + batch].to(device).float()622present = (x2.abs().flatten(1).sum(1) > 0) # (B,)623l2 = model(x2)624logits = torch.where(present[:, None], torch.maximum(logits, l2), logits)625correct += (logits.argmax(1).cpu() == Y[b0: b0 + batch]).sum().item()626return correct / len(Y)
SHA-256 of complete original file: bd735aff62734666b09d145b6b6b1f17a2797d711c418a6aabe870805b095f43
3435# COCO-17 keypoint order (HRNet / DETRPose).36COCO17_JOINTS = [37"nose", "L-eye", "R-eye", "L-ear", "R-ear",38"L-shoulder", "R-shoulder", "L-elbow", "R-elbow", "L-wrist", "R-wrist",39"L-hip", "R-hip", "L-knee", "R-knee", "L-ankle", "R-ankle",40]41COCO17_EDGES = [42(0, 1), (0, 2), (1, 3), (2, 4), # head43(0, 5), (0, 6), # nose->shoulders: LINKS head to body
Unrelated source lines omitted.
105106107def annotation_to_bodies(ann, t_fix):108"""One PYSKL annotation -> two (C_IN, t_fix, 17) float32 bodies in their109ORIGINAL tracked order (absent 2nd body = zeros).110111Normalization EXACTLY matches PYSKL's PreNormalize2D (mode 'fix') — the112reference recipe: center on the IMAGE center and divide by half-image-size,113then zero the x,y of any joint whose HRNet confidence <= 0.01. Crucially it114does NOT hip-center — keeping absolute in-frame position preserves the spatial115relationship between the two actors, which NTU-120's many two-person actions116depend on. Bones/velocities derive from these image-centered coords.117"""118kp = ann["keypoint"].astype(np.float32) # (M, T, 17, 2) pixels119sc = ann["keypoint_score"].astype(np.float32) # (M, T, 17)120h, w = ann["img_shape"]121kp = kp.copy()122kp[..., 0] = (kp[..., 0] - w / 2.0) / (w / 2.0) # image-center, -> ~[-1,1]123kp[..., 1] = (kp[..., 1] - h / 2.0) / (h / 2.0)124low = sc <= 0.01 # zero out low-confidence joints125kp[..., 0][low] = 0.0126kp[..., 1][low] = 0.0127128outs = []129for m in range(kp.shape[0]):130coords = _uniform_sample(kp[m], t_fix) # (t_fix, 17, 2)131conf = _uniform_sample(sc[m], t_fix) # (t_fix, 17)132vel = np.zeros_like(coords); vel[1:] = coords[1:] - coords[:-1]133bone = coords - coords[:, BONE_PARENT, :]134bvel = np.zeros_like(bone); bvel[1:] = bone[1:] - bone[:-1]135feat = np.concatenate(136[coords, conf[..., None], vel, bone, bvel], axis=2137).transpose(2, 0, 1).astype(np.float32) # (9, t_fix, 17)138outs.append(feat)139140bodies = outs[:2]141while len(bodies) < 2:142bodies.append(np.zeros((C_IN, t_fix, V), dtype=np.float32))143return bodies[0], bodies[1]144145146def _uniform_sample(arr, t_fix):147"""(T, ...) -> (t_fix, ...) uniform index sampling; short clips repeat."""148t = arr.shape[0]149idx = np.linspace(0, t - 1, t_fix).round().astype(int)150return arr[idx]
SHA-256 of complete original file: 657a1e9aee900bfa903cc0498a95a41cc0c10bf2cc796ff4318a1fb0d37a57c9
118CH = {"x": 0, "y": 1, "conf": 2, "dx": 3, "dy": 4, "bx": 5, "by": 6, "dbx": 7, "dby": 8}119STREAM_CHANNELS = {120"joint": ["x", "y", "conf", "dx", "dy"], # positions + motion (the original)121"bone": ["bx", "by", "conf", "dbx", "dby"], # local limb geometry + its motion122"motion": ["dx", "dy", "dbx", "dby"], # pure velocity123"all": ["x", "y", "conf", "dx", "dy", "bx", "by", "dbx", "dby"], # input-level fusion124# The canonical 4-stream fusion (each 3ch: spatial pair + conf), the literature125# standard. augment_batch rotates channels [0,1], correct for all four.126"j": ["x", "y", "conf"], # joint positions127"b": ["bx", "by", "conf"], # bone vectors128"jm": ["dx", "dy", "conf"], # joint motion129"bm": ["dbx", "dby", "conf"], # bone motion130}131FOUR_STREAM = "j,b,jm,bm"132133134def resolve_channels(spec):135names = STREAM_CHANNELS[spec] if spec in STREAM_CHANNELS else spec.split(",")136return [CH[n] for n in names], names137138139def parse_dilations(spec):140return [int(x) for x in spec.split(",")] if spec else None141142143def predict_probs(model, X, device, batch=256):144model.eval()145out = []146with torch.no_grad():147for b0 in range(0, len(X), batch):148xb = X[b0:b0 + batch].to(device).float()149out.append(torch.softmax(model(xb), 1).cpu())150return torch.cat(out)151152153def train_stream(*, seed, d, cfg, args, device, ch_idx, dilations, hidden, log, tag=""):154"""Train one LegibleTCN on a channel subset; return (probs_test, nparam, secs)."""155torch.manual_seed(seed); np.random.seed(seed)156model = build_model(arch=args.arch, c_in=len(ch_idx),157num_classes=cfg["num_classes"], args=args,
Unrelated source lines omitted.
300data = load_dataset(pkl_path=pkl_path, dataset=dataset, t_fix=t_fix, test_len=tl,301limit=limit, log=log)302log(f"[bench] adapted in {time.time()-t0:.0f}s; caching -> {cache}")303np.savez_compressed(cache, **data)304return data305306307def run_one_seed(*, seed, d, cfg, args, device, log):308torch.manual_seed(seed)309np.random.seed(seed)310ch_idx, _ = resolve_channels(args.channels)311model = LegibleTCN(v=V, c_in=len(ch_idx), num_classes=cfg["num_classes"],312filters=args.filters, hidden=args.hidden,313temporal_dilations=parse_dilations(args.dilations)).to(device)314nparam = sum(p.numel() for p in model.parameters())315X = torch.from_numpy(d["x1_train"][:, ch_idx])316Y = torch.from_numpy(d["y_train"])317Xte = torch.from_numpy(d["x1_test"][:, ch_idx])318Yte = torch.from_numpy(d["y_test"])319log(f"[bench] seed={seed} train={len(Y)} test={len(Yte)} "320f"classes={cfg['num_classes']} params={nparam/1e3:.1f}k "321f"filters={args.filters} hidden={args.hidden} lr={args.lr} epochs={args.epochs}")322t0 = time.time()323train_classifier(model=model, X=X, Y=Y, epochs=args.epochs, batch=args.batch,324device=device, lr=args.lr, log=lambda m: log(f"[bench] s{seed} {m}"),325log_every=max(1, args.epochs // 8), val=(Xte, Yte))326acc1 = eval_top1(model=model, X=Xte, Y=Yte, device=device)327row = {"seed": seed, "acc1": acc1, "params": nparam, "secs": time.time() - t0}328predicted = predict_probs(model, Xte, device).argmax(1)329class_counts = torch.bincount(Yte, minlength=cfg["num_classes"])330assert torch.all(class_counts > 0), "Every class must be represented in the test split"331class_correct = torch.bincount(Yte[predicted == Yte], minlength=cfg["num_classes"])332row["mean_class_accuracy"] = (class_correct / class_counts).mean().item()333if args.results_json:334checkpoint = os.path.splitext(args.results_json)[0] + f"-seed{seed}.pt"335torch.save(model.state_dict(), checkpoint)336row["checkpoint_bytes"] = os.path.getsize(checkpoint)337if cfg["two_body"]:338Xte2 = torch.from_numpy(d["x2_test"][:, ch_idx])339row["acc2"] = eval_top1(model=model, X=Xte, Y=Yte, device=device, two_body_max=Xte2)340log(f"[bench] seed={seed} top-1 body1={100*acc1:.2f}%"341+ (f" max2body={100*row['acc2']:.2f}%" if 'acc2' in row else "")342+ f" [{row['secs']:.0f}s]")343return row344345346def dump_errors(*, dataset, d, cfg, args, device, out_path, log):
Unrelated source lines omitted.
440ap.add_argument("--data-dir", default="~/Datasets/pyskl")441ap.add_argument("--cache-dir", default="~/Datasets/pyskl/cache")442ap.add_argument("--epochs", type=int, default=80)443ap.add_argument("--batch", type=int, default=64)444ap.add_argument("--lr", type=float, default=2e-3)445ap.add_argument("--filters", type=int, default=8)446ap.add_argument("--hidden", type=int, default=96)447ap.add_argument("--t-fix", type=int, default=64)448ap.add_argument("--seeds", default="42,43,44")449ap.add_argument("--limit", type=int, default=0, help="pilot: random subset of N annotations")450ap.add_argument("--adapt-only", action="store_true", help="build the cache and exit (no training)")451ap.add_argument("--dump-errors", default=None, help="train 1 seed, write per-class/confusion JSON to this path")452# levers 1-3453ap.add_argument("--channels", default="joint", help="joint|bone|motion|all or comma channel names (single-stream)")454ap.add_argument("--streams", default=None, help="comma stream names for multi-stream ensemble, e.g. joint,bone,motion")455ap.add_argument("--dilations", default=None, help="temporal dilations, e.g. 1,2,4 (bigger receptive field)")456ap.add_argument("--experiment", action="store_true", help="run the channels/streams/capacity experiment path")457ap.add_argument("--arch", default="legible", choices=["legible", "graph"], help="legible per-joint TCN or compact ST-GCN")458ap.add_argument("--gchannels", default="64,64,128,128", help="GraphTCN block channels")459ap.add_argument("--partitioned", action="store_true", help="ST-GCN 3-partition adjacency (self/centripetal/centrifugal)")460# full-recipe knobs461ap.add_argument("--optimizer", default="adamw", choices=["adamw", "sgd"])462ap.add_argument("--weight-decay", type=float, default=1e-4)463ap.add_argument("--warmup", type=int, default=0, help="linear warmup epochs before cosine")464ap.add_argument("--augment", action="store_true", help="on-the-fly skeleton augmentation (rotation/scale/jitter)")465ap.add_argument("--four-stream", action="store_true", help="shortcut for --streams j,b,jm,bm (canonical 4-stream)")466ap.add_argument("--reference", action="store_true", help="PYSKL reference recipe: image-norm, no aug")467ap.add_argument("--num-person", type=int, default=2, help="1 = single actor (single-person case); 2 = both actors")468ap.add_argument("--test-len", type=int, default=None, help="frames stored per test seq (>= t_fix) so test clips can differ")469ap.add_argument("--test-clips", type=int, default=1, help="N-clip test-time averaging (PYSKL uses 10)")470ap.add_argument("--stgcnpp", action="store_true", help="SOTA grafts: learnable adaptive graph + multi-scale temporal conv (ST-GCN++)")471ap.add_argument("--ctrgc", action="store_true", help="CTR-GCN channel-wise topology refinement (+ MS-TCN)")472ap.add_argument("--dct", action="store_true", help="NOVEL: DCT frequency-domain temporal residual with learnable band gains")473ap.add_argument("--antisym", action="store_true", help="NOVEL: time-reversal antisymmetry feature (motion direction) for reversal-pair classes")474args = ap.parse_args()475if args.four_stream:476args.streams = FOUR_STREAM477478data_dir = os.path.expanduser(args.data_dir)
SHA-256 of complete original file: e4655a8691afef33713e9b0e63ce7fe8ff3cee4b893947dcd0aa74022dde65cd
43class TimestepGroupNorm(nn.Module):44"""GroupNorm with per-timestep statistics: each frame is normalized over45its channel group only. Standard GroupNorm on (N, C, T) pools stats over46the WHOLE clip, so a causal conv stack followed by it is not actually47causal — this is."""4849def __init__(self, num_groups, num_channels):50super().__init__()51self.num_groups = num_groups52self.weight = nn.Parameter(torch.ones(num_channels))53self.bias = nn.Parameter(torch.zeros(num_channels))5455def forward(self, x): # (N, C, T)56n, c, t = x.shape57z = x.permute(0, 2, 1).reshape(n * t, c)58z = F.group_norm(z, self.num_groups, self.weight, self.bias)59return z.reshape(n, t, c).permute(0, 2, 1)606162class LegibleTCN(nn.Module):63"""Same architecture as the BoxingVI model, with V/C_IN/classes/K as args.6465causal=True: left-pad only — every conv sees ONLY past frames, so the model66can emit a verdict at every frame with zero lookahead. The head is linear,67so applying it before or after temporal mean-pooling is identical.68"""6970def __init__(self, *, v, c_in, num_classes, filters=8, hidden=64, k=9,71causal=False, attn=False, per_timestep_norm=None,72temporal_dilations=None):73super().__init__()74self.v = v75self.c_in = c_in76self.filters = filters77self.hidden = hidden78self.k = k79self.causal = causal80self.attn = attn81self.last_a = None82# temporal_dilations: WaveNet-style dilated depthwise stack replacing the83# single l3, to see WHOLE actions. Receptive field (causal) =84# (k-1) [l1] + (k-1)*Σ dilations [stack] + 1.85# PKU-MMD actions median 98 frames; [1,2,4,8] k9 → ~129-frame (4.3s)86# field. Each depthwise layer stays a NAMED per-unit temporal template87# at its dilation, so legibility survives. None → original single l3.88self.temporal_dilations = temporal_dilations89pad = 0 if causal else k // 290# (N, V*C_IN, T) grouped by joint: each group reads one joint's channels.91self.l1 = nn.Conv1d(v * c_in, v * filters, k, padding=pad, groups=v)92# GroupNorm, not BatchNorm: BN's fold-dependent batch statistics caused93# a deterministic training collapse on the BoxingVI V4 holdout.94# Per-timestep GroupNorm is REQUIRED whenever activations are consumed95# per-frame at a different sequence length than training:96# - causal streaming (else the norm leaks the future — 1.14 logit delta)97# - framewise DETECTION trained on short windows but run on long98# sequences (plain GroupNorm pools over T, so its stats depend on99# sequence length → train/test mismatch → mAP collapses to ~0.01).100# Clip classification (fixed T, mean-pooled) is fine with plain GN.101# Default: tie to causal; detectors pass per_timestep_norm=True explicitly.102use_ts = causal if per_timestep_norm is None else per_timestep_norm103norm = TimestepGroupNorm if use_ts else nn.GroupNorm104self.bn1 = norm(v, v * filters)105self.mix = nn.Conv1d(v * filters, hidden, 1)106self.bn2 = norm(8, hidden)107if temporal_dilations:108self._norm_cls = norm109self.l3_stack = nn.ModuleList([110nn.Conv1d(hidden, hidden, k, padding=(0 if causal else (k // 2) * d),111dilation=d, groups=hidden)112for d in temporal_dilations113])114self.l3_norms = nn.ModuleList([norm(8, hidden) for _ in temporal_dilations])115else:116self.l3 = nn.Conv1d(hidden, hidden, k, padding=pad, groups=hidden)117self.bn3 = norm(8, hidden)118self.head = nn.Linear(hidden, num_classes)119self.att = nn.Conv1d(hidden, 1, 1) if attn else None120121def _trunk(self, x): # x: (N, C_IN, T, V)122n, c, t, v = x.shape123z = x.permute(0, 3, 1, 2).reshape(n, v * c, t) # joint-major124if self.causal:125z = F.pad(z, (self.k - 1, 0))126z = F.relu(self.bn1(self.l1(z)))127z = F.relu(self.bn2(self.mix(z)))128if self.temporal_dilations:129# Dilated depthwise TCN stack with residual connections; each layer130# left-pads (k-1)*d in causal mode so it stays zero-lookahead.131for conv, bn, d in zip(self.l3_stack, self.l3_norms, self.temporal_dilations):132res = z133if self.causal:134z = F.pad(z, ((self.k - 1) * d, 0))135z = F.relu(bn(conv(z)) + res)136return z137if self.causal:138z = F.pad(z, (self.k - 1, 0))139return F.relu(self.bn3(self.l3(z))) # (N, H, T)140141def forward(self, x):142z = self._trunk(x)143if self.att is not None:144a = torch.sigmoid(self.att(z))145self.last_a = a.squeeze(1)146fw = self.head(z.transpose(1, 2))147return (a.transpose(1, 2) * fw).sum(1) / (1 + a.sum(-1))148return self.head(z.mean(-1))149150def framewise_logits(self, x): # (N, T, classes), zero lookahead when causal151return self.head(self._trunk(x).transpose(1, 2))
Unrelated source lines omitted.
551def train_classifier(*, model, X, Y, epochs, batch, device, lr=2e-3,552log=print, log_every=1, val=None, optimizer="adamw",553weight_decay=1e-4, warmup_epochs=0, augment=False):554"""X: (N, C, T, V) float32 CPU tensor; Y: (N,) int64 CPU tensor. Batches555are moved to `device` per step so NTU-scale data stays in host RAM.556557Full-recipe knobs (default off → original BoxingVI behavior): optimizer558'sgd' (momentum 0.9) or 'adamw'; weight_decay; linear warmup_epochs then559cosine; augment=True enables on-the-fly skeleton augmentation."""560num_classes = model.head.out_features561counts = np.bincount(Y.numpy(), minlength=num_classes).astype(np.float32)562counts[counts == 0] = 1563w = torch.from_numpy((1.0 / counts)[Y.numpy()])564if optimizer == "sgd":565opt = torch.optim.SGD(model.parameters(), lr=lr, momentum=0.9,566weight_decay=weight_decay, nesterov=True)567else:568opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=weight_decay)569# linear warmup for warmup_epochs, then cosine to 0 over the remainder570def lr_at(ep):571if warmup_epochs and ep < warmup_epochs:572return (ep + 1) / warmup_epochs573prog = (ep - warmup_epochs) / max(1, epochs - warmup_epochs)574return 0.5 * (1 + np.cos(np.pi * prog))575sched = torch.optim.lr_scheduler.LambdaLR(opt, lr_lambda=lr_at)576n = len(Y)577import time as _time578for ep in range(epochs):579t0 = _time.time()580model.train()581idx = torch.multinomial(w, n, replacement=True)582tot = 0.0583for b0 in range(0, n, batch):584bidx = idx[b0: b0 + batch]585# NO non_blocking=True: async copies from unpinned host memory race586# on MPS — the labels arrive as garbage (observed: loss pinned at587# the label-smoothing floor while accuracy stays at chance).588# .float(): host X may be float16 (PYSKL benchmarks keep it half to589# fit Kinetics-scale data in 16GB RAM); the model runs in float32.590xb = X[bidx].to(device).float()591if augment:592xb = augment_batch(xb)593yb = Y[bidx].to(device)594loss = F.cross_entropy(model(xb), yb, label_smoothing=0.1)595opt.zero_grad()596loss.backward()597opt.step()598tot += loss.item() * len(bidx)599sched.step()600if (ep + 1) % log_every == 0 or ep == 0:601msg = f"epoch {ep+1}/{epochs} loss={tot/n:.4f} [{_time.time()-t0:.0f}s]"602if val is not None and ((ep + 1) % (log_every * 5) == 0 or ep + 1 == epochs):603msg += f" val={eval_top1(model=model, X=val[0], Y=val[1], device=device)*100:.1f}%"604log(msg)605return model606607608def eval_top1(*, model, X, Y, device, batch=256, two_body_max=None):609"""Top-1 accuracy. two_body_max: (N, C, T, V) second-body tensor — logits610are elementwise-maxed over the two single-body passes, but ONLY for samples611whose second body is actually present. An all-zero second body (the common612single-actor case) still produces finite logits that would otherwise win613the max and wreck accuracy, so those samples keep body-1 logits alone."""614model.eval()615correct = 0616with torch.no_grad():617for b0 in range(0, len(Y), batch):618xb = X[b0: b0 + batch].to(device).float()619logits = model(xb)620if two_body_max is not None:621x2 = two_body_max[b0: b0 + batch].to(device).float()622present = (x2.abs().flatten(1).sum(1) > 0) # (B,)623l2 = model(x2)624logits = torch.where(present[:, None], torch.maximum(logits, l2), logits)625correct += (logits.argmax(1).cpu() == Y[b0: b0 + batch]).sum().item()626return correct / len(Y)
SHA-256 of complete original file: 83ab7c94a16349d5fe369113244f76fbb708fd77c8fbe619943e0155083c311e
3435# COCO-17 keypoint order (HRNet / DETRPose).36COCO17_JOINTS = [37"nose", "L-eye", "R-eye", "L-ear", "R-ear",38"L-shoulder", "R-shoulder", "L-elbow", "R-elbow", "L-wrist", "R-wrist",39"L-hip", "R-hip", "L-knee", "R-knee", "L-ankle", "R-ankle",40]41COCO17_EDGES = [42(0, 1), (0, 2), (1, 3), (2, 4), # head43(0, 5), (0, 6), # nose->shoulders: LINKS head to body
Unrelated source lines omitted.
105106107def annotation_to_bodies(ann, t_fix):108"""One PYSKL annotation -> two (C_IN, t_fix, 17) float32 bodies in their109ORIGINAL tracked order (absent 2nd body = zeros).110111Normalization EXACTLY matches PYSKL's PreNormalize2D (mode 'fix') — the112reference recipe: center on the IMAGE center and divide by half-image-size,113then zero the x,y of any joint whose HRNet confidence <= 0.01. Crucially it114does NOT hip-center — keeping absolute in-frame position preserves the spatial115relationship between the two actors, which NTU-120's many two-person actions116depend on. Bones/velocities derive from these image-centered coords.117"""118kp = ann["keypoint"].astype(np.float32) # (M, T, 17, 2) pixels119sc = ann["keypoint_score"].astype(np.float32) # (M, T, 17)120h, w = ann["img_shape"]121kp = kp.copy()122kp[..., 0] = (kp[..., 0] - w / 2.0) / (w / 2.0) # image-center, -> ~[-1,1]123kp[..., 1] = (kp[..., 1] - h / 2.0) / (h / 2.0)124low = sc <= 0.01 # zero out low-confidence joints125kp[..., 0][low] = 0.0126kp[..., 1][low] = 0.0127128outs = []129for m in range(kp.shape[0]):130coords = _uniform_sample(kp[m], t_fix) # (t_fix, 17, 2)131conf = _uniform_sample(sc[m], t_fix) # (t_fix, 17)132vel = np.zeros_like(coords); vel[1:] = coords[1:] - coords[:-1]133bone = coords - coords[:, BONE_PARENT, :]134bvel = np.zeros_like(bone); bvel[1:] = bone[1:] - bone[:-1]135feat = np.concatenate(136[coords, conf[..., None], vel, bone, bvel], axis=2137).transpose(2, 0, 1).astype(np.float32) # (9, t_fix, 17)138outs.append(feat)139140bodies = outs[:2]141while len(bodies) < 2:142bodies.append(np.zeros((C_IN, t_fix, V), dtype=np.float32))143return bodies[0], bodies[1]144145146def _uniform_sample(arr, t_fix):147"""(T, ...) -> (t_fix, ...) uniform index sampling; short clips repeat."""148t = arr.shape[0]149idx = np.linspace(0, t - 1, t_fix).round().astype(int)150return arr[idx]
SHA-256 of complete original file: 98f3b41e8bf3e7614ebfc5d52e8dae79ac9a2e4a4b2ce985e16d52f0e258f156
118CH = {"x": 0, "y": 1, "conf": 2, "dx": 3, "dy": 4, "bx": 5, "by": 6, "dbx": 7, "dby": 8}119STREAM_CHANNELS = {120"joint": ["x", "y", "conf", "dx", "dy"], # positions + motion (the original)121"bone": ["bx", "by", "conf", "dbx", "dby"], # local limb geometry + its motion122"motion": ["dx", "dy", "dbx", "dby"], # pure velocity123"all": ["x", "y", "conf", "dx", "dy", "bx", "by", "dbx", "dby"], # input-level fusion124# The canonical 4-stream fusion (each 3ch: spatial pair + conf), the literature125# standard. augment_batch rotates channels [0,1], correct for all four.126"j": ["x", "y", "conf"], # joint positions127"b": ["bx", "by", "conf"], # bone vectors128"jm": ["dx", "dy", "conf"], # joint motion129"bm": ["dbx", "dby", "conf"], # bone motion130}131FOUR_STREAM = "j,b,jm,bm"132133134def resolve_channels(spec):135names = STREAM_CHANNELS[spec] if spec in STREAM_CHANNELS else spec.split(",")136return [CH[n] for n in names], names137138139def parse_dilations(spec):140return [int(x) for x in spec.split(",")] if spec else None141142143def predict_probs(model, X, device, batch=256):144model.eval()145out = []146with torch.no_grad():147for b0 in range(0, len(X), batch):148xb = X[b0:b0 + batch].to(device).float()149out.append(torch.softmax(model(xb), 1).cpu())150return torch.cat(out)151152153def train_stream(*, seed, d, cfg, args, device, ch_idx, dilations, hidden, log, tag=""):154"""Train one LegibleTCN on a channel subset; return (probs_test, nparam, secs)."""155torch.manual_seed(seed); np.random.seed(seed)156model = build_model(arch=args.arch, c_in=len(ch_idx),157num_classes=cfg["num_classes"], args=args,
Unrelated source lines omitted.
300data = load_dataset(pkl_path=pkl_path, dataset=dataset, t_fix=t_fix, test_len=tl,301limit=limit, log=log)302log(f"[bench] adapted in {time.time()-t0:.0f}s; caching -> {cache}")303np.savez_compressed(cache, **data)304return data305306307def run_one_seed(*, seed, d, cfg, args, device, log):308torch.manual_seed(seed)309np.random.seed(seed)310ch_idx, _ = resolve_channels(args.channels)311model = LegibleTCN(v=V, c_in=len(ch_idx), num_classes=cfg["num_classes"],312filters=args.filters, hidden=args.hidden,313temporal_dilations=parse_dilations(args.dilations)).to(device)314nparam = sum(p.numel() for p in model.parameters())315X = torch.from_numpy(d["x1_train"][:, ch_idx])316Y = torch.from_numpy(d["y_train"])317Xte = torch.from_numpy(d["x1_test"][:, ch_idx])318Yte = torch.from_numpy(d["y_test"])319log(f"[bench] seed={seed} train={len(Y)} test={len(Yte)} "320f"classes={cfg['num_classes']} params={nparam/1e3:.1f}k "321f"filters={args.filters} hidden={args.hidden} lr={args.lr} epochs={args.epochs}")322t0 = time.time()323train_classifier(model=model, X=X, Y=Y, epochs=args.epochs, batch=args.batch,324device=device, lr=args.lr, log=lambda m: log(f"[bench] s{seed} {m}"),325log_every=max(1, args.epochs // 8), val=(Xte, Yte))326acc1 = eval_top1(model=model, X=Xte, Y=Yte, device=device)327row = {"seed": seed, "acc1": acc1, "params": nparam, "secs": time.time() - t0}328predicted = predict_probs(model, Xte, device).argmax(1)329class_counts = torch.bincount(Yte, minlength=cfg["num_classes"])330class_correct = torch.bincount(Yte[predicted == Yte], minlength=cfg["num_classes"])331supported = class_counts > 0332row["mean_class_accuracy"] = (class_correct[supported] / class_counts[supported]).mean().item()333row["test_class_counts"] = class_counts.tolist()334row["test_class_correct"] = class_correct.tolist()335row["test_classes_present"] = int(supported.sum())336if args.results_json:337checkpoint = os.path.splitext(args.results_json)[0] + f"-seed{seed}.pt"338torch.save(model.state_dict(), checkpoint)339row["checkpoint_bytes"] = os.path.getsize(checkpoint)340if cfg["two_body"]:341Xte2 = torch.from_numpy(d["x2_test"][:, ch_idx])342row["acc2"] = eval_top1(model=model, X=Xte, Y=Yte, device=device, two_body_max=Xte2)343log(f"[bench] seed={seed} top-1 body1={100*acc1:.2f}%"344+ (f" max2body={100*row['acc2']:.2f}%" if 'acc2' in row else "")345+ f" [{row['secs']:.0f}s]")346return row
Unrelated source lines omitted.
440ap = argparse.ArgumentParser()441ap.add_argument("--dataset", required=True, choices=list(SPLITS.keys()))442ap.add_argument("--results-json", help="Save final-epoch metrics, recipe, sizes and per-seed checkpoints")443ap.add_argument("--inspect-data", action="store_true", help="Print annotation schema and split counts without adapting or training")444ap.add_argument("--data-dir", default="~/Datasets/pyskl")445ap.add_argument("--cache-dir", default="~/Datasets/pyskl/cache")446ap.add_argument("--epochs", type=int, default=80)447ap.add_argument("--batch", type=int, default=64)448ap.add_argument("--lr", type=float, default=2e-3)449ap.add_argument("--filters", type=int, default=8)450ap.add_argument("--hidden", type=int, default=96)451ap.add_argument("--t-fix", type=int, default=64)452ap.add_argument("--seeds", default="42,43,44")453ap.add_argument("--limit", type=int, default=0, help="pilot: random subset of N annotations")454ap.add_argument("--adapt-only", action="store_true", help="build the cache and exit (no training)")455ap.add_argument("--dump-errors", default=None, help="train 1 seed, write per-class/confusion JSON to this path")456# levers 1-3457ap.add_argument("--channels", default="joint", help="joint|bone|motion|all or comma channel names (single-stream)")458ap.add_argument("--streams", default=None, help="comma stream names for multi-stream ensemble, e.g. joint,bone,motion")459ap.add_argument("--dilations", default=None, help="temporal dilations, e.g. 1,2,4 (bigger receptive field)")460ap.add_argument("--experiment", action="store_true", help="run the channels/streams/capacity experiment path")461ap.add_argument("--arch", default="legible", choices=["legible", "graph"], help="legible per-joint TCN or compact ST-GCN")462ap.add_argument("--gchannels", default="64,64,128,128", help="GraphTCN block channels")463ap.add_argument("--partitioned", action="store_true", help="ST-GCN 3-partition adjacency (self/centripetal/centrifugal)")464# full-recipe knobs465ap.add_argument("--optimizer", default="adamw", choices=["adamw", "sgd"])466ap.add_argument("--weight-decay", type=float, default=1e-4)467ap.add_argument("--warmup", type=int, default=0, help="linear warmup epochs before cosine")468ap.add_argument("--augment", action="store_true", help="on-the-fly skeleton augmentation (rotation/scale/jitter)")469ap.add_argument("--four-stream", action="store_true", help="shortcut for --streams j,b,jm,bm (canonical 4-stream)")470ap.add_argument("--reference", action="store_true", help="PYSKL reference recipe: image-norm, no aug")471ap.add_argument("--num-person", type=int, default=2, help="1 = single actor (single-person case); 2 = both actors")472ap.add_argument("--test-len", type=int, default=None, help="frames stored per test seq (>= t_fix) so test clips can differ")473ap.add_argument("--test-clips", type=int, default=1, help="N-clip test-time averaging (PYSKL uses 10)")474ap.add_argument("--stgcnpp", action="store_true", help="SOTA grafts: learnable adaptive graph + multi-scale temporal conv (ST-GCN++)")475ap.add_argument("--ctrgc", action="store_true", help="CTR-GCN channel-wise topology refinement (+ MS-TCN)")476ap.add_argument("--dct", action="store_true", help="NOVEL: DCT frequency-domain temporal residual with learnable band gains")477ap.add_argument("--antisym", action="store_true", help="NOVEL: time-reversal antisymmetry feature (motion direction) for reversal-pair classes")478args = ap.parse_args()