- Updated `export.py` to include model size as a parameter for ONNX graph exports, improving flexibility in model loading. - Refactored the `load` function to accept a checkpoint string, allowing dynamic model loading based on specified size. - Introduced new methods in `tracker.ts` for holding and previewing frames, enhancing the user experience during object segmentation. - Updated type exports in `video.ts` to include `FrameRequest`, facilitating better integration with the tracking system. - Improved documentation and comments throughout the codebase for clarity on new features and functionality.
328 lines
15 KiB
Python
328 lines
15 KiB
Python
# /// script
|
|
# requires-python = ">=3.10"
|
|
# dependencies = ["numpy", "onnx==1.23.0", "onnxslim==0.1.96", "torch==2.14.0", "transformers==5.17.0"]
|
|
# ///
|
|
"""
|
|
Exports the graphs the package loads: SAM 2.1 video tracking as five
|
|
fixed-shape ONNX graphs, from the `transformers` port of a checkpoint.
|
|
|
|
The model size (tiny, small, base-plus or large), the input resolution and
|
|
the number of frame memories attended to are parameters; the package's
|
|
constants must match the ones exported. The
|
|
graphs compute in fp16 but take and return fp32, with a cast at each boundary.
|
|
Constants are folded in fp32 first, so positional encodings are computed at
|
|
full precision before they are stored as halves.
|
|
|
|
uv run packages/sam2/scripts/export.py <model-size> <image-size> <memory-frames> <out-dir>
|
|
"""
|
|
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import onnx
|
|
import onnxslim
|
|
import torch
|
|
import torch.nn.functional as F
|
|
from onnx import TensorProto, helper, numpy_helper
|
|
from torch import nn
|
|
from transformers import Sam2VideoConfig, Sam2VideoModel
|
|
from transformers.models.sam2_video.modeling_sam2_video import NO_OBJ_SCORE, get_1d_sine_pe
|
|
|
|
# Masks come out at this size whatever the input resolution, upsampled from the decoder's.
|
|
MASK_SIZE = 256
|
|
MAX_POINTERS = 16
|
|
|
|
# Inputs the ONNX spec types as float32 whatever the op computes in, by (op, input index).
|
|
FLOAT32_INPUTS = {('Resize', 1), ('Resize', 2)}
|
|
|
|
|
|
def load(checkpoint: str, image_size: int) -> Sam2VideoModel:
|
|
config = Sam2VideoConfig.from_pretrained(checkpoint)
|
|
feat = image_size // 16
|
|
config.image_size = image_size
|
|
config.vision_config.backbone_config.image_size = [image_size, image_size]
|
|
config.vision_config.backbone_feature_sizes = [[feat * 4, feat * 4], [feat * 2, feat * 2], [feat, feat]]
|
|
config.prompt_encoder_config.image_size = image_size
|
|
config.mask_decoder_config.image_size = image_size
|
|
config.memory_attention_rope_feat_sizes = [feat, feat]
|
|
model = Sam2VideoModel.from_pretrained(checkpoint, config=config, attn_implementation='eager').eval()
|
|
|
|
# The backbone's position embedding depends only on the input size; exported as the
|
|
# computation it is, it brings `If` nodes that the runtime evaluates on the CPU.
|
|
backbone = model.vision_encoder.backbone
|
|
with torch.no_grad():
|
|
pos_embed = backbone._get_pos_embed((image_size // 4, image_size // 4))
|
|
backbone._get_pos_embed = lambda hw: pos_embed
|
|
return model
|
|
|
|
|
|
class VisionEncoder(nn.Module):
|
|
"""A frame's features at three scales, the smallest also with the prompted frame's no-memory embedding."""
|
|
|
|
def __init__(self, model: Sam2VideoModel):
|
|
super().__init__()
|
|
self.model = model
|
|
# Depends only on the input size, so it is a constant rather than a computation in fp16.
|
|
size = model.image_size
|
|
with torch.no_grad():
|
|
self.register_buffer('pos_embed', model.vision_encoder(torch.zeros(1, 3, size, size)).fpn_position_encoding[2])
|
|
|
|
def forward(self, pixel_values):
|
|
m = self.model
|
|
feats = m.vision_encoder(pixel_values, return_dict=True).fpn_hidden_states
|
|
feats0 = m.mask_decoder.conv_s0(feats[0])
|
|
feats1 = m.mask_decoder.conv_s1(feats[1])
|
|
feats2_no_mem = feats[2] + m.no_memory_embedding.view(1, -1, 1, 1)
|
|
return feats0, feats1, feats[2], feats2_no_mem, self.pos_embed
|
|
|
|
|
|
class MaskDecoder(nn.Module):
|
|
"""Prompt encoder and mask decoder, picking the mask and object pointer as the video predictor does."""
|
|
|
|
def __init__(self, model: Sam2VideoModel):
|
|
super().__init__()
|
|
self.model = model
|
|
|
|
def forward(self, feats0, feats1, feats2_cond, input_points, input_labels):
|
|
m = self.model
|
|
d = m.mask_decoder
|
|
sparse, dense = m.prompt_encoder(input_points=input_points, input_labels=input_labels, input_boxes=None, input_masks=None)
|
|
|
|
_, channels, height, width = feats2_cond.shape
|
|
tokens = torch.cat([d.obj_score_token.weight, d.iou_token.weight, d.mask_tokens.weight], dim=0)
|
|
tokens = torch.cat([tokens[None, None], sparse], dim=2)
|
|
point_embeddings, image_embeddings = d.transformer(
|
|
point_embeddings=tokens,
|
|
image_embeddings=feats2_cond + dense,
|
|
image_positional_embeddings=m.get_image_wide_positional_embeddings(),
|
|
attention_similarity=None,
|
|
target_embedding=None,
|
|
)
|
|
iou_token_out = point_embeddings[:, :, 1, :]
|
|
mask_tokens_out = point_embeddings[:, :, 2 : 2 + d.num_mask_tokens, :]
|
|
|
|
image_embeddings = image_embeddings.transpose(2, 3).reshape(1, channels, height, width)
|
|
upscaled = d.activation(d.upscale_layer_norm(d.upscale_conv1(image_embeddings) + feats1))
|
|
upscaled = d.activation(d.upscale_conv2(upscaled) + feats0)
|
|
hyper_in = torch.stack([mlp(mask_tokens_out[:, :, i, :]) for i, mlp in enumerate(d.output_hypernetworks_mlps)], dim=2)
|
|
_, c, h, w = upscaled.shape
|
|
masks = (hyper_in @ upscaled.view(1, 1, c, h * w)).view(1, 1, -1, h, w)
|
|
iou_pred = d.iou_prediction_head(iou_token_out)
|
|
object_score_logits = d.pred_obj_score_head(point_embeddings[:, :, 0, :])
|
|
|
|
# Several candidate masks when there is at most one real point: every tracked frame, and a single click.
|
|
multimask = (input_labels != -1).sum() <= m.config.multimask_max_pt_num
|
|
best = torch.argmax(iou_pred[0, 0, 1:]) + 1
|
|
multi_mask, multi_iou, multi_token = masks[0, 0, best], iou_pred[0, 0, best], mask_tokens_out[0, 0, best]
|
|
|
|
# One mask: token 0's, unless it is unstable and the best candidate's is taken instead.
|
|
single = masks[0, 0, 0]
|
|
delta = d.dynamic_multimask_stability_delta
|
|
area_i = (single > delta).sum().float()
|
|
area_u = (single > -delta).sum().float()
|
|
stability = torch.where(area_u > 0, area_i / area_u, torch.ones_like(area_u))
|
|
stable = stability >= d.dynamic_multimask_stability_thresh
|
|
single_mask = torch.where(stable, single, multi_mask)
|
|
single_iou = torch.where(stable, iou_pred[0, 0, 0], multi_iou)
|
|
|
|
low_res = torch.where(multimask, multi_mask, single_mask)
|
|
iou = torch.where(multimask, multi_iou, single_iou)
|
|
token = torch.where(multimask, multi_token, mask_tokens_out[0, 0, 0])
|
|
|
|
appearing = object_score_logits[0, 0, 0] > 0
|
|
low_res = torch.where(appearing, low_res, torch.full_like(low_res, NO_OBJ_SCORE))[None, None]
|
|
high_res = F.interpolate(low_res, size=(m.image_size, m.image_size), mode='bilinear', align_corners=False)
|
|
mask = low_res if w == MASK_SIZE else F.interpolate(low_res, size=(MASK_SIZE, MASK_SIZE), mode='bilinear', align_corners=False)
|
|
|
|
weight = appearing.float()
|
|
pointer = weight * m.object_pointer_proj(token[None, None]) + (1 - weight) * m.no_object_pointer
|
|
return mask, high_res, iou.view(1, 1), object_score_logits, pointer.view(1, 1, -1)
|
|
|
|
|
|
class MemoryEncoder(nn.Module):
|
|
"""A frame's memory from its features and mask; `binarize` is 1 for a prompted frame."""
|
|
|
|
def __init__(self, model: Sam2VideoModel):
|
|
super().__init__()
|
|
self.model = model
|
|
|
|
def forward(self, feats2, high_res_mask, object_score_logits, binarize):
|
|
m = self.model
|
|
mask = torch.where(binarize > 0.5, (high_res_mask > 0).float(), torch.sigmoid(high_res_mask))
|
|
mask = mask * m.config.sigmoid_scale_for_mem_enc + m.config.sigmoid_bias_for_mem_enc
|
|
features, pos = m.memory_encoder(feats2, mask)
|
|
occluded = 1 - (object_score_logits > 0).float()
|
|
features = features + occluded.view(1, 1, 1, 1) * m.occlusion_spatial_embedding_parameter.view(1, -1, 1, 1)
|
|
return features.flatten(2).permute(2, 0, 1), pos.flatten(2).permute(2, 0, 1)
|
|
|
|
|
|
class MemoryAttention(nn.Module):
|
|
"""The frame's features conditioned on the memory bank: frame memories, then object pointer tokens."""
|
|
|
|
def __init__(self, model: Sam2VideoModel):
|
|
super().__init__()
|
|
self.model = model
|
|
|
|
def forward(self, current_vision_features, current_vision_position_embeddings, memory, memory_pos):
|
|
m = self.model
|
|
feat = m.backbone_feature_sizes[-1][0]
|
|
out = m.memory_attention(
|
|
current_vision_features=current_vision_features,
|
|
current_vision_position_embeddings=current_vision_position_embeddings,
|
|
memory=memory,
|
|
memory_posision_embeddings=memory_pos,
|
|
num_object_pointer_tokens=MAX_POINTERS * (m.hidden_dim // m.mem_dim),
|
|
)
|
|
return out.squeeze(1).transpose(1, 2).reshape(1, m.hidden_dim, feat, feat)
|
|
|
|
|
|
class PointerTpos(nn.Module):
|
|
"""Temporal positions of the object pointers, from their normalized frame distances."""
|
|
|
|
def __init__(self, model: Sam2VideoModel):
|
|
super().__init__()
|
|
self.model = model
|
|
|
|
def forward(self, normalized_diffs):
|
|
m = self.model
|
|
return m.temporal_positional_encoding_projection_layer(get_1d_sine_pe(normalized_diffs, dim=m.hidden_dim))
|
|
|
|
|
|
def export_fp32(model: Sam2VideoModel, memory_frames: int, out: Path) -> None:
|
|
s = model.image_size
|
|
f = s // 16
|
|
rows = memory_frames * f * f + MAX_POINTERS * (model.hidden_dim // model.mem_dim)
|
|
|
|
def save(module, name, args, inputs, outputs, dynamic_axes=None):
|
|
with torch.no_grad():
|
|
torch.onnx.export(
|
|
module, args, str(out / f'{name}.onnx'), input_names=inputs, output_names=outputs,
|
|
opset_version=17, dynamo=False, dynamic_axes=dynamic_axes,
|
|
)
|
|
|
|
save(VisionEncoder(model), 'vision_encoder', (torch.randn(1, 3, s, s),),
|
|
['pixel_values'], ['feats0', 'feats1', 'feats2', 'feats2_no_mem', 'vision_pos_embed'])
|
|
save(MaskDecoder(model), 'mask_decoder',
|
|
(torch.randn(1, 32, 4 * f, 4 * f), torch.randn(1, 64, 2 * f, 2 * f), torch.randn(1, 256, f, f),
|
|
torch.rand(1, 1, 2, 2) * s, torch.tensor([[[1, 0]]], dtype=torch.int32)),
|
|
['feats0', 'feats1', 'feats2_cond', 'input_points', 'input_labels'],
|
|
['low_res_mask', 'high_res_mask', 'iou', 'object_score_logits', 'object_pointer'],
|
|
{'input_points': {2: 'num_points'}, 'input_labels': {2: 'num_points'}})
|
|
save(MemoryEncoder(model), 'memory_encoder',
|
|
(torch.randn(1, 256, f, f), torch.randn(1, 1, s, s), torch.randn(1, 1), torch.tensor(1.0)),
|
|
['feats2', 'high_res_mask', 'object_score_logits', 'binarize'], ['memory_tokens', 'memory_pos'])
|
|
save(MemoryAttention(model), 'memory_attention',
|
|
(torch.randn(f * f, 1, 256), torch.randn(f * f, 1, 256), torch.randn(rows, 1, 64), torch.randn(rows, 1, 64)),
|
|
['current_vision_features', 'current_vision_position_embeddings', 'memory', 'memory_pos'], ['conditioned_feats'])
|
|
save(PointerTpos(model), 'pointer_tpos', (torch.rand(MAX_POINTERS),), ['normalized_diffs'], ['pointer_pos'])
|
|
|
|
|
|
def to_fp16(model: onnx.ModelProto) -> onnx.ModelProto:
|
|
graph = model.graph
|
|
keep_float = {
|
|
node.input[i]
|
|
for node in graph.node
|
|
for op, i in FLOAT32_INPUTS
|
|
if node.op_type == op and len(node.input) > i
|
|
}
|
|
|
|
for init in graph.initializer:
|
|
if init.data_type == TensorProto.FLOAT and init.name not in keep_float:
|
|
init.CopyFrom(numpy_helper.from_array(to_half(numpy_helper.to_array(init)), init.name))
|
|
|
|
for node in graph.node:
|
|
for attr in node.attribute:
|
|
if node.op_type == 'Cast' and attr.name == 'to' and attr.i == TensorProto.FLOAT:
|
|
attr.i = TensorProto.FLOAT16
|
|
elif (
|
|
attr.type == onnx.AttributeProto.TENSOR
|
|
and attr.t.data_type == TensorProto.FLOAT
|
|
and node.op_type in ('Constant', 'ConstantOfShape')
|
|
and node.output[0] not in keep_float
|
|
):
|
|
attr.t.CopyFrom(numpy_helper.from_array(to_half(numpy_helper.to_array(attr.t)), attr.t.name))
|
|
|
|
# Float inputs are cast down on entry and float outputs cast back up on exit.
|
|
entry, exit = [], []
|
|
for vi in graph.input:
|
|
if vi.type.tensor_type.elem_type == TensorProto.FLOAT:
|
|
half = f'{vi.name}__fp16'
|
|
for node in graph.node:
|
|
node.input[:] = [half if name == vi.name else name for name in node.input]
|
|
entry.append(helper.make_node('Cast', [vi.name], [half], name=f'{vi.name}__to_fp16', to=TensorProto.FLOAT16))
|
|
for vi in graph.output:
|
|
if vi.type.tensor_type.elem_type == TensorProto.FLOAT:
|
|
half = f'{vi.name}__fp16'
|
|
for node in graph.node:
|
|
node.output[:] = [half if name == vi.name else name for name in node.output]
|
|
node.input[:] = [half if name == vi.name else name for name in node.input]
|
|
exit.append(helper.make_node('Cast', [half], [vi.name], name=f'{vi.name}__to_fp32', to=TensorProto.FLOAT))
|
|
|
|
nodes = entry + list(graph.node) + exit
|
|
del graph.node[:]
|
|
graph.node.extend(nodes)
|
|
del graph.value_info[:]
|
|
return model
|
|
|
|
|
|
def to_half(array: np.ndarray) -> np.ndarray:
|
|
return np.clip(array, -65504, 65504).astype(np.float16)
|
|
|
|
|
|
def dedupe_constants(model: onnx.ModelProto) -> onnx.ModelProto:
|
|
"""Stores each distinct constant once: folding leaves a copy of the RoPE tables in every attention block."""
|
|
graph = model.graph
|
|
for node in [n for n in graph.node if n.op_type == 'Constant' and n.attribute[0].name == 'value']:
|
|
tensor = onnx.TensorProto()
|
|
tensor.CopyFrom(node.attribute[0].t)
|
|
tensor.name = node.output[0]
|
|
graph.initializer.append(tensor)
|
|
graph.node.remove(node)
|
|
|
|
first: dict[str, str] = {}
|
|
rename: dict[str, str] = {}
|
|
kept = []
|
|
for init in graph.initializer:
|
|
array = numpy_helper.to_array(init)
|
|
key = hashlib.sha1(f'{array.dtype}{array.shape}'.encode() + array.tobytes()).hexdigest()
|
|
if key in first:
|
|
rename[init.name] = first[key]
|
|
else:
|
|
first[key] = init.name
|
|
kept.append(init)
|
|
del graph.initializer[:]
|
|
graph.initializer.extend(kept)
|
|
for node in graph.node:
|
|
node.input[:] = [rename.get(name, name) for name in node.input]
|
|
return model
|
|
|
|
|
|
def main(model_size: str, image_size: int, memory_frames: int, out: Path) -> None:
|
|
model = load(f'facebook/sam2.1-hiera-{model_size}', image_size)
|
|
(out / 'onnx').mkdir(parents=True, exist_ok=True)
|
|
with tempfile.TemporaryDirectory() as fp32:
|
|
export_fp32(model, memory_frames, Path(fp32))
|
|
for path in sorted(Path(fp32).glob('*.onnx')):
|
|
graph = dedupe_constants(to_fp16(onnxslim.slim(onnx.load(path))))
|
|
onnx.checker.check_model(graph)
|
|
onnx.save(graph, out / 'onnx' / path.name)
|
|
print(f'{path.stem}: {(out / "onnx" / path.name).stat().st_size:,} bytes')
|
|
|
|
constants = {
|
|
'model': f'sam2.1-hiera-{model_size}',
|
|
'image_size': image_size,
|
|
'memory_frames': memory_frames,
|
|
'image_mean': [0.485, 0.456, 0.406],
|
|
'image_std': [0.229, 0.224, 0.225],
|
|
'memory_temporal_positional_encoding': model.memory_temporal_positional_encoding.detach().reshape(model.num_maskmem, -1).tolist(),
|
|
}
|
|
(out / 'constants.json').write_text(json.dumps(constants))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main(sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), Path(sys.argv[4]))
|