Duplicate from lwanming/depth-anything-v2-small

Co-authored-by: Wanming Lin <lwanming@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+onnx/model_fp16.onnx_data filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,6 @@
+---
+license: apache-2.0
+---
+The model (onnx/model_fp16.onnx) is sourced from onnx-community/depth-anything-v2-small. It is a Depth-Anything-V2 (with a DINOv2 backbone) ONNX model.
+
+To resolve the issue where the original model constructs 5D tensors in its fused-QKV attention, we used convert_5d_to_4d.py to convert it into an equivalent model that maintains 4D attention tensors for better compatibility.

config.json

@@ -0,0 +1,56 @@
+{
+  "_commit_hash": null,
+  "architectures": [
+    "DepthAnythingForDepthEstimation"
+  ],
+  "backbone": null,
+  "backbone_config": {
+    "architectures": [
+      "Dinov2Model"
+    ],
+    "hidden_size": 384,
+    "image_size": 518,
+    "model_type": "dinov2",
+    "num_attention_heads": 6,
+    "out_features": [
+      "stage3",
+      "stage6",
+      "stage9",
+      "stage12"
+    ],
+    "out_indices": [
+      3,
+      6,
+      9,
+      12
+    ],
+    "patch_size": 14,
+    "reshape_hidden_states": false,
+    "torch_dtype": "float32"
+  },
+  "fusion_hidden_size": 64,
+  "head_hidden_size": 32,
+  "head_in_index": -1,
+  "initializer_range": 0.02,
+  "model_type": "depth_anything",
+  "neck_hidden_sizes": [
+    48,
+    96,
+    192,
+    384
+  ],
+  "patch_size": 14,
+  "reassemble_factors": [
+    4,
+    2,
+    1,
+    0.5
+  ],
+  "reassemble_hidden_size": 384,
+  "torch_dtype": "float32",
+  "transformers_version": null,
+  "transformers.js_config": {
+    "use_external_data_format": true
+  },
+  "use_pretrained_backbone": false
+}

convert_5d_to_4d.py

@@ -0,0 +1,250 @@
+"""
+Convert a Depth-Anything-V2 (DINOv2 backbone) ONNX model that builds 5D tensors
+in its fused-QKV attention into an equivalent model whose attention stays 4D,
+so it can fully offload to the WebNN execution provider.
+
+Per transformer block the original (LiheYoung repo) attention does:
+
+    qkv Add  [B, N, 3*C]
+      -> Reshape   -> [B, N, 3, H, Dh]      (5D)
+      -> Transpose -> [3, B, H, N, Dh]      (5D), perm [2,0,3,1,4]
+      -> Gather idx 0/1/2 (axis 0) -> q,k,v [B, H, N, Dh]
+
+We replace that 5D path with the HuggingFace-style 4D path that produces the
+SAME q/k/v output tensor names the rest of the block already consumes:
+
+    qkv Add  [B, N, 3*C]
+      -> Split(axis=2, 3)  -> q_lin,k_lin,v_lin  [B, N, C]
+      -> Reshape [0,0,H,Dh] -> [B, N, H, Dh]
+      -> Transpose [0,2,1,3] -> [B, H, N, Dh]   (named as the old Gather outputs)
+
+This is algebraically identical, preserves dynamic B/N (so
+freeDimensionOverrides still apply), and keeps every other node untouched.
+"""
+
+import os
+import sys
+import re
+from collections import Counter, defaultdict
+
+import onnx
+from onnx import helper, TensorProto, shape_inference
+
+HOME = os.path.expanduser("~")
+SRC = os.path.join(HOME, "Downloads", "model_fp16-5d.onnx")
+DST = os.path.join(HOME, "Downloads", "model_fp16-5d-to-4d.onnx")
+
+HEADS = 6
+HEAD_DIM = 64
+# q, k, v are Gather idx 0, 1, 2 -> map to output-name suffix
+GATHER_FOR = {0: "Gather_3", 1: "Gather_4", 2: "Gather_5"}
+
+
+def tensor_rank(vi):
+    t = vi.type.tensor_type
+    if not t.HasField("shape"):
+        return None
+    return len(t.shape.dim)
+
+
+def count_high_rank(model):
+    """Return number of value_info/input/output tensors with rank > 4.
+
+    Clears any pre-existing value_info first so stale shapes from deleted nodes
+    are not counted -- shape inference repopulates from the live graph.
+    """
+    m = onnx.ModelProto()
+    m.CopyFrom(model)
+    del m.graph.value_info[:]
+    m = shape_inference.infer_shapes(m, strict_mode=False, data_prop=True)
+    g = m.graph
+    n = 0
+    for coll in (g.value_info, g.input, g.output):
+        for vi in coll:
+            r = tensor_rank(vi)
+            if r is not None and r > 4:
+                n += 1
+    return n
+
+
+def discover_blocks(graph):
+    """Find attention block indices that use the fused-qkv 5D reshape pattern."""
+    blocks = set()
+    pat = re.compile(r"^/blocks\.(\d+)/attn/Reshape$")
+    have_reshape = {}
+    for node in graph.node:
+        mo = pat.match(node.name or "")
+        if mo:
+            have_reshape[int(mo.group(1))] = True
+    # Only keep blocks that also have the 3 gather outputs we rewire onto.
+    node_names = {n.name for n in graph.node}
+    for i in have_reshape:
+        needed = [f"/blocks.{i}/attn/{g}" for g in GATHER_FOR.values()]
+        needed += [f"/blocks.{i}/attn/Transpose"]
+        if all(nm in node_names for nm in needed):
+            blocks.add(i)
+    return sorted(blocks)
+
+
+def dead_code_eliminate(graph):
+    """Iteratively drop nodes whose outputs are consumed by nothing."""
+    graph_outputs = {o.name for o in graph.output}
+    removed_total = 0
+    while True:
+        consumed = set()
+        for n in graph.node:
+            for i in n.input:
+                consumed.add(i)
+        consumed |= graph_outputs
+        keep = []
+        removed = 0
+        for n in graph.node:
+            if any(o in consumed for o in n.output if o):
+                keep.append(n)
+            else:
+                removed += 1
+        if removed == 0:
+            break
+        del graph.node[:]
+        graph.node.extend(keep)
+        removed_total += removed
+    return removed_total
+
+
+def topo_sort(graph, available):
+    """Stable topological sort of graph.node. `available` = names known before
+    any node runs (graph inputs + initializers)."""
+    ready = set(available)
+    pending = list(graph.node)
+    ordered = []
+    progressed = True
+    while pending and progressed:
+        progressed = False
+        still = []
+        for n in pending:
+            deps = [i for i in n.input if i]
+            if all(d in ready for d in deps):
+                ordered.append(n)
+                for o in n.output:
+                    if o:
+                        ready.add(o)
+                progressed = True
+            else:
+                still.append(n)
+        pending = still
+    if pending:
+        # Should not happen; keep leftovers to surface a checker error rather
+        # than silently dropping nodes.
+        ordered.extend(pending)
+    del graph.node[:]
+    graph.node.extend(ordered)
+
+
+def main():
+    if not os.path.exists(SRC):
+        sys.exit(f"source model not found: {SRC}")
+
+    model = onnx.load(SRC, load_external_data=True)
+    graph = model.graph
+
+    before_hist = Counter(n.op_type for n in graph.node)
+    before_high = count_high_rank(model)
+    print(f"loaded {SRC}")
+    print(f"  nodes={len(graph.node)}  rank>4 tensors={before_high}")
+
+    blocks = discover_blocks(graph)
+    print(f"  attention blocks to rewrite: {blocks}")
+    if not blocks:
+        sys.exit("no fused-qkv 5D attention blocks found; nothing to do")
+
+    # shared reshape target [0,0,H,Dh]
+    reshape_shape_name = "qkv_reshape_shape_0_0_H_Dh"
+    graph.initializer.append(
+        helper.make_tensor(
+            reshape_shape_name, TensorProto.INT64, [4], [0, 0, HEADS, HEAD_DIM]
+        )
+    )
+
+    remove_suffixes = {"Reshape", "Transpose", "Gather_3", "Gather_4", "Gather_5"}
+    remove_names = set()
+    for i in blocks:
+        for s in remove_suffixes:
+            remove_names.add(f"/blocks.{i}/attn/{s}")
+
+    # index nodes by name for reference / sanity
+    kept_nodes = [n for n in graph.node if n.name not in remove_names]
+    removed_count = len(graph.node) - len(kept_nodes)
+    del graph.node[:]
+    graph.node.extend(kept_nodes)
+
+    new_nodes = []
+    for i in blocks:
+        p = f"/blocks.{i}/attn"
+        qkv_add = f"{p}/qkv/Add_output_0"
+        q_lin, k_lin, v_lin = f"{p}/n_q_lin", f"{p}/n_k_lin", f"{p}/n_v_lin"
+        # Split [B,N,3C] -> 3 x [B,N,C] on axis 2
+        new_nodes.append(
+            helper.make_node(
+                "Split",
+                inputs=[qkv_add],
+                outputs=[q_lin, k_lin, v_lin],
+                name=f"{p}/n_Split",
+                axis=2,
+            )
+        )
+        for idx, lin in ((0, q_lin), (1, k_lin), (2, v_lin)):
+            rout = f"{p}/n_reshape_{idx}"
+            gather_out = f"{p}/{GATHER_FOR[idx]}_output_0"
+            new_nodes.append(
+                helper.make_node(
+                    "Reshape",
+                    inputs=[lin, reshape_shape_name],
+                    outputs=[rout],
+                    name=f"{p}/n_Reshape_{idx}",
+                    allowzero=0,
+                )
+            )
+            new_nodes.append(
+                helper.make_node(
+                    "Transpose",
+                    inputs=[rout],
+                    outputs=[gather_out],  # feed the exact name downstream expects
+                    name=f"{p}/n_Transpose_{idx}",
+                    perm=[0, 2, 1, 3],
+                )
+            )
+    graph.node.extend(new_nodes)
+    print(f"  removed {removed_count} nodes, added {len(new_nodes)} nodes")
+
+    # dead-code eliminate the now-orphan 5D shape builders (Concat, Unsqueeze_2, ...)
+    dce = dead_code_eliminate(graph)
+    print(f"  dead-code eliminated {dce} orphan nodes")
+
+    # topo-sort so producers precede consumers
+    available = {i.name for i in graph.initializer} | {i.name for i in graph.input}
+    topo_sort(graph, available)
+
+    # drop stale value_info (deleted 5D tensors); ORT re-infers at load time
+    del graph.value_info[:]
+
+    onnx.checker.check_model(model, full_check=True)
+    after_high = count_high_rank(model)
+    after_hist = Counter(n.op_type for n in graph.node)
+    print(f"  after: nodes={len(graph.node)}  rank>4 tensors={after_high}")
+
+    # op histogram diff
+    diff = {}
+    for op in set(before_hist) | set(after_hist):
+        d = after_hist.get(op, 0) - before_hist.get(op, 0)
+        if d:
+            diff[op] = f"{before_hist.get(op,0)}->{after_hist.get(op,0)}"
+    print(f"  op changes: {diff}")
+
+    assert after_high == 0, f"still have {after_high} rank>4 tensors after surgery"
+
+    onnx.save(model, DST, save_as_external_data=False)
+    print(f"saved {DST}  ({os.path.getsize(DST)/1e6:.1f} MB)")
+
+
+if __name__ == "__main__":
+    main()

onnx/model_fp16.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09aba7b8af9535e11c0bf4b93da21ef5a2394e27487729ae624eae15473ef6ad
+size 1768360

onnx/model_fp16.onnx_data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d60cae2c173c758fe9b8ece018de5c5dee55ee6c34a878e8701a918d1a38f02
+size 48892416

preprocessor_config.json

@@ -0,0 +1,26 @@
+{
+  "do_normalize": true,
+  "do_pad": false,
+  "do_rescale": true,
+  "do_resize": true,
+  "ensure_multiple_of": 14,
+  "image_mean": [
+    0.485,
+    0.456,
+    0.406
+  ],
+  "image_processor_type": "DPTImageProcessor",
+  "image_std": [
+    0.229,
+    0.224,
+    0.225
+  ],
+  "keep_aspect_ratio": true,
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 518,
+    "width": 518
+  },
+  "size_divisor": null
+}