| { |
| "schema": "cortiq-decision-gpu-evidence/1", |
| "created_utc": "2026-09-28T05:26:36.972640+00:00", |
| "model_sha256": "ed9b8ec2bbfe9e9fd30f14a5eaf82314f38bc7e7510a39772baa2de3801d79b1", |
| "oracle_calls": 0, |
| "model_bytes_changed": false, |
| "precision": "FP32; no FP16/TF32; unchanged 1e-5 embedding tolerance", |
| "tests": [ |
| { |
| "backend": "metal", |
| "mode": "paired", |
| "run": 1, |
| "source_sha256": "8b5d7b02728504308e209b21928965322a2dd1de30c2c37f8a774f8081a9924b", |
| "schema": "cortiq-decision-metal-comparison/2", |
| "model_sha": "386b6e43fd35d5aa9e9710529dfdf30aa868df98ab4434d8e45dae9829898f85", |
| "skill": "banking77", |
| "n": 3080, |
| "dataset_sha256": "649607fa7b124293c80b455d2effe5e41da713e1836ee936073fe34ab9dc6fba", |
| "device": "metal: Apple M4", |
| "warmup": 50, |
| "order": "paired serial; CPU-first/GPU-first alternates per row; same process, model, scorer and texts; no oracle", |
| "golden_max_abs": { |
| "cpu": 0.0, |
| "metal": 1.1920928955078125e-07 |
| }, |
| "max_embedding_abs": 1.6391277313232422e-07, |
| "max_reconstruction_error_abs": 1.1742115020751953e-05, |
| "max_reconstruction_error_scaled": 6.425947049137903e-06, |
| "winner_changes": 0, |
| "gate_changes": 0, |
| "hash_changes": 0, |
| "different_rows": [], |
| "correct": { |
| "cpu": 2875, |
| "metal": 2875 |
| }, |
| "accepted": { |
| "cpu": 2791, |
| "metal": 2791 |
| }, |
| "metal_command_buffers": 3080, |
| "cpu_command_buffers": 0, |
| "metal_resonance_dispatches": 3080, |
| "ms": { |
| "cpu_total": { |
| "p50": 3.458792, |
| "p95": 4.6185, |
| "p99": 6.479583 |
| }, |
| "metal_total": { |
| "p50": 2.5725409999999997, |
| "p95": 4.429583, |
| "p99": 7.538375 |
| }, |
| "cpu_encoder": { |
| "p50": 2.071917, |
| "p95": 3.203542, |
| "p99": 5.040916 |
| }, |
| "metal_split_encoder": { |
| "p50": 0.0, |
| "p95": 0.0, |
| "p99": 0.0 |
| }, |
| "cpu_resonance": { |
| "p50": 1.350666, |
| "p95": 1.4649159999999999, |
| "p99": 1.538958 |
| }, |
| "metal_ranking_cpu": { |
| "p50": 0.006333, |
| "p95": 0.010209, |
| "p99": 0.019042000000000003 |
| }, |
| "metal_joint_pipeline": { |
| "p50": 2.554542, |
| "p95": 4.40325, |
| "p99": 7.522084 |
| } |
| }, |
| "by_tokens": [ |
| { |
| "tokens": "1-8", |
| "n": 230, |
| "cpu_ms": { |
| "p50": 3.093625, |
| "p95": 3.267084, |
| "p99": 3.378875 |
| }, |
| "gpu_ms": { |
| "p50": 2.420125, |
| "p95": 3.592583, |
| "p99": 5.5611250000000005 |
| } |
| }, |
| { |
| "tokens": "9-16", |
| "n": 2084, |
| "cpu_ms": { |
| "p50": 3.398666, |
| "p95": 3.682708, |
| "p99": 3.81625 |
| }, |
| "gpu_ms": { |
| "p50": 2.523584, |
| "p95": 3.9925409999999997, |
| "p99": 6.467167 |
| } |
| }, |
| { |
| "tokens": "17-32", |
| "n": 632, |
| "cpu_ms": { |
| "p50": 3.750125, |
| "p95": 4.5125, |
| "p99": 4.7405 |
| }, |
| "gpu_ms": { |
| "p50": 2.7897920000000003, |
| "p95": 4.739041, |
| "p99": 8.428542 |
| } |
| }, |
| { |
| "tokens": "33-64", |
| "n": 129, |
| "cpu_ms": { |
| "p50": 5.745417, |
| "p95": 7.550709, |
| "p99": 8.154875 |
| }, |
| "gpu_ms": { |
| "p50": 3.735917, |
| "p95": 6.810416, |
| "p99": 9.454334 |
| } |
| }, |
| { |
| "tokens": "65+", |
| "n": 5, |
| "cpu_ms": { |
| "p50": 10.160499999999999, |
| "p95": 11.220374999999999, |
| "p99": 11.220374999999999 |
| }, |
| "gpu_ms": { |
| "p50": 5.704416, |
| "p95": 15.93725, |
| "p99": 15.93725 |
| } |
| } |
| ], |
| "gpu_scope": "One command buffer: embedding lookup, transformer layers, row-order pooling and FP32 L2, reconstruction errors; tokenization, hashing and ranking/gates on CPU. Joint GPU time is not split into fictitious encoder/resonance wall times." |
| }, |
| { |
| "backend": "metal", |
| "mode": "paired", |
| "run": 2, |
| "source_sha256": "17972babb78332434ae98acddc9ceb5b3635aa60811fa9863ad636a0bae95015", |
| "schema": "cortiq-decision-metal-comparison/2", |
| "model_sha": "386b6e43fd35d5aa9e9710529dfdf30aa868df98ab4434d8e45dae9829898f85", |
| "skill": "banking77", |
| "n": 3080, |
| "dataset_sha256": "649607fa7b124293c80b455d2effe5e41da713e1836ee936073fe34ab9dc6fba", |
| "device": "metal: Apple M4", |
| "warmup": 50, |
| "order": "paired serial; CPU-first/GPU-first alternates per row; same process, model, scorer and texts; no oracle", |
| "golden_max_abs": { |
| "cpu": 0.0, |
| "metal": 1.1920928955078125e-07 |
| }, |
| "max_embedding_abs": 1.6391277313232422e-07, |
| "max_reconstruction_error_abs": 1.1742115020751953e-05, |
| "max_reconstruction_error_scaled": 6.425947049137903e-06, |
| "winner_changes": 0, |
| "gate_changes": 0, |
| "hash_changes": 0, |
| "different_rows": [], |
| "correct": { |
| "cpu": 2875, |
| "metal": 2875 |
| }, |
| "accepted": { |
| "cpu": 2791, |
| "metal": 2791 |
| }, |
| "metal_command_buffers": 3080, |
| "cpu_command_buffers": 0, |
| "metal_resonance_dispatches": 3080, |
| "ms": { |
| "cpu_total": { |
| "p50": 3.4517089999999997, |
| "p95": 4.600292, |
| "p99": 6.439208 |
| }, |
| "metal_total": { |
| "p50": 2.583042, |
| "p95": 3.750917, |
| "p99": 6.0215 |
| }, |
| "cpu_encoder": { |
| "p50": 2.066834, |
| "p95": 3.2210419999999997, |
| "p99": 5.067708 |
| }, |
| "metal_split_encoder": { |
| "p50": 0.0, |
| "p95": 0.0, |
| "p99": 0.0 |
| }, |
| "cpu_resonance": { |
| "p50": 1.3495409999999999, |
| "p95": 1.467708, |
| "p99": 1.539374 |
| }, |
| "metal_ranking_cpu": { |
| "p50": 0.006416, |
| "p95": 0.015667, |
| "p99": 0.01925 |
| }, |
| "metal_joint_pipeline": { |
| "p50": 2.563375, |
| "p95": 3.721, |
| "p99": 6.00075 |
| } |
| }, |
| "by_tokens": [ |
| { |
| "tokens": "1-8", |
| "n": 230, |
| "cpu_ms": { |
| "p50": 3.1007919999999998, |
| "p95": 3.277709, |
| "p99": 3.465416 |
| }, |
| "gpu_ms": { |
| "p50": 2.448209, |
| "p95": 3.439, |
| "p99": 4.4012080000000005 |
| } |
| }, |
| { |
| "tokens": "9-16", |
| "n": 2084, |
| "cpu_ms": { |
| "p50": 3.3943749999999997, |
| "p95": 3.7052080000000003, |
| "p99": 3.836292 |
| }, |
| "gpu_ms": { |
| "p50": 2.548833, |
| "p95": 3.523375, |
| "p99": 5.749334 |
| } |
| }, |
| { |
| "tokens": "17-32", |
| "n": 632, |
| "cpu_ms": { |
| "p50": 3.763, |
| "p95": 4.530416, |
| "p99": 4.754292 |
| }, |
| "gpu_ms": { |
| "p50": 2.768875, |
| "p95": 3.757083, |
| "p99": 6.398375000000001 |
| } |
| }, |
| { |
| "tokens": "33-64", |
| "n": 129, |
| "cpu_ms": { |
| "p50": 5.747959000000001, |
| "p95": 7.2992919999999994, |
| "p99": 8.021875 |
| }, |
| "gpu_ms": { |
| "p50": 3.626416, |
| "p95": 4.8097080000000005, |
| "p99": 5.3585 |
| } |
| }, |
| { |
| "tokens": "65+", |
| "n": 5, |
| "cpu_ms": { |
| "p50": 10.039708, |
| "p95": 11.134333, |
| "p99": 11.134333 |
| }, |
| "gpu_ms": { |
| "p50": 5.726041, |
| "p95": 6.274583, |
| "p99": 6.274583 |
| } |
| } |
| ], |
| "gpu_scope": "One command buffer: embedding lookup, transformer layers, row-order pooling and FP32 L2, reconstruction errors; tokenization, hashing and ranking/gates on CPU. Joint GPU time is not split into fictitious encoder/resonance wall times." |
| }, |
| { |
| "backend": "metal", |
| "mode": "paired", |
| "run": 1, |
| "source_sha256": "fc7a10ec0068a37b983f205d3eecfa4efe3460a4cb66301cc60f1735ca7864ce", |
| "schema": "cortiq-decision-metal-comparison/2", |
| "model_sha": "386b6e43fd35d5aa9e9710529dfdf30aa868df98ab4434d8e45dae9829898f85", |
| "skill": "clinc150", |
| "n": 4500, |
| "dataset_sha256": "2bf61b64ef682646a8f942e636ee73aad240f04dbd50c8db2a413b44a8b64871", |
| "device": "metal: Apple M4", |
| "warmup": 50, |
| "order": "paired serial; CPU-first/GPU-first alternates per row; same process, model, scorer and texts; no oracle", |
| "golden_max_abs": { |
| "cpu": 0.0, |
| "metal": 1.1920928955078125e-07 |
| }, |
| "max_embedding_abs": 1.6391277313232422e-07, |
| "max_reconstruction_error_abs": 1.621246337890625e-05, |
| "max_reconstruction_error_scaled": 7.066538728395244e-06, |
| "winner_changes": 0, |
| "gate_changes": 0, |
| "hash_changes": 0, |
| "different_rows": [], |
| "correct": { |
| "cpu": 4328, |
| "metal": 4328 |
| }, |
| "accepted": { |
| "cpu": 4145, |
| "metal": 4145 |
| }, |
| "metal_command_buffers": 4500, |
| "cpu_command_buffers": 0, |
| "metal_resonance_dispatches": 4500, |
| "ms": { |
| "cpu_total": { |
| "p50": 3.267958, |
| "p95": 3.6562919999999997, |
| "p99": 3.939334 |
| }, |
| "metal_total": { |
| "p50": 2.515084, |
| "p95": 3.380667, |
| "p99": 4.091292 |
| }, |
| "cpu_encoder": { |
| "p50": 1.924042, |
| "p95": 2.301792, |
| "p99": 2.5448329999999997 |
| }, |
| "metal_split_encoder": { |
| "p50": 0.0, |
| "p95": 0.0, |
| "p99": 0.0 |
| }, |
| "cpu_resonance": { |
| "p50": 1.313542, |
| "p95": 1.432668, |
| "p99": 1.493917 |
| }, |
| "metal_ranking_cpu": { |
| "p50": 0.01225, |
| "p95": 0.022333000000000002, |
| "p99": 0.026250000000000002 |
| }, |
| "metal_joint_pipeline": { |
| "p50": 2.492792, |
| "p95": 3.3617920000000003, |
| "p99": 4.070749999999999 |
| } |
| }, |
| "by_tokens": [ |
| { |
| "tokens": "1-8", |
| "n": 1122, |
| "cpu_ms": { |
| "p50": 3.035875, |
| "p95": 3.2303330000000003, |
| "p99": 3.267417 |
| }, |
| "gpu_ms": { |
| "p50": 2.480708, |
| "p95": 3.425959, |
| "p99": 4.00425 |
| } |
| }, |
| { |
| "tokens": "9-16", |
| "n": 3076, |
| "cpu_ms": { |
| "p50": 3.316917, |
| "p95": 3.6136660000000003, |
| "p99": 3.753667 |
| }, |
| "gpu_ms": { |
| "p50": 2.5173330000000003, |
| "p95": 3.2338340000000003, |
| "p99": 4.145417 |
| } |
| }, |
| { |
| "tokens": "17-32", |
| "n": 302, |
| "cpu_ms": { |
| "p50": 3.617875, |
| "p95": 4.072334000000001, |
| "p99": 4.338167 |
| }, |
| "gpu_ms": { |
| "p50": 2.7420410000000004, |
| "p95": 3.61625, |
| "p99": 3.951833 |
| } |
| } |
| ], |
| "gpu_scope": "One command buffer: embedding lookup, transformer layers, row-order pooling and FP32 L2, reconstruction errors; tokenization, hashing and ranking/gates on CPU. Joint GPU time is not split into fictitious encoder/resonance wall times." |
| }, |
| { |
| "backend": "metal", |
| "mode": "paired", |
| "run": 2, |
| "source_sha256": "681627e6ba06747e7e277329fe47e0ea4264899ddc57359d95062159f8885101", |
| "schema": "cortiq-decision-metal-comparison/2", |
| "model_sha": "386b6e43fd35d5aa9e9710529dfdf30aa868df98ab4434d8e45dae9829898f85", |
| "skill": "clinc150", |
| "n": 4500, |
| "dataset_sha256": "2bf61b64ef682646a8f942e636ee73aad240f04dbd50c8db2a413b44a8b64871", |
| "device": "metal: Apple M4", |
| "warmup": 50, |
| "order": "paired serial; CPU-first/GPU-first alternates per row; same process, model, scorer and texts; no oracle", |
| "golden_max_abs": { |
| "cpu": 0.0, |
| "metal": 1.1920928955078125e-07 |
| }, |
| "max_embedding_abs": 1.6391277313232422e-07, |
| "max_reconstruction_error_abs": 1.621246337890625e-05, |
| "max_reconstruction_error_scaled": 7.066538728395244e-06, |
| "winner_changes": 0, |
| "gate_changes": 0, |
| "hash_changes": 0, |
| "different_rows": [], |
| "correct": { |
| "cpu": 4328, |
| "metal": 4328 |
| }, |
| "accepted": { |
| "cpu": 4145, |
| "metal": 4145 |
| }, |
| "metal_command_buffers": 4500, |
| "cpu_command_buffers": 0, |
| "metal_resonance_dispatches": 4500, |
| "ms": { |
| "cpu_total": { |
| "p50": 3.262333, |
| "p95": 3.689833, |
| "p99": 4.154291 |
| }, |
| "metal_total": { |
| "p50": 2.554791, |
| "p95": 3.5370000000000004, |
| "p99": 6.369834 |
| }, |
| "cpu_encoder": { |
| "p50": 1.922083, |
| "p95": 2.319333, |
| "p99": 2.65525 |
| }, |
| "metal_split_encoder": { |
| "p50": 0.0, |
| "p95": 0.0, |
| "p99": 0.0 |
| }, |
| "cpu_resonance": { |
| "p50": 1.312, |
| "p95": 1.4382080000000002, |
| "p99": 1.594958 |
| }, |
| "metal_ranking_cpu": { |
| "p50": 0.012209, |
| "p95": 0.022541, |
| "p99": 0.025791 |
| }, |
| "metal_joint_pipeline": { |
| "p50": 2.532959, |
| "p95": 3.5140000000000002, |
| "p99": 6.349708 |
| } |
| }, |
| "by_tokens": [ |
| { |
| "tokens": "1-8", |
| "n": 1122, |
| "cpu_ms": { |
| "p50": 3.0227090000000003, |
| "p95": 3.213959, |
| "p99": 3.4175 |
| }, |
| "gpu_ms": { |
| "p50": 2.498291, |
| "p95": 3.451, |
| "p99": 4.900708 |
| } |
| }, |
| { |
| "tokens": "9-16", |
| "n": 3076, |
| "cpu_ms": { |
| "p50": 3.307833, |
| "p95": 3.652208, |
| "p99": 4.212333 |
| }, |
| "gpu_ms": { |
| "p50": 2.574583, |
| "p95": 3.5756669999999997, |
| "p99": 6.6290000000000004 |
| } |
| }, |
| { |
| "tokens": "17-32", |
| "n": 302, |
| "cpu_ms": { |
| "p50": 3.616209, |
| "p95": 4.066125, |
| "p99": 4.446791999999999 |
| }, |
| "gpu_ms": { |
| "p50": 2.780833, |
| "p95": 3.815208, |
| "p99": 6.182792 |
| } |
| } |
| ], |
| "gpu_scope": "One command buffer: embedding lookup, transformer layers, row-order pooling and FP32 L2, reconstruction errors; tokenization, hashing and ranking/gates on CPU. Joint GPU time is not split into fictitious encoder/resonance wall times." |
| }, |
| { |
| "backend": "metal", |
| "mode": "paired", |
| "run": 1, |
| "source_sha256": "fba875dad32c2f41d1bb7ec4270de68a3d1d012c09a86f8f2e8ff115f97161ad", |
| "schema": "cortiq-decision-metal-comparison/2", |
| "model_sha": "386b6e43fd35d5aa9e9710529dfdf30aa868df98ab4434d8e45dae9829898f85", |
| "skill": "massive", |
| "n": 2974, |
| "dataset_sha256": "5517f3de57d9ec9f73885dbd01b26a55a5cd806b9e7ff10c8873f5998e72dbd3", |
| "device": "metal: Apple M4", |
| "warmup": 50, |
| "order": "paired serial; CPU-first/GPU-first alternates per row; same process, model, scorer and texts; no oracle", |
| "golden_max_abs": { |
| "cpu": 0.0, |
| "metal": 1.1920928955078125e-07 |
| }, |
| "max_embedding_abs": 1.7881393432617188e-07, |
| "max_reconstruction_error_abs": 2.0384788513183594e-05, |
| "max_reconstruction_error_scaled": 8.684849490236957e-06, |
| "winner_changes": 0, |
| "gate_changes": 0, |
| "hash_changes": 0, |
| "different_rows": [], |
| "correct": { |
| "cpu": 2562, |
| "metal": 2562 |
| }, |
| "accepted": { |
| "cpu": 1615, |
| "metal": 1615 |
| }, |
| "metal_command_buffers": 2974, |
| "cpu_command_buffers": 0, |
| "metal_resonance_dispatches": 2974, |
| "ms": { |
| "cpu_total": { |
| "p50": 2.660333, |
| "p95": 3.0798330000000003, |
| "p99": 3.3184590000000003 |
| }, |
| "metal_total": { |
| "p50": 2.281333, |
| "p95": 3.068542, |
| "p99": 3.3704579999999997 |
| }, |
| "cpu_encoder": { |
| "p50": 1.8122500000000001, |
| "p95": 2.230708, |
| "p99": 2.445875 |
| }, |
| "metal_split_encoder": { |
| "p50": 0.0, |
| "p95": 0.0, |
| "p99": 0.0 |
| }, |
| "cpu_resonance": { |
| "p50": 0.816874, |
| "p95": 0.8975, |
| "p99": 0.923208 |
| }, |
| "metal_ranking_cpu": { |
| "p50": 0.005667, |
| "p95": 0.016125, |
| "p99": 0.018917 |
| }, |
| "metal_joint_pipeline": { |
| "p50": 2.265208, |
| "p95": 3.047834, |
| "p99": 3.353167 |
| } |
| }, |
| "by_tokens": [ |
| { |
| "tokens": "1-8", |
| "n": 1368, |
| "cpu_ms": { |
| "p50": 2.500333, |
| "p95": 2.691542, |
| "p99": 2.740958 |
| }, |
| "gpu_ms": { |
| "p50": 2.241958, |
| "p95": 2.847834, |
| "p99": 3.29425 |
| } |
| }, |
| { |
| "tokens": "9-16", |
| "n": 1487, |
| "cpu_ms": { |
| "p50": 2.7821249999999997, |
| "p95": 3.0707500000000003, |
| "p99": 3.2034580000000004 |
| }, |
| "gpu_ms": { |
| "p50": 2.320292, |
| "p95": 3.112666, |
| "p99": 3.3556250000000003 |
| } |
| }, |
| { |
| "tokens": "17-32", |
| "n": 117, |
| "cpu_ms": { |
| "p50": 3.137833, |
| "p95": 3.7797080000000003, |
| "p99": 4.111084 |
| }, |
| "gpu_ms": { |
| "p50": 2.556792, |
| "p95": 3.4589999999999996, |
| "p99": 3.741084 |
| } |
| }, |
| { |
| "tokens": "33-64", |
| "n": 2, |
| "cpu_ms": { |
| "p50": 7.734584, |
| "p95": 7.734584, |
| "p99": 7.734584 |
| }, |
| "gpu_ms": { |
| "p50": 4.539750000000001, |
| "p95": 4.539750000000001, |
| "p99": 4.539750000000001 |
| } |
| } |
| ], |
| "gpu_scope": "One command buffer: embedding lookup, transformer layers, row-order pooling and FP32 L2, reconstruction errors; tokenization, hashing and ranking/gates on CPU. Joint GPU time is not split into fictitious encoder/resonance wall times." |
| }, |
| { |
| "backend": "metal", |
| "mode": "paired", |
| "run": 2, |
| "source_sha256": "05265e40e4b03cf628040b026cd4cec81ae2cc9119f5267c361e88cac7937dd7", |
| "schema": "cortiq-decision-metal-comparison/2", |
| "model_sha": "386b6e43fd35d5aa9e9710529dfdf30aa868df98ab4434d8e45dae9829898f85", |
| "skill": "massive", |
| "n": 2974, |
| "dataset_sha256": "5517f3de57d9ec9f73885dbd01b26a55a5cd806b9e7ff10c8873f5998e72dbd3", |
| "device": "metal: Apple M4", |
| "warmup": 50, |
| "order": "paired serial; CPU-first/GPU-first alternates per row; same process, model, scorer and texts; no oracle", |
| "golden_max_abs": { |
| "cpu": 0.0, |
| "metal": 1.1920928955078125e-07 |
| }, |
| "max_embedding_abs": 1.7881393432617188e-07, |
| "max_reconstruction_error_abs": 2.0384788513183594e-05, |
| "max_reconstruction_error_scaled": 8.684849490236957e-06, |
| "winner_changes": 0, |
| "gate_changes": 0, |
| "hash_changes": 0, |
| "different_rows": [], |
| "correct": { |
| "cpu": 2562, |
| "metal": 2562 |
| }, |
| "accepted": { |
| "cpu": 1615, |
| "metal": 1615 |
| }, |
| "metal_command_buffers": 2974, |
| "cpu_command_buffers": 0, |
| "metal_resonance_dispatches": 2974, |
| "ms": { |
| "cpu_total": { |
| "p50": 2.684167, |
| "p95": 3.127583, |
| "p99": 3.5339579999999997 |
| }, |
| "metal_total": { |
| "p50": 2.3083750000000003, |
| "p95": 3.864667, |
| "p99": 7.1722920000000006 |
| }, |
| "cpu_encoder": { |
| "p50": 1.8290419999999998, |
| "p95": 2.255833, |
| "p99": 2.546667 |
| }, |
| "metal_split_encoder": { |
| "p50": 0.0, |
| "p95": 0.0, |
| "p99": 0.0 |
| }, |
| "cpu_resonance": { |
| "p50": 0.822208, |
| "p95": 0.911333, |
| "p99": 1.023625 |
| }, |
| "metal_ranking_cpu": { |
| "p50": 0.005542, |
| "p95": 0.015958, |
| "p99": 0.019292 |
| }, |
| "metal_joint_pipeline": { |
| "p50": 2.291833, |
| "p95": 3.8492919999999997, |
| "p99": 7.164042 |
| } |
| }, |
| "by_tokens": [ |
| { |
| "tokens": "1-8", |
| "n": 1368, |
| "cpu_ms": { |
| "p50": 2.521625, |
| "p95": 2.758834, |
| "p99": 3.1563749999999997 |
| }, |
| "gpu_ms": { |
| "p50": 2.2763750000000003, |
| "p95": 3.9769999999999994, |
| "p99": 7.167666 |
| } |
| }, |
| { |
| "tokens": "9-16", |
| "n": 1487, |
| "cpu_ms": { |
| "p50": 2.797, |
| "p95": 3.109041, |
| "p99": 3.3707089999999997 |
| }, |
| "gpu_ms": { |
| "p50": 2.336875, |
| "p95": 3.387958, |
| "p99": 7.853667 |
| } |
| }, |
| { |
| "tokens": "17-32", |
| "n": 117, |
| "cpu_ms": { |
| "p50": 3.1375, |
| "p95": 3.769041, |
| "p99": 3.985417 |
| }, |
| "gpu_ms": { |
| "p50": 2.574042, |
| "p95": 3.8526670000000003, |
| "p99": 6.3545 |
| } |
| }, |
| { |
| "tokens": "33-64", |
| "n": 2, |
| "cpu_ms": { |
| "p50": 7.529083, |
| "p95": 7.529083, |
| "p99": 7.529083 |
| }, |
| "gpu_ms": { |
| "p50": 4.361125, |
| "p95": 4.361125, |
| "p99": 4.361125 |
| } |
| } |
| ], |
| "gpu_scope": "One command buffer: embedding lookup, transformer layers, row-order pooling and FP32 L2, reconstruction errors; tokenization, hashing and ranking/gates on CPU. Joint GPU time is not split into fictitious encoder/resonance wall times." |
| }, |
| { |
| "backend": "metal", |
| "mode": "separate", |
| "run": 1, |
| "source_sha256": "508fade6a9d905b58ecd96c29f259775c9fc5c1e66faab9fca39bbd5a2149a6b", |
| "schema": "cortiq-decision-metal-comparison/2", |
| "model_sha": "386b6e43fd35d5aa9e9710529dfdf30aa868df98ab4434d8e45dae9829898f85", |
| "skill": "banking77", |
| "n": 3080, |
| "dataset_sha256": "649607fa7b124293c80b455d2effe5e41da713e1836ee936073fe34ab9dc6fba", |
| "device": "metal: Apple M4", |
| "warmup": 50, |
| "order": "separate serial phases: all CPU references, 50 additional GPU warmups, all GPU rows. Same process/model/texts and parity checks; no inserted CPU inference between GPU requests; no oracle.", |
| "golden_max_abs": { |
| "cpu": 0.0, |
| "metal": 1.1920928955078125e-07 |
| }, |
| "max_embedding_abs": 1.6391277313232422e-07, |
| "max_reconstruction_error_abs": 1.1742115020751953e-05, |
| "max_reconstruction_error_scaled": 6.425947049137903e-06, |
| "winner_changes": 0, |
| "gate_changes": 0, |
| "hash_changes": 0, |
| "different_rows": [], |
| "correct": { |
| "cpu": 2875, |
| "metal": 2875 |
| }, |
| "accepted": { |
| "cpu": 2791, |
| "metal": 2791 |
| }, |
| "metal_command_buffers": 3080, |
| "cpu_command_buffers": 0, |
| "metal_resonance_dispatches": 3080, |
| "ms": { |
| "cpu_total": { |
| "p50": 3.426791, |
| "p95": 4.608875, |
| "p99": 6.514125 |
| }, |
| "metal_total": { |
| "p50": 2.082292, |
| "p95": 3.145417, |
| "p99": 3.803208 |
| }, |
| "cpu_encoder": { |
| "p50": 2.053667, |
| "p95": 3.216875, |
| "p99": 5.132416 |
| }, |
| "metal_split_encoder": { |
| "p50": 0.0, |
| "p95": 0.0, |
| "p99": 0.0 |
| }, |
| "cpu_resonance": { |
| "p50": 1.3502919999999998, |
| "p95": 1.425459, |
| "p99": 1.4637499999999999 |
| }, |
| "metal_ranking_cpu": { |
| "p50": 0.006167000000000001, |
| "p95": 0.007875, |
| "p99": 0.008541 |
| }, |
| "metal_joint_pipeline": { |
| "p50": 2.054458, |
| "p95": 3.111166, |
| "p99": 3.762542 |
| } |
| }, |
| "by_tokens": [ |
| { |
| "tokens": "1-8", |
| "n": 230, |
| "cpu_ms": { |
| "p50": 3.0661669999999996, |
| "p95": 3.2113329999999998, |
| "p99": 3.249959 |
| }, |
| "gpu_ms": { |
| "p50": 1.9797500000000001, |
| "p95": 2.824375, |
| "p99": 2.939917 |
| } |
| }, |
| { |
| "tokens": "9-16", |
| "n": 2084, |
| "cpu_ms": { |
| "p50": 3.3712500000000003, |
| "p95": 3.651292, |
| "p99": 3.739666 |
| }, |
| "gpu_ms": { |
| "p50": 2.05275, |
| "p95": 2.918917, |
| "p99": 3.126792 |
| } |
| }, |
| { |
| "tokens": "17-32", |
| "n": 632, |
| "cpu_ms": { |
| "p50": 3.7390000000000003, |
| "p95": 4.504708, |
| "p99": 4.871083 |
| }, |
| "gpu_ms": { |
| "p50": 2.3177499999999998, |
| "p95": 3.191875, |
| "p99": 3.535459 |
| } |
| }, |
| { |
| "tokens": "33-64", |
| "n": 129, |
| "cpu_ms": { |
| "p50": 5.708208, |
| "p95": 7.52575, |
| "p99": 7.977583 |
| }, |
| "gpu_ms": { |
| "p50": 3.325958, |
| "p95": 4.169416, |
| "p99": 4.547917 |
| } |
| }, |
| { |
| "tokens": "65+", |
| "n": 5, |
| "cpu_ms": { |
| "p50": 10.176041, |
| "p95": 10.944333, |
| "p99": 10.944333 |
| }, |
| "gpu_ms": { |
| "p50": 4.924333000000001, |
| "p95": 6.196083, |
| "p99": 6.196083 |
| } |
| } |
| ], |
| "gpu_scope": "One command buffer: embedding lookup, transformer layers, row-order pooling and FP32 L2, reconstruction errors; tokenization, hashing and ranking/gates on CPU. Joint GPU time is not split into fictitious encoder/resonance wall times." |
| }, |
| { |
| "backend": "metal", |
| "mode": "separate", |
| "run": 2, |
| "source_sha256": "77dd75120ca679112906d122f26f7abf15666dd3d0b9d0fe5c7134f79ee457d3", |
| "schema": "cortiq-decision-metal-comparison/2", |
| "model_sha": "386b6e43fd35d5aa9e9710529dfdf30aa868df98ab4434d8e45dae9829898f85", |
| "skill": "banking77", |
| "n": 3080, |
| "dataset_sha256": "649607fa7b124293c80b455d2effe5e41da713e1836ee936073fe34ab9dc6fba", |
| "device": "metal: Apple M4", |
| "warmup": 50, |
| "order": "separate serial phases: all CPU references, 50 additional GPU warmups, all GPU rows. Same process/model/texts and parity checks; no inserted CPU inference between GPU requests; no oracle.", |
| "golden_max_abs": { |
| "cpu": 0.0, |
| "metal": 1.1920928955078125e-07 |
| }, |
| "max_embedding_abs": 1.6391277313232422e-07, |
| "max_reconstruction_error_abs": 1.1742115020751953e-05, |
| "max_reconstruction_error_scaled": 6.425947049137903e-06, |
| "winner_changes": 0, |
| "gate_changes": 0, |
| "hash_changes": 0, |
| "different_rows": [], |
| "correct": { |
| "cpu": 2875, |
| "metal": 2875 |
| }, |
| "accepted": { |
| "cpu": 2791, |
| "metal": 2791 |
| }, |
| "metal_command_buffers": 3080, |
| "cpu_command_buffers": 0, |
| "metal_resonance_dispatches": 3080, |
| "ms": { |
| "cpu_total": { |
| "p50": 3.42625, |
| "p95": 4.570374999999999, |
| "p99": 6.416209 |
| }, |
| "metal_total": { |
| "p50": 2.0866249999999997, |
| "p95": 3.125875, |
| "p99": 3.751917 |
| }, |
| "cpu_encoder": { |
| "p50": 2.0503750000000003, |
| "p95": 3.180292, |
| "p99": 5.027209 |
| }, |
| "metal_split_encoder": { |
| "p50": 0.0, |
| "p95": 0.0, |
| "p99": 0.0 |
| }, |
| "cpu_resonance": { |
| "p50": 1.349958, |
| "p95": 1.421834, |
| "p99": 1.4535 |
| }, |
| "metal_ranking_cpu": { |
| "p50": 0.006292, |
| "p95": 0.008, |
| "p99": 0.008624999999999999 |
| }, |
| "metal_joint_pipeline": { |
| "p50": 2.061583, |
| "p95": 3.097584, |
| "p99": 3.718375 |
| } |
| }, |
| "by_tokens": [ |
| { |
| "tokens": "1-8", |
| "n": 230, |
| "cpu_ms": { |
| "p50": 3.058416, |
| "p95": 3.202792, |
| "p99": 3.234083 |
| }, |
| "gpu_ms": { |
| "p50": 1.978125, |
| "p95": 2.8625, |
| "p99": 3.101292 |
| } |
| }, |
| { |
| "tokens": "9-16", |
| "n": 2084, |
| "cpu_ms": { |
| "p50": 3.364125, |
| "p95": 3.64475, |
| "p99": 3.7355829999999997 |
| }, |
| "gpu_ms": { |
| "p50": 2.059875, |
| "p95": 2.921125, |
| "p99": 3.14525 |
| } |
| }, |
| { |
| "tokens": "17-32", |
| "n": 632, |
| "cpu_ms": { |
| "p50": 3.732167, |
| "p95": 4.517417, |
| "p99": 4.661958 |
| }, |
| "gpu_ms": { |
| "p50": 2.3240000000000003, |
| "p95": 3.290917, |
| "p99": 3.5608329999999997 |
| } |
| }, |
| { |
| "tokens": "33-64", |
| "n": 129, |
| "cpu_ms": { |
| "p50": 5.674583, |
| "p95": 7.4869580000000004, |
| "p99": 8.160708 |
| }, |
| "gpu_ms": { |
| "p50": 3.256042, |
| "p95": 4.225541, |
| "p99": 4.588667 |
| } |
| }, |
| { |
| "tokens": "65+", |
| "n": 5, |
| "cpu_ms": { |
| "p50": 10.207834, |
| "p95": 11.088707999999999, |
| "p99": 11.088707999999999 |
| }, |
| "gpu_ms": { |
| "p50": 5.067416, |
| "p95": 6.178833, |
| "p99": 6.178833 |
| } |
| } |
| ], |
| "gpu_scope": "One command buffer: embedding lookup, transformer layers, row-order pooling and FP32 L2, reconstruction errors; tokenization, hashing and ranking/gates on CPU. Joint GPU time is not split into fictitious encoder/resonance wall times." |
| }, |
| { |
| "backend": "metal", |
| "mode": "separate", |
| "run": 1, |
| "source_sha256": "be41b7ac8fcd001f24a2f96e98481d1a8e79d548ca77d3c37e58781df31a7d2e", |
| "schema": "cortiq-decision-metal-comparison/2", |
| "model_sha": "386b6e43fd35d5aa9e9710529dfdf30aa868df98ab4434d8e45dae9829898f85", |
| "skill": "clinc150", |
| "n": 4500, |
| "dataset_sha256": "2bf61b64ef682646a8f942e636ee73aad240f04dbd50c8db2a413b44a8b64871", |
| "device": "metal: Apple M4", |
| "warmup": 50, |
| "order": "separate serial phases: all CPU references, 50 additional GPU warmups, all GPU rows. Same process/model/texts and parity checks; no inserted CPU inference between GPU requests; no oracle.", |
| "golden_max_abs": { |
| "cpu": 0.0, |
| "metal": 1.1920928955078125e-07 |
| }, |
| "max_embedding_abs": 1.6391277313232422e-07, |
| "max_reconstruction_error_abs": 1.621246337890625e-05, |
| "max_reconstruction_error_scaled": 7.066538728395244e-06, |
| "winner_changes": 0, |
| "gate_changes": 0, |
| "hash_changes": 0, |
| "different_rows": [], |
| "correct": { |
| "cpu": 4328, |
| "metal": 4328 |
| }, |
| "accepted": { |
| "cpu": 4145, |
| "metal": 4145 |
| }, |
| "metal_command_buffers": 4500, |
| "cpu_command_buffers": 0, |
| "metal_resonance_dispatches": 4500, |
| "ms": { |
| "cpu_total": { |
| "p50": 3.2464999999999997, |
| "p95": 3.627459, |
| "p99": 3.8617920000000003 |
| }, |
| "metal_total": { |
| "p50": 2.060625, |
| "p95": 2.932292, |
| "p99": 3.153084 |
| }, |
| "cpu_encoder": { |
| "p50": 1.9132909999999999, |
| "p95": 2.273292, |
| "p99": 2.485417 |
| }, |
| "metal_split_encoder": { |
| "p50": 0.0, |
| "p95": 0.0, |
| "p99": 0.0 |
| }, |
| "cpu_resonance": { |
| "p50": 1.313833, |
| "p95": 1.392667, |
| "p99": 1.432625 |
| }, |
| "metal_ranking_cpu": { |
| "p50": 0.011833, |
| "p95": 0.014959, |
| "p99": 0.015666 |
| }, |
| "metal_joint_pipeline": { |
| "p50": 2.0349169999999996, |
| "p95": 2.903167, |
| "p99": 3.123041 |
| } |
| }, |
| "by_tokens": [ |
| { |
| "tokens": "1-8", |
| "n": 1122, |
| "cpu_ms": { |
| "p50": 3.014959, |
| "p95": 3.1492080000000002, |
| "p99": 3.220209 |
| }, |
| "gpu_ms": { |
| "p50": 2.003458, |
| "p95": 2.833, |
| "p99": 2.982958 |
| } |
| }, |
| { |
| "tokens": "9-16", |
| "n": 3076, |
| "cpu_ms": { |
| "p50": 3.292125, |
| "p95": 3.584625, |
| "p99": 3.674208 |
| }, |
| "gpu_ms": { |
| "p50": 2.071125, |
| "p95": 2.93775, |
| "p99": 3.148125 |
| } |
| }, |
| { |
| "tokens": "17-32", |
| "n": 302, |
| "cpu_ms": { |
| "p50": 3.615583, |
| "p95": 4.034333, |
| "p99": 4.19475 |
| }, |
| "gpu_ms": { |
| "p50": 2.321459, |
| "p95": 3.160875, |
| "p99": 3.412791 |
| } |
| } |
| ], |
| "gpu_scope": "One command buffer: embedding lookup, transformer layers, row-order pooling and FP32 L2, reconstruction errors; tokenization, hashing and ranking/gates on CPU. Joint GPU time is not split into fictitious encoder/resonance wall times." |
| }, |
| { |
| "backend": "metal", |
| "mode": "separate", |
| "run": 2, |
| "source_sha256": "243c6f11d09dce45245c73a99386f3c86c9c3532a86963d25ca6255d9d380b9d", |
| "schema": "cortiq-decision-metal-comparison/2", |
| "model_sha": "386b6e43fd35d5aa9e9710529dfdf30aa868df98ab4434d8e45dae9829898f85", |
| "skill": "clinc150", |
| "n": 4500, |
| "dataset_sha256": "2bf61b64ef682646a8f942e636ee73aad240f04dbd50c8db2a413b44a8b64871", |
| "device": "metal: Apple M4", |
| "warmup": 50, |
| "order": "separate serial phases: all CPU references, 50 additional GPU warmups, all GPU rows. Same process/model/texts and parity checks; no inserted CPU inference between GPU requests; no oracle.", |
| "golden_max_abs": { |
| "cpu": 0.0, |
| "metal": 1.1920928955078125e-07 |
| }, |
| "max_embedding_abs": 1.6391277313232422e-07, |
| "max_reconstruction_error_abs": 1.621246337890625e-05, |
| "max_reconstruction_error_scaled": 7.066538728395244e-06, |
| "winner_changes": 0, |
| "gate_changes": 0, |
| "hash_changes": 0, |
| "different_rows": [], |
| "correct": { |
| "cpu": 4328, |
| "metal": 4328 |
| }, |
| "accepted": { |
| "cpu": 4145, |
| "metal": 4145 |
| }, |
| "metal_command_buffers": 4500, |
| "cpu_command_buffers": 0, |
| "metal_resonance_dispatches": 4500, |
| "ms": { |
| "cpu_total": { |
| "p50": 3.241291, |
| "p95": 3.6175420000000003, |
| "p99": 3.849542 |
| }, |
| "metal_total": { |
| "p50": 2.1338749999999997, |
| "p95": 4.1122499999999995, |
| "p99": 5.328208 |
| }, |
| "cpu_encoder": { |
| "p50": 1.910292, |
| "p95": 2.270959, |
| "p99": 2.518958 |
| }, |
| "metal_split_encoder": { |
| "p50": 0.0, |
| "p95": 0.0, |
| "p99": 0.0 |
| }, |
| "cpu_resonance": { |
| "p50": 1.314292, |
| "p95": 1.387333, |
| "p99": 1.422625 |
| }, |
| "metal_ranking_cpu": { |
| "p50": 0.011958, |
| "p95": 0.017041, |
| "p99": 0.019833 |
| }, |
| "metal_joint_pipeline": { |
| "p50": 2.104209, |
| "p95": 4.0625, |
| "p99": 5.26075 |
| } |
| }, |
| "by_tokens": [ |
| { |
| "tokens": "1-8", |
| "n": 1122, |
| "cpu_ms": { |
| "p50": 3.017375, |
| "p95": 3.155667, |
| "p99": 3.195166 |
| }, |
| "gpu_ms": { |
| "p50": 2.039625, |
| "p95": 3.83825, |
| "p99": 5.165209 |
| } |
| }, |
| { |
| "tokens": "9-16", |
| "n": 3076, |
| "cpu_ms": { |
| "p50": 3.2857089999999998, |
| "p95": 3.5790420000000003, |
| "p99": 3.672209 |
| }, |
| "gpu_ms": { |
| "p50": 2.132625, |
| "p95": 4.120041, |
| "p99": 5.41925 |
| } |
| }, |
| { |
| "tokens": "17-32", |
| "n": 302, |
| "cpu_ms": { |
| "p50": 3.606833, |
| "p95": 4.023, |
| "p99": 4.379667 |
| }, |
| "gpu_ms": { |
| "p50": 2.3708750000000003, |
| "p95": 4.5940829999999995, |
| "p99": 6.212083 |
| } |
| } |
| ], |
| "gpu_scope": "One command buffer: embedding lookup, transformer layers, row-order pooling and FP32 L2, reconstruction errors; tokenization, hashing and ranking/gates on CPU. Joint GPU time is not split into fictitious encoder/resonance wall times." |
| }, |
| { |
| "backend": "metal", |
| "mode": "separate", |
| "run": 1, |
| "source_sha256": "35fb0c3d2645926355cc1f76c42f5d972fecc1f611639eb4b90727b9f1f72ecc", |
| "schema": "cortiq-decision-metal-comparison/2", |
| "model_sha": "386b6e43fd35d5aa9e9710529dfdf30aa868df98ab4434d8e45dae9829898f85", |
| "skill": "massive", |
| "n": 2974, |
| "dataset_sha256": "5517f3de57d9ec9f73885dbd01b26a55a5cd806b9e7ff10c8873f5998e72dbd3", |
| "device": "metal: Apple M4", |
| "warmup": 50, |
| "order": "separate serial phases: all CPU references, 50 additional GPU warmups, all GPU rows. Same process/model/texts and parity checks; no inserted CPU inference between GPU requests; no oracle.", |
| "golden_max_abs": { |
| "cpu": 0.0, |
| "metal": 1.1920928955078125e-07 |
| }, |
| "max_embedding_abs": 1.7881393432617188e-07, |
| "max_reconstruction_error_abs": 2.0384788513183594e-05, |
| "max_reconstruction_error_scaled": 8.684849490236957e-06, |
| "winner_changes": 0, |
| "gate_changes": 0, |
| "hash_changes": 0, |
| "different_rows": [], |
| "correct": { |
| "cpu": 2562, |
| "metal": 2562 |
| }, |
| "accepted": { |
| "cpu": 1615, |
| "metal": 1615 |
| }, |
| "metal_command_buffers": 2974, |
| "cpu_command_buffers": 0, |
| "metal_resonance_dispatches": 2974, |
| "ms": { |
| "cpu_total": { |
| "p50": 2.633041, |
| "p95": 3.0664580000000004, |
| "p99": 3.293333 |
| }, |
| "metal_total": { |
| "p50": 1.809375, |
| "p95": 2.696959, |
| "p99": 2.923583 |
| }, |
| "cpu_encoder": { |
| "p50": 1.795, |
| "p95": 2.2252919999999996, |
| "p99": 2.460125 |
| }, |
| "metal_split_encoder": { |
| "p50": 0.0, |
| "p95": 0.0, |
| "p99": 0.0 |
| }, |
| "cpu_resonance": { |
| "p50": 0.816125, |
| "p95": 0.8720000000000001, |
| "p99": 0.910791 |
| }, |
| "metal_ranking_cpu": { |
| "p50": 0.005332999999999999, |
| "p95": 0.00675, |
| "p99": 0.007084 |
| }, |
| "metal_joint_pipeline": { |
| "p50": 1.786291, |
| "p95": 2.6687909999999997, |
| "p99": 2.8969579999999997 |
| } |
| }, |
| "by_tokens": [ |
| { |
| "tokens": "1-8", |
| "n": 1368, |
| "cpu_ms": { |
| "p50": 2.482125, |
| "p95": 2.622125, |
| "p99": 2.686292 |
| }, |
| "gpu_ms": { |
| "p50": 1.771291, |
| "p95": 2.626583, |
| "p99": 2.81875 |
| } |
| }, |
| { |
| "tokens": "9-16", |
| "n": 1487, |
| "cpu_ms": { |
| "p50": 2.753584, |
| "p95": 3.064, |
| "p99": 3.174083 |
| }, |
| "gpu_ms": { |
| "p50": 1.845625, |
| "p95": 2.712209, |
| "p99": 2.9455839999999998 |
| } |
| }, |
| { |
| "tokens": "17-32", |
| "n": 117, |
| "cpu_ms": { |
| "p50": 3.114834, |
| "p95": 3.8297499999999998, |
| "p99": 4.0048330000000005 |
| }, |
| "gpu_ms": { |
| "p50": 2.079334, |
| "p95": 2.985791, |
| "p99": 3.344291 |
| } |
| }, |
| { |
| "tokens": "33-64", |
| "n": 2, |
| "cpu_ms": { |
| "p50": 7.607125, |
| "p95": 7.607125, |
| "p99": 7.607125 |
| }, |
| "gpu_ms": { |
| "p50": 3.9274169999999997, |
| "p95": 3.9274169999999997, |
| "p99": 3.9274169999999997 |
| } |
| } |
| ], |
| "gpu_scope": "One command buffer: embedding lookup, transformer layers, row-order pooling and FP32 L2, reconstruction errors; tokenization, hashing and ranking/gates on CPU. Joint GPU time is not split into fictitious encoder/resonance wall times." |
| }, |
| { |
| "backend": "metal", |
| "mode": "separate", |
| "run": 2, |
| "source_sha256": "d49be65a3e5f4ee0c1165ce6c6d7829e4d6eff7af4ec2719097e7e19c1464767", |
| "schema": "cortiq-decision-metal-comparison/2", |
| "model_sha": "386b6e43fd35d5aa9e9710529dfdf30aa868df98ab4434d8e45dae9829898f85", |
| "skill": "massive", |
| "n": 2974, |
| "dataset_sha256": "5517f3de57d9ec9f73885dbd01b26a55a5cd806b9e7ff10c8873f5998e72dbd3", |
| "device": "metal: Apple M4", |
| "warmup": 50, |
| "order": "separate serial phases: all CPU references, 50 additional GPU warmups, all GPU rows. Same process/model/texts and parity checks; no inserted CPU inference between GPU requests; no oracle.", |
| "golden_max_abs": { |
| "cpu": 0.0, |
| "metal": 1.1920928955078125e-07 |
| }, |
| "max_embedding_abs": 1.7881393432617188e-07, |
| "max_reconstruction_error_abs": 2.0384788513183594e-05, |
| "max_reconstruction_error_scaled": 8.684849490236957e-06, |
| "winner_changes": 0, |
| "gate_changes": 0, |
| "hash_changes": 0, |
| "different_rows": [], |
| "correct": { |
| "cpu": 2562, |
| "metal": 2562 |
| }, |
| "accepted": { |
| "cpu": 1615, |
| "metal": 1615 |
| }, |
| "metal_command_buffers": 2974, |
| "cpu_command_buffers": 0, |
| "metal_resonance_dispatches": 2974, |
| "ms": { |
| "cpu_total": { |
| "p50": 2.636959, |
| "p95": 3.060417, |
| "p99": 3.2911669999999997 |
| }, |
| "metal_total": { |
| "p50": 3.2125830000000004, |
| "p95": 5.442749999999999, |
| "p99": 5.974334000000001 |
| }, |
| "cpu_encoder": { |
| "p50": 1.7979580000000002, |
| "p95": 2.215167, |
| "p99": 2.4351659999999997 |
| }, |
| "metal_split_encoder": { |
| "p50": 0.0, |
| "p95": 0.0, |
| "p99": 0.0 |
| }, |
| "cpu_resonance": { |
| "p50": 0.820249, |
| "p95": 0.877958, |
| "p99": 0.925708 |
| }, |
| "metal_ranking_cpu": { |
| "p50": 0.006334, |
| "p95": 0.009082999999999999, |
| "p99": 0.010458 |
| }, |
| "metal_joint_pipeline": { |
| "p50": 3.169042, |
| "p95": 5.399167, |
| "p99": 5.931625 |
| } |
| }, |
| "by_tokens": [ |
| { |
| "tokens": "1-8", |
| "n": 1368, |
| "cpu_ms": { |
| "p50": 2.4871659999999998, |
| "p95": 2.633291, |
| "p99": 2.705 |
| }, |
| "gpu_ms": { |
| "p50": 2.913458, |
| "p95": 5.177166000000001, |
| "p99": 5.509708000000001 |
| } |
| }, |
| { |
| "tokens": "9-16", |
| "n": 1487, |
| "cpu_ms": { |
| "p50": 2.763167, |
| "p95": 3.060709, |
| "p99": 3.167125 |
| }, |
| "gpu_ms": { |
| "p50": 3.3691250000000004, |
| "p95": 5.669917, |
| "p99": 5.914667000000001 |
| } |
| }, |
| { |
| "tokens": "17-32", |
| "n": 117, |
| "cpu_ms": { |
| "p50": 3.1175, |
| "p95": 3.743042, |
| "p99": 3.994167 |
| }, |
| "gpu_ms": { |
| "p50": 4.251916, |
| "p95": 6.402959, |
| "p99": 7.223292 |
| } |
| }, |
| { |
| "tokens": "33-64", |
| "n": 2, |
| "cpu_ms": { |
| "p50": 7.465333, |
| "p95": 7.465333, |
| "p99": 7.465333 |
| }, |
| "gpu_ms": { |
| "p50": 6.928792, |
| "p95": 6.928792, |
| "p99": 6.928792 |
| } |
| } |
| ], |
| "gpu_scope": "One command buffer: embedding lookup, transformer layers, row-order pooling and FP32 L2, reconstruction errors; tokenization, hashing and ranking/gates on CPU. Joint GPU time is not split into fictitious encoder/resonance wall times." |
| }, |
| { |
| "backend": "vulkan", |
| "mode": "separate", |
| "run": 1, |
| "source_sha256": "f785183f2c6e75116dec3510313a2fe2220d6e4f839ec5f53f50455081b1e974", |
| "schema": "cortiq-decision-vulkan-comparison/2", |
| "model_sha": "386b6e43fd35d5aa9e9710529dfdf30aa868df98ab4434d8e45dae9829898f85", |
| "skill": "banking77", |
| "n": 3080, |
| "dataset_sha256": "649607fa7b124293c80b455d2effe5e41da713e1836ee936073fe34ab9dc6fba", |
| "device": "vulkan: NVIDIA RTX PRO 4000 Blackwell", |
| "warmup": 50, |
| "order": "separate serial phases: all CPU references, 50 additional GPU warmups, all GPU rows. Same process/model/texts and parity checks; no inserted CPU inference between GPU requests; no oracle.", |
| "golden_max_abs": { |
| "cpu": 1.1920928955078125e-07, |
| "vulkan": 1.341104507446289e-07 |
| }, |
| "max_embedding_abs": 1.4901161193847656e-07, |
| "max_reconstruction_error_abs": 1.1682510375976562e-05, |
| "max_reconstruction_error_scaled": 6.459768428612733e-06, |
| "winner_changes": 0, |
| "gate_changes": 0, |
| "hash_changes": 0, |
| "different_rows": [], |
| "correct": { |
| "cpu": 2875, |
| "vulkan": 2875 |
| }, |
| "accepted": { |
| "cpu": 2791, |
| "vulkan": 2791 |
| }, |
| "vulkan_command_buffers": 3080, |
| "cpu_command_buffers": 0, |
| "vulkan_resonance_dispatches": 3080, |
| "ms": { |
| "cpu_total": { |
| "p50": 24.300031999999998, |
| "p95": 47.229065, |
| "p99": 72.325288 |
| }, |
| "vulkan_total": { |
| "p50": 1.243007, |
| "p95": 1.779005, |
| "p99": 2.225192 |
| }, |
| "cpu_encoder": { |
| "p50": 17.429967, |
| "p95": 40.324829, |
| "p99": 65.381701 |
| }, |
| "vulkan_split_encoder": { |
| "p50": 0.0, |
| "p95": 0.0, |
| "p99": 0.0 |
| }, |
| "cpu_resonance": { |
| "p50": 6.8515440000000005, |
| "p95": 6.980182, |
| "p99": 7.111081 |
| }, |
| "vulkan_ranking_cpu": { |
| "p50": 0.005383, |
| "p95": 0.022862, |
| "p99": 0.026559 |
| }, |
| "vulkan_joint_pipeline": { |
| "p50": 1.205752, |
| "p95": 1.714144, |
| "p99": 2.136114 |
| } |
| }, |
| "by_tokens": [ |
| { |
| "tokens": "1-8", |
| "n": 230, |
| "cpu_ms": { |
| "p50": 17.484273, |
| "p95": 18.781585, |
| "p99": 19.075535000000002 |
| }, |
| "gpu_ms": { |
| "p50": 1.036284, |
| "p95": 1.187726, |
| "p99": 1.273517 |
| } |
| }, |
| { |
| "tokens": "9-16", |
| "n": 2084, |
| "cpu_ms": { |
| "p50": 23.02284, |
| "p95": 27.872968, |
| "p99": 28.654959 |
| }, |
| "gpu_ms": { |
| "p50": 1.216743, |
| "p95": 1.3710110000000002, |
| "p99": 1.451524 |
| } |
| }, |
| { |
| "tokens": "17-32", |
| "n": 632, |
| "cpu_ms": { |
| "p50": 33.219012, |
| "p95": 45.601294, |
| "p99": 50.627319 |
| }, |
| "gpu_ms": { |
| "p50": 1.453273, |
| "p95": 1.7577289999999999, |
| "p99": 1.838818 |
| } |
| }, |
| { |
| "tokens": "33-64", |
| "n": 129, |
| "cpu_ms": { |
| "p50": 60.365970999999995, |
| "p95": 83.40327, |
| "p99": 89.708682 |
| }, |
| "gpu_ms": { |
| "p50": 2.007216, |
| "p95": 2.5339579999999997, |
| "p99": 2.688999 |
| } |
| }, |
| { |
| "tokens": "65+", |
| "n": 5, |
| "cpu_ms": { |
| "p50": 111.20082000000001, |
| "p95": 113.755177, |
| "p99": 113.755177 |
| }, |
| "gpu_ms": { |
| "p50": 3.074434, |
| "p95": 3.303453, |
| "p99": 3.303453 |
| } |
| } |
| ], |
| "gpu_scope": "One command buffer: embedding lookup, transformer layers, row-order pooling and FP32 L2, reconstruction errors; tokenization, hashing and ranking/gates on CPU. Joint GPU time is not split into fictitious encoder/resonance wall times." |
| }, |
| { |
| "backend": "vulkan", |
| "mode": "separate", |
| "run": 2, |
| "source_sha256": "94a0c4678f95e2adf44c79ceb2c35d34eb45630dbc0fb87ca9f8ae48a860007f", |
| "schema": "cortiq-decision-vulkan-comparison/2", |
| "model_sha": "386b6e43fd35d5aa9e9710529dfdf30aa868df98ab4434d8e45dae9829898f85", |
| "skill": "banking77", |
| "n": 3080, |
| "dataset_sha256": "649607fa7b124293c80b455d2effe5e41da713e1836ee936073fe34ab9dc6fba", |
| "device": "vulkan: NVIDIA RTX PRO 4000 Blackwell", |
| "warmup": 50, |
| "order": "separate serial phases: all CPU references, 50 additional GPU warmups, all GPU rows. Same process/model/texts and parity checks; no inserted CPU inference between GPU requests; no oracle.", |
| "golden_max_abs": { |
| "cpu": 1.1920928955078125e-07, |
| "vulkan": 1.341104507446289e-07 |
| }, |
| "max_embedding_abs": 1.4901161193847656e-07, |
| "max_reconstruction_error_abs": 1.1682510375976562e-05, |
| "max_reconstruction_error_scaled": 6.459768428612733e-06, |
| "winner_changes": 0, |
| "gate_changes": 0, |
| "hash_changes": 0, |
| "different_rows": [], |
| "correct": { |
| "cpu": 2875, |
| "vulkan": 2875 |
| }, |
| "accepted": { |
| "cpu": 2791, |
| "vulkan": 2791 |
| }, |
| "vulkan_command_buffers": 3080, |
| "cpu_command_buffers": 0, |
| "vulkan_resonance_dispatches": 3080, |
| "ms": { |
| "cpu_total": { |
| "p50": 24.053228, |
| "p95": 47.818072, |
| "p99": 71.431764 |
| }, |
| "vulkan_total": { |
| "p50": 1.2399229999999999, |
| "p95": 1.772518, |
| "p99": 2.296392 |
| }, |
| "cpu_encoder": { |
| "p50": 17.044211999999998, |
| "p95": 40.826451, |
| "p99": 64.435344 |
| }, |
| "vulkan_split_encoder": { |
| "p50": 0.0, |
| "p95": 0.0, |
| "p99": 0.0 |
| }, |
| "cpu_resonance": { |
| "p50": 6.954758, |
| "p95": 7.078664, |
| "p99": 7.189576 |
| }, |
| "vulkan_ranking_cpu": { |
| "p50": 0.005202, |
| "p95": 0.014052, |
| "p99": 0.025805 |
| }, |
| "vulkan_joint_pipeline": { |
| "p50": 1.203881, |
| "p95": 1.717145, |
| "p99": 2.198691 |
| } |
| }, |
| "by_tokens": [ |
| { |
| "tokens": "1-8", |
| "n": 230, |
| "cpu_ms": { |
| "p50": 17.640131, |
| "p95": 18.821365999999998, |
| "p99": 18.949503999999997 |
| }, |
| "gpu_ms": { |
| "p50": 1.051358, |
| "p95": 1.200305, |
| "p99": 1.249962 |
| } |
| }, |
| { |
| "tokens": "9-16", |
| "n": 2084, |
| "cpu_ms": { |
| "p50": 23.347106999999998, |
| "p95": 28.079639999999998, |
| "p99": 28.775907 |
| }, |
| "gpu_ms": { |
| "p50": 1.214836, |
| "p95": 1.369686, |
| "p99": 1.4417259999999998 |
| } |
| }, |
| { |
| "tokens": "17-32", |
| "n": 632, |
| "cpu_ms": { |
| "p50": 33.129235, |
| "p95": 46.115206, |
| "p99": 50.701332 |
| }, |
| "gpu_ms": { |
| "p50": 1.454658, |
| "p95": 1.7448590000000002, |
| "p99": 1.828859 |
| } |
| }, |
| { |
| "tokens": "33-64", |
| "n": 129, |
| "cpu_ms": { |
| "p50": 58.644467, |
| "p95": 85.296793, |
| "p99": 93.428545 |
| }, |
| "gpu_ms": { |
| "p50": 1.9888150000000002, |
| "p95": 2.478187, |
| "p99": 2.61321 |
| } |
| }, |
| { |
| "tokens": "65+", |
| "n": 5, |
| "cpu_ms": { |
| "p50": 104.71356800000001, |
| "p95": 128.376401, |
| "p99": 128.376401 |
| }, |
| "gpu_ms": { |
| "p50": 3.02045, |
| "p95": 3.336854, |
| "p99": 3.336854 |
| } |
| } |
| ], |
| "gpu_scope": "One command buffer: embedding lookup, transformer layers, row-order pooling and FP32 L2, reconstruction errors; tokenization, hashing and ranking/gates on CPU. Joint GPU time is not split into fictitious encoder/resonance wall times." |
| }, |
| { |
| "backend": "vulkan", |
| "mode": "separate", |
| "run": 1, |
| "source_sha256": "ec20b3839eb265c842fe880f9347de9ba34793a625b509fcd6490dff8b6aa8a8", |
| "schema": "cortiq-decision-vulkan-comparison/2", |
| "model_sha": "386b6e43fd35d5aa9e9710529dfdf30aa868df98ab4434d8e45dae9829898f85", |
| "skill": "clinc150", |
| "n": 4500, |
| "dataset_sha256": "2bf61b64ef682646a8f942e636ee73aad240f04dbd50c8db2a413b44a8b64871", |
| "device": "vulkan: NVIDIA RTX PRO 4000 Blackwell", |
| "warmup": 50, |
| "order": "separate serial phases: all CPU references, 50 additional GPU warmups, all GPU rows. Same process/model/texts and parity checks; no inserted CPU inference between GPU requests; no oracle.", |
| "golden_max_abs": { |
| "cpu": 1.1920928955078125e-07, |
| "vulkan": 1.341104507446289e-07 |
| }, |
| "max_embedding_abs": 1.6391277313232422e-07, |
| "max_reconstruction_error_abs": 1.6450881958007812e-05, |
| "max_reconstruction_error_scaled": 7.17045804776717e-06, |
| "winner_changes": 0, |
| "gate_changes": 0, |
| "hash_changes": 0, |
| "different_rows": [], |
| "correct": { |
| "cpu": 4328, |
| "vulkan": 4328 |
| }, |
| "accepted": { |
| "cpu": 4145, |
| "vulkan": 4145 |
| }, |
| "vulkan_command_buffers": 4500, |
| "cpu_command_buffers": 0, |
| "vulkan_resonance_dispatches": 4500, |
| "ms": { |
| "cpu_total": { |
| "p50": 21.070990000000002, |
| "p95": 28.693944, |
| "p99": 35.540867 |
| }, |
| "vulkan_total": { |
| "p50": 1.183739, |
| "p95": 1.4106889999999999, |
| "p99": 1.555227 |
| }, |
| "cpu_encoder": { |
| "p50": 14.319863999999999, |
| "p95": 21.939096000000003, |
| "p99": 28.740892 |
| }, |
| "vulkan_split_encoder": { |
| "p50": 0.0, |
| "p95": 0.0, |
| "p99": 0.0 |
| }, |
| "cpu_resonance": { |
| "p50": 6.732084, |
| "p95": 6.846308, |
| "p99": 6.971 |
| }, |
| "vulkan_ranking_cpu": { |
| "p50": 0.010095000000000002, |
| "p95": 0.026825, |
| "p99": 0.037598999999999994 |
| }, |
| "vulkan_joint_pipeline": { |
| "p50": 1.142588, |
| "p95": 1.3502889999999999, |
| "p99": 1.4805000000000001 |
| } |
| }, |
| "by_tokens": [ |
| { |
| "tokens": "1-8", |
| "n": 1122, |
| "cpu_ms": { |
| "p50": 17.321112, |
| "p95": 18.650374, |
| "p99": 18.874518 |
| }, |
| "gpu_ms": { |
| "p50": 1.0571860000000002, |
| "p95": 1.184428, |
| "p99": 1.2439479999999998 |
| } |
| }, |
| { |
| "tokens": "9-16", |
| "n": 3076, |
| "cpu_ms": { |
| "p50": 21.709161, |
| "p95": 27.078848999999998, |
| "p99": 28.25304 |
| }, |
| "gpu_ms": { |
| "p50": 1.205003, |
| "p95": 1.356696, |
| "p99": 1.442542 |
| } |
| }, |
| { |
| "tokens": "17-32", |
| "n": 302, |
| "cpu_ms": { |
| "p50": 30.710203, |
| "p95": 38.634969, |
| "p99": 44.367315 |
| }, |
| "gpu_ms": { |
| "p50": 1.428216, |
| "p95": 1.618438, |
| "p99": 1.7441170000000001 |
| } |
| } |
| ], |
| "gpu_scope": "One command buffer: embedding lookup, transformer layers, row-order pooling and FP32 L2, reconstruction errors; tokenization, hashing and ranking/gates on CPU. Joint GPU time is not split into fictitious encoder/resonance wall times." |
| }, |
| { |
| "backend": "vulkan", |
| "mode": "separate", |
| "run": 2, |
| "source_sha256": "133a985c0a007cd45d8fcc23333d2edc319a721bd8d1b7048986e61768f9a87d", |
| "schema": "cortiq-decision-vulkan-comparison/2", |
| "model_sha": "386b6e43fd35d5aa9e9710529dfdf30aa868df98ab4434d8e45dae9829898f85", |
| "skill": "clinc150", |
| "n": 4500, |
| "dataset_sha256": "2bf61b64ef682646a8f942e636ee73aad240f04dbd50c8db2a413b44a8b64871", |
| "device": "vulkan: NVIDIA RTX PRO 4000 Blackwell", |
| "warmup": 50, |
| "order": "separate serial phases: all CPU references, 50 additional GPU warmups, all GPU rows. Same process/model/texts and parity checks; no inserted CPU inference between GPU requests; no oracle.", |
| "golden_max_abs": { |
| "cpu": 1.1920928955078125e-07, |
| "vulkan": 1.341104507446289e-07 |
| }, |
| "max_embedding_abs": 1.6391277313232422e-07, |
| "max_reconstruction_error_abs": 1.6450881958007812e-05, |
| "max_reconstruction_error_scaled": 7.17045804776717e-06, |
| "winner_changes": 0, |
| "gate_changes": 0, |
| "hash_changes": 0, |
| "different_rows": [], |
| "correct": { |
| "cpu": 4328, |
| "vulkan": 4328 |
| }, |
| "accepted": { |
| "cpu": 4145, |
| "vulkan": 4145 |
| }, |
| "vulkan_command_buffers": 4500, |
| "cpu_command_buffers": 0, |
| "vulkan_resonance_dispatches": 4500, |
| "ms": { |
| "cpu_total": { |
| "p50": 21.108217, |
| "p95": 30.122833, |
| "p99": 35.055634000000005 |
| }, |
| "vulkan_total": { |
| "p50": 1.20068, |
| "p95": 1.428602, |
| "p99": 1.585988 |
| }, |
| "cpu_encoder": { |
| "p50": 14.355258, |
| "p95": 23.349382, |
| "p99": 28.311857 |
| }, |
| "vulkan_split_encoder": { |
| "p50": 0.0, |
| "p95": 0.0, |
| "p99": 0.0 |
| }, |
| "cpu_resonance": { |
| "p50": 6.738755, |
| "p95": 6.857221, |
| "p99": 6.979973 |
| }, |
| "vulkan_ranking_cpu": { |
| "p50": 0.016079999999999997, |
| "p95": 0.027441999999999998, |
| "p99": 0.037886 |
| }, |
| "vulkan_joint_pipeline": { |
| "p50": 1.1596270000000002, |
| "p95": 1.369136, |
| "p99": 1.508872 |
| } |
| }, |
| "by_tokens": [ |
| { |
| "tokens": "1-8", |
| "n": 1122, |
| "cpu_ms": { |
| "p50": 17.345407, |
| "p95": 18.718383000000003, |
| "p99": 18.832627000000002 |
| }, |
| "gpu_ms": { |
| "p50": 1.06457, |
| "p95": 1.2041160000000002, |
| "p99": 1.2711560000000002 |
| } |
| }, |
| { |
| "tokens": "9-16", |
| "n": 3076, |
| "cpu_ms": { |
| "p50": 21.634235, |
| "p95": 27.603956999999998, |
| "p99": 28.299501000000003 |
| }, |
| "gpu_ms": { |
| "p50": 1.221998, |
| "p95": 1.374075, |
| "p99": 1.44901 |
| } |
| }, |
| { |
| "tokens": "17-32", |
| "n": 302, |
| "cpu_ms": { |
| "p50": 31.161271, |
| "p95": 38.58855, |
| "p99": 42.460954 |
| }, |
| "gpu_ms": { |
| "p50": 1.44708, |
| "p95": 1.641224, |
| "p99": 1.728692 |
| } |
| } |
| ], |
| "gpu_scope": "One command buffer: embedding lookup, transformer layers, row-order pooling and FP32 L2, reconstruction errors; tokenization, hashing and ranking/gates on CPU. Joint GPU time is not split into fictitious encoder/resonance wall times." |
| }, |
| { |
| "backend": "vulkan", |
| "mode": "separate", |
| "run": 1, |
| "source_sha256": "f3c2a40a056decc6b55db1f7b24b6db8c210a238ed9a0e36ba198a40b58cd138", |
| "schema": "cortiq-decision-vulkan-comparison/2", |
| "model_sha": "386b6e43fd35d5aa9e9710529dfdf30aa868df98ab4434d8e45dae9829898f85", |
| "skill": "massive", |
| "n": 2974, |
| "dataset_sha256": "5517f3de57d9ec9f73885dbd01b26a55a5cd806b9e7ff10c8873f5998e72dbd3", |
| "device": "vulkan: NVIDIA RTX PRO 4000 Blackwell", |
| "warmup": 50, |
| "order": "separate serial phases: all CPU references, 50 additional GPU warmups, all GPU rows. Same process/model/texts and parity checks; no inserted CPU inference between GPU requests; no oracle.", |
| "golden_max_abs": { |
| "cpu": 1.1920928955078125e-07, |
| "vulkan": 1.341104507446289e-07 |
| }, |
| "max_embedding_abs": 1.4901161193847656e-07, |
| "max_reconstruction_error_abs": 2.0503997802734375e-05, |
| "max_reconstruction_error_scaled": 8.735638402868062e-06, |
| "winner_changes": 0, |
| "gate_changes": 0, |
| "hash_changes": 0, |
| "different_rows": [], |
| "correct": { |
| "cpu": 2562, |
| "vulkan": 2562 |
| }, |
| "accepted": { |
| "cpu": 1615, |
| "vulkan": 1615 |
| }, |
| "vulkan_command_buffers": 2974, |
| "cpu_command_buffers": 0, |
| "vulkan_resonance_dispatches": 2974, |
| "ms": { |
| "cpu_total": { |
| "p50": 16.259952000000002, |
| "p95": 25.118388, |
| "p99": 31.614088 |
| }, |
| "vulkan_total": { |
| "p50": 1.1542169999999998, |
| "p95": 1.373807, |
| "p99": 1.563799 |
| }, |
| "cpu_encoder": { |
| "p50": 12.123259000000001, |
| "p95": 21.019095, |
| "p99": 27.428562 |
| }, |
| "vulkan_split_encoder": { |
| "p50": 0.0, |
| "p95": 0.0, |
| "p99": 0.0 |
| }, |
| "cpu_resonance": { |
| "p50": 4.093815, |
| "p95": 4.180759, |
| "p99": 4.279031 |
| }, |
| "vulkan_ranking_cpu": { |
| "p50": 0.010161, |
| "p95": 0.011192, |
| "p99": 0.025403000000000002 |
| }, |
| "vulkan_joint_pipeline": { |
| "p50": 1.116078, |
| "p95": 1.32156, |
| "p99": 1.492097 |
| } |
| }, |
| "by_tokens": [ |
| { |
| "tokens": "1-8", |
| "n": 1368, |
| "cpu_ms": { |
| "p50": 13.768594, |
| "p95": 15.994110999999998, |
| "p99": 16.164708 |
| }, |
| "gpu_ms": { |
| "p50": 1.060067, |
| "p95": 1.202129, |
| "p99": 1.256436 |
| } |
| }, |
| { |
| "tokens": "9-16", |
| "n": 1487, |
| "cpu_ms": { |
| "p50": 18.553100999999998, |
| "p95": 23.654970000000002, |
| "p99": 25.654531000000002 |
| }, |
| "gpu_ms": { |
| "p50": 1.2158609999999999, |
| "p95": 1.363216, |
| "p99": 1.435772 |
| } |
| }, |
| { |
| "tokens": "17-32", |
| "n": 117, |
| "cpu_ms": { |
| "p50": 28.220928, |
| "p95": 40.57731, |
| "p99": 42.695208 |
| }, |
| "gpu_ms": { |
| "p50": 1.448512, |
| "p95": 1.744253, |
| "p99": 1.797189 |
| } |
| }, |
| { |
| "tokens": "33-64", |
| "n": 2, |
| "cpu_ms": { |
| "p50": 92.568872, |
| "p95": 92.568872, |
| "p99": 92.568872 |
| }, |
| "gpu_ms": { |
| "p50": 2.741677, |
| "p95": 2.741677, |
| "p99": 2.741677 |
| } |
| } |
| ], |
| "gpu_scope": "One command buffer: embedding lookup, transformer layers, row-order pooling and FP32 L2, reconstruction errors; tokenization, hashing and ranking/gates on CPU. Joint GPU time is not split into fictitious encoder/resonance wall times." |
| }, |
| { |
| "backend": "vulkan", |
| "mode": "separate", |
| "run": 2, |
| "source_sha256": "08cc059ac15ca13c03b88491852f551b6dfe4e0641e7c0f113e1843d8da2d03e", |
| "schema": "cortiq-decision-vulkan-comparison/2", |
| "model_sha": "386b6e43fd35d5aa9e9710529dfdf30aa868df98ab4434d8e45dae9829898f85", |
| "skill": "massive", |
| "n": 2974, |
| "dataset_sha256": "5517f3de57d9ec9f73885dbd01b26a55a5cd806b9e7ff10c8873f5998e72dbd3", |
| "device": "vulkan: NVIDIA RTX PRO 4000 Blackwell", |
| "warmup": 50, |
| "order": "separate serial phases: all CPU references, 50 additional GPU warmups, all GPU rows. Same process/model/texts and parity checks; no inserted CPU inference between GPU requests; no oracle.", |
| "golden_max_abs": { |
| "cpu": 1.1920928955078125e-07, |
| "vulkan": 1.341104507446289e-07 |
| }, |
| "max_embedding_abs": 1.4901161193847656e-07, |
| "max_reconstruction_error_abs": 2.0503997802734375e-05, |
| "max_reconstruction_error_scaled": 8.735638402868062e-06, |
| "winner_changes": 0, |
| "gate_changes": 0, |
| "hash_changes": 0, |
| "different_rows": [], |
| "correct": { |
| "cpu": 2562, |
| "vulkan": 2562 |
| }, |
| "accepted": { |
| "cpu": 1615, |
| "vulkan": 1615 |
| }, |
| "vulkan_command_buffers": 2974, |
| "cpu_command_buffers": 0, |
| "vulkan_resonance_dispatches": 2974, |
| "ms": { |
| "cpu_total": { |
| "p50": 16.226368, |
| "p95": 25.084632, |
| "p99": 31.197762 |
| }, |
| "vulkan_total": { |
| "p50": 1.104685, |
| "p95": 1.333802, |
| "p99": 1.501266 |
| }, |
| "cpu_encoder": { |
| "p50": 12.084242999999999, |
| "p95": 20.98241, |
| "p99": 27.027615 |
| }, |
| "vulkan_split_encoder": { |
| "p50": 0.0, |
| "p95": 0.0, |
| "p99": 0.0 |
| }, |
| "cpu_resonance": { |
| "p50": 4.08907, |
| "p95": 4.173007999999999, |
| "p99": 4.284597000000001 |
| }, |
| "vulkan_ranking_cpu": { |
| "p50": 0.004288, |
| "p95": 0.010778000000000001, |
| "p99": 0.02296 |
| }, |
| "vulkan_joint_pipeline": { |
| "p50": 1.070387, |
| "p95": 1.281504, |
| "p99": 1.449374 |
| } |
| }, |
| "by_tokens": [ |
| { |
| "tokens": "1-8", |
| "n": 1368, |
| "cpu_ms": { |
| "p50": 13.77318, |
| "p95": 15.957989999999999, |
| "p99": 16.201877 |
| }, |
| "gpu_ms": { |
| "p50": 1.0107110000000001, |
| "p95": 1.145669, |
| "p99": 1.198661 |
| } |
| }, |
| { |
| "tokens": "9-16", |
| "n": 1487, |
| "cpu_ms": { |
| "p50": 18.513498, |
| "p95": 24.04242, |
| "p99": 25.511760000000002 |
| }, |
| "gpu_ms": { |
| "p50": 1.168246, |
| "p95": 1.323223, |
| "p99": 1.401163 |
| } |
| }, |
| { |
| "tokens": "17-32", |
| "n": 117, |
| "cpu_ms": { |
| "p50": 28.784033, |
| "p95": 38.288958, |
| "p99": 46.601112 |
| }, |
| "gpu_ms": { |
| "p50": 1.387599, |
| "p95": 1.630614, |
| "p99": 1.743193 |
| } |
| }, |
| { |
| "tokens": "33-64", |
| "n": 2, |
| "cpu_ms": { |
| "p50": 83.40803600000001, |
| "p95": 83.40803600000001, |
| "p99": 83.40803600000001 |
| }, |
| "gpu_ms": { |
| "p50": 2.7668329999999997, |
| "p95": 2.7668329999999997, |
| "p99": 2.7668329999999997 |
| } |
| } |
| ], |
| "gpu_scope": "One command buffer: embedding lookup, transformer layers, row-order pooling and FP32 L2, reconstruction errors; tokenization, hashing and ranking/gates on CPU. Joint GPU time is not split into fictitious encoder/resonance wall times." |
| } |
| ], |
| "vulkan_final_alternating_diagnostic": [ |
| { |
| "schema": "cortiq-decision-vulkan-comparison/2", |
| "model_sha": "386b6e43fd35d5aa9e9710529dfdf30aa868df98ab4434d8e45dae9829898f85", |
| "skill": "banking77", |
| "n": 400, |
| "dataset_sha256": "9b521ef5a55ec3065e70bccd2b30fa12e5b999406008f44338ef91ddbbaf1764", |
| "device": "vulkan: NVIDIA RTX PRO 4000 Blackwell", |
| "warmup": 50, |
| "mode": "paired", |
| "order": "paired serial; CPU-first/GPU-first alternates per row; same process, model, scorer and texts; no oracle", |
| "golden_max_abs": { |
| "cpu": 1.1920928955078125e-07, |
| "vulkan": 1.341104507446289e-07 |
| }, |
| "max_embedding_abs": 1.341104507446289e-07, |
| "max_reconstruction_error_abs": 7.748603820800781e-06, |
| "max_reconstruction_error_scaled": 4.2701681195467245e-06, |
| "winner_changes": 0, |
| "gate_changes": 0, |
| "hash_changes": 0, |
| "different_rows": [], |
| "correct": { |
| "cpu": 378, |
| "vulkan": 378 |
| }, |
| "accepted": { |
| "cpu": 368, |
| "vulkan": 368 |
| }, |
| "vulkan_command_buffers": 400, |
| "cpu_command_buffers": 0, |
| "vulkan_resonance_dispatches": 400, |
| "ms": { |
| "cpu_total": { |
| "p50": 25.294016, |
| "p95": 50.685048, |
| "p99": 71.975504 |
| }, |
| "vulkan_total": { |
| "p50": 8.570200999999999, |
| "p95": 9.157026, |
| "p99": 10.475235 |
| }, |
| "cpu_encoder": { |
| "p50": 18.249418, |
| "p95": 43.703988, |
| "p99": 64.88609600000001 |
| }, |
| "vulkan_split_encoder": { |
| "p50": 0.0, |
| "p95": 0.0, |
| "p99": 0.0 |
| }, |
| "cpu_resonance": { |
| "p50": 7.033309, |
| "p95": 7.121943000000001, |
| "p99": 7.204884 |
| }, |
| "vulkan_ranking_cpu": { |
| "p50": 0.005862, |
| "p95": 0.014029000000000002, |
| "p99": 0.022748 |
| }, |
| "vulkan_joint_pipeline": { |
| "p50": 8.532857, |
| "p95": 9.114592, |
| "p99": 10.426771 |
| } |
| }, |
| "by_tokens": [ |
| { |
| "tokens": "1-8", |
| "n": 33, |
| "cpu_ms": { |
| "p50": 18.183600000000002, |
| "p95": 19.627952, |
| "p99": 19.660096 |
| }, |
| "gpu_ms": { |
| "p50": 8.504406000000001, |
| "p95": 9.157026, |
| "p99": 10.475235 |
| } |
| }, |
| { |
| "tokens": "9-16", |
| "n": 254, |
| "cpu_ms": { |
| "p50": 23.578274, |
| "p95": 27.982298999999998, |
| "p99": 28.295987 |
| }, |
| "gpu_ms": { |
| "p50": 8.541694000000001, |
| "p95": 8.830613, |
| "p99": 10.490369 |
| } |
| }, |
| { |
| "tokens": "17-32", |
| "n": 97, |
| "cpu_ms": { |
| "p50": 32.586508, |
| "p95": 50.685048, |
| "p99": 52.140518 |
| }, |
| "gpu_ms": { |
| "p50": 8.737691, |
| "p95": 9.156465, |
| "p99": 10.988516 |
| } |
| }, |
| { |
| "tokens": "33-64", |
| "n": 15, |
| "cpu_ms": { |
| "p50": 59.938963, |
| "p95": 86.883251, |
| "p99": 86.883251 |
| }, |
| "gpu_ms": { |
| "p50": 9.268764000000001, |
| "p95": 9.999656000000002, |
| "p99": 9.999656000000002 |
| } |
| }, |
| { |
| "tokens": "65+", |
| "n": 1, |
| "cpu_ms": { |
| "p50": 96.077608, |
| "p95": 96.077608, |
| "p99": 96.077608 |
| }, |
| "gpu_ms": { |
| "p50": 2.9224379999999996, |
| "p95": 2.9224379999999996, |
| "p99": 2.9224379999999996 |
| } |
| } |
| ], |
| "gpu_scope": "One command buffer: embedding lookup, transformer layers, row-order pooling and FP32 L2, reconstruction errors; tokenization, hashing and ranking/gates on CPU. Joint GPU time is not split into fictitious encoder/resonance wall times." |
| }, |
| { |
| "schema": "cortiq-decision-vulkan-comparison/2", |
| "model_sha": "386b6e43fd35d5aa9e9710529dfdf30aa868df98ab4434d8e45dae9829898f85", |
| "skill": "clinc150", |
| "n": 400, |
| "dataset_sha256": "94ca69865b628e19e0b6c220e6749197888bdc7cb648cd1f72a20305bc13d9e5", |
| "device": "vulkan: NVIDIA RTX PRO 4000 Blackwell", |
| "warmup": 50, |
| "mode": "paired", |
| "order": "paired serial; CPU-first/GPU-first alternates per row; same process, model, scorer and texts; no oracle", |
| "golden_max_abs": { |
| "cpu": 1.1920928955078125e-07, |
| "vulkan": 1.341104507446289e-07 |
| }, |
| "max_embedding_abs": 1.4901161193847656e-07, |
| "max_reconstruction_error_abs": 1.3113021850585938e-05, |
| "max_reconstruction_error_scaled": 6.0342304095684085e-06, |
| "winner_changes": 0, |
| "gate_changes": 0, |
| "hash_changes": 0, |
| "different_rows": [], |
| "correct": { |
| "cpu": 385, |
| "vulkan": 385 |
| }, |
| "accepted": { |
| "cpu": 371, |
| "vulkan": 371 |
| }, |
| "vulkan_command_buffers": 400, |
| "cpu_command_buffers": 0, |
| "vulkan_resonance_dispatches": 400, |
| "ms": { |
| "cpu_total": { |
| "p50": 21.496622, |
| "p95": 30.177698, |
| "p99": 35.863288 |
| }, |
| "vulkan_total": { |
| "p50": 5.015118, |
| "p95": 8.837807, |
| "p99": 9.597926 |
| }, |
| "cpu_encoder": { |
| "p50": 14.484449999999999, |
| "p95": 23.357059, |
| "p99": 28.846717 |
| }, |
| "vulkan_split_encoder": { |
| "p50": 0.0, |
| "p95": 0.0, |
| "p99": 0.0 |
| }, |
| "cpu_resonance": { |
| "p50": 6.933250999999999, |
| "p95": 7.025986, |
| "p99": 7.164185 |
| }, |
| "vulkan_ranking_cpu": { |
| "p50": 0.010098999999999999, |
| "p95": 0.025981999999999998, |
| "p99": 0.031384 |
| }, |
| "vulkan_joint_pipeline": { |
| "p50": 4.972818, |
| "p95": 8.779129, |
| "p99": 9.560920000000001 |
| } |
| }, |
| "by_tokens": [ |
| { |
| "tokens": "1-8", |
| "n": 110, |
| "cpu_ms": { |
| "p50": 17.64624, |
| "p95": 19.207981999999998, |
| "p99": 19.309098 |
| }, |
| "gpu_ms": { |
| "p50": 4.989846, |
| "p95": 8.768485, |
| "p99": 10.341634 |
| } |
| }, |
| { |
| "tokens": "9-16", |
| "n": 257, |
| "cpu_ms": { |
| "p50": 22.010488000000002, |
| "p95": 27.188233999999998, |
| "p99": 28.56827 |
| }, |
| "gpu_ms": { |
| "p50": 5.015118, |
| "p95": 8.818614, |
| "p99": 9.548928 |
| } |
| }, |
| { |
| "tokens": "17-32", |
| "n": 33, |
| "cpu_ms": { |
| "p50": 30.596008, |
| "p95": 40.529081000000005, |
| "p99": 40.800326999999996 |
| }, |
| "gpu_ms": { |
| "p50": 5.370083999999999, |
| "p95": 8.975268999999999, |
| "p99": 8.983433000000002 |
| } |
| } |
| ], |
| "gpu_scope": "One command buffer: embedding lookup, transformer layers, row-order pooling and FP32 L2, reconstruction errors; tokenization, hashing and ranking/gates on CPU. Joint GPU time is not split into fictitious encoder/resonance wall times." |
| }, |
| { |
| "schema": "cortiq-decision-vulkan-comparison/2", |
| "model_sha": "386b6e43fd35d5aa9e9710529dfdf30aa868df98ab4434d8e45dae9829898f85", |
| "skill": "massive", |
| "n": 400, |
| "dataset_sha256": "1662fff070a896417bec73752f8b90e40085f174b38fdfa4690ee472fe1b86d3", |
| "device": "vulkan: NVIDIA RTX PRO 4000 Blackwell", |
| "warmup": 50, |
| "mode": "paired", |
| "order": "paired serial; CPU-first/GPU-first alternates per row; same process, model, scorer and texts; no oracle", |
| "golden_max_abs": { |
| "cpu": 1.1920928955078125e-07, |
| "vulkan": 1.341104507446289e-07 |
| }, |
| "max_embedding_abs": 1.1920928955078125e-07, |
| "max_reconstruction_error_abs": 1.6808509826660156e-05, |
| "max_reconstruction_error_scaled": 7.22503500583116e-06, |
| "winner_changes": 0, |
| "gate_changes": 0, |
| "hash_changes": 0, |
| "different_rows": [], |
| "correct": { |
| "cpu": 341, |
| "vulkan": 341 |
| }, |
| "accepted": { |
| "cpu": 208, |
| "vulkan": 208 |
| }, |
| "vulkan_command_buffers": 400, |
| "cpu_command_buffers": 0, |
| "vulkan_resonance_dispatches": 400, |
| "ms": { |
| "cpu_total": { |
| "p50": 16.558957999999997, |
| "p95": 24.45713, |
| "p99": 30.835259 |
| }, |
| "vulkan_total": { |
| "p50": 3.3972960000000003, |
| "p95": 7.495115, |
| "p99": 7.610043 |
| }, |
| "cpu_encoder": { |
| "p50": 12.411912000000001, |
| "p95": 20.365925, |
| "p99": 26.537292 |
| }, |
| "vulkan_split_encoder": { |
| "p50": 0.0, |
| "p95": 0.0, |
| "p99": 0.0 |
| }, |
| "cpu_resonance": { |
| "p50": 4.108808000000001, |
| "p95": 4.267616, |
| "p99": 4.309318 |
| }, |
| "vulkan_ranking_cpu": { |
| "p50": 0.0046099999999999995, |
| "p95": 0.011302, |
| "p99": 0.023187 |
| }, |
| "vulkan_joint_pipeline": { |
| "p50": 3.3793870000000004, |
| "p95": 7.45397, |
| "p99": 7.584391 |
| } |
| }, |
| "by_tokens": [ |
| { |
| "tokens": "1-8", |
| "n": 178, |
| "cpu_ms": { |
| "p50": 14.953128, |
| "p95": 16.197547999999998, |
| "p99": 16.479971 |
| }, |
| "gpu_ms": { |
| "p50": 3.356858, |
| "p95": 7.449042, |
| "p99": 8.81558 |
| } |
| }, |
| { |
| "tokens": "9-16", |
| "n": 210, |
| "cpu_ms": { |
| "p50": 18.80779, |
| "p95": 24.283237, |
| "p99": 26.044375000000002 |
| }, |
| "gpu_ms": { |
| "p50": 3.452348, |
| "p95": 7.5117780000000005, |
| "p99": 7.610043 |
| } |
| }, |
| { |
| "tokens": "17-32", |
| "n": 11, |
| "cpu_ms": { |
| "p50": 28.205081, |
| "p95": 33.755212, |
| "p99": 33.755212 |
| }, |
| "gpu_ms": { |
| "p50": 3.871171, |
| "p95": 7.600968, |
| "p99": 7.600968 |
| } |
| }, |
| { |
| "tokens": "33-64", |
| "n": 1, |
| "cpu_ms": { |
| "p50": 48.965486000000006, |
| "p95": 48.965486000000006, |
| "p99": 48.965486000000006 |
| }, |
| "gpu_ms": { |
| "p50": 1.903141, |
| "p95": 1.903141, |
| "p99": 1.903141 |
| } |
| } |
| ], |
| "gpu_scope": "One command buffer: embedding lookup, transformer layers, row-order pooling and FP32 L2, reconstruction errors; tokenization, hashing and ranking/gates on CPU. Joint GPU time is not split into fictitious encoder/resonance wall times." |
| } |
| ], |
| "http": { |
| "apple_m4": { |
| "cpu": { |
| "requests": 11, |
| "rss_after_requests_MiB": 478.46875, |
| "temporary_server_stopped": true |
| }, |
| "metal": { |
| "requests": 11, |
| "rss_after_requests_MiB": 808.09375, |
| "temporary_server_stopped": true |
| } |
| }, |
| "xeon_rtx": { |
| "cpu": { |
| "requests": 11, |
| "rss_after_requests_MiB": 437.578125, |
| "temporary_server_stopped": true |
| }, |
| "vulkan": { |
| "requests": 11, |
| "rss_after_requests_MiB": 694.8125, |
| "temporary_server_stopped": true |
| } |
| } |
| }, |
| "vulkan_process_vram_snapshot_MiB": 522, |
| "artifact_sha256": { |
| "cortiq-linux-x86_64-vulkan": "005881d968e99a5444f92d7835e5aff8d93926cb570ec1ced9c7c3c2d8b31781", |
| "cortiq-macos-arm64": "02aed7010feb10706a9238050487050b144afe1ff2090e4bc496301e784b2ff7" |
| }, |
| "gpu_release_status_at_measurement": "Validated development build; GPU decision backend is not in the currently published crates.io package.", |
| "release_validation": { |
| "version": "0.8.0", |
| "commit": "2db7916d8ece61d8f521940cf650d59eb140133b", |
| "numerical_sources_unchanged_from_benchmarks": true, |
| "release_archives_verified": 10, |
| "crate_versions_verified": 7, |
| "cargo_install_verified": true, |
| "http": { |
| "metal-http": { |
| "cpu": { |
| "requests": 11, |
| "concurrent_clients": 4, |
| "rss_mib": 478.265625, |
| "adapter": "cpu", |
| "server_stopped": true |
| }, |
| "metal": { |
| "requests": 11, |
| "concurrent_clients": 4, |
| "rss_mib": 808.109375, |
| "adapter": "metal: Apple M4", |
| "server_stopped": true, |
| "gpu_submissions": 11 |
| } |
| }, |
| "vulkan-release-http": { |
| "cpu": { |
| "requests": 11, |
| "concurrent_clients": 4, |
| "rss_mib": 437.1953125, |
| "adapter": "cpu", |
| "server_stopped": true |
| }, |
| "vulkan": { |
| "requests": 11, |
| "concurrent_clients": 4, |
| "rss_mib": 648.3046875, |
| "adapter": "vulkan: NVIDIA RTX PRO 4000 Blackwell", |
| "server_stopped": true, |
| "gpu_submissions": 11, |
| "vram_snapshot": "522 MiB" |
| } |
| }, |
| "vulkan-cargo-http": { |
| "cpu": { |
| "requests": 11, |
| "concurrent_clients": 4, |
| "rss_mib": 439.14453125, |
| "adapter": "cpu", |
| "server_stopped": true |
| }, |
| "vulkan": { |
| "requests": 11, |
| "concurrent_clients": 4, |
| "rss_mib": 650.41796875, |
| "adapter": "vulkan: NVIDIA RTX PRO 4000 Blackwell", |
| "server_stopped": true, |
| "gpu_submissions": 11, |
| "vram_snapshot": "522 MiB" |
| } |
| } |
| } |
| } |
| } |
|
|
