{
  "name": "ANNex benchmarks",
  "canonical": "https://annexsearch.dev/benchmarks",
  "markdown": "https://annexsearch.dev/benchmarks.md",
  "policy": "https://github.com/rosharma719/ANNex/blob/master/BENCHMARK_POLICY.md",
  "annexVersion": "0.3.0",
  "note": "No result establishes production superiority or launch readiness. See the benchmark policy.",
  "methodology": [
    "Recall is measured against exact nearest neighbors computed independently. All ANNex runs use 3 rounds; reported latencies are medians.",
    "Timing measures end-to-end search latency only. Embedding generation, index build, training and recovery are excluded.",
    "nDCG@10 uses linear relevance gains matching BEIR's evaluator and trec_eval, verified against pytrec_eval within 1e-12.",
    "Interleaved runs share OS memory state between engines: realistic, but not fully isolated per-engine timing."
  ],
  "tables": {
    "beir_ndcg10": {
      "id": "beir-ndcg10",
      "title": "BEIR nDCG@10: ANNex vs LanceDB and Qdrant on five held-out corpora",
      "metric": "nDCG@10",
      "implementation": "f68f970",
      "hardware": "Apple M2 / 16 GiB",
      "date": "2026-09-29",
      "embeddings": "Shared pinned MiniLM vectors, exact cosine, 100 candidates per channel",
      "corpora": [
        {
          "key": "nf",
          "name": "NFCorpus",
          "queries": 162
        },
        {
          "key": "sci",
          "name": "SciFact",
          "queries": 150
        },
        {
          "key": "argu",
          "name": "ArguAna",
          "queries": 703
        },
        {
          "key": "fiqa",
          "name": "FiQA",
          "queries": 324
        },
        {
          "key": "scid",
          "name": "SCIDOCS",
          "queries": 500
        }
      ],
      "note": "Every per-corpus 95% bootstrap interval between hybrid systems includes zero: parity, not a statistically established advantage.",
      "rows": [
        {
          "system": "ANNex hybrid",
          "nf": 0.35742,
          "sci": 0.72639,
          "argu": 0.52384,
          "fiqa": 0.37977,
          "scid": 0.21329,
          "macro": 0.44014,
          "annex": true
        },
        {
          "system": "LanceDB native hybrid",
          "nf": 0.35425,
          "sci": 0.72762,
          "argu": 0.52646,
          "fiqa": 0.37838,
          "scid": 0.21324,
          "macro": 0.43999,
          "annex": false
        },
        {
          "system": "Qdrant Server hybrid",
          "nf": 0.35797,
          "sci": 0.72572,
          "argu": 0.52074,
          "fiqa": 0.37983,
          "scid": 0.21357,
          "macro": 0.43957,
          "annex": false
        },
        {
          "system": "ANNex dense",
          "nf": 0.31485,
          "sci": 0.6642,
          "argu": 0.48305,
          "fiqa": 0.37728,
          "scid": 0.22429,
          "macro": 0.41273,
          "annex": true
        },
        {
          "system": "ANNex BM25",
          "nf": 0.32849,
          "sci": 0.67408,
          "argu": 0.46676,
          "fiqa": 0.2435,
          "scid": 0.16247,
          "macro": 0.37506,
          "annex": true
        }
      ]
    },
    "nyt256_head_to_head": {
      "id": "nyt256-head-to-head",
      "title": "NYT256 head-to-head: ANNex vs FAISS, hnswlib and USearch at three recall tiers",
      "dataset": {
        "name": "NYT256",
        "documents": 290000,
        "dimensions": 256,
        "metric": "angular"
      },
      "hardware": "Apple M2",
      "offset": 4000,
      "interleaved": true,
      "versions": {
        "annex": "0.1.0",
        "faiss": "1.15.1",
        "hnswlib": "0.8.0",
        "usearch": "2.26.2"
      },
      "note": "Interleaved run: all engines share OS memory state, so per-engine timing is realistic but not fully isolated.",
      "rows": [
        {
          "label": "annex_screen",
          "ef": 32,
          "recall": 0.864,
          "p50": 0.252,
          "p99": 1.042,
          "annex": true,
          "tier": 1
        },
        {
          "label": "faiss_hnsw16",
          "ef": 128,
          "recall": 0.868,
          "p50": 0.356,
          "p99": 0.598,
          "annex": false,
          "tier": 1
        },
        {
          "label": "faiss_hnsw32",
          "ef": 32,
          "recall": 0.834,
          "p50": 0.208,
          "p99": 0.373,
          "annex": false,
          "tier": 1
        },
        {
          "label": "usearch_f16",
          "ef": 128,
          "recall": 0.864,
          "p50": 0.54,
          "p99": 0.941,
          "annex": false,
          "tier": 1
        },
        {
          "label": "hnswlib_m16",
          "ef": 128,
          "recall": 0.861,
          "p50": 0.684,
          "p99": 1.127,
          "annex": false,
          "tier": 1
        },
        {
          "label": "annex_screen",
          "ef": 128,
          "recall": 0.918,
          "p50": 0.58,
          "p99": 0.972,
          "annex": true,
          "tier": 2
        },
        {
          "label": "faiss_hnsw16",
          "ef": 512,
          "recall": 0.925,
          "p50": 1.457,
          "p99": 2.758,
          "annex": false,
          "tier": 2
        },
        {
          "label": "faiss_hnsw32",
          "ef": 128,
          "recall": 0.904,
          "p50": 0.627,
          "p99": 1.009,
          "annex": false,
          "tier": 2
        },
        {
          "label": "usearch_f16",
          "ef": 512,
          "recall": 0.922,
          "p50": 2.059,
          "p99": 4.246,
          "annex": false,
          "tier": 2
        },
        {
          "label": "hnswlib_m16",
          "ef": 512,
          "recall": 0.918,
          "p50": 2.562,
          "p99": 7.01,
          "annex": false,
          "tier": 2
        },
        {
          "label": "annex_screen",
          "ef": 512,
          "recall": 0.961,
          "p50": 2.205,
          "p99": 5.99,
          "annex": true,
          "tier": 3
        },
        {
          "label": "faiss_hnsw32",
          "ef": 512,
          "recall": 0.957,
          "p50": 2.489,
          "p99": 4.344,
          "annex": false,
          "tier": 3
        },
        {
          "label": "faiss_ivf1024",
          "ef": 512,
          "recall": 0.964,
          "p50": 3.753,
          "p99": 9.103,
          "annex": false,
          "tier": 3
        },
        {
          "label": "usearch_f32",
          "ef": 1024,
          "recall": 0.948,
          "p50": 6.261,
          "p99": 9.592,
          "annex": false,
          "tier": 3
        },
        {
          "label": "hnswlib_m16",
          "ef": 1024,
          "recall": 0.946,
          "p50": 5.043,
          "p99": 10.5,
          "annex": false,
          "tier": 3
        }
      ]
    },
    "nyt256_pareto": {
      "id": "nyt256-pareto",
      "title": "NYT256 recall–latency Pareto: fastest ANNex config per recall threshold vs hnswlib",
      "dataset": {
        "name": "NYT256",
        "documents": 290000,
        "dimensions": 256,
        "metric": "angular"
      },
      "hardware": "Apple M2 / 16 GiB",
      "rounds": 3,
      "offset": 1000,
      "commit": "4b11157",
      "baseline": "hnswlib 0.8.0",
      "note": "Offset 1000 excludes query vectors from the corpus. The 0.97 row is where hnswlib takes the Pareto front.",
      "rows": [
        {
          "threshold": 0.8,
          "label": "combined_stop0.01",
          "ef": 32,
          "recall": 0.827,
          "p50": 0.225,
          "p99": 0.517,
          "note": ""
        },
        {
          "threshold": 0.85,
          "label": "pq_screen0.1",
          "ef": 128,
          "recall": 0.871,
          "p50": 0.3,
          "p99": 0.564,
          "note": ""
        },
        {
          "threshold": 0.87,
          "label": "pq_screen0.1",
          "ef": 128,
          "recall": 0.871,
          "p50": 0.3,
          "p99": 0.564,
          "note": ""
        },
        {
          "threshold": 0.9,
          "label": "pq_screen0.15",
          "ef": 128,
          "recall": 0.903,
          "p50": 0.466,
          "p99": 1.122,
          "note": ""
        },
        {
          "threshold": 0.92,
          "label": "pq_screen0.1",
          "ef": 256,
          "recall": 0.925,
          "p50": 0.629,
          "p99": 1.108,
          "note": ""
        },
        {
          "threshold": 0.94,
          "label": "pq_screen0.1",
          "ef": 512,
          "recall": 0.956,
          "p50": 1.412,
          "p99": 2.219,
          "note": ""
        },
        {
          "threshold": 0.96,
          "label": "pq_screen0.15",
          "ef": 512,
          "recall": 0.961,
          "p50": 2.699,
          "p99": 4.239,
          "note": ""
        },
        {
          "threshold": 0.97,
          "label": "hnswlib",
          "ef": 2048,
          "recall": 0.971,
          "p50": 9.162,
          "p99": 11.065,
          "note": "hnswlib wins Pareto here"
        }
      ]
    },
    "nyt256_ec2_topk10": {
      "id": "nyt256-ec2-topk10",
      "title": "NYT256 throughput on EC2, top-k=10",
      "dataset": {
        "name": "NYT256",
        "documents": 290000,
        "dimensions": 256,
        "metric": "angular"
      },
      "hardware": "EC2",
      "date": "2026-10-01",
      "topK": 10,
      "threads": 1,
      "note": "sq8 = SQ8 quantization; rcm = residual compression map. Serial requests, no warmup. Timing excludes embedding generation.",
      "rows": [
        {
          "config": "m16",
          "ef": 32,
          "recall": 0.836,
          "p50": 0.203,
          "p99": 0.331,
          "qps": 4706
        },
        {
          "config": "m16+sq8",
          "ef": 32,
          "recall": 0.836,
          "p50": 0.182,
          "p99": 0.282,
          "qps": 5324
        },
        {
          "config": "m16+rcm",
          "ef": 32,
          "recall": 0.836,
          "p50": 0.202,
          "p99": 0.326,
          "qps": 4763
        },
        {
          "config": "m16+rcm+sq8",
          "ef": 32,
          "recall": 0.836,
          "p50": 0.179,
          "p99": 0.282,
          "qps": 5424
        },
        {
          "config": "m16",
          "ef": 128,
          "recall": 0.902,
          "p50": 0.559,
          "p99": 0.894,
          "qps": 1769
        },
        {
          "config": "m16+sq8",
          "ef": 128,
          "recall": 0.902,
          "p50": 0.5,
          "p99": 0.792,
          "qps": 1971
        },
        {
          "config": "m16+rcm+sq8",
          "ef": 128,
          "recall": 0.902,
          "p50": 0.484,
          "p99": 0.765,
          "qps": 2037
        },
        {
          "config": "m16+sq8",
          "ef": 256,
          "recall": 0.925,
          "p50": 0.914,
          "p99": 1.313,
          "qps": 1104
        },
        {
          "config": "m16+rcm+sq8",
          "ef": 256,
          "recall": 0.925,
          "p50": 0.897,
          "p99": 1.308,
          "qps": 1127
        }
      ]
    }
  }
}
