{
  "status": "confirmed",
  "source": "https://siper.gurubase.ai/benchmark",
  "read": "2026-11-02",
  "dataset": {
    "documents": 403,
    "language": "Turkish",
    "period": "October 2026",
    "temporalCoverage": "2026-10",
    "falseAlarmDocuments": 321,
    "falseAlarmHits": 46,
    "resamples": 1000
  },
  "siperSetting": "\"Turkish and English\" option",
  "siperMeasured": "2 November 2026",
  "dataCategories": 32,
  "siperCategories": [
    {
      "label": "Person name",
      "recall": 0.904
    },
    {
      "label": "Address",
      "recall": 0.778
    },
    {
      "label": "Health",
      "recall": 0.807
    },
    {
      "label": "Religion",
      "recall": 0.86
    },
    {
      "label": "Ethnic origin",
      "recall": 0.912
    },
    {
      "label": "Political opinion",
      "recall": 0.881
    },
    {
      "label": "Union membership",
      "recall": 0.94
    },
    {
      "label": "Criminal conviction",
      "recall": 0.774
    },
    {
      "label": "Sex life",
      "recall": 0.896
    }
  ],
  "latencyTest": "On-prem systems: median per document on one 8-vCPU server without GPU, unless the row states another setup. Cloud services: API response time including network. Systems ran on different infrastructure, so this is not a direct hardware comparison.",
  "systems": [
    {
      "name": "Gurubase Siper",
      "self": true,
      "onPrem": true,
      "recall": 0.857,
      "recallCi": [
        0.839,
        0.877
      ],
      "precision": 0.561,
      "f1": 0.678,
      "f1Ci": [
        0.652,
        0.7
      ],
      "falseAlarm": 0.143,
      "latencyMs": 176,
      "latencyP95Ms": null,
      "latencySetup": "One 8-vCPU server, no GPU",
      "latencyMeasured": "October 2026"
    },
    {
      "name": "GPT-5-mini",
      "onPrem": false,
      "recall": 0.693,
      "recallCi": [
        0.653,
        0.73
      ],
      "precision": 0.628,
      "f1": 0.659,
      "f1Ci": [
        0.632,
        0.684
      ],
      "falseAlarm": 0.121,
      "latencyMs": 18115,
      "latencyP95Ms": 19621,
      "latencySetup": "Cloud API",
      "latencyMeasured": "July 2026"
    },
    {
      "name": "Gemini 2.5 Flash",
      "onPrem": false,
      "recall": 0.522,
      "recallCi": [
        0.47,
        0.571
      ],
      "precision": 0.615,
      "f1": 0.565,
      "f1Ci": [
        0.526,
        0.6
      ],
      "falseAlarm": 0.019,
      "latencyMs": 1790,
      "latencyP95Ms": 2072,
      "latencySetup": "Cloud API",
      "latencyMeasured": "July 2026"
    },
    {
      "name": "Limina (Private AI)",
      "onPrem": false,
      "recall": 0.515,
      "recallCi": [
        0.477,
        0.552
      ],
      "precision": 0.652,
      "f1": 0.575,
      "f1Ci": [
        0.543,
        0.605
      ],
      "falseAlarm": 0.486,
      "latencyMs": 561,
      "latencyP95Ms": 629,
      "latencySetup": "Cloud API",
      "latencyMeasured": "July 2026"
    },
    {
      "name": "Qwen2.5-32B",
      "onPrem": true,
      "recall": 0.483,
      "recallCi": [
        0.446,
        0.521
      ],
      "precision": 0.585,
      "f1": 0.529,
      "f1Ci": [
        0.496,
        0.561
      ],
      "falseAlarm": null,
      "latencyMs": 19242,
      "latencyP95Ms": 19367,
      "latencySetup": "Local GPU",
      "latencyMeasured": "July 2026"
    },
    {
      "name": "Azure AI Language",
      "onPrem": false,
      "recall": 0.267,
      "recallCi": [
        0.22,
        0.311
      ],
      "precision": 0.918,
      "f1": 0.414,
      "f1Ci": [
        0.355,
        0.466
      ],
      "falseAlarm": 0.034,
      "latencyMs": 221,
      "latencyP95Ms": 264,
      "latencySetup": "Cloud API",
      "latencyMeasured": "July 2026"
    },
    {
      "name": "Google Cloud DLP",
      "onPrem": false,
      "recall": 0.245,
      "recallCi": [
        0.202,
        0.288
      ],
      "precision": 0.426,
      "f1": 0.311,
      "f1Ci": [
        0.265,
        0.354
      ],
      "falseAlarm": 0.402,
      "latencyMs": 220,
      "latencyP95Ms": 289,
      "latencySetup": "Cloud API",
      "latencyMeasured": "July 2026"
    },
    {
      "name": "Tonic Textual",
      "onPrem": false,
      "recall": 0.222,
      "recallCi": [
        0.18,
        0.265
      ],
      "precision": 0.241,
      "f1": 0.231,
      "f1Ci": [
        0.193,
        0.264
      ],
      "falseAlarm": 0.427,
      "latencyMs": 491,
      "latencyP95Ms": 568,
      "latencySetup": "Cloud API",
      "latencyMeasured": "July 2026"
    },
    {
      "name": "Microsoft Presidio",
      "onPrem": true,
      "recall": 0.192,
      "recallCi": [
        0.156,
        0.229
      ],
      "precision": 0.09,
      "f1": 0.122,
      "f1Ci": [
        0.098,
        0.146
      ],
      "falseAlarm": 0.981,
      "latencyMs": 7,
      "latencyP95Ms": 8,
      "latencySetup": "Local CPU",
      "latencyMeasured": "July 2026"
    },
    {
      "name": "OpenAI Privacy Filter",
      "onPrem": true,
      "recall": 0.089,
      "recallCi": [
        0.059,
        0.12
      ],
      "precision": 0.489,
      "f1": 0.15,
      "f1Ci": [
        0.103,
        0.198
      ],
      "falseAlarm": 0.184,
      "latencyMs": 4600,
      "latencyP95Ms": null,
      "latencySetup": "One 8-vCPU server, no GPU",
      "latencyMeasured": "2026"
    }
  ],
  "categories": {
    "labels": [
      "Health",
      "Religion",
      "Ethnic origin",
      "Political opinion",
      "Union membership",
      "Criminal conviction",
      "Sex life"
    ],
    "allAboveThreshold": 0.65,
    "rows": [
      {
        "name": "Gurubase Siper",
        "self": true,
        "values": [
          0.81,
          0.86,
          0.91,
          0.88,
          0.94,
          0.77,
          0.9
        ]
      },
      {
        "name": "GPT-5-mini",
        "values": [
          0.51,
          0.88,
          0.96,
          0.81,
          0.89,
          0.53,
          0.71
        ]
      },
      {
        "name": "Limina (Private AI)",
        "values": [
          0.42,
          0.56,
          0.91,
          0.37,
          null,
          0,
          0.04
        ]
      },
      {
        "name": "Gemini 2.5 Flash",
        "values": [
          0.24,
          0.78,
          0.68,
          0.39,
          0.77,
          0.15,
          0.52
        ]
      },
      {
        "name": "Qwen2.5-32B",
        "values": [
          0.37,
          0.3,
          0.79,
          0.47,
          0.85,
          0.17,
          0.56
        ]
      },
      {
        "name": "Azure AI Language",
        "values": [
          null,
          null,
          null,
          null,
          null,
          null,
          null
        ]
      },
      {
        "name": "Google Cloud DLP",
        "values": [
          0,
          0.3,
          0.6,
          0,
          0,
          0.06,
          null
        ]
      },
      {
        "name": "Tonic Textual",
        "values": [
          0,
          null,
          0.09,
          null,
          null,
          null,
          null
        ]
      },
      {
        "name": "OpenAI Privacy Filter",
        "values": [
          null,
          null,
          null,
          null,
          null,
          null,
          null
        ]
      },
      {
        "name": "Microsoft Presidio",
        "values": [
          null,
          null,
          null,
          null,
          null,
          null,
          null
        ]
      }
    ]
  },
  "englishResults": null,
  "notEvaluated": [
    {
      "name": "Global cloud provider C",
      "reason": "Its official documentation states that it does not support Turkish personal data detection."
    }
  ],
  "pairwise": {
    "total": 135,
    "favourable": 113,
    "noDifference": 20,
    "unfavourable": 2,
    "alternatives": 15,
    "notListed": 6,
    "classes": 9,
    "classesDescription": "person name, address and seven special-category classes"
  },
  "url": "https://siper.io/benchmark",
  "disclosure": "The test set is Siper's own. This is not an independent audit. The tables list the alternatives we can name. The pairwise counts cover all of them.",
  "scales": {
    "recall": "Share of real personal data found, from 0 to 1.",
    "precision": "Share of detections that were correct, from 0 to 1.",
    "f1": "Micro-F1 of recall and precision, from 0 to 1.",
    "recallCi": "95% confidence interval from document resampling, as [low, high].",
    "f1Ci": "95% confidence interval from document resampling, as [low, high].",
    "falseAlarm": "Share of documents with no personal data that received at least one unnecessary mask, from 0 to 1. null means not measured.",
    "latencyMs": "Median response time in milliseconds. null means not measured. On-prem systems: median per document on one 8-vCPU server without GPU, unless the row states another setup. Cloud services: API response time including network. Systems ran on different infrastructure, so this is not a direct hardware comparison.",
    "latencyP95Ms": "95th percentile response time in milliseconds. null means not measured.",
    "categoryValues": "Recall in that category, from 0 to 1. null means out of scope: the system has no label for the category. 0 is a measured zero.",
    "englishResults": "Results on English text. null means not published."
  }
}
