{
  "altLabels": [
    "LLM Judge",
    "LLM-as-a-Judge"
  ],
  "definition": "A method that uses a large language model to evaluate, label, score, or compare outputs or query-item pairs.",
  "distinctions": [
    {
      "reason": "LLM-as-judge is a judging method; qrels are a structured set of relevance labels.",
      "statement": "LLM-as-Judge is not Qrels.",
      "target": "qrels",
      "targetLabel": "Qrels",
      "targetUri": "https://id.searchplex.net/qrels/"
    },
    {
      "reason": "LLM-as-judge may estimate or label relevance, but it is not the relevance relationship itself.",
      "statement": "LLM-as-Judge is not Relevance.",
      "target": "relevance",
      "targetLabel": "Relevance",
      "targetUri": "https://id.searchplex.net/relevance/"
    },
    {
      "reason": "LLM-as-judge is a method for producing or assisting judgments; a relevance judgment is the assessment itself.",
      "statement": "LLM-as-Judge is not Relevance Judgment.",
      "target": "relevance-judgment",
      "targetLabel": "Relevance Judgment",
      "targetUri": "https://id.searchplex.net/relevance-judgment/"
    }
  ],
  "id": "llm-as-judge",
  "kind": "Method",
  "license": "https://creativecommons.org/publicdomain/zero/1.0/",
  "links": [
    {
      "label": "LLMJudge: LLMs for Relevance Judgments",
      "type": "canonicalSurvey",
      "url": "https://arxiv.org/abs/2408.08896"
    },
    {
      "label": "Don't Use LLMs to Make Relevance Judgments",
      "type": "canonicalSurvey",
      "url": "https://irrj.org/article/id/2564/"
    }
  ],
  "modifiedAt": "2026-09-14T06:43:46Z",
  "prefLabel": "LLM-as-Judge",
  "publisher": {
    "name": "Searchplex",
    "url": "https://searchplex.net/"
  },
  "relations": [
    {
      "note": "LLM-as-judge is a judging method; qrels are a structured set of relevance labels.",
      "target": "qrels",
      "type": "notEquivalentTo"
    },
    {
      "note": "LLM-as-judge may estimate or label relevance, but it is not the relevance relationship itself.",
      "target": "relevance",
      "type": "notEquivalentTo"
    },
    {
      "note": "LLM-as-judge is a method for producing or assisting judgments; a relevance judgment is the assessment itself.",
      "target": "relevance-judgment",
      "type": "notEquivalentTo"
    },
    {
      "target": "qrels",
      "type": "related"
    },
    {
      "target": "relevance",
      "type": "related"
    },
    {
      "target": "relevance-judgment",
      "type": "related"
    },
    {
      "target": "test-collection",
      "type": "related"
    }
  ],
  "representations": {
    "html": "https://id.searchplex.net/llm-as-judge/",
    "json": "https://id.searchplex.net/llm-as-judge.json",
    "jsonld": "https://id.searchplex.net/llm-as-judge.jsonld"
  },
  "scheme": "https://id.searchplex.net/scheme/",
  "scopeNote": "In retrieval evaluation, LLM-as-judge methods may be used to generate or assist relevance assessments. They are not the same as human relevance judgments, qrels, or the relevance relationship being judged.",
  "status": "published",
  "uri": "https://id.searchplex.net/llm-as-judge/"
}