feat(text-metrics): split oneig_reasoning

pyproject.toml

@@ -67,41 +67,33 @@ name = "pruna_internal"
 url = "https://prunaai.pythonanywhere.com/simple/"
 explicit = true
 
-[[tool.uv.index]]
-name = "intel-pytorch-extension"
-url = "https://pytorch-extension.intel.com/release-whl/stable/cpu/cn/"
-explicit = true
-
 [tool.uv]
 index-strategy = "first-index"
+exclude-newer = "1 week"  # protection against compromised dependencies
+# trusted dev wheels that are missing an upload date
+exclude-newer-package = { gptqmodel = false, "stable-fast-pruna" = false }
 
 conflicts = [
     [{ extra = "awq" }, { extra = "vbench" }],
     [{ extra = "vllm" }, { extra = "vbench" }],
-    [{ extra = "intel" }, { extra = "awq" }],
     [{ extra = "gptq" }, { extra = "awq" }],
-    # intel is incompatible with all stable-fast variants and vllm
-    [{ extra = "intel" }, { extra = "stable-fast" }, { extra = "stable-fast-extraindex" }],
-    [{ extra = "intel" }, { extra = "full" }, { extra = "stable-fast-extraindex" }],
-    [{ extra = "intel" }, { extra = "vllm" }],
     [{ extra = "kvpress" }, { extra = "vbench" }],
 ]
 
 [tool.uv.sources]
 gptqmodel = { index = "pruna_internal", marker = "sys_platform != 'darwin' or platform_machine != 'arm64'" }
-intel-extension-for-pytorch = { index = "intel-pytorch-extension" }
 stable-fast-pruna = { index = "pruna_internal", extra = "stable-fast-extraindex" }
 
 [project]
 name = "pruna"
-version = "0.3.2"
+version = "0.3.3"
 description = "Smash your AI models"
 authors = [
     {name = "Pruna AI", email = "hello@pruna.ai"}
 ]
 license = {file = "LICENSE"}
 readme = "README.md"
-requires-python = ">=3.10,<3.13"
+requires-python = ">=3.10,<3.14"
 keywords = ["AI", "machine learning", "model optimization", "pruning"]
 classifiers = [
     "Development Status :: 4 - Beta",
@@ -246,12 +238,6 @@ lmharness = [
     "lm-eval>=0.4.0"
 ]
 
-# Intel extension is tightly coupled with the torch version
-intel = [
-    "intel-extension-for-pytorch>=2.7.0",
-    "torch>=2.7.0,<2.9.0",
-    "torchvision>=0.22.0,<0.24.0",
-]
 kvpress = [
     "kvpress>=0.5.2",
 ]

src/pruna/evaluation/benchmarks.py

@@ -66,19 +66,19 @@ class BenchmarkRegistry:
     paper (see reference URL). All entries verified from paper evaluation
     sections (ar5iv/HTML or PDF) as of verification pass:
 
-    - Parti Prompts (2206.10789 §5.2, §5.4): human side-by-side only on P222.
-    - DrawBench (2205.11487 §4.3): human raters only; COCO uses FID + CLIP.
+    - Parti Prompts (2206.10789 ?5.2, ?5.4): human side-by-side only on P222.
+    - DrawBench (2205.11487 ?4.3): human raters only; COCO uses FID + CLIP.
     - GenAI Bench (2406.13743): VQAScore only (web/PWC; ar5iv failed).
     - VBench (2311.17982): 16 dimension-specific methods; no single Pruna metric.
-    - COCO (2205.11487 §4.1): FID and CLIP score for fidelity and alignment.
-    - ImageNet (1409.0575 §4): top-1/top-5 classification accuracy.
-    - WikiText (1609.07843 §5): perplexity on validation/test.
-    - GenEval (2310.11513 §3.2): Mask2Former + CLIP color pipeline, binary score.
+    - COCO (2205.11487 ?4.1): FID and CLIP score for fidelity and alignment.
+    - ImageNet (1409.0575 ?4): top-1/top-5 classification accuracy.
+    - WikiText (1609.07843 ?5): perplexity on validation/test.
+    - GenEval (2310.11513 ?3.2): Mask2Former + CLIP color pipeline, binary score.
     - HPS (2306.09341): HPS v2 scoring model (CLIP fine-tuned on HPD v2).
-    - ImgEdit (2505.20275 §4.2): GPT-4o 1���5 ratings and ImgEdit-Judge.
-    - Long Text Bench (2507.22058 §4): Text Accuracy (OCR, Qwen2.5-VL-7B).
-    - GEditBench (2504.17761 §4.2): VIEScore (SQ, PQ, O via GPT-4.1/Qwen2.5-VL).
-    - OneIG (2506.07977 §4.1): per-dimension metrics (semantic alignment, ED, etc.).
+    - ImgEdit (2505.20275 ?4.2): GPT-4o 1���5 ratings and ImgEdit-Judge.
+    - Long Text Bench (2507.22058 ?4): Text Accuracy (OCR, Qwen2.5-VL-7B).
+    - GEditBench (2504.17761 ?4.2): VIEScore (SQ, PQ, O via GPT-4.1/Qwen2.5-VL).
+    - OneIG (2506.07977 ?4.1): per-dimension metrics (semantic alignment, ED, etc.).
     - DPG (2403.05135): DSG-style graph score, mPLUG-large adjudicator.
     """
 
@@ -195,7 +195,7 @@ def list(cls, task_type: str | None = None) -> list[str]:
             "MS-COCO for text-to-image evaluation (Imagen, 2205.11487). Paper reports "
             "FID for fidelity and CLIP score for image-text alignment."
         ),
-        metrics=["fid", "clip_score"],  # §4.1: FID + CLIP score
+        metrics=["fid", "clip_score"],  # ?4.1: FID + CLIP score
         task_type="text_to_image",
         reference="https://arxiv.org/abs/2205.11487",
     ),

src/pruna/evaluation/metrics/__init__.py

@@ -22,12 +22,13 @@
 from pruna.evaluation.metrics.metric_evalharness import LMEvalMetric
 from pruna.evaluation.metrics.metric_memory import DiskMemoryMetric, InferenceMemoryMetric, TrainingMemoryMetric
 from pruna.evaluation.metrics.metric_model_architecture import TotalMACsMetric, TotalParamsMetric
-from pruna.evaluation.metrics.metric_pairwise_clip import PairwiseClipScore
 from pruna.evaluation.metrics.metric_oneig_alignment import OneIGAlignmentMetric
+from pruna.evaluation.metrics.metric_oneig_reasoning import OneIGReasoningMetric
+from pruna.evaluation.metrics.metric_pairwise_clip import PairwiseClipScore
 from pruna.evaluation.metrics.metric_qa_accuracy import QAAccuracyMetric
-from pruna.evaluation.metrics.metric_text_score import OneIGTextScoreMetric, TextScoreMetric
 from pruna.evaluation.metrics.metric_rapiddata import RapidataMetric as RapidataMetric
 from pruna.evaluation.metrics.metric_sharpness import SharpnessMetric
+from pruna.evaluation.metrics.metric_text_score import OneIGTextScoreMetric, TextScoreMetric
 from pruna.evaluation.metrics.metric_torch import TorchMetricWrapper
 from pruna.evaluation.metrics.vlm_base import (
     BaseVLM,
@@ -57,6 +58,7 @@
     "AestheticLAION",
     "LMEvalMetric",
     "OneIGAlignmentMetric",
+    "OneIGReasoningMetric",
     "OneIGTextScoreMetric",
     "QAAccuracyMetric",
     "RapidataMetric",

src/pruna/evaluation/metrics/metric_oneig_alignment.py

@@ -151,8 +151,6 @@ class OneIGAlignmentMetric(QAAccuracyMetric):
     (default ``2 x 2``), score **one question per VLM call** across all cells, apply
     dependency masking per cell, then average cell scores.
 
-    Scoring semantics
-    -----------------
     OneIG Q_D probes are phrased so **Yes = aligned**. Each call requests
     :meth:`~pruna.evaluation.metrics.vlm_base.BaseVLM.score` with expected answer
     ``"Yes"`` (probability of Yes). Low scores act as semantic **No** for dependency
@@ -178,11 +176,9 @@ class OneIGAlignmentMetric(QAAccuracyMetric):
     api_key : str | None, optional
         API key for litellm.
     call_type : str, optional
-        Call type for the metric.
-    aggregation : str, optional
-        Unused; kept for registry compatibility with :class:`QAAccuracyMetric`.
+        Call type for the metric (``"single"`` or ``"pairwise"``).
     **kwargs : Any
-        Additional keyword arguments for :class:`QAAccuracyMetric`.
+        Forwarded to :class:`QAAccuracyMetric` (e.g. ``aggregation``).
 
     Examples
     --------
@@ -199,7 +195,6 @@ class OneIGAlignmentMetric(QAAccuracyMetric):
 
     def __init__(
         self,
-        *args: Any,
         grid_size: tuple[int, int] = (2, 2),
         vlm: Any | None = None,
         vlm_type: Literal["litellm", "transformers"] = "transformers",
@@ -212,18 +207,18 @@ def __init__(
         **kwargs: Any,
     ) -> None:
         super().__init__(
-            *args,
             vlm=vlm,
             vlm_type=vlm_type,
             model_name=model_name,
             vlm_kwargs=vlm_kwargs,
             structured_output=structured_output,
             device=device,
             api_key=api_key,
-            call_type=call_type if call_type is not None else "y_gt",
+            call_type=call_type,
             **kwargs,
         )
         self.grid_size = (int(grid_size[0]), int(grid_size[1]))
+        self.metric_units = type(self).metric_units
 
     def _score_sample(self, image: Any, aux: dict[str, Any]) -> float:
         if not isinstance(image, Image.Image):