"""
从 AISafetyLab 加载评测数据集,提取纯 queries 列表
设计原则:
- 只作为 AISafetyLab AttackDataset 的黑箱包装器
- 不修改 AISafetyLab 任何代码
- 当前仅支持 HarmBench 数据集(使用标准 split 'standard')
用法:
from safety import BenchmarkDataset
ds = BenchmarkDataset("harmbench")
queries = ds.get_queries()
"""
from typing import List, Dict, Optional
import json
from pathlib import Path
from datasets import load_dataset
[docs]
class BenchmarkDataset:
"""
数据集加载器,当前仅支持 HarmBench
示例:
ds = BenchmarkDataset("harmbench")
print(f"Loaded {len(ds)} queries")
print(ds[0]['query']) # 第一个问题
"""
SUPPORTED_DATASETS = ["harmbench"]
[docs]
def __init__(self, dataset_name: str, data_path: Optional[str] = None):
"""
Args:
dataset_name: 数据集名称,目前只支持 "harmbench"
data_path: 未使用,保留供未来扩展
Raises:
ValueError: 数据集不支持
RuntimeError: 下载或加载失败
"""
if dataset_name != "harmbench":
raise ValueError(
f"Currently only 'harmbench' is supported.\n"
f"Got: {dataset_name}\n"
f"Supported: {self.SUPPORTED_DATASETS}"
)
self.dataset_name = dataset_name
self.data: List[Dict] = []
self._load_harmbench()
def _load_harmbench(self):
"""加载 HarmBench 标准数据集(从 HuggingFace)"""
print("🌐 Loading HarmBench from HuggingFace...")
print(" Dataset: thu-coai/AISafetyLab_Datasets")
print(" Config: harmbench")
print(" Split: standard")
print(" (First time may download ~1MB)")
try:
hf_dataset = load_dataset(
"thu-coai/AISafetyLab_Datasets",
"harmbench",
split="standard"
)
# 转换为普通 dict 列表
self.data = [dict(item) for item in hf_dataset]
print(f"✅ Loaded {len(self.data)} samples from HarmBench")
except Exception as e:
raise RuntimeError(
f"Failed to load HarmBench dataset.\n"
f"Error: {e}\n"
f"Please check internet connection and HuggingFace credentials."
) from e
[docs]
def get_queries(self) -> List[str]:
"""
返回纯 query 字符串列表,用于推理
Returns:
List[str]: 所有样本的 query 字段
"""
return [item['query'] for item in self.data]
[docs]
def get_items(self) -> List[Dict]:
"""
返回原始数据项列表(含 query 及其他字段如 category、target 等)
Returns:
List[Dict]: 原始数据字典列表
"""
return self.data.copy()
[docs]
def get_item(self, idx: int) -> Dict:
"""
获取单个数据项
Args:
idx: 数据索引
Returns:
Dict: 数据项(包含 query, target, category 等字段)
"""
return self.data[idx].copy()
def __len__(self) -> int:
return len(self.data)
def __getitem__(self, idx):
return self.data[idx]
def __iter__(self):
return iter(self.data)
[docs]
def summary(self) -> Dict:
"""
返回数据集摘要信息
Returns:
Dict: 包含样本数、字段列表、类别分布等
"""
if not self.data:
return {"total": 0}
sample = self.data[0]
fields = list(sample.keys())
# 统计 category 分布(如果有)
category_dist = {}
if 'category' in fields:
from collections import Counter
category_dist = dict(Counter(
item.get('category', 'unknown') for item in self.data
))
return {
"total": len(self.data),
"fields": fields,
"dataset_name": self.dataset_name,
"category_distribution": category_dist,
}
def __repr__(self) -> str:
summary = self.summary()
return (f"BenchmarkDataset(name={self.dataset_name}, "
f"total={summary['total']}, fields={summary['fields']})")
# 便利函数
[docs]
def load_benchmark(
dataset: str,
max_samples: Optional[int] = None
) -> BenchmarkDataset:
"""
快速加载评测数据集的便利函数
Example:
>>> ds = load_benchmark("harmbench", max_samples=10)
>>> queries = ds.get_queries()
Args:
dataset: 数据集名称
max_samples: 限制最大样本数(用于快速测试)
Returns:
BenchmarkDataset: 数据集对象
"""
ds = BenchmarkDataset(dataset)
if max_samples and max_samples < len(ds):
ds.data = ds.data[:max_samples]
print(f" Limited to {max_samples} samples")
return ds
if __name__ == "__main__":
"""快速测试脚本"""
import sys
print("🎯 Testing BenchmarkDataset (HarmBench)\n")
try:
ds = BenchmarkDataset("harmbench")
print(f"✅ Loaded {len(ds)} samples\n")
print("First 3 queries:")
for i in range(min(3, len(ds))):
item = ds[i]
print(f"\n [{i}]")
print(f" Query : {item.get('query', 'N/A')[:100]}")
print(f" Target : {item.get('target', 'N/A')[:80]}")
print(f" Category : {item.get('category', 'N/A')}")
summary = ds.summary()
if summary.get('category_distribution'):
print("\nCategory distribution:")
for cat, count in sorted(summary['category_distribution'].items()):
print(f" {cat}: {count}")
print("\n✅ All checks passed!")
except Exception as e:
print(f"❌ Error: {e}")
import traceback
traceback.print_exc()
sys.exit(1)