Source code for safety.dataset

"""
从 AISafetyLab 加载评测数据集,提取纯 queries 列表

设计原则:
- 只作为 AISafetyLab AttackDataset 的黑箱包装器
- 不修改 AISafetyLab 任何代码
- 当前仅支持 HarmBench 数据集(使用标准 split 'standard')

用法:
    from safety import BenchmarkDataset
    ds = BenchmarkDataset("harmbench")
    queries = ds.get_queries()
"""

from typing import List, Dict, Optional
import json
from pathlib import Path
from datasets import load_dataset


[docs] class BenchmarkDataset: """ 数据集加载器,当前仅支持 HarmBench 示例: ds = BenchmarkDataset("harmbench") print(f"Loaded {len(ds)} queries") print(ds[0]['query']) # 第一个问题 """ SUPPORTED_DATASETS = ["harmbench"]
[docs] def __init__(self, dataset_name: str, data_path: Optional[str] = None): """ Args: dataset_name: 数据集名称,目前只支持 "harmbench" data_path: 未使用,保留供未来扩展 Raises: ValueError: 数据集不支持 RuntimeError: 下载或加载失败 """ if dataset_name != "harmbench": raise ValueError( f"Currently only 'harmbench' is supported.\n" f"Got: {dataset_name}\n" f"Supported: {self.SUPPORTED_DATASETS}" ) self.dataset_name = dataset_name self.data: List[Dict] = [] self._load_harmbench()
def _load_harmbench(self): """加载 HarmBench 标准数据集(从 HuggingFace)""" print("🌐 Loading HarmBench from HuggingFace...") print(" Dataset: thu-coai/AISafetyLab_Datasets") print(" Config: harmbench") print(" Split: standard") print(" (First time may download ~1MB)") try: hf_dataset = load_dataset( "thu-coai/AISafetyLab_Datasets", "harmbench", split="standard" ) # 转换为普通 dict 列表 self.data = [dict(item) for item in hf_dataset] print(f"✅ Loaded {len(self.data)} samples from HarmBench") except Exception as e: raise RuntimeError( f"Failed to load HarmBench dataset.\n" f"Error: {e}\n" f"Please check internet connection and HuggingFace credentials." ) from e
[docs] def get_queries(self) -> List[str]: """ 返回纯 query 字符串列表,用于推理 Returns: List[str]: 所有样本的 query 字段 """ return [item['query'] for item in self.data]
[docs] def get_items(self) -> List[Dict]: """ 返回原始数据项列表(含 query 及其他字段如 category、target 等) Returns: List[Dict]: 原始数据字典列表 """ return self.data.copy()
[docs] def get_item(self, idx: int) -> Dict: """ 获取单个数据项 Args: idx: 数据索引 Returns: Dict: 数据项(包含 query, target, category 等字段) """ return self.data[idx].copy()
def __len__(self) -> int: return len(self.data) def __getitem__(self, idx): return self.data[idx] def __iter__(self): return iter(self.data)
[docs] def summary(self) -> Dict: """ 返回数据集摘要信息 Returns: Dict: 包含样本数、字段列表、类别分布等 """ if not self.data: return {"total": 0} sample = self.data[0] fields = list(sample.keys()) # 统计 category 分布(如果有) category_dist = {} if 'category' in fields: from collections import Counter category_dist = dict(Counter( item.get('category', 'unknown') for item in self.data )) return { "total": len(self.data), "fields": fields, "dataset_name": self.dataset_name, "category_distribution": category_dist, }
def __repr__(self) -> str: summary = self.summary() return (f"BenchmarkDataset(name={self.dataset_name}, " f"total={summary['total']}, fields={summary['fields']})")
# 便利函数
[docs] def load_benchmark( dataset: str, max_samples: Optional[int] = None ) -> BenchmarkDataset: """ 快速加载评测数据集的便利函数 Example: >>> ds = load_benchmark("harmbench", max_samples=10) >>> queries = ds.get_queries() Args: dataset: 数据集名称 max_samples: 限制最大样本数(用于快速测试) Returns: BenchmarkDataset: 数据集对象 """ ds = BenchmarkDataset(dataset) if max_samples and max_samples < len(ds): ds.data = ds.data[:max_samples] print(f" Limited to {max_samples} samples") return ds
if __name__ == "__main__": """快速测试脚本""" import sys print("🎯 Testing BenchmarkDataset (HarmBench)\n") try: ds = BenchmarkDataset("harmbench") print(f"✅ Loaded {len(ds)} samples\n") print("First 3 queries:") for i in range(min(3, len(ds))): item = ds[i] print(f"\n [{i}]") print(f" Query : {item.get('query', 'N/A')[:100]}") print(f" Target : {item.get('target', 'N/A')[:80]}") print(f" Category : {item.get('category', 'N/A')}") summary = ds.summary() if summary.get('category_distribution'): print("\nCategory distribution:") for cat, count in sorted(summary['category_distribution'].items()): print(f" {cat}: {count}") print("\n✅ All checks passed!") except Exception as e: print(f"❌ Error: {e}") import traceback traceback.print_exc() sys.exit(1)