#!/usr/bin/env python
"""
Build a mixed Arrow dataset from multiple source datasets with configurable sampling ratios.
Usage:
python data/build_mixed_arrow_dataset.py --config configs/sft_llada.yaml
The script:
1. Loads each Arrow dataset from config.dataset.arrow_dirs
2. Applies sampling (downsample/upsample) based on config.dataset.sampling_ratios
3. Concatenates all sampled datasets
4. Shuffles globally with the seed from config.training.seed
5. Saves to a single output directory (config.experiment.output_dir/mixed_arrow)
6. Saves metadata (mixing_meta.json) with statistics
Requirements:
- datasets (Hugging Face)
- omegaconf
"""
import os
import sys
import json
import random
import logging
from pathlib import Path
from typing import Dict, List, Tuple
# Add project root to path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from omegaconf import OmegaConf
from datasets import load_from_disk, concatenate_datasets, Dataset
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO,
)
logger = logging.getLogger(__name__)
[docs]
def sample_dataset(dataset: Dataset, target_ratio: float, seed: int) -> Dataset:
"""
Adjust dataset size by downsampling or upsampling.
Args:
dataset: HuggingFace Dataset
target_ratio: float (e.g., 0.5, 2.0, 1.0)
- 1.0: use all samples
- < 1.0: downsample without replacement
- > 1.0: upsample with replacement (duplicate)
seed: random seed for reproducibility
Returns:
Dataset with adjusted size
"""
n_original = len(dataset)
if target_ratio == 1.0:
logger.info(f" Ratio = 1.0 → using all {n_original:,} samples")
return dataset
n_target = int(n_original * target_ratio)
if target_ratio < 1.0:
# Downsample: random subset without replacement
logger.info(f" Downsampling: {n_original:,} → {n_target:,} samples (ratio={target_ratio})")
indices = list(range(n_original))
random.Random(seed).shuffle(indices)
keep_indices = indices[:n_target]
return dataset.select(keep_indices)
else: # target_ratio > 1.0
# Upsample: duplicate with replacement
logger.info(f" Upsampling: {n_original:,} → {n_target:,} samples (ratio={target_ratio})")
sampled_indices = random.Random(seed).choices(range(n_original), k=n_target)
return dataset.select(sampled_indices)
[docs]
def build_mixed_dataset(dataset_specs: List[Tuple[str, float]], seed: int, output_dir: str) -> str:
"""
Load, sample, concatenate, shuffle, and save mixed dataset.
Args:
dataset_specs: List of (arrow_dir, ratio) tuples
seed: random seed for reproducibility
output_dir: where to save the mixed dataset
Returns:
Path to the saved mixed Arrow dataset
"""
logger.info(f"Building mixed dataset from {len(dataset_specs)} sources:")
all_sampled_datasets = []
stats = {}
for idx, (dir_path, ratio) in enumerate(dataset_specs):
name = Path(dir_path).name
logger.info(f"\n[{idx+1}/{len(dataset_specs)}] {name}")
logger.info(f" Path: {dir_path}")
logger.info(f" Ratio: {ratio}")
# Load Arrow dataset
ds = load_from_disk(dir_path)
n_original = len(ds)
logger.info(f" Original chunks: {n_original:,}")
# Filter out invalid samples (None values in critical fields)
def is_valid_sample(example):
return (example.get("input_ids") is not None and
example.get("diffusion_mask") is not None and
example.get("document_ids") is not None)
before_filter = len(ds)
ds = ds.filter(is_valid_sample, num_proc=4, desc=f"Filtering {name}")
after_filter = len(ds)
if after_filter < before_filter:
logger.warning(f" Filtered out {before_filter - after_filter:,} invalid samples ({(before_filter-after_filter)/before_filter*100:.1f}%)")
logger.info(f" After filtering: {after_filter:,} valid samples")
# Apply sampling
sampled_ds = sample_dataset(ds, ratio, seed)
n_sampled = len(sampled_ds)
logger.info(f" Final chunks: {n_sampled:,}")
stats[name] = {
"original_chunks": n_original,
"ratio": ratio,
"final_chunks": n_sampled
}
all_sampled_datasets.append(sampled_ds)
# Concatenate all sampled datasets
logger.info(f"\nConcatenating {len(all_sampled_datasets)} datasets...")
mixed_dataset = concatenate_datasets(all_sampled_datasets)
logger.info(f"Total before shuffle: {len(mixed_dataset):,}")
# Global shuffle (important for training!)
logger.info("Shuffling globally...")
mixed_dataset = mixed_dataset.shuffle(seed=seed)
logger.info(f"Final mixed dataset size: {len(mixed_dataset):,} chunks")
# Save to disk
output_path = Path(output_dir)
output_path.mkdir(parents=True, exist_ok=True)
logger.info(f"\nSaving mixed dataset to: {output_path}")
mixed_dataset.save_to_disk(str(output_path))
logger.info("✓ Saved successfully")
# Save metadata
meta = {
"total_chunks": len(mixed_dataset),
"dataset_stats": stats,
"config": {
"seed": seed,
"output_dir": str(output_path),
"dataset_specs": [(str(p), r) for p, r in dataset_specs]
}
}
meta_path = output_path / "mixing_meta.json"
with open(meta_path, "w") as f:
json.dump(meta, f, indent=2)
logger.info(f"✓ Saved metadata to: {meta_path}")
# Print summary
logger.info("\n" + "="*60)
logger.info("MIXING SUMMARY")
logger.info("="*60)
for name, stat in stats.items():
pct = stat['final_chunks'] / meta['total_chunks'] * 100
logger.info(f"{name:50s} {stat['final_chunks']:8,} chunks ({pct:5.1f}%)")
logger.info("="*60)
logger.info(f"TOTAL: {meta['total_chunks']:,} chunks")
logger.info("="*60)
return str(output_path)
[docs]
def main():
# Parse command line args
import argparse
parser = argparse.ArgumentParser(
description="Build mixed Arrow dataset with configurable sampling ratios",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
# Mix two datasets with different ratios
python data/build_mixed_arrow_dataset.py \\
--output-dir /scratch/.../mixed_arrow \\
--dataset /path/to/OpenThoughts-114k-pyarrow 1.0 \\
--dataset /path/to/wildguardmix-safe-prompts-pyarrow 2.0 \\
--seed 42
# Mix all datasets
python data/build_mixed_arrow_dataset.py \\
--output-dir /scratch/.../mixed_arrow \\
--dataset /path/to/OpenThoughts-114k-pyarrow 0.3 \\
--dataset /path/to/MathInstruct-pyarrow 1.5 \\
--dataset /path/to/CodeFeedback-pyarrow 1.0 \\
--dataset /path/to/HelpSteer2-pyarrow 1.0 \\
--dataset /path/to/Magpie-pyarrow 0.5 \\
--dataset /path/to/wildguardmix-safe-prompts-pyarrow 1.0 \\
--dataset /path/to/wildguardmix-harmful-prompt-safe-response-pyarrow 1.0 \\
--dataset /path/to/wildguardmix-harmful-prompt-harmful-response-pyarrow 2.0 \\
--seed 42
"""
)
parser.add_argument("--output-dir", type=str, required=True, help="Output directory for mixed Arrow dataset")
parser.add_argument("--dataset", action="append", nargs=2, metavar=("PATH", "RATIO"),
help="Dataset path and ratio (can specify multiple times)")
parser.add_argument("--seed", type=int, default=42, help="Random seed for shuffling and sampling (default: 42)")
args = parser.parse_args()
if not args.dataset:
parser.error("You must specify at least one --dataset")
# Convert to list of (path, ratio) tuples
dataset_specs = []
for path, ratio_str in args.dataset:
try:
ratio = float(ratio_str)
if ratio <= 0:
raise ValueError("Ratio must be positive")
except ValueError:
parser.error(f"Invalid ratio: {ratio_str}. Must be a positive number.")
dataset_specs.append((path, ratio))
# Build mixed dataset
output_path = build_mixed_dataset(dataset_specs, seed=args.seed, output_dir=args.output_dir)
logger.info(f"\n✓ Mixed dataset ready at: {output_path}")
logger.info("Next steps:")
logger.info(" 1. Update config: dataset.arrow_dirs to point to this mixed_arrow directory")
logger.info(" 2. Run training as usual")
if __name__ == "__main__":
main()