Source code for data.build_mixed_arrow_dataset

#!/usr/bin/env python
"""
Build a mixed Arrow dataset from multiple source datasets with configurable sampling ratios.

Usage:
    python data/build_mixed_arrow_dataset.py --config configs/sft_llada.yaml

The script:
1. Loads each Arrow dataset from config.dataset.arrow_dirs
2. Applies sampling (downsample/upsample) based on config.dataset.sampling_ratios
3. Concatenates all sampled datasets
4. Shuffles globally with the seed from config.training.seed
5. Saves to a single output directory (config.experiment.output_dir/mixed_arrow)
6. Saves metadata (mixing_meta.json) with statistics

Requirements:
    - datasets (Hugging Face)
    - omegaconf
"""

import os
import sys
import json
import random
import logging
from pathlib import Path
from typing import Dict, List, Tuple

# Add project root to path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from omegaconf import OmegaConf
from datasets import load_from_disk, concatenate_datasets, Dataset

logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
    datefmt="%m/%d/%Y %H:%M:%S",
    level=logging.INFO,
)
logger = logging.getLogger(__name__)


[docs] def sample_dataset(dataset: Dataset, target_ratio: float, seed: int) -> Dataset: """ Adjust dataset size by downsampling or upsampling. Args: dataset: HuggingFace Dataset target_ratio: float (e.g., 0.5, 2.0, 1.0) - 1.0: use all samples - < 1.0: downsample without replacement - > 1.0: upsample with replacement (duplicate) seed: random seed for reproducibility Returns: Dataset with adjusted size """ n_original = len(dataset) if target_ratio == 1.0: logger.info(f" Ratio = 1.0 → using all {n_original:,} samples") return dataset n_target = int(n_original * target_ratio) if target_ratio < 1.0: # Downsample: random subset without replacement logger.info(f" Downsampling: {n_original:,} → {n_target:,} samples (ratio={target_ratio})") indices = list(range(n_original)) random.Random(seed).shuffle(indices) keep_indices = indices[:n_target] return dataset.select(keep_indices) else: # target_ratio > 1.0 # Upsample: duplicate with replacement logger.info(f" Upsampling: {n_original:,} → {n_target:,} samples (ratio={target_ratio})") sampled_indices = random.Random(seed).choices(range(n_original), k=n_target) return dataset.select(sampled_indices)
[docs] def build_mixed_dataset(dataset_specs: List[Tuple[str, float]], seed: int, output_dir: str) -> str: """ Load, sample, concatenate, shuffle, and save mixed dataset. Args: dataset_specs: List of (arrow_dir, ratio) tuples seed: random seed for reproducibility output_dir: where to save the mixed dataset Returns: Path to the saved mixed Arrow dataset """ logger.info(f"Building mixed dataset from {len(dataset_specs)} sources:") all_sampled_datasets = [] stats = {} for idx, (dir_path, ratio) in enumerate(dataset_specs): name = Path(dir_path).name logger.info(f"\n[{idx+1}/{len(dataset_specs)}] {name}") logger.info(f" Path: {dir_path}") logger.info(f" Ratio: {ratio}") # Load Arrow dataset ds = load_from_disk(dir_path) n_original = len(ds) logger.info(f" Original chunks: {n_original:,}") # Filter out invalid samples (None values in critical fields) def is_valid_sample(example): return (example.get("input_ids") is not None and example.get("diffusion_mask") is not None and example.get("document_ids") is not None) before_filter = len(ds) ds = ds.filter(is_valid_sample, num_proc=4, desc=f"Filtering {name}") after_filter = len(ds) if after_filter < before_filter: logger.warning(f" Filtered out {before_filter - after_filter:,} invalid samples ({(before_filter-after_filter)/before_filter*100:.1f}%)") logger.info(f" After filtering: {after_filter:,} valid samples") # Apply sampling sampled_ds = sample_dataset(ds, ratio, seed) n_sampled = len(sampled_ds) logger.info(f" Final chunks: {n_sampled:,}") stats[name] = { "original_chunks": n_original, "ratio": ratio, "final_chunks": n_sampled } all_sampled_datasets.append(sampled_ds) # Concatenate all sampled datasets logger.info(f"\nConcatenating {len(all_sampled_datasets)} datasets...") mixed_dataset = concatenate_datasets(all_sampled_datasets) logger.info(f"Total before shuffle: {len(mixed_dataset):,}") # Global shuffle (important for training!) logger.info("Shuffling globally...") mixed_dataset = mixed_dataset.shuffle(seed=seed) logger.info(f"Final mixed dataset size: {len(mixed_dataset):,} chunks") # Save to disk output_path = Path(output_dir) output_path.mkdir(parents=True, exist_ok=True) logger.info(f"\nSaving mixed dataset to: {output_path}") mixed_dataset.save_to_disk(str(output_path)) logger.info("✓ Saved successfully") # Save metadata meta = { "total_chunks": len(mixed_dataset), "dataset_stats": stats, "config": { "seed": seed, "output_dir": str(output_path), "dataset_specs": [(str(p), r) for p, r in dataset_specs] } } meta_path = output_path / "mixing_meta.json" with open(meta_path, "w") as f: json.dump(meta, f, indent=2) logger.info(f"✓ Saved metadata to: {meta_path}") # Print summary logger.info("\n" + "="*60) logger.info("MIXING SUMMARY") logger.info("="*60) for name, stat in stats.items(): pct = stat['final_chunks'] / meta['total_chunks'] * 100 logger.info(f"{name:50s} {stat['final_chunks']:8,} chunks ({pct:5.1f}%)") logger.info("="*60) logger.info(f"TOTAL: {meta['total_chunks']:,} chunks") logger.info("="*60) return str(output_path)
[docs] def main(): # Parse command line args import argparse parser = argparse.ArgumentParser( description="Build mixed Arrow dataset with configurable sampling ratios", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: # Mix two datasets with different ratios python data/build_mixed_arrow_dataset.py \\ --output-dir /scratch/.../mixed_arrow \\ --dataset /path/to/OpenThoughts-114k-pyarrow 1.0 \\ --dataset /path/to/wildguardmix-safe-prompts-pyarrow 2.0 \\ --seed 42 # Mix all datasets python data/build_mixed_arrow_dataset.py \\ --output-dir /scratch/.../mixed_arrow \\ --dataset /path/to/OpenThoughts-114k-pyarrow 0.3 \\ --dataset /path/to/MathInstruct-pyarrow 1.5 \\ --dataset /path/to/CodeFeedback-pyarrow 1.0 \\ --dataset /path/to/HelpSteer2-pyarrow 1.0 \\ --dataset /path/to/Magpie-pyarrow 0.5 \\ --dataset /path/to/wildguardmix-safe-prompts-pyarrow 1.0 \\ --dataset /path/to/wildguardmix-harmful-prompt-safe-response-pyarrow 1.0 \\ --dataset /path/to/wildguardmix-harmful-prompt-harmful-response-pyarrow 2.0 \\ --seed 42 """ ) parser.add_argument("--output-dir", type=str, required=True, help="Output directory for mixed Arrow dataset") parser.add_argument("--dataset", action="append", nargs=2, metavar=("PATH", "RATIO"), help="Dataset path and ratio (can specify multiple times)") parser.add_argument("--seed", type=int, default=42, help="Random seed for shuffling and sampling (default: 42)") args = parser.parse_args() if not args.dataset: parser.error("You must specify at least one --dataset") # Convert to list of (path, ratio) tuples dataset_specs = [] for path, ratio_str in args.dataset: try: ratio = float(ratio_str) if ratio <= 0: raise ValueError("Ratio must be positive") except ValueError: parser.error(f"Invalid ratio: {ratio_str}. Must be a positive number.") dataset_specs.append((path, ratio)) # Build mixed dataset output_path = build_mixed_dataset(dataset_specs, seed=args.seed, output_dir=args.output_dir) logger.info(f"\n✓ Mixed dataset ready at: {output_path}") logger.info("Next steps:") logger.info(" 1. Update config: dataset.arrow_dirs to point to this mixed_arrow directory") logger.info(" 2. Run training as usual")
if __name__ == "__main__": main()