run_specs.conf

entries: [
    #bigbench

    #analytic_entailment: https://github.com/google/BIG-bench/blob/main/bigbench/benchmark_tasks/analytic_entailment
    {description: "big_bench:model=neurips/local,max_train_instances=3,task=analytic_entailment,subtask=", priority: 1}

    #causal_judgment: https://github.com/google/BIG-bench/blob/main/bigbench/benchmark_tasks/causal_judgment
    {description: "big_bench:model=neurips/local,max_train_instances=3,task=causal_judgment,subtask=", priority: 1}

    #emoji_movie: https://github.com/google/big-bench/tree/main/bigbench/benchmark_tasks/emoji_movie
    {description: "big_bench:model=neurips/local,max_train_instances=3,task=emoji_movie,subtask=", priority: 1}

    #empirical_judgments: https://github.com/google/big-bench/tree/main/bigbench/benchmark_tasks/empirical_judgments
    {description: "big_bench:model=neurips/local,max_train_instances=3,task=empirical_judgments,subtask=", priority: 1}

    #known_unknowns: https://github.com/google/big-bench/tree/main/bigbench/benchmark_tasks/known_unknowns
    {description: "big_bench:model=neurips/local,max_train_instances=3,task=known_unknowns,subtask=", priority: 1}

    # logical_deduction: https://github.com/google/big-bench/tree/main/bigbench/benchmark_tasks/logical_deduction
    {description: "big_bench:model=neurips/local,max_train_instances=3,task=logical_deduction,subtask=three_objects", priority: 1}

    #strange_stories: https://github.com/google/big-bench/tree/main/bigbench/benchmark_tasks/strange_stories
    {description: "big_bench:model=neurips/local,max_train_instances=3,task=strange_stories,subtask=multiple_choice", priority: 1}

    #snarks: https://github.com/google/BIG-bench/tree/main/bigbench/benchmark_tasks/snarks
    {description: "big_bench:model=neurips/local,max_train_instances=3,task=snarks,subtask=", priority: 1}

    #dark_humor_detection: https://github.com/google/BIG-bench/tree/main/bigbench/benchmark_tasks/dark_humor_detection
    {description: "big_bench:model=neurips/local,max_train_instances=3,task=dark_humor_detection,subtask=", priority: 1}

    
    #mmlu
    {description: "mmlu:model=neurips/local,subject=philosophy,data_augmentation=canonical", priority: 1}
    {description: "mmlu:model=neurips/local,subject=high_school_biology,data_augmentation=canonical", priority: 1}
    {description: "mmlu:model=neurips/local,subject=high_school_chemistry,data_augmentation=canonical", priority: 1}
    {description: "mmlu:model=neurips/local,subject=high_school_computer_science,data_augmentation=canonical", priority: 1}
    {description: "mmlu:model=neurips/local,subject=high_school_european_history,data_augmentation=canonical", priority: 1}
    {description: "mmlu:model=neurips/local,subject=high_school_geography,data_augmentation=canonical", priority: 1}
    {description: "mmlu:model=neurips/local,subject=high_school_government_and_politics,data_augmentation=canonical", priority: 1}
    {description: "mmlu:model=neurips/local,subject=high_school_macroeconomics,data_augmentation=canonical", priority: 1}
    {description: "mmlu:model=neurips/local,subject=high_school_mathematics,data_augmentation=canonical", priority: 1}
    {description: "mmlu:model=neurips/local,subject=high_school_microeconomics,data_augmentation=canonical", priority: 1}
    {description: "mmlu:model=neurips/local,subject=high_school_physics,data_augmentation=canonical", priority: 1}
    {description: "mmlu:model=neurips/local,subject=high_school_psychology,data_augmentation=canonical", priority: 1}
    {description: "mmlu:model=neurips/local,subject=high_school_statistics,data_augmentation=canonical", priority: 1}
    {description: "mmlu:model=neurips/local,subject=high_school_us_history,data_augmentation=canonical", priority: 1}
    {description: "mmlu:model=neurips/local,subject=high_school_world_history,data_augmentation=canonical", priority: 1}
    {description: "mmlu:model=neurips/local,subject=moral_disputes,data_augmentation=canonical", priority: 1}
    {description: "mmlu:model=neurips/local,subject=moral_scenarios,data_augmentation=canonical", priority: 1}


    #truthful QA
    {description: "truthful_qa:task=mc_single,model=neurips/local", priority: 1},

    #CNN/daily mail
    {description: "summarization_cnndm:model=neurips/local", priority: 1},
    #GSM
    {description: "gsm:model=neurips/local", priority: 1}
    #BBQ
    {description: "bbq:subject=all,model=neurips/local", priority: 1},

]