complete vlm

.github/workflows/black.yaml

@@ -0,0 +1,18 @@
+name: Run black
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install venv
+        run: |
+          sudo apt-get -y install python3.10-venv
+      - uses: psf/black@stable
+        with:
+          options: "--check --verbose -l88"
+          src: "."

.github/workflows/test-build.yaml

@@ -0,0 +1,22 @@
+name: Build package
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.10
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install .

.gitignore

@@ -158,3 +158,4 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+.vscode

README.md

@@ -1,2 +1,56 @@
-# evolving-merged-models
-Evolutionary Optimization of Model Merging Recipes
+# Evolutionary Optimization of Model Merging Recipes
+
+This is an official repository of [Evolutionary Optimization of Model Merging Recipes](https://arxiv.org/).
+
+## Installation
+
+### 1. Clone the repo
+
+```bash
+git clone https://github.com/SakanaAI/evolving-merged-models.git
+cd evolving-merged-models
+```
+
+### 2. Install necesarry libraries
+
+```bash
+pip install -r requirements.txt
+```
+
+* We tested under the following environment:
+  * Python Version: 3.10
+  * CUDA Version: 12.3
+
+## Models
+
+### EvoLLM-JP
+
+| Model | MGSM (Accuracy) | [lm-eval-harness](https://github.com/Stability-AI/lm-evaluation-harness/tree/jp-stable) (Average) |
+| :-- | --: | --: |
+| [shisa-gamma-7b-v1](https://huggingface.co/augmxnt/shisa-gamma-7b-v1) | 0.10 | 66.06 |
+| [WizardMath-7B-V1.1](https://huggingface.co/WizardLM/WizardMath-7B-V1.1) | 0.18 | 60.07 |
+| [Abel-7B-002](https://huggingface.co/GAIR/Abel-7B-002) | 0.31 | 56.51 |
+| [Ours: EvoLLM-JP](https://huggingface.co/SakanaAI/evollm-jp) | **0.52** | **70.51** |
+
+### EvoVLM-JP
+
+| Model | Ja-VG-VQA-500 (Ja-R-L) | JaVLM-Bench-In-the-Wild (Ja-R-L) |
+| :-- | --: | --: |
+| [LLaVA-1.6-Mistral-7B](https://llava-vl.github.io/blog/2024-01-30-llava-next/) | 14.32 | 41.10 |
+| [JSVLM](https://huggingface.co/stabilityai/japanese-stable-vlm) | - | 40.50 |
+| [Ours: EvoVLM-JP](https://huggingface.co/SakanaAI/evovlm-jp) | **19.70** | **51.94** |
+
+## Evaluation
+
+To launch evaluation, run the following script with a certain config. All configs used for the paper are in `configs`.
+
+```bash
+python evaluate.py --config_path {path-to-config}
+```
+
+## Citation
+
+```bibtex
+
+
+```

configs/llm/evollm-jp.yaml

@@ -0,0 +1,7 @@
+model:
+  target: evomerge.CausalLM
+  params:
+    model_path: SakanaAI/EvoLLM-JP
+    template: ja-shisa-vqa
+    model_kwargs:
+      torch_dtype: torch.float16

configs/vlm/evovlm-jp.yaml

@@ -0,0 +1,23 @@
+model:
+  target: evomerge.LLaVA
+  params:
+    model_path: SakanaAI/EvoVLM-JP
+    template: ja-shisa-vqa
+    model_kwargs:
+      torch_dtype: torch.float16
+    generation_config:
+      max_new_tokens: 512
+      do_sample: false
+      num_beams: 5
+      repetition_penalty: 1.5
+eval:
+    - target: evomerge.eval.JaVGVQA
+      params:
+        loader_kwargs:
+          batch_size: 4
+          num_workers: 2
+    - target: evomerge.eval.JaVLMBenchIntheWild
+      params:
+        loader_kwargs:
+          batch_size: 4
+          num_workers: 2

configs/vlm/jsvlm.yaml

@@ -0,0 +1,16 @@
+model:
+  target: evomerge.JSVLM
+  params:
+    model_kwargs:
+      torch_dtype: torch.float16
+    generation_config:
+      do_sample: false
+      max_new_tokens: 512
+      num_beams: 5
+      repetition_penalty: 1.5
+eval:
+    - target: evomerge.eval.JaVLMBenchIntheWild
+      params:
+        loader_kwargs:
+          batch_size: 4
+          num_workers: 2

configs/vlm/llava-1-6-mistral-7b.yaml

@@ -0,0 +1,23 @@
+model:
+  target: evomerge.LLaVA
+  params:
+    model_path: llava-hf/llava-v1.6-mistral-7b-hf
+    template: ja-shisa-vqa
+    model_kwargs:
+      torch_dtype: torch.float16
+    generation_config:
+      max_new_tokens: 512
+      do_sample: false
+      num_beams: 5
+      repetition_penalty: 1.5
+eval:
+    - target: evomerge.eval.JaVGVQA
+      params:
+        loader_kwargs:
+          batch_size: 4
+          num_workers: 2
+    - target: evomerge.eval.JaVLMBenchIntheWild
+      params:
+        loader_kwargs:
+          batch_size: 4
+          num_workers: 2

evaluate.py

@@ -0,0 +1,70 @@
+import os
+import argparse
+import gc
+import json
+import logging
+import os
+from dataclasses import asdict
+
+import torch
+
+from evomerge import instantiate_from_config, load_config, set_seed
+
+logger = logging.getLogger(__name__)
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config_path", type=str, required=True, help="config path")
+    parser.add_argument("--output_path", type=str, default=None)
+    args = parser.parse_args()
+    # validation
+    if args.output_path is None:
+        args.output_path = (
+            os.path.splitext(os.path.basename(args.config_path))[0] + ".json"
+        )
+        args.output_path = f"results/{args.output_path}"
+        os.makedirs("results", exist_ok=True)
+    assert args.output_path.endswith(".json"), "`output_path` must be json file"
+    return args
+
+
+def main(args):
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        level=logging.INFO,
+    )
+    config = load_config(args.config_path)
+    logger.info(f"Config:\n{json.dumps(config, indent=2, ensure_ascii=False)}")
+    set_seed(42)
+
+    # 1. load model (it's already moved to device)
+    model = instantiate_from_config(config["model"])
+    logger.info(f"Model: {model.__class__.__name__}")
+
+    eval_configs = config["eval"]
+    if isinstance(eval_configs, str):
+        eval_configs = [eval_configs]
+
+    results = {}
+    for eval_config in eval_configs:
+        # 2. load evaluator
+        evaluator = instantiate_from_config(eval_config)
+        logger.info(f"Evaluator: {evaluator.__class__.__name__}")
+        # 3. Run!
+        outputs = evaluator(model)
+        logger.info(f"Result:\n{outputs.metrics}")
+        results[evaluator.name] = asdict(outputs)
+        del evaluator
+        torch.cuda.empty_cache()
+        gc.collect()
+
+    with open(args.output_path, "w") as f:
+        json.dump(results, f, indent=2, ensure_ascii=False)
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    main(args)

evomerge/__init__.py

@@ -0,0 +1,3 @@
+from .eval import *
+from .models import *
+from .utils import *

evomerge/eval/__init__.py

@@ -0,0 +1,2 @@
+from .ja_vg_vqa import JaVGVQA
+from .ja_vlm_wild import JaVLMBenchIntheWild