Add gen reg tests

.github/workflows/ci.yml

@@ -3,7 +3,7 @@ name: Continuous integration
 on:
   push:
     branches:
-    - main 
+    - main
     paths-ignore:
     - '**.md'
     - 'CITATION.cff'
@@ -12,7 +12,7 @@ on:
     - 'docs/**'
   pull_request:
     branches:
-    - main 
+    - main
     paths-ignore:
     - '**.md'
     - 'CITATION.cff'
@@ -29,8 +29,9 @@ on:
         type: string
 
 env:
-  REVISION_REFERENCE: v2.8.2
+  REVISION_REFERENCE: v2.14.0
   #9d31b2ec4df6d8228f370ff20c8267ec6ba39383 earliest compatible v2.7.0 + pretrained_hf param
+  GEN_REVISION_REFERENCE: 7b8dd2cbaf9cca13ca5b1defa6a321a145eb166c
 
 jobs:
   Tests:
@@ -79,17 +80,30 @@ jobs:
           --splitting-algorithm least_duration \
           --splits ${{ matrix.job_num }} \
           --group ${{ matrix.job }} \
-          -m regression_test \
+          -m "regression_test" \
           tests \
           | head -n -2 | grep -Po 'test_inference_with_data\[\K[^]]*(?=-False]|-True])' \
           > models_gh_runner.txt
+        python -m pytest \
+          --quiet --co \
+          --splitting-algorithm least_duration \
+          --splits ${{ matrix.job_num }} \
+          --group ${{ matrix.job }} \
+          -m "generative_regression_test" \
+          tests \
+          | head -n -2 | grep -Po 'test_generative_with_data\[\K[^]]*' \
+          > generative_models_gh_runner.txt
         if [ -n "${{ inputs.manual_revision_reference }}" ]; then
           REVISION_REFERENCE=${{ inputs.manual_revision_reference }}
         fi
         python tests/util_test.py \
           --save_model_list models_gh_runner.txt \
           --model_list models_gh_runner.txt \
           --git_revision $REVISION_REFERENCE
+        python tests/util_test.py \
+          --save_model_list generative_models_gh_runner.txt \
+          --generative_model_list generative_models_gh_runner.txt \
+          --git_revision $GEN_REVISION_REFERENCE
     - name: Unit tests
       run: |
         source .env/bin/activate
@@ -113,7 +127,7 @@ jobs:
           --store-durations \
           --durations-path durations_2 \
           --clean-durations \
-          -m "regression_test" \
+          -m "regression_test and generative_regression_test" \
           tests
         jq -s -S 'add' durations_* > .test_durations
     - name: Collect pytest durations

pytest.ini

@@ -1,3 +1,4 @@
 [pytest]
 markers =
     regression_test
+    generative_regression_test

src/open_clip/__init__.py

@@ -1,7 +1,7 @@
 from .coca_model import CoCa
 from .constants import OPENAI_DATASET_MEAN, OPENAI_DATASET_STD
 from .factory import create_model, create_model_and_transforms, create_model_from_pretrained, get_tokenizer, create_loss
-from .factory import list_models, add_model_config, get_model_config, load_checkpoint
+from .factory import list_models, list_generative_models, add_model_config, get_model_config, load_checkpoint
 from .loss import ClipLoss, DistillClipLoss, CoCaLoss
 from .model import CLIP, CustomTextCLIP, CLIPTextCfg, CLIPVisionCfg, \
     convert_weights_to_lp, convert_weights_to_fp16, trace_model, get_cast_dtype, get_input_dtype, \