fix(collector): fix duplicate color detect (#189)

.github/workflows/collect.yml

@@ -12,18 +12,8 @@ jobs:
     if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }}
     runs-on: ubuntu-latest
     steps:
-      - shell: bash
-        run: |
-          echo " start at"
-          echo $PWD
       - uses: actions/checkout@v4
-      - name: Install Python3
-        shell: bash
-        run: |
-          sudo apt-get install -y python3 python3-dev python3-venv python3-pip python3-docutils
-          python3 -m pip install --user -r requirements.txt
       - name: Install Google-Chrome
-        shell: bash
         run: |
           wget -q https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
           sudo dpkg -i google-chrome-stable_current_amd64.deb
@@ -38,9 +28,12 @@ jobs:
           chromedriver --url-base=/wd/hub &
           sudo Xvfb -ac :99 -screen 0 1280x1024x24 > /dev/null 2>&1 & # optional
       - name: Collect Colors
-        shell: bash
         run: |
           export PYTHONPATH="$PWD:$PYTHONPATH"
+          sudo apt-get install -y python3 python3-dev python3-venv python3-pip python3-docutils
+          python3 -m venv venv
+          . venv/bin/activate
+          python3 -m pip install -r requirements.txt
           python3 collect.py
       - name: Auto Commit
         uses: stefanzweifel/git-auto-commit-action@v4

COLORSCHEMES.md

@@ -1,5 +1,5 @@
 
-ColorSchemes List (72)
+ColorSchemes List (74)
 ======================
 
 - [folke/tokyonight.nvim](https://github.com/folke/tokyonight.nvim) (stars: 4800, last update: 2024-01-31)  
@@ -19,6 +19,14 @@ ColorSchemes List (72)
   - kanagawa-dragon  
   - kanagawa-lotus  
   - kanagawa-wave  
+- [edeneast/nightfox.nvim](https://github.com/edeneast/nightfox.nvim) (stars: 2600, last update: 2024-01-31)  
+  - carbonfox  
+  - dawnfox  
+  - dayfox  
+  - duskfox  
+  - nightfox  
+  - nordfox  
+  - terafox  
 - [nlknguyen/papercolor-theme](https://github.com/nlknguyen/papercolor-theme) (stars: 2591, last update: 2022-06-08)  
   - PaperColor  
 - [sainnhe/everforest](https://github.com/sainnhe/everforest) (stars: 2400, last update: 2024-02-10)  
@@ -113,17 +121,14 @@ ColorSchemes List (72)
   - falcon  
 - [drewtempelmeyer/palenight.vim](https://github.com/drewtempelmeyer/palenight.vim) (stars: 648, last update: 2023-04-27)  
   - palenight  
-- [olimorris/onedarkpro.nvim](https://github.com/olimorris/onedarkpro.nvim) (stars: 642, last update: 2024-02-24)  
-  - onedark  
-  - onedark_dark  
-  - onedark_vivid  
-  - onelight  
 - [embark-theme/vim](https://github.com/embark-theme/vim) (stars: 607, last update: 2024-02-14)  
   - embark  
 - [preservim/vim-colors-pencil](https://github.com/preservim/vim-colors-pencil) (stars: 596, last update: 2022-05-07)  
   - pencil  
 - [savq/melange-nvim](https://github.com/savq/melange-nvim) (stars: 582, last update: 2024-02-14)  
   - melange  
+- [mofiqul/vscode.nvim](https://github.com/mofiqul/vscode.nvim) (stars: 577, last update: 2024-02-25)  
+  - vscode  
 - [challenger-deep-theme/vim](https://github.com/challenger-deep-theme/vim) (stars: 577, last update: 2022-04-25)  
   - challenger_deep  
 - [liuchengxu/space-vim-dark](https://github.com/liuchengxu/space-vim-dark) (stars: 568, last update: 2023-11-04)  
@@ -155,6 +160,8 @@ ColorSchemes List (72)
   - hybrid_reverse  
 - [rigellute/rigel](https://github.com/rigellute/rigel) (stars: 505, last update: 2024-01-11)  
   - rigel  
+- [alexvzyl/nordic.nvim](https://github.com/alexvzyl/nordic.nvim) (stars: 500, last update: 2024-02-08)  
+  - nordic  
 - [wadackel/vim-dogrun](https://github.com/wadackel/vim-dogrun) (stars: 499, last update: 2024-02-20)  
   - dogrun  
 - [lifepillar/vim-gruvbox8](https://github.com/lifepillar/vim-gruvbox8) (stars: 495, last update: 2023-08-04)  

collect.py

@@ -41,9 +41,6 @@
 DATE_FORMAT = "%Y-%m-%d"
 CANDIDATE_DIR = "candidate"
 
-if os.path.exists("autoload/db.json"):
-    os.remove("autoload/db.json")
-
 
 def init_logging(level: typing.Optional[int]) -> None:
     assert isinstance(level, int) or level is None
@@ -202,7 +199,7 @@ def _init_url(self, handle: str) -> str:
             handle = handle[1:]
         while handle.endswith("/"):
             handle = handle[:-1]
-        return handle
+        return handle.lower()
 
     def _init_config(self, handle: str) -> typing.Optional[ColorSpecConfig]:
         return REPO_META_CONFIG[handle] if handle in REPO_META_CONFIG else None
@@ -507,7 +504,7 @@ def filter_color_specs() -> None:
     # logging.debug(f"before filter:{[str(s) for s in specs]}")
 
     def blacklist(sp: ColorSpec) -> bool:
-        return any([True for b in BLACKLIST if sp.url.find(b) >= 0])
+        return any([True for b in BLACKLIST if sp.url.lower().find(b.lower()) >= 0])
 
     for spec in ColorSpec.all():
         logging.debug(f"process filtering on spec:{spec}")
@@ -572,6 +569,9 @@ def greater_than(a: ColorSpec, b: ColorSpec) -> bool:
         for spec in ColorSpec.all():
             color_names = spec.get_vim_color_names()
             logging.debug(f"process dedup spec:{spec}, color names:{color_names}")
+            found_duplicate = False
+            drop_target = None
+            replace_target = None
             for color in color_names:
                 # detect duplicated color
                 if color in specs_map:
@@ -581,14 +581,40 @@ def greater_than(a: ColorSpec, b: ColorSpec) -> bool:
                     )
                     # replace old repo if new repo has higher priority
                     if greater_than(spec, old_spec):
-                        logging.info(f"replace old spec({old_spec}) with new({spec})")
+                        logging.info(
+                            f"replace old spec({old_spec}) with new spec({spec})"
+                        )
                         specs_map[color] = spec
                         specs_set.add(spec)
                         specs_set.remove(old_spec)
+
+                        replace_target = spec
+                        drop_target = old_spec
+                    else:
+                        logging.info(
+                            f"keep old spec({old_spec}), drop new spec({spec})"
+                        )
+
+                        replace_target = old_spec
+                        drop_target = spec
+
+                    found_duplicate = True
+                    break
                 else:
                     # add new color
                     specs_map[color] = spec
                     specs_set.add(spec)
+            if found_duplicate and drop_target and replace_target:
+                if drop_target in specs_set:
+                    specs_set.remove(drop_target)
+                if replace_target not in specs_set:
+                    specs_set.add(replace_target)
+                for color in drop_target.get_vim_color_names():
+                    if color in specs_map:
+                        specs_map.pop(color)
+                for color in replace_target.get_vim_color_names():
+                    specs_map[color] = replace_target
+
         logging.debug(f"after dedup: {[str(s) for s in specs_set]}")
         return list(specs_set)
 
@@ -636,6 +662,7 @@ def collect(debug_opt, no_headless_opt, skip_fetch_opt):
     if no_headless_opt:
         WEBDRIVER_HEADLESS = False
     if not skip_fetch_opt:
+        ColorSpec.truncate()
         vcs = VimColorSchemes()
         vcs.fetch()
         asn = AwesomeNeovimColorScheme()