initial commit

Author: frankli0324

config.ini

@@ -0,0 +1,9 @@
+[general]
+logdir           = /home/mirror/mirror/logs.d
+chdir            = /home/mirror/mirror
+timeformat       = %%Y-%%m-%%d %%H:%%M (UTC+8)
+vars[tuna]       = mirrors.tuna.tsinghua.edu.cn
+vars[ustc]       = rsync.mirrors.ustc.edu.cn
+vars[neusoft]    = mirrors.neusoft.edu.cn
+vars[rsync_args] = -6azH --delete --delete-after --delete-excluded --progress --info=progress2 --timeout=300
+vars[excludes]   = --exclude-from=excludes.d/

excludes.d/.gitkeep

@@ -0,0 +1,40 @@
+{
+    "version": 1.5,
+    "site": {
+        "url": "https://linux.xidian.edu.cn/mirrors",
+        "logo": "https://linux.xidian.edu.cn/img/xdosc.svg",
+        "logo_darkmode": "https://linux.xidian.edu.cn/img/xdosc.svg",
+        "abbr": "Xidian",
+        "name": "西电开源软件镜像站",
+        "homepage": "https://linux.xidian.edu.cn",
+        "issue": "https://linux.xidian.edu.cn/git/xdlinux/issues",
+        "group": "Telegram: https://t.me/xdosc",
+        "note": "仅面向校内使用"
+    },
+    "mirrors": [],
+    "info": [],
+    "extension": "D",
+    "endpoints": [
+        {
+            "label": "xidian",
+            "public": false,
+            "resolve": "linux.xidian.edu.cn/mirrors",
+            "filter": [
+                "V4",
+                "SSL",
+                "NOSSL"
+            ],
+            "range": [
+                "222.25.128.0/18",
+                "219.245.64.0/18",
+                "219.244.112.0/20",
+                "210.27.0.0/20",
+                "202.117.112.0/20",
+                "115.155.32.0/19",
+                "115.155.0.0/19",
+                "113.200.174.0/24",
+                "113.140.11.0/24"
+            ]
+        }
+    ]
+}

logs.d/.gitkeep

@@ -0,0 +1,108 @@
+#!env python3
+from pathlib import Path
+from datetime import datetime
+import subprocess
+import json
+import os
+import fcntl
+
+from utils import get_logger, parse_config, update_status_json, parse_state
+
+general, mirrors = parse_config()
+os.chdir(general['chdir'])
+log = get_logger(general['logdir'], 'sync')
+
+
+def do_sync(mirror, lastsuccess=None):
+    status['state'] = f'Y{int(datetime.now().timestamp())}'
+    if lastsuccess:
+        status['state'] += f'O{int(lastsuccess.timestamp())}'
+    log.info(f'syncing {mirror}')
+    with open(path, 'w') as f:
+        f.write(json.dumps(status))
+    update_status_json(mirrors)
+
+    proc_output_path = Path('logs.d', 'output', f'{mirror}-{status["state"]}')
+    proc_output = open(proc_output_path, 'w')
+    proc = subprocess.Popen(
+        executable='/bin/sh',
+        args=["sh", "-c", status['command']],
+        stderr=subprocess.STDOUT,
+        stdout=proc_output,
+    )
+    proc_output.close()
+
+    try:
+        code = proc.wait()
+    except KeyboardInterrupt:
+        code = -1
+
+    if code == 0:
+        log.info(f'successfully synced {mirror}')
+        message = f'Successfully Synced {mirror}.'
+        proc_output_path.unlink()
+        status['state'] = f'S{int(datetime.now().timestamp())}'
+    elif code == -1:
+        message = f'Paused Syncing {mirror}.'
+        status['state'] = f'P{int(datetime.now().timestamp())}'
+        if lastsuccess:
+            message += f' Last Successful Sync: {lastsuccess.strftime(general["timeformat"])}.'
+            status['state'] += f'O{int(lastsuccess.timestamp())}'
+    else:
+        message = f'Error Occured Syncing {mirror}.'
+        log.error(f'error syncing {mirror}')
+        status['state'] = f'F{int(datetime.now().timestamp())}'
+        if lastsuccess:
+            message += f' Last Successful Sync: {lastsuccess.strftime(general["timeformat"])}.'
+            status['state'] += f'O{int(lastsuccess.timestamp())}'
+
+    with open(path, 'w') as f:
+        f.write(json.dumps(status))
+    update_status_json(mirrors)
+
+
+if os.path.exists('sync.lock'):
+    print('sync.py already running...')
+    exit()
+
+with open('sync.lock', 'w+') as f:
+    try:
+        fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except IOError:
+        print('sync.py already running...')
+        exit()
+
+    for mirror in mirrors.sections():
+        path = Path('status.d', mirror)
+        path.touch()
+        with open(path, 'r') as f:
+            status = f.read()
+            try:
+                status = json.loads(status)
+            except json.JSONDecodeError:
+                status = {}
+            if not status:
+                log.warning(f'{mirror} never synced, syncing for the first time...')
+        status['name'] = mirror
+        status['command'] = mirrors[mirror]['command'].format(**general['vars'])
+        lastsuccess = None
+        for state, time in parse_state(status.get('state') or ''):  # follows mirrorz rules
+            if state == 'S':
+                if (datetime.now() - time).total_seconds() < 28800:
+                    log.info(f'skipping {mirror}, less than 8 hours since last sync')
+                    break
+                lastsuccess = time
+            elif state == 'Y':
+                break
+            elif state == 'O' and lastsuccess is None:
+                lastsuccess = time
+        else:
+            do_sync(mirror, lastsuccess)
+
+    # execute this unconditionally on exit
+    # to update status.json when mirrorz.meta.json changes
+    update_status_json(mirrors)
+    try:
+        fcntl.flock(f, fcntl.LOCK_UN)
+    finally:
+        os.remove('sync.lock')

mirrorz.meta.json

@@ -0,0 +1,77 @@
+from configparser import ConfigParser
+from logging import Logger, handlers, Formatter
+from pathlib import Path
+from datetime import datetime
+import re
+import json
+from typing import Tuple
+
+
+def parse_config(file='config.ini'):
+    config = ConfigParser()
+    config.read(file)
+
+    general = {}
+    for k, v in config['general'].items():
+        if res := re.match(r'^(.*)\[(.*)\]$', k):
+            if res[1] not in general:
+                general[res[1]] = {}
+            general[res[1]][res[2]] = v
+        else:
+            general[k] = v
+    config.remove_section('general')
+    return general, config
+
+
+def get_logger(logdir, name):
+    log = Logger(name)
+    handler = handlers.RotatingFileHandler(Path(logdir, f'{name}.log'), maxBytes=1024 * 128)
+    handler.setFormatter(Formatter('[%(asctime)s] - %(message)s'))
+    log.addHandler(handler)
+    return log
+
+
+def timespan_fromtimestamp(ts):
+    return (datetime.now() - datetime.fromtimestamp(ts))
+
+
+def update_status_json(mirrors):
+    with open('mirrorz.meta.json', 'r') as f:
+        status = json.loads(f.read())
+    for mirror in mirrors.sections():
+        path = Path('status.d', mirror)
+        path.touch()
+        obj = {
+            "cname": mirror,
+            "desc": mirrors[mirror].get('desc') or '',
+            "url": mirrors[mirror].get('url') or f'/{mirror}',
+        }
+        with open(path, 'r') as f:
+            try:
+                job = json.loads(f.read())
+                obj['status'] = job['state']
+            except json.JSONDecodeError:
+                obj['status'] = f'N{int(datetime.now().timestamp())}'
+        status['mirrors'].append(obj)
+
+    status['mirrors'].sort(key=lambda x: x['cname'])
+    with open('/srv/http/status.json.root/mirrors/status.json', 'w') as f:
+        f.write(json.dumps(status))
+
+
+def parse_state(state: str) -> Tuple[str, datetime]:
+    def next_ts(s: str, i: int):
+        for j in range(i, len(s)):
+            if not s[j].isdigit():
+                return s[i:j - 1]
+        return s[i:]
+    i = 0
+    while i < len(state):
+        if state[i] in 'SYFPXNO':
+            ts = next_ts(state, i + 1)
+            time = datetime.fromtimestamp(int(ts))
+            yield state[i], time
+            i += len(ts)
+        else:
+            yield state[i], datetime.fromtimestamp(0)
+        i += 1