Add Nudgis-to-Nudgis mass transfer tools

examples/backup_channel_recursive.py

@@ -166,7 +166,7 @@ def download_media_best_resource(msc, item, media_backup_dir, file_prefix, local
         url_resource = msc.api(
             'download/',
             method='get',
-            params=dict(oid=item['oid'], url=best_quality['file'], redirect='no')
+            params=dict(oid=item['oid'], url=best_quality['path'], redirect='no')
         )['url']
         if os.path.exists(destination_resource):
             local_size = os.path.getsize(destination_resource)

examples/backup_media.py

@@ -230,7 +230,7 @@ def download_media_best_resource(msc, item, media_backup_dir, file_prefix, local
         # download resource
         url_resource = msc.api(
             'download/',
-            params=dict(oid=item['oid'], url=best_quality['file'], redirect='no')
+            params=dict(oid=item['oid'], url=best_quality['path'], redirect='no')
         )['url']
         if os.path.exists(destination_resource):
             local_size = os.path.getsize(destination_resource)

examples/dump_oids_legacy.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+'''
+Script to dump the entire catalog on legacy servers that to not support the catalog API
+'''
+
+import datetime
+import argparse
+import os
+import sys
+from pathlib import Path
+
+
+def dump_voids(msc):
+    oids = set()
+
+    more = True
+    start = (datetime.datetime.today() + datetime.timedelta(days=1)).strftime('%Y-%m-%d 00:00:00')
+    index = 0
+    while more:
+        print('//// Making request on latest (start=%s)' % start)
+        response = msc.api(
+            'latest/',
+            params=dict(start=start, order_by='added', content='v', count=100),
+        )
+        for item in response['items']:
+            index += 1
+            oids.add(item["oid"])
+
+        start = response['max_date']
+        more = response['more']
+
+    print(f"Found total of {len(oids)}")
+    return oids
+
+
+if __name__ == '__main__':
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    from ms_client.client import MediaServerClient
+
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        '--conf',
+        help='Path to the configuration file.',
+        required=True,
+        type=str,
+    )
+    parser.add_argument(
+        '--file',
+        default='oids.txt',
+        help='Text file to store the list of oids into.',
+        type=str,
+    )
+
+    args = parser.parse_args()
+    msc = MediaServerClient(args.conf)
+
+    oids = dump_voids(msc)
+
+    print(f"Writing oids in to {args.file}")
+    Path(args.file).open('w').write("\n".join(oids))

examples/dump_users_with_personal_media.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+'''
+Script to dump all users with an email and with data in their personal channel into a CSV file
+'''
+import os
+import sys
+import csv
+
+
+if __name__ == '__main__':
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    from ms_client.client import MediaServerClient
+
+    local_conf = sys.argv[1] if len(sys.argv) > 1 else None
+    msc = MediaServerClient(local_conf)
+    # ping
+    users = msc.api('/users/', params={"limit": 0})["users"]
+    print(f"Found {len(users)} users")
+
+    users_with_data = list()
+
+    for user in users:
+        user_id = user["id"]
+        try:
+            oid = msc.api(
+                '/channels/personal/', params={"create": "no", "id": user_id}
+            )["oid"]
+            size = msc.api('/channels/get/', params={"oid": oid, "full": "yes"})[
+                "info"
+            ]["storage_used"]
+            if size > 0 and user.get("email"):
+                users_with_data.append(user)
+        except Exception as e:
+            if "403" in str(e):
+                pass
+
+    csv_path = "users_with_data.csv"
+    print(f"Found {len(users_with_data)} users with data, writing to {csv_path}")
+
+    if len(users_with_data):
+        with open(csv_path, "w") as csvfile:
+            fieldnames = ["email", "first_name", "last_name"]
+            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+
+            writer.writeheader()
+            for user in users_with_data:
+                user_simple = dict()
+                for key, value in user.items():
+                    if key in fieldnames:
+                        user_simple[key] = value
+                writer.writerow(user_simple)

examples/sync_transferred_media_permissions.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+'''
+Script to sync access permissions on media migrated from nudgis to nudgis using the transfer_media.py script
+'''
+
+import argparse
+import os
+import sys
+
+
+def sync_group_permissions(msc_src, oid_src, msc_dst, oid_dst):
+    groups = msc_src.api(
+        '/perms/get/default/',
+        params={'oid': oid_src},
+    )['groups']
+
+    edit_params = {'oid': oid_dst, 'prefix': 'reference'}
+    for g in groups:
+        ref = g['ref']
+        for perm in ['can_access_media']:
+            access_perms = g['permissions'][perm]
+            if access_perms.get('val') or access_perms.get('inherit_val'):
+                edit_params[f'{ref}-{perm}'] = 'True'
+
+    print(f'Synchronizing access permissions with params {edit_params}')
+
+    r = msc_dst.api('/perms/edit/default/', method='post', data=edit_params)
+    print(r)
+
+
+if __name__ == '__main__':
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    from ms_client.client import MediaServerClient
+
+    parser = argparse.ArgumentParser(description=__doc__.strip())
+
+    parser.add_argument(
+        '--conf-src',
+        help='Path to the configuration file for the source platform.',
+        required=True,
+        type=str,
+    )
+
+    parser.add_argument(
+        '--conf-dest',
+        help='Path to the configuration file for the destination platform.',
+        required=True,
+        type=str,
+    )
+
+    parser.add_argument(
+        '--apply',
+        help='Whether to apply changes',
+        action='store_true',
+    )
+
+    args = parser.parse_args()
+
+    msc_src = MediaServerClient(args.conf_src)
+    src_domain = msc_src.conf['SERVER_URL'].split('/')[2]
+
+    msc_dest = MediaServerClient(args.conf_dest)
+
+    dest_videos = msc_dest.get_catalog(fmt='flat').get('videos', list())
+    dest_external_refs = dict()
+    for v in dest_videos:
+        if f'nudgis:{src_domain}' in v['external_ref']:
+            dest_external_refs[v['external_ref']] = v['oid']
+
+    print(f"Found {len(dest_external_refs)} oids to sync")
+
+    index = 0
+    for external_ref, oid_dest in dest_external_refs.items():
+        print(f'Processing {index + 1}/{len(dest_external_refs)}')
+        index += 1
+
+        # external_ref_prefix = f'nudgis:{src_domain}'
+        # external_ref = f'{external_ref_prefix}:{oid_src}'
+
+        oid_src = external_ref.split(':')[-1]
+
+        if args.apply:
+            sync_group_permissions(msc_src, oid_src, msc_dest, oid_dest)
+        else:
+            print(f"Would sync {oid_src} > {oid_dest} perms")