From 3659434f1efcb82231a4e41073187c596ccb3bf5 Mon Sep 17 00:00:00 2001 From: Brian Konzman Date: Sat, 12 May 2018 21:05:01 -0400 Subject: [PATCH] initial commit --- scripts/weight_skewness/compute_skewness.py | 76 +++++++++++++++++++++ scripts/weight_skewness/download_weights.py | 57 ++++++++++++++++ scripts/weight_skewness/weights_file.py | 44 ++++++++++++ 3 files changed, 177 insertions(+) create mode 100644 scripts/weight_skewness/compute_skewness.py create mode 100644 scripts/weight_skewness/download_weights.py create mode 100644 scripts/weight_skewness/weights_file.py diff --git a/scripts/weight_skewness/compute_skewness.py b/scripts/weight_skewness/compute_skewness.py new file mode 100644 index 000000000..7cd8de251 --- /dev/null +++ b/scripts/weight_skewness/compute_skewness.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +# This file is part of Leela Chess. +# Copyright (C) 2018 Brian Konzman +# +# Leela Chess is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Leela Chess is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Leela Chess. If not, see . + + +# This script only works for 15x196 networks + +from weights_file import read_weights_file +from scipy.stats import skew +from scipy.stats import kurtosis +import numpy as np +import os + +import matplotlib.pyplot as plt + + +directory = os.getcwd() + +data_points = list() +ids = list() + +files = os.listdir(directory) +for file in files: + id = int(file.split('_')[1].split('.')[0]) + ids.append(id) + +ids.sort() + +data_points = list() +policy_skews = list() +value_skews = list() +ids_used = list() +first_conv_inputs_skews = list() +second_conv_inputs_skews = list() +middle_conv_inputs_skews = list() +last_conv_inputs_skews = list() + +FIRST_15x196_ID = 227 +POLICY_FCL_INDEX = 128 + +for id_number in ids: + filename = 'weights_' + str(id_number) + '.txt.gz' + if filename.endswith(".gz") and id_number >= FIRST_15x196_ID: + file = os.path.join(directory, filename) + filters, blocks, weights = read_weights_file(file) + + if type(weights) == list: + policy_skews.append(skew(weights[POLICY_FCL_INDEX])) + + ids_used.append(id_number) + print('ID' + str(id_number) + ' complete') + else: + print(type(weights)) + print(str(id_number)) + + continue + else: + continue + +plt.plot(ids_used, policy_skews) + +plt.show() diff --git a/scripts/weight_skewness/download_weights.py b/scripts/weight_skewness/download_weights.py new file mode 100644 index 000000000..c628b618f --- /dev/null +++ b/scripts/weight_skewness/download_weights.py @@ -0,0 +1,57 @@ +# This file is part of Leela Chess. +# Copyright (C) 2018 github username so-much-meta +# Copyright (C) 2018 Brian Konzman +# +# Leela Chess is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Leela Chess is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Leela Chess. If not, see . + +# Downloads all weights files from lczero.org, does not try to download files already present in destination folder + +from bs4 import BeautifulSoup +import os +import requests +import shutil + +page = requests.get("http://lczero.org/networks") +soup = BeautifulSoup(page.content, 'html.parser') +network_table = soup.find_all('tr') + +download_links = dict() + +for i in range(1, len(network_table)): + row = list(network_table[i]) + net_number = str(row[1]).split('>')[1].split('<')[0] + download_links[net_number] = 'http://lczero.org' + str(row[3]).split('href="')[1].split('"')[0] + +most_recent_net_number = str(list(network_table[1])[1]).split('>')[1].split('<')[0] + +directory = os.getcwd() +all_weights = list() +id_numbers = list() + +# remove links that we have already downloaded +for filename in os.listdir(directory): + if filename.endswith(".gz"): + net_id = filename.split('_')[1].split('.')[0] + if net_id in download_links.keys(): + download_links.pop(net_id) + + +keys = download_links.keys() +for k in keys: + link = download_links.get(k) + r = requests.get(link, stream=True) + if r.status_code == 200: + with open(directory + 'weights_' + k + '.txt.gz', 'wb') as f: + r.raw.decode_content = True + shutil.copyfileobj(r.raw, f) \ No newline at end of file diff --git a/scripts/weight_skewness/weights_file.py b/scripts/weight_skewness/weights_file.py new file mode 100644 index 000000000..f2b20948d --- /dev/null +++ b/scripts/weight_skewness/weights_file.py @@ -0,0 +1,44 @@ +# This file is part of Leela Chess. +# Copyright (C) 2018 Brian Konzman +# +# Leela Chess is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Leela Chess is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Leela Chess. If not, see . + +# Only supports weights version 2 + +import gzip + +LEELA_WEIGHTS_VERSION = '2' + + +def read_weights_file(filename): + if '.gz' in filename: + opener = gzip.open + else: + opener = open + with opener(filename, 'r') as f: + version = f.readline().decode('ascii') + if version != '{}\n'.format(LEELA_WEIGHTS_VERSION): + raise ValueError("Invalid version {}".format(version.strip())) + weights = [] + for e, line in enumerate(f): + line = line.decode('ascii') + weight = list(map(float, line.split(' '))) + weights.append(weight) + if e == 1: + filters = len(line.split(' ')) + blocks = e - (3 + 14) + if blocks % 8 != 0: + raise ValueError("Inconsistent number of weights in the file") + blocks //= 8 + return (filters, blocks, weights) \ No newline at end of file