From 8a5efb16a8bf25656d1fb515ac43202542f1dcec Mon Sep 17 00:00:00 2001 From: juga0 Date: Mon, 6 May 2019 05:51:43 +0000 Subject: [PATCH 1/6] chg: Add stem's bandwidth_file.py part to start refactoring using stem's code and don't keep duplicated code. For now, including only the header constant. --- sbws/lib/stem_bandwidth_file.py | 67 +++++++++++++++++++++++++++++++++ tox.ini | 3 ++ 2 files changed, 70 insertions(+) create mode 100644 sbws/lib/stem_bandwidth_file.py diff --git a/sbws/lib/stem_bandwidth_file.py b/sbws/lib/stem_bandwidth_file.py new file mode 100644 index 00000000..a7fefe5f --- /dev/null +++ b/sbws/lib/stem_bandwidth_file.py @@ -0,0 +1,67 @@ +"""Stem's Bandwidth file parser part. +To be removed on Stem's release 1.8.0.""" + +# Converts header attributes to a given type. Malformed fields should be +# ignored according to the spec. + +def _str(val): + return val # already a str + + +def _int(val): + return int(val) if (val and val.isdigit()) else None + + +def _date(val): + try: + return stem.util.str_tools._parse_iso_timestamp(val) + except ValueError: + return None # not an iso formatted date + + +def _csv(val): + return map(lambda v: v.strip(), val.split(',')) if val is not None else None + + +# mapping of attributes => (header, type) + +HEADER_ATTR = { + # version 1.1.0 introduced headers + + 'version': ('version', _str), + + 'software': ('software', _str), + 'software_version': ('software_version', _str), + + 'earliest_bandwidth': ('earliest_bandwidth', _date), + 'latest_bandwidth': ('latest_bandwidth', _date), + 'created_at': ('file_created', _date), + 'generated_at': ('generator_started', _date), + + # version 1.2.0 additions + + 'consensus_size': ('number_consensus_relays', _int), + 'eligible_count': ('number_eligible_relays', _int), + 'eligible_percent': ('percent_eligible_relays', _int), + 'min_count': ('minimum_number_eligible_relays', _int), + 'min_percent': ('minimum_percent_eligible_relays', _int), + + # version 1.3.0 additions + + 'scanner_country': ('scanner_country', _str), + 'destinations_countries': ('destinations_countries', _csv), + + # version 1.4.0 additions + + 'time_to_report_half_network': ('time_to_report_half_network', _int), + + 'recent_stats.consensus_count': ('recent_consensus_count', _int), + 'recent_stats.prioritized_relay_lists': ('recent_priority_list_count', _int), + 'recent_stats.prioritized_relays': ('recent_priority_relay_count', _int), + 'recent_stats.measurement_attempts': ('recent_measurement_attempt_count', _int), + 'recent_stats.measurement_failures': ('recent_measurement_failure_count', _int), + 'recent_stats.relay_failures.no_measurement': ('recent_measurements_excluded_error_count', _int), + 'recent_stats.relay_failures.insuffient_period': ('recent_measurements_excluded_near_count', _int), + 'recent_stats.relay_failures.insufficient_measurements': ('recent_measurements_excluded_few_count', _int), + 'recent_stats.relay_failures.stale': ('recent_measurements_excluded_old_count', _int), +} diff --git a/tox.ini b/tox.ini index 6eaf894e..77c2ec55 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,9 @@ python = 3.5: py35, inst, setup, unit, integration, lint, doc 3.6: py36, inst, setup, unit, integration +[flake8] +exclude = sbws/lib/stem_bandwidth_file.py + [testenv] # install_command can be removed when --process-dependency-links is not # needed anymore, and this section commented From ebbb9ffcf3ee28d48c68b7ef45463ed5f534e2ff Mon Sep 17 00:00:00 2001 From: juga0 Date: Mon, 6 May 2019 07:24:51 +0000 Subject: [PATCH 2/6] chg: v3bwfile: !refactor. Replace constants Replace when possible, all header constants to use Stem's one, not changing functionality. Tests don't need modifications, since functionality isn't changed. Closes #30406 --- sbws/lib/v3bwfile.py | 165 ++++++++++++++++++++++++------------------- 1 file changed, 92 insertions(+), 73 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 3e2c6464..9a95072c 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -15,6 +15,9 @@ TORFLOW_SCALING, SBWS_SCALING, TORFLOW_BW_MARGIN, TORFLOW_OBS_LAST, TORFLOW_OBS_MEAN, PROP276_ROUND_DIG, MIN_REPORT, MAX_BW_DIFF_PERC) + +# Using Stem's code header constant to don't maintain 2 versions. +from .stem_bandwidth_file import HEADER_ATTR, _int from sbws.lib.resultdump import ResultSuccess, _ResultType from sbws.util.filelock import DirectoryLock from sbws.util.timestamp import (now_isodt_str, unixts_to_isodt_str, @@ -27,15 +30,14 @@ KEYVALUE_SEP_V1 = '=' KEYVALUE_SEP_V2 = ' ' -# NOTE: in a future refactor make make all the KeyValues be a dictionary -# with their type, so that it's more similar to stem parser. - +# NOTE: Start using Stem's header constant in 1.1.1-dev0. +# # Header KeyValues # ================= -# List of the extra KeyValues accepted by the class -EXTRA_ARG_KEYVALUES = ['software', 'software_version', 'file_created', - 'earliest_bandwidth', 'generator_started', - 'scanner_country', 'destinations_countries'] + +# XXX: These constants need to be kept even if using Stem's header one. +# Unless Stem's will add a property type to the attributes. + # number_eligible_relays is the number that ends in the bandwidth file # ie, have not been excluded by one of the filters in 4. below # They should be call recent_measurement_included_count to be congruent @@ -44,74 +46,82 @@ 'number_consensus_relays', 'percent_eligible_relays', 'minimum_percent_eligible_relays'] -# KeyValues that count the number of relays that are in the bandwidth file, -# but ignored by Tor when voting, because they do not have a -# measured bandwidth. -BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED = [ - # Number of relays that were measured but all the measurements failed - # because of network failures or it was - # not found a suitable helper relay - 'recent_measurements_excluded_error_count', - # Number of relays that have successful measurements but the measurements - # were not away from each other in X time (by default 1 day). - 'recent_measurements_excluded_near_count', - # Number of relays that have successful measurements and they are away from - # each other but they are not X time recent. - # By default this is 5 days, which is the same time the older - # the measurements can be by default. - 'recent_measurements_excluded_old_count', - # Number of relays that have successful measurements and they are away from - # each other and recent - # but the number of measurements are less than X (by default 2). - 'recent_measurements_excluded_few_count', -] -# Added in #29591 -# NOTE: recent_consensus_count, recent_priority_list_count, -# recent_measurement_attempt_count and recent_priority_relay_count -# are not reset when the scanner is stop. -# They will accumulate the values since the scanner was ever started. -BW_HEADER_KEYVALUES_MONITOR = [ - # 1.1 header: the number of different consensuses, that sbws has seen, - # since the last 5 days - 'recent_consensus_count', - # 2.4 Number of times a priority list has been created - 'recent_priority_list_count', - # 2.5 Number of relays that there were in a priority list - # [50, number of relays in the network * 0.05] - 'recent_priority_relay_count', - # 3.6 header: the number of times that sbws has tried to measure any relay, - # since the last 5 days - # This would be the number of times a relays were in a priority list - 'recent_measurement_attempt_count', - # 3.7 header: the number of times that sbws has tried to measure any relay, - # since the last 5 days, but it didn't work - # This should be the number of attempts - number of ResultSuccess - - # something else we don't know yet - # So far is the number of ResultError - 'recent_measurement_failure_count', - # The time it took to report about half of the network. - 'time_to_report_half_network', -] + BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED -BANDWIDTH_HEADER_KEY_VALUES_INIT = \ - ['earliest_bandwidth', 'generator_started', - 'scanner_country', 'destinations_countries']\ - + STATS_KEYVALUES \ - + BW_HEADER_KEYVALUES_MONITOR - -KEYVALUES_INT = STATS_KEYVALUES + BW_HEADER_KEYVALUES_MONITOR -# List of all unordered KeyValues currently being used to generate the file -UNORDERED_KEYVALUES = EXTRA_ARG_KEYVALUES + STATS_KEYVALUES + \ - ['latest_bandwidth'] + \ - BW_HEADER_KEYVALUES_MONITOR -# List of all the KeyValues currently being used to generate the file -ALL_KEYVALUES = ['version'] + UNORDERED_KEYVALUES +# # KeyValues that count the number of relays that are in the bandwidth file, +# # but ignored by Tor when voting, because they do not have a +# # measured bandwidth. +# BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED = [ +# # Number of relays that were measured but all the measurements failed +# # because of network failures or it was +# # not found a suitable helper relay +# 'recent_measurements_excluded_error_count', +# # Number of relays that have successful measurements but the measurements +# # were not away from each other in X time (by default 1 day). +# 'recent_measurements_excluded_near_count', +# # Number of relays that have successful measurements and they are away +# # from each other but they are not X time recent. +# # By default this is 5 days, which is the same time the older +# # the measurements can be by default. +# 'recent_measurements_excluded_old_count', +# # Number of relays that have successful measurements and they are away +# # from each other and recent +# # but the number of measurements are less than X (by default 2). +# 'recent_measurements_excluded_few_count', +# ] + +# The stem's attribute name for these Keys starts with this string. +BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED = \ + [k for n, (k, _) in HEADER_ATTR.items() + if n.startswith("recent_stats.relay_failures.")] + +# The commented code is kept so that +# it can be known what each Key means. + +# # Added in #29591 +# # NOTE: recent_consensus_count, recent_priority_list_count, +# # recent_measurement_attempt_count and recent_priority_relay_count +# # are not reset when the scanner is stop. +# # They will accumulate the values since the scanner was ever started. +# BW_HEADER_KEYVALUES_MONITOR = [ +# # 1.1 header: the number of different consensuses, that sbws has seen, +# # since the last 5 days +# 'recent_consensus_count', +# # 2.4 Number of times a priority list has been created +# 'recent_priority_list_count', +# # 2.5 Number of relays that there were in a priority list +# # [50, number of relays in the network * 0.05] +# 'recent_priority_relay_count', +# # 3.6 header: the number of times that sbws has tried to measure any +# # relay,s ince the last 5 days +# # This would be the number of times a relays were in a priority list +# 'recent_measurement_attempt_count', +# # 3.7 header: the number of times that sbws has tried to measure any +# # relay, since the last 5 days, but it didn't work +# # This should be the number of attempts - number of ResultSuccess - +# # something else we don't know yet +# # So far is the number of ResultError +# 'recent_measurement_failure_count', +# # The time it took to report about half of the network. +# 'time_to_report_half_network', +# ] + BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED +# BANDWIDTH_HEADER_KEY_VALUES_INIT = \ +# ['earliest_bandwidth', 'generator_started', +# 'scanner_country', 'destinations_countries']\ +# + STATS_KEYVALUES \ +# + BW_HEADER_KEYVALUES_MONITOR + +# XXX: The following constants are kept for compatibility with the generator/ +# parser here. They should be replaced by Stem's one. +KEYVALUES_INT = [k for k, t in HEADER_ATTR.values() if t is _int] +# # List of all unordered KeyValues currently being used to generate the file +UNORDERED_KEYVALUES = [k for k, _ in HEADER_ATTR.values()] +UNORDERED_KEYVALUES.remove('version') TERMINATOR = '=====' # Bandwidth Lines KeyValues # ========================= # Num header lines in v1.X.X using all the KeyValues -NUM_LINES_HEADER_V1 = len(ALL_KEYVALUES) + 2 +NUM_LINES_HEADER_V1 = len(HEADER_ATTR) + 2 LINE_TERMINATOR = TERMINATOR + LINE_SEP # KeyValue separator in Bandwidth Lines @@ -250,11 +260,21 @@ class V3BWHeader(object): when the generator started """ def __init__(self, timestamp, **kwargs): + # XXX: do not convert all the arguments to string here, convert them + # only in __str__ and check their actual types. assert isinstance(timestamp, str) for v in kwargs.values(): assert isinstance(v, str) self.timestamp = timestamp - # KeyValues with default value when not given by kwargs + + # Initialize all defined attrs, + # XXX: Since using the stem's name for the attributes will require + # other refactoring here, use the name in the actual file. + header_keys = [k for k, _ in HEADER_ATTR.values()] + [setattr(self, k, v) for k, v in kwargs.items() if k in header_keys] + + # Then initialize the ones with defaults, if they weren't already. + # Ideally these defaults could be defined in the HEADER_ATTR constant. self.version = kwargs.get('version', SPEC_VERSION) self.software = kwargs.get('software', 'sbws') self.software_version = kwargs.get('software_version', __version__) @@ -262,8 +282,6 @@ def __init__(self, timestamp, **kwargs): # latest_bandwidth should not be in kwargs, since it MUST be the # same as timestamp self.latest_bandwidth = unixts_to_isodt_str(timestamp) - [setattr(self, k, v) for k, v in kwargs.items() - if k in BANDWIDTH_HEADER_KEY_VALUES_INIT] def __str__(self): if self.version.startswith('1.'): @@ -337,9 +355,10 @@ def from_lines_v1(cls, lines): log.warn('Terminator is not in lines') return None ts = lines[0] + header_keys = [k for k, _ in HEADER_ATTR.values()] kwargs = dict([l.split(KEYVALUE_SEP_V1) for l in lines[:index_terminator] - if l.split(KEYVALUE_SEP_V1)[0] in ALL_KEYVALUES]) + if l.split(KEYVALUE_SEP_V1)[0] in header_keys]) h = cls(ts, **kwargs) # last line is new line return h, lines[index_terminator + 1:-1] From c2dfe268bcc1fb68932d57001feddb70e35d525a Mon Sep 17 00:00:00 2001 From: juga0 Date: Sun, 19 May 2019 16:03:39 +0000 Subject: [PATCH 3/6] fixup! chg: Add stem's bandwidth_file.py part --- sbws/lib/stem_bandwidth_file.py | 10 +++++++--- sbws/lib/v3bwfile.py | 4 +++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/sbws/lib/stem_bandwidth_file.py b/sbws/lib/stem_bandwidth_file.py index a7fefe5f..5f12047d 100644 --- a/sbws/lib/stem_bandwidth_file.py +++ b/sbws/lib/stem_bandwidth_file.py @@ -1,5 +1,9 @@ -"""Stem's Bandwidth file parser part. -To be removed on Stem's release 1.8.0.""" +"""Stem's Bandwidth file HEADER_ATTR part as it is in stem's commit +658dd5281604eb9c63a91e529501947ecc65ef6b, which will be included in the next +Stem's release, 1.8.0, except ``_date`` because depends on other stem's module. +""" +# XXX: Remove this file when stem releases 1.8.0. +from ..util.timestamp import isostr_to_dt_obj # Converts header attributes to a given type. Malformed fields should be # ignored according to the spec. @@ -14,7 +18,7 @@ def _int(val): def _date(val): try: - return stem.util.str_tools._parse_iso_timestamp(val) + return isostr_to_dt_obj(val) except ValueError: return None # not an iso formatted date diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 9a95072c..ef6bef64 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -16,7 +16,9 @@ TORFLOW_OBS_LAST, TORFLOW_OBS_MEAN, PROP276_ROUND_DIG, MIN_REPORT, MAX_BW_DIFF_PERC) -# Using Stem's code header constant to don't maintain 2 versions. +# XXX: replace this line by: +# from stem.descriptor.bandwidth_file import HEADER_ATTR, _int +# when stem releases 1.8.0. More information in stem_bandwidth_file.py from .stem_bandwidth_file import HEADER_ATTR, _int from sbws.lib.resultdump import ResultSuccess, _ResultType from sbws.util.filelock import DirectoryLock From 2d92fbde78432232cf5a2ccbffc39fc7245fe3dd Mon Sep 17 00:00:00 2001 From: juga0 Date: Sun, 19 May 2019 16:11:33 +0000 Subject: [PATCH 4/6] fixup! chg: v3bwfile: !refactor. Replace constants --- sbws/lib/v3bwfile.py | 74 +++++--------------------------------------- 1 file changed, 7 insertions(+), 67 deletions(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index ef6bef64..46516167 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -1,6 +1,10 @@ # -*- coding: utf-8 -*- """Classes and functions that create the bandwidth measurements document -(v3bw) used by bandwidth authorities.""" +(v3bw) used by bandwidth authorities. + +See https://gitweb.torproject.org/torspec.git/tree/bandwidth-file-spec.txt for +more information on the meaning of the ``KeyValues`` +""" import copy import logging @@ -32,87 +36,23 @@ KEYVALUE_SEP_V1 = '=' KEYVALUE_SEP_V2 = ' ' -# NOTE: Start using Stem's header constant in 1.1.1-dev0. -# # Header KeyValues # ================= # XXX: These constants need to be kept even if using Stem's header one. # Unless Stem's will add a property type to the attributes. - -# number_eligible_relays is the number that ends in the bandwidth file -# ie, have not been excluded by one of the filters in 4. below -# They should be call recent_measurement_included_count to be congruent -# with the other KeyValues. STATS_KEYVALUES = ['number_eligible_relays', 'minimum_number_eligible_relays', 'number_consensus_relays', 'percent_eligible_relays', 'minimum_percent_eligible_relays'] -# # KeyValues that count the number of relays that are in the bandwidth file, -# # but ignored by Tor when voting, because they do not have a -# # measured bandwidth. -# BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED = [ -# # Number of relays that were measured but all the measurements failed -# # because of network failures or it was -# # not found a suitable helper relay -# 'recent_measurements_excluded_error_count', -# # Number of relays that have successful measurements but the measurements -# # were not away from each other in X time (by default 1 day). -# 'recent_measurements_excluded_near_count', -# # Number of relays that have successful measurements and they are away -# # from each other but they are not X time recent. -# # By default this is 5 days, which is the same time the older -# # the measurements can be by default. -# 'recent_measurements_excluded_old_count', -# # Number of relays that have successful measurements and they are away -# # from each other and recent -# # but the number of measurements are less than X (by default 2). -# 'recent_measurements_excluded_few_count', -# ] - # The stem's attribute name for these Keys starts with this string. BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED = \ [k for n, (k, _) in HEADER_ATTR.items() if n.startswith("recent_stats.relay_failures.")] -# The commented code is kept so that -# it can be known what each Key means. - -# # Added in #29591 -# # NOTE: recent_consensus_count, recent_priority_list_count, -# # recent_measurement_attempt_count and recent_priority_relay_count -# # are not reset when the scanner is stop. -# # They will accumulate the values since the scanner was ever started. -# BW_HEADER_KEYVALUES_MONITOR = [ -# # 1.1 header: the number of different consensuses, that sbws has seen, -# # since the last 5 days -# 'recent_consensus_count', -# # 2.4 Number of times a priority list has been created -# 'recent_priority_list_count', -# # 2.5 Number of relays that there were in a priority list -# # [50, number of relays in the network * 0.05] -# 'recent_priority_relay_count', -# # 3.6 header: the number of times that sbws has tried to measure any -# # relay,s ince the last 5 days -# # This would be the number of times a relays were in a priority list -# 'recent_measurement_attempt_count', -# # 3.7 header: the number of times that sbws has tried to measure any -# # relay, since the last 5 days, but it didn't work -# # This should be the number of attempts - number of ResultSuccess - -# # something else we don't know yet -# # So far is the number of ResultError -# 'recent_measurement_failure_count', -# # The time it took to report about half of the network. -# 'time_to_report_half_network', -# ] + BW_HEADER_KEYVALUES_RECENT_MEASUREMENTS_EXCLUDED -# BANDWIDTH_HEADER_KEY_VALUES_INIT = \ -# ['earliest_bandwidth', 'generator_started', -# 'scanner_country', 'destinations_countries']\ -# + STATS_KEYVALUES \ -# + BW_HEADER_KEYVALUES_MONITOR - # XXX: The following constants are kept for compatibility with the generator/ -# parser here. They should be replaced by Stem's one. +# parser here. Use stem's parser/generator when possible and refactor this +# code when stem releases 1.8.0. KEYVALUES_INT = [k for k, t in HEADER_ATTR.values() if t is _int] # # List of all unordered KeyValues currently being used to generate the file UNORDERED_KEYVALUES = [k for k, _ in HEADER_ATTR.values()] From 92148777d632911fed8718ef6921e32995dee6c5 Mon Sep 17 00:00:00 2001 From: juga0 Date: Sun, 19 May 2019 16:14:45 +0000 Subject: [PATCH 5/6] new: tests: Add test network data and v3bwfile test to test generating a bandwidth file from a whole network measurements data. --- tests/conftest.py | 65 ++++++++++++++ tests/data/.sbws.ini | 30 +++++++ tests/data/.sbws/datadir/2019-05-11.txt | 15 ++++ tests/data/.sbws/state.dat | 10 +-- tests/data/.sbws/tor/cached-consensus | 98 ++++++++++----------- tests/data/.sbws/tor/cached-descriptors.new | 54 ++++++++++++ tests/data/.sbws/tor/state | 17 ++++ tests/data/.sbws/v3bw/20190511_063232.v3bw | 40 +++++++++ tests/data/.sbws/v3bw/latest.v3bw | 1 + 9 files changed, 276 insertions(+), 54 deletions(-) create mode 100644 tests/data/.sbws.ini create mode 100644 tests/data/.sbws/datadir/2019-05-11.txt create mode 100644 tests/data/.sbws/tor/cached-descriptors.new create mode 100644 tests/data/.sbws/tor/state create mode 100644 tests/data/.sbws/v3bw/20190511_063232.v3bw create mode 120000 tests/data/.sbws/v3bw/latest.v3bw diff --git a/tests/conftest.py b/tests/conftest.py index 1d5da5ba..3c000e39 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,17 @@ """Common pytest configuration for unit and integration tests.""" +import argparse +import os.path import pytest +from unittest import mock + +from sbws.lib import resultdump +from sbws.util import config from sbws.util.parser import create_parser +FAKE_TIME = 1557556118 + + @pytest.fixture(scope='session') def parser(): return create_parser() @@ -29,3 +38,59 @@ def readlines(self, name): with self.open(name, "r") as f: return f.readlines() return D(request.fspath.dirpath("data")) + + +@pytest.fixture() +def tests_data_dir(scope="global"): + """Root data directory, common to all the tests.""" + here = os.path.abspath(os.path.dirname(__file__)) + return os.path.join(here, "data") + + +@pytest.fixture() +def sbws_dir(tests_data_dir): + return os.path.join(tests_data_dir, ".sbws") + + +# The following fixtures are similar to the ones defined in unit and +# integration sub-directories, but they are for the tests data to be read, +# not to write. +@pytest.fixture() +def sbws_args(tests_data_dir): + config_fpath = os.path.join(tests_data_dir, ".sbws.ini") + sbws_args = argparse.Namespace(**{'config': config_fpath}) + return sbws_args + + +@pytest.fixture() +def sbws_conf(sbws_args, sbws_dir): + """Default configuration with sbws home in the tmp test dir.""" + conf = config.get_config(sbws_args) + conf['paths']['sbws_home'] = sbws_dir + conf['paths']['v3bw_fname'] = "${v3bw_dname}/latest.v3bw" + return conf + + +@pytest.fixture() +@mock.patch('time.time') +def measurements(mock_time, sbws_conf): + # Because load_recent_results_in_datadir will use time.time() + # to decide which results are recent. + mock_time.return_value = FAKE_TIME + print("measurements", sbws_conf.getpath('paths', 'datadir')) + measurements = resultdump \ + .load_recent_results_in_datadir( + sbws_conf.getint('general', 'data_period'), + sbws_conf.getpath('paths', 'datadir')) + return measurements + + +@pytest.fixture +def bandwidth_file_headers(): + d = { + 'earliest_bandwidth': "2019-05-11T06:26:41", + 'file_created': "2019-05-11T06:32:32", + 'generator_started': "2019-05-11T06:26:28", + 'latest_bandwidth': "2019-05-11T06:28:38", + } + return d diff --git a/tests/data/.sbws.ini b/tests/data/.sbws.ini new file mode 100644 index 00000000..01000ea2 --- /dev/null +++ b/tests/data/.sbws.ini @@ -0,0 +1,30 @@ +[general] +# Days into the past that measurements are considered valid +data_period = 1 + +[paths] +sbws_home = .sbws + +[scanner] +country = ZZ + +[destinations] +local = on + +[destinations.local] +; url = https://localhost:28888/sbws.bin +url = http://127.0.0.1:28888/sbws.bin +verify = False +country = ZZ + +[tor] +extra_lines = + DirAuthority auth1 orport=2002 no-v2 v3ident=D7DBC517EFD2BA1A5012CF1BD0BB38F17C8160BD 127.10.0.1:2003 AA45C13025C037F056E734169891878ED0880231 + DirAuthority auth2 orport=2002 no-v2 v3ident=4EE103A081F400E6622F5461D51782B876BB5C24 127.10.0.2:2003 E7B3C9A0040D628DAC88B0251AE6334D28E8F531 + DirAuthority auth3 orport=2002 no-v2 v3ident=8B85069C7FC0593801E6491A34100264FCE28980 127.10.0.3:2003 35E3B8BB71C81355649AEC5862ECB7ED7EFDBC5C + TestingTorNetwork 1 + NumCPUs 1 + +[logging] +level = debug +to_stdout_level = ${level} \ No newline at end of file diff --git a/tests/data/.sbws/datadir/2019-05-11.txt b/tests/data/.sbws/datadir/2019-05-11.txt new file mode 100644 index 00000000..42143fc1 --- /dev/null +++ b/tests/data/.sbws/datadir/2019-05-11.txt @@ -0,0 +1,15 @@ +{"master_key_ed25519": "gnREYdkUN0uYI3zF8oB+Wwm4MhM6ETPX1hEyt2m2a+Y", "fingerprint": "C7C5094677013F5BC124183C71A482D0156CDCFE", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556001.1935074, "circ": ["C7C5094677013F5BC124183C71A482D0156CDCFE", "270A861ABED22EC2B625198BCCD7B2B9DBFFC93C"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 5 [relay6 (C7C50946) -> exit1 (270A861A)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "relay6", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.9"} +{"master_key_ed25519": "zOZX+yyD7EN1W/Y2wgjuvObpFOOWK+LZIWlKW0AOcIE", "fingerprint": "2ABFBACE61167A1019A56CB35B2E3362B97D8570", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556011.233926, "circ": ["2ABFBACE61167A1019A56CB35B2E3362B97D8570", "270A861ABED22EC2B625198BCCD7B2B9DBFFC93C"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 6 [relay1 (2ABFBACE) -> exit1 (270A861A)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "relay1", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.4"} +{"master_key_ed25519": "NQspfAK/xkywFHV5LsQ5lLi2BmTKx2Imu4jacJ0d9os", "fingerprint": "4D664E247E530CA5CD5176B8C1A6DABC9531F0B0", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556021.2721329, "circ": ["4D664E247E530CA5CD5176B8C1A6DABC9531F0B0", "FC264325EA99D597FF94DA88379DABB64304DD9D"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 7 [relay4 (4D664E24) -> exit3 (FC264325)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "relay4", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.7"} +{"master_key_ed25519": "2fXiF4T993i+vVwZZEQ4fYee+N8OThzCGeacvnVaNbo", "relay_observed_bandwidth": 0, "fingerprint": "117A456C911114076BEB4E757AC48B16CC0CCC5F", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "consensus_bandwidth": 0, "relay_in_recent_consensus_count": 1, "time": 1557556022.519254, "circ": ["117A456C911114076BEB4E757AC48B16CC0CCC5F", "270A861ABED22EC2B625198BCCD7B2B9DBFFC93C"], "rtts": [], "relay_burst_bandwidth": 1073741824, "nickname": "relay1mbyteMAB", "consensus_bandwidth_is_unmeasured": false, "type": "success", "dest_url": "http://127.0.0.1:28888/sbws.bin", "relay_average_bandwidth": 1048576, "address": "127.10.0.14", "downloads": [{"duration": 5.801189422607422, "amount": 33454356}, {"duration": 5.437012195587158, "amount": 33454356}, {"duration": 5.97099232673645, "amount": 33454356}, {"duration": 5.786560535430908, "amount": 33454356}, {"duration": 5.766895532608032, "amount": 33454356}]} +{"master_key_ed25519": "cj7V+PYPJvSANsvBOjZSiCvXuGHXFrpSnPqC46I6DgU", "fingerprint": "E894C65997F8EC96558B554176EEEA39C6A43EF6", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556032.5962188, "circ": ["E894C65997F8EC96558B554176EEEA39C6A43EF6", "C0606B414423F9A2BBA2679B440056E3B07FEC85"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 11 [relay7 (E894C659) -> exit2 (C0606B41)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "relay7", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.10"} +{"master_key_ed25519": "Wiv9uOemlcFzRY1gZfBOpbQ+aJn6NG0Z/IArZFCdSdk", "fingerprint": "FC264325EA99D597FF94DA88379DABB64304DD9D", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556042.6297724, "circ": ["32B7178F7201F76411A99D3552F340D3597D5629", "FC264325EA99D597FF94DA88379DABB64304DD9D"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 12 [relay5 (32B7178F) -> exit3 (FC264325)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "exit3", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.13"} +{"master_key_ed25519": "eR0HnYlzpOEGwxuFjZkGJZ6pu2eV1i6fd9lhF4UOMno", "fingerprint": "934E06F38A391CB71DF83ECDE05DFF5CDE3AC49D", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556052.6754208, "circ": ["934E06F38A391CB71DF83ECDE05DFF5CDE3AC49D", "FC264325EA99D597FF94DA88379DABB64304DD9D"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 14 [relay1mbyteRBR (934E06F3) -> exit3 (FC264325)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "relay1mbyteRBR", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.15"} +{"master_key_ed25519": "Pymu1Z1eZRWgkE42xzDCYFLVKNtY743GKZzt6Im0OUw", "relay_observed_bandwidth": 0, "fingerprint": "8E687E91DCAB967F6E4EE8E46E66F6AD05C7C625", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "consensus_bandwidth": 0, "relay_in_recent_consensus_count": 1, "time": 1557556054.3331225, "circ": ["8E687E91DCAB967F6E4EE8E46E66F6AD05C7C625", "FC264325EA99D597FF94DA88379DABB64304DD9D"], "rtts": [], "relay_burst_bandwidth": 1073741824, "nickname": "relay2", "consensus_bandwidth_is_unmeasured": false, "type": "success", "dest_url": "http://127.0.0.1:28888/sbws.bin", "relay_average_bandwidth": 1073741824, "address": "127.10.0.5", "downloads": [{"duration": 6.025712728500366, "amount": 33632847}, {"duration": 5.685051441192627, "amount": 33632847}, {"duration": 5.941658973693848, "amount": 33632847}, {"duration": 5.831774950027466, "amount": 33632847}, {"duration": 5.711573600769043, "amount": 33632847}]} +{"master_key_ed25519": "uPz8ZZNm2Gcra7BauJP5PH+7uANRraYpCj7NFtp1KdM", "fingerprint": "E7B3C9A0040D628DAC88B0251AE6334D28E8F531", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556064.441996, "circ": ["E7B3C9A0040D628DAC88B0251AE6334D28E8F531", "C0606B414423F9A2BBA2679B440056E3B07FEC85"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 17 [auth2 (E7B3C9A0) -> exit2 (C0606B41)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "auth2", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.2"} +{"master_key_ed25519": "r29RWlFMIdU5GUvKsWGdhQIKjIpUzqgw0yNx/7IpPFM", "fingerprint": "35E3B8BB71C81355649AEC5862ECB7ED7EFDBC5C", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556074.4799252, "circ": ["35E3B8BB71C81355649AEC5862ECB7ED7EFDBC5C", "270A861ABED22EC2B625198BCCD7B2B9DBFFC93C"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 18 [auth3 (35E3B8BB) -> exit1 (270A861A)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "auth3", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.3"} +{"master_key_ed25519": "C506YEdasDDQqidu4G2VLMFOwaqMX28BYkuxr1+wI9o", "fingerprint": "270A861ABED22EC2B625198BCCD7B2B9DBFFC93C", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556084.5123796, "circ": ["C7C5094677013F5BC124183C71A482D0156CDCFE", "270A861ABED22EC2B625198BCCD7B2B9DBFFC93C"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 20 [relay6 (C7C50946) -> exit1 (270A861A)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "exit1", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.11"} +{"master_key_ed25519": "wLglSEw9/DHfpNrlrqjVRSnGLVWfnm0vYxkryH4aT6Q", "relay_observed_bandwidth": 0, "fingerprint": "AA45C13025C037F056E734169891878ED0880231", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "consensus_bandwidth": 0, "relay_in_recent_consensus_count": 1, "time": 1557556088.1481652, "circ": ["AA45C13025C037F056E734169891878ED0880231", "FC264325EA99D597FF94DA88379DABB64304DD9D"], "rtts": [], "relay_burst_bandwidth": 1073741824, "nickname": "auth1", "consensus_bandwidth_is_unmeasured": false, "type": "success", "dest_url": "http://127.0.0.1:28888/sbws.bin", "relay_average_bandwidth": 1073741824, "address": "127.10.0.1", "downloads": [{"duration": 6.3723015785217285, "amount": 35748124}, {"duration": 6.351818323135376, "amount": 35748124}, {"duration": 5.8544604778289795, "amount": 35748124}, {"duration": 6.5015482902526855, "amount": 35748124}, {"duration": 6.196664094924927, "amount": 35748124}]} +{"master_key_ed25519": "/PrTbpen3BrKNgiNRhAa93JQtnrT3LJX3Ka1+jvbWj4", "fingerprint": "693F73187624BE760AAD2A12C5ED89DB1DE044F5", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556098.237717, "circ": ["693F73187624BE760AAD2A12C5ED89DB1DE044F5", "FC264325EA99D597FF94DA88379DABB64304DD9D"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 23 [relay3 (693F7318) -> exit3 (FC264325)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "relay3", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.6"} +{"master_key_ed25519": "iriD8sIKS25WGc6mLesQ2okT1Tcn81AuqnEbItJeuvY", "fingerprint": "C0606B414423F9A2BBA2679B440056E3B07FEC85", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556108.2805848, "circ": ["E7B3C9A0040D628DAC88B0251AE6334D28E8F531", "C0606B414423F9A2BBA2679B440056E3B07FEC85"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 24 [auth2 (E7B3C9A0) -> exit2 (C0606B41)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "exit2", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.12"} +{"master_key_ed25519": "5O+uCpFoBzsey33+Zzlgyy18/McmV1mpPJZ+DrZMKRc", "relay_observed_bandwidth": 0, "fingerprint": "32B7178F7201F76411A99D3552F340D3597D5629", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "consensus_bandwidth": 0, "relay_in_recent_consensus_count": 1, "time": 1557556118.4493086, "circ": ["32B7178F7201F76411A99D3552F340D3597D5629", "FC264325EA99D597FF94DA88379DABB64304DD9D"], "rtts": [], "relay_burst_bandwidth": 1073741824, "nickname": "relay5", "consensus_bandwidth_is_unmeasured": false, "type": "success", "dest_url": "http://127.0.0.1:28888/sbws.bin", "relay_average_bandwidth": 1073741824, "address": "127.10.0.8", "downloads": [{"duration": 5.454082012176514, "amount": 31498448}, {"duration": 5.785161018371582, "amount": 31498448}, {"duration": 5.272622108459473, "amount": 31498448}, {"duration": 5.410377025604248, "amount": 31498448}, {"duration": 5.6312878131866455, "amount": 31498448}]} diff --git a/tests/data/.sbws/state.dat b/tests/data/.sbws/state.dat index 38acf6c6..81bcd3f4 100644 --- a/tests/data/.sbws/state.dat +++ b/tests/data/.sbws/state.dat @@ -1,9 +1,9 @@ { - "uuid": "806218a0-3ce5-4778-b839-d6faf6798405", - "scanner_started": "2019-03-25T13:03:06", - "recent_consensus_count": 1, - "recent_priority_list_count": 1, "recent_measurement_attempt_count": 15, + "recent_priority_relay_count": 15, + "uuid": "714d1bf4-b08b-4d68-a56e-b917cda848f8", + "recent_priority_list_count": 1, + "recent_consensus_count": 1, "min_perc_reached": null, - "recent_priority_relay_count": 15 + "scanner_started": "2019-05-11T06:26:28" } \ No newline at end of file diff --git a/tests/data/.sbws/tor/cached-consensus b/tests/data/.sbws/tor/cached-consensus index 32f12ed2..4462d692 100644 --- a/tests/data/.sbws/tor/cached-consensus +++ b/tests/data/.sbws/tor/cached-consensus @@ -1,9 +1,9 @@ network-status-version 3 vote-status consensus consensus-method 28 -valid-after 2019-03-25 13:08:50 -fresh-until 2019-03-25 13:09:00 -valid-until 2019-03-25 13:09:20 +valid-after 2019-05-11 06:32:10 +fresh-until 2019-05-11 06:32:20 +valid-until 2019-05-11 06:32:40 voting-delay 2 2 client-versions server-versions @@ -13,102 +13,102 @@ recommended-relay-protocols Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRen required-client-protocols Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=4 Microdesc=1-2 Relay=2 required-relay-protocols Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 Link=3-4 Microdesc=1 Relay=1-2 shared-rand-previous-value 0 zxJao+gBmFMSezvz/VXkEWEQJD5b/z+7AXNCGoLFVW0= -shared-rand-current-value 3 zOVHVZs8fQfIcs1HSjWa8nwr4kls3aj+LNhJcE4vUPw= +shared-rand-current-value 3 DERm8OHphlayJXT47ipdDWTvAPtzKuRGp3/BEY+w7qk= dir-source auth2 4EE103A081F400E6622F5461D51782B876BB5C24 127.10.0.2 127.10.0.2 2003 2002 contact pastly@torproject.org -vote-digest 9147EE32EA7B7FEE195EFFF8DD61BCBC044ACD1D +vote-digest 0EC1E4518ACC446F2C2C9E462FA56E99FF5338F0 dir-source auth3 8B85069C7FC0593801E6491A34100264FCE28980 127.10.0.3 127.10.0.3 2003 2002 contact pastly@torproject.org -vote-digest 21747518E052E648242156E67793D2B75A8462EF +vote-digest F3EA44D4F1C9D38966F1B365090041FE25650843 dir-source auth1 D7DBC517EFD2BA1A5012CF1BD0BB38F17C8160BD 127.10.0.1 127.10.0.1 2003 2002 contact pastly@torproject.org -vote-digest 9735B1C401CEED00AA4FAF84348F0E2DB88002A5 -r relay1mbyteMAB EXpFbJERFAdr6051esSLFswMzF8 g041Vrpm/01ukZ5oAW+KAcudmTI 2019-03-25 13:02:42 127.10.0.14 2002 2003 +vote-digest B00ADEF37EEAEADCDB7A06417F15DF7C5D1254B0 +r relay1mbyteMAB EXpFbJERFAdr6051esSLFswMzF8 3SS7LKBsCfyK8hDao1siISnwbsU 2019-05-11 06:26:02 127.10.0.14 2002 2003 s Fast Guard HSDir Running Stable V2Dir Valid v Tor 0.3.5.8 pr Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-5 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 w Bandwidth=0 Unmeasured=1 p reject 1-65535 -r exit1 JwqGGr7SLsK2JRmLzNeyudv/yTw t7/OM287ZDaeItUVISJ9PisVHi0 2019-03-25 13:02:42 127.10.0.11 2002 2003 +r exit1 JwqGGr7SLsK2JRmLzNeyudv/yTw RL21Z7jJw6Ci2B3djpDlKUhVkOc 2019-05-11 06:26:02 127.10.0.11 2002 2003 s Exit Fast Guard HSDir Running Stable V2Dir Valid v Tor 0.3.5.8 pr Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-5 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 w Bandwidth=0 Unmeasured=1 p accept 1-65535 -r relay1 Kr+6zmEWehAZpWyzWy4zYrl9hXA baev0r8l0sbx5foNr6L7jMQBz7c 2019-03-25 13:02:42 127.10.0.4 2002 2003 -s Fast Guard HSDir Running Stable V2Dir Valid +r relay1 Kr+6zmEWehAZpWyzWy4zYrl9hXA B+GR3oAJ59CPgTWmuJsEZpQoBbg 2019-05-11 06:26:03 127.10.0.4 2002 2003 +s Fast Running V2Dir Valid v Tor 0.3.5.8 pr Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-5 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 w Bandwidth=0 Unmeasured=1 p reject 1-65535 -r relay5 MrcXj3IB92QRqZ01UvNA01l9Vik 17SIRm6SJrDZDwBnb8FPqzqXD5E 2019-03-25 13:02:42 127.10.0.8 2002 2003 -s Fast Guard HSDir Running Stable V2Dir Valid +r relay5 MrcXj3IB92QRqZ01UvNA01l9Vik IihK4P79tr2GBmYluw6zQzs1Ctk 2019-05-11 06:26:03 127.10.0.8 2002 2003 +s Fast Running V2Dir Valid v Tor 0.3.5.8 pr Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-5 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 w Bandwidth=0 Unmeasured=1 p reject 1-65535 -r auth3 NeO4u3HIE1VkmuxYYuy37X79vFw DWMUpmsayhRW35zrevN7h1AFd2U 2019-03-25 13:02:56 127.10.0.3 2002 2003 -s Authority Fast Running V2Dir Valid +r auth3 NeO4u3HIE1VkmuxYYuy37X79vFw ne9nL3A1XWBjtQIx+97ruAOy/d4 2019-05-11 06:26:11 127.10.0.3 2002 2003 +s Authority Fast Guard HSDir Running Stable V2Dir Valid v Tor 0.3.5.8 pr Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-5 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 w Bandwidth=0 Unmeasured=1 p reject 1-65535 -r relay4 TWZOJH5TDKXNUXa4wabavJUx8LA xVCoP9aGJrtbCj39wMNsTTzufek 2019-03-25 13:02:42 127.10.0.7 2002 2003 +r relay4 TWZOJH5TDKXNUXa4wabavJUx8LA djl2OVbIyhOkys+9VigRr2yqYcQ 2019-05-11 06:26:02 127.10.0.7 2002 2003 s Fast Guard HSDir Running Stable V2Dir Valid v Tor 0.3.5.8 pr Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-5 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 w Bandwidth=0 Unmeasured=1 p reject 1-65535 -r relay3 aT9zGHYkvnYKrSoSxe2J2x3gRPU PRQtdZg7vi0s5AuBP2hvVwa+Hg4 2019-03-25 13:02:42 127.10.0.6 2002 2003 +r relay3 aT9zGHYkvnYKrSoSxe2J2x3gRPU cJzsCGONWI7Q0e950GAWr/AMEPQ 2019-05-11 06:26:01 127.10.0.6 2002 2003 s Fast Guard HSDir Running Stable V2Dir Valid v Tor 0.3.5.8 pr Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-5 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 w Bandwidth=0 Unmeasured=1 p reject 1-65535 -r relay2 jmh+kdyrln9uTujkbmb2rQXHxiU ZyXj9jg668CPFfNGzP3M6R7W5eA 2019-03-25 13:02:43 127.10.0.5 2002 2003 -s Fast Running V2Dir Valid +r relay2 jmh+kdyrln9uTujkbmb2rQXHxiU Uz9JbLxVpNqcTrtozQ94QqKRzIo 2019-05-11 06:26:02 127.10.0.5 2002 2003 +s Fast Guard HSDir Running Stable V2Dir Valid v Tor 0.3.5.8 pr Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-5 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 w Bandwidth=0 Unmeasured=1 p reject 1-65535 -r relay1mbyteRBR k04G84o5HLcd+D7N4F3/XN46xJ0 AvNHG0R8ux15D/4pfDRafr96avM 2019-03-25 13:02:42 127.10.0.15 2002 2003 -s Fast Guard HSDir Running Stable V2Dir Valid +r relay1mbyteRBR k04G84o5HLcd+D7N4F3/XN46xJ0 ysFLY+txT12fERZKOJetzvNUNdM 2019-05-11 06:26:03 127.10.0.15 2002 2003 +s Fast Running V2Dir Valid v Tor 0.3.5.8 pr Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-5 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 w Bandwidth=0 Unmeasured=1 p reject 1-65535 -r auth1 qkXBMCXAN/BW5zQWmJGHjtCIAjE aB5bSTH8rgroGvljDElcFN6Eauo 2019-03-25 13:02:47 127.10.0.1 2002 2003 +r auth1 qkXBMCXAN/BW5zQWmJGHjtCIAjE srJdsKt+yToOjQs+Kn4MznLlegc 2019-05-11 06:26:13 127.10.0.1 2002 2003 s Authority Fast Guard HSDir Running Stable V2Dir Valid v Tor 0.3.5.8 pr Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-5 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 w Bandwidth=0 Unmeasured=1 p reject 1-65535 -r exit2 wGBrQUQj+aK7omebRABW47B/7IU dI5io9KalNyLFOFjvEoUbWvDB2U 2019-03-25 13:02:43 127.10.0.12 2002 2003 -s Exit Fast Running V2Dir Valid +r exit2 wGBrQUQj+aK7omebRABW47B/7IU /kTDPb7311lUBLXYoeN07eMCWrw 2019-05-11 06:26:01 127.10.0.12 2002 2003 +s Exit Fast Guard HSDir Running Stable V2Dir Valid v Tor 0.3.5.8 pr Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-5 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 w Bandwidth=0 Unmeasured=1 p accept 1-65535 -r relay6 x8UJRncBP1vBJBg8caSC0BVs3P4 X8bEx5uGoC+0zI/P1k+H8md7e/g 2019-03-25 13:02:43 127.10.0.9 2002 2003 -s Fast Running V2Dir Valid +r relay6 x8UJRncBP1vBJBg8caSC0BVs3P4 jpNzMahtVBsh6udeHfHQmFY+VXg 2019-05-11 06:26:02 127.10.0.9 2002 2003 +s Fast Guard HSDir Running Stable V2Dir Valid v Tor 0.3.5.8 pr Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-5 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 w Bandwidth=0 Unmeasured=1 p reject 1-65535 -r auth2 57PJoAQNYo2siLAlGuYzTSjo9TE KDl2076V5kyGxbuYGsROm2LZKAA 2019-03-25 13:02:56 127.10.0.2 2002 2003 +r auth2 57PJoAQNYo2siLAlGuYzTSjo9TE 4OhRKWL3wj/aFfbfLT4LrZLeGKo 2019-05-11 06:26:14 127.10.0.2 2002 2003 s Authority Fast Running V2Dir Valid v Tor 0.3.5.8 pr Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-5 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 w Bandwidth=0 Unmeasured=1 p reject 1-65535 -r relay7 6JTGWZf47JZVi1VBdu7qOcakPvY f4N5R25SZPF8qjpCYOQluKK1hjw 2019-03-25 13:02:43 127.10.0.10 2002 2003 -s Fast Running V2Dir Valid +r relay7 6JTGWZf47JZVi1VBdu7qOcakPvY Xb2TkaBB9RC9PQPHqaKFOLDuIfE 2019-05-11 06:26:02 127.10.0.10 2002 2003 +s Fast Guard HSDir Running Stable V2Dir Valid v Tor 0.3.5.8 pr Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-5 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 w Bandwidth=0 Unmeasured=1 p reject 1-65535 -r exit3 /CZDJeqZ1Zf/lNqIN52rtkME3Z0 BgiTPSUaddDaGl1hWS9Vw6/jJTg 2019-03-25 13:02:43 127.10.0.13 2002 2003 -s Exit Fast Running V2Dir Valid +r exit3 /CZDJeqZ1Zf/lNqIN52rtkME3Z0 FQGPryri2QY0regrUgwewDWVGak 2019-05-11 06:26:02 127.10.0.13 2002 2003 +s Exit Fast Guard HSDir Running Stable V2Dir Valid v Tor 0.3.5.8 pr Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-5 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 w Bandwidth=0 Unmeasured=1 @@ -117,28 +117,28 @@ directory-footer bandwidth-weights Wbd=3333 Wbe=0 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=3333 Wee=10000 Weg=3333 Wem=10000 Wgb=10000 Wgd=3333 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=3333 Wme=0 Wmg=0 Wmm=10000 directory-signature 4EE103A081F400E6622F5461D51782B876BB5C24 9734950A2091BD2A6892A6AA0B53BEF3D85E2D2E -----BEGIN SIGNATURE----- -VQZNfgZ81EmCqJUTTNf9JyphsOX4JaMtMSuHQx0Kr25jtznLZ3IGYssptgDWeD9F -mCt3xHKQd1mtF9cxPtRKMhAb6Tn9qL3vI55lzYinxJ2/RQ0aCpJyeMaeVhd0Rgq6 -zz/tV9iCLBltksk3jZy7tl/Yo7NbIOV97oQNSRXHRtYtRr37ADvW/hC8jegtDEfc -Nw3u/NMJnWTaO/2L4Mxh3/xBQP+Yhy0OQC6korX1HT4SnC9xVEl8xjV2/ShdObBx -m4FLHd4qqoCXLfVHEMcAb/x2BO/GWdrREiSKX0//UpnTLzOFdz3+Al2m8+qDzSrO -7D9qaS8IohSEoIvVPqa+CA== +SPbOg9xMtYkpzauon+r/Osfi9mEV+B+8SiFI7cnxyCztp9KQDkXCTfWtd0wURO2m +mU02kVhAQ5vnJzGjBpHA9nyso4+Grvg2gb20HbFs6PHtywpBhWfL9Trf3JzY7/rt +2hzy/Lax9wB9qigdrn142m6Zie315b5H/1eOi/uIjwcsL+jbDpccfUQe837dS6Z0 +IOL7b7iimebQCMJ2faES5fJvVz3PaVy8hMR+tflfZRmfM8AbMLY3bR/szg9epF4Q +fVGPq1YU+N6Lr2BtRCtuUHnaz/qEoPdDih37hiMuPtZFz32BYGm19UT3hnZAKt0b +AVgZXa+y5V7KI2W6WGK/ig== -----END SIGNATURE----- directory-signature 8B85069C7FC0593801E6491A34100264FCE28980 EA74AA3BF805F776E2BBC0C81BFDE45E74D3281E -----BEGIN SIGNATURE----- -ZoSZOhNf6dqDe2OAIXTD8WV+BX7hHX3XfX9uCAC0LTwlr4bNRXHvV15I1NIK6x05 -Ptzz3Hgs9xaBqwa/VtcTyapxXy5uGNwAWCVbIrXL6LCbILRI5qh6vKy+CTXtY5Pt -lmitMjstIMXTQ79WmGiIQsPlWM4w5zeD+T6g5E0RYdKylY1U3wDBXBc4jokrCG64 -mcY4FYQgn1S6VrAJZLAvQ2H8l6P7QqSmVQk9T4EDkwNsbYsm1oIwkTVw0d+pEZLb -nZ/n525HDWDcWjeAhsicn9ptMMxBjpRqjxUnKZPPsQ95pWJ9a7T7/U+upuIuq7V/ -1cmRHWdBJwGHp0JISuzAJA== +cf+CQImUklrrbJxv66etJVf6vBdR4Rahz8RLp0UCVihefv5du4WD5SXhVqhuo/29 +hO5acKfNeJtr04rZMLAghK6sw0QsX3mzFxD70L46i68iQoLxNSgCi6Cd+f/cct9Y +crari5WpAoB2beFURRRQXl1HEb2DxZdaKfKWBKuSwBWsORtHLaHLi9sNwuykaz4Y +rYGbV4Te2+X508XzBblcBTdfDIEhRWfn70tvfnxSed/HOos0HCFuw1FgeStQzLvy +aCCePMGz1KrYsEmXGLOICvqnSzUgh+7xly3TRKKy0zlfN70FCAcg6uxaMoLOsAmJ +Z9VoPx6pDHMuq+k3W7D1ig== -----END SIGNATURE----- directory-signature D7DBC517EFD2BA1A5012CF1BD0BB38F17C8160BD 8E164C0799EA8FF145C8631FED8A4F512A3022CA -----BEGIN SIGNATURE----- -Okb5iAC6le5ERMTOMxt8EqLNlLGJP7E5a/YdIJcjB6i33EXskIp5UQX69OQmEia8 -B6coFmof0UTJjR/KpacgG91+aE7mN6HNFogfWC8XIz+XFGlvgL14mQsvhv3aIXXx -NCpGh1FSn3G/M2MyPVSaJhtCWfGBXrIzk3twkPsztWl4jOnSG+Tjsvp2PMnZXoXH -gYhJjVDG625Y6oKbwVwRlCzidQrzFkX9CGrcp4o8UopOAm0V0W088LT8IcPLgvTH -E/zY+w75r4jQXbBFUlfy2EAZ1wbP7CSBW3DVA6LZdIcO7domb+5P3aeZLYtmGulO -4FLyzRArnIAdNEFKsE6pGg== +IuOZx8P1KmnNvf1WMuyy5xLPCBtdUf3DJyoxld6FTrHv5u2myG+7Cv1r0jLYRrnt ++9gyeZzIc0p/1CFzzAvC/6OFTM+ieAMjICLH15XjS6fB89sK0JoiDku8suJPxrfb +X/8mO1XKF9/3Sjyja4SjYM7A302kCQmWBU4pMR82xM57/Lw1ELQFVU27N1vh11g8 +x3yzLm9qGaBG/h/IlhcHJwUEtGn0QX08nwO6QlCwxnourIKluzbCbCCL2MgghPUd +4XBJyw/ZItZ/bbLNWI4R5VtOUo6xsGyMnEpQbH1Y43U8heYNV/pfsdKptt7NsgIY +w3zIhvZkPQUgL6FFjKsc5w== -----END SIGNATURE----- diff --git a/tests/data/.sbws/tor/cached-descriptors.new b/tests/data/.sbws/tor/cached-descriptors.new new file mode 100644 index 00000000..e7bf2afd --- /dev/null +++ b/tests/data/.sbws/tor/cached-descriptors.new @@ -0,0 +1,54 @@ +@downloaded-at 2019-05-11 06:27:40 +@source "127.10.0.14" +router auth1 127.10.0.1 2002 0 2003 +identity-ed25519 +-----BEGIN ED25519 CERT----- +AQQABpzfAd9qtXV5sGTKnEuUanBnllxAupyMDWHoVYtsoheIPfRBAQAgBADAuCVI +TD38Md+k2uWuqNVFKcYtVZ+ebS9jGSvIfhpPpNHIbdPioboV4/HMQGC1ikYkGlYW +6JzaxsK30b+wNQE+yfF1cYOkmaqp18C3e2joV00e5PAiI6qeMyQmmjS6Rg0= +-----END ED25519 CERT----- +master-key-ed25519 wLglSEw9/DHfpNrlrqjVRSnGLVWfnm0vYxkryH4aT6Q +platform Tor 0.3.5.8 on Linux +proto Cons=1-2 Desc=1-2 DirCache=1-2 HSDir=1-2 HSIntro=3-4 HSRend=1-2 Link=1-5 LinkAuth=1,3 Microdesc=1-2 Relay=1-2 +published 2019-05-11 06:26:13 +fingerprint AA45 C130 25C0 37F0 56E7 3416 9891 878E D088 0231 +uptime 11 +bandwidth 1073741824 1073741824 0 +extra-info-digest F54F06B095E1285B9B841E5082A6577D9983DA39 fXmZk3yp36F1QMB3a6keLl1et7JDQ4BagejTBY3VOc8 +caches-extra-info +onion-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAM82kmbl7IWKNfwbEfi2lfZgqFPFVPBl7MjufSMOlDnOad0tXbbMcHY5 +jVQVL7ku2PM8SbbDvWZprbPqY9LbWoM/H3QTPgYATd+RQ+eoIuaABBTE10bsx4VC +2WjsztC4xNStZlhuBBVMYNaf8ctHGKTSSCl12PjZUIjnXoBqPp43AgMBAAE= +-----END RSA PUBLIC KEY----- +signing-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAKnUlZwRSWLcmzgq0nGkHMkKGHwkH7R5jWKNQpdW8pLEWGEz+gIZrc5Q +OcFrFA1I/G+9rYjVTjbL1ZfgfWmz304xlhcqsNkTkCZZ0BdKBss4pvSiL0LASHNF +1ykYWj9ZTMhQHWlenqnaauxQY70gUpteltHs3eI+BpQznzqt51GxAgMBAAE= +-----END RSA PUBLIC KEY----- +onion-key-crosscert +-----BEGIN CROSSCERT----- +BlWMFdfFy+GUoe4jwNSYOgqFnqyeedhLeOygmEVC2StyJ5CZzK+aq4NFeDrU7r0p +Q5qLSqBfE9FwhRgoj9mbMgLtFtPW9i5f48tEvB10N20Cf6SZSFPW5ugWu5iwJ1/F +lHIcyVRmdII0ZF0n7IhDxmtXn/Y2TILJR4tYZXJczNQ= +-----END CROSSCERT----- +ntor-onion-key-crosscert 1 +-----BEGIN ED25519 CERT----- +AQoABpyvAcC4JUhMPfwx36Ta5a6o1UUpxi1Vn55tL2MZK8h+Gk+kAF+rL+la5E58 +e0kf5hmH0r+wYdvy8QfntWkGPvIU33Mr4Dd9JGXTYVnqGLlNT5NKmJ+D/ugCXyrA +6Jf5ZrxMKAM= +-----END ED25519 CERT----- +hidden-service-dir +contact pastly@torproject.org +ntor-onion-key 6Bn3JO/LbYmTcgQcMD/GJgW/A7ogap06F3eZ2d85X1Y= +reject *:* +tunnelled-dir-server +router-sig-ed25519 BXD/VV0LNI4PHkBXW/tzslK7GlYe++MnzTbR1NNT5bdgSCQDJ98YOgFfZOXv1SlOufskejICajFhwJnpYxj/Bw +router-signature +-----BEGIN SIGNATURE----- +lDWNgAZX4MBeVhj1J8VKAnSBDyeMi5dr4eXJGsAGxsOTG45wG1bYb15EAqZr8gro +4UjcOdH/daptA65XH9NK057UIwVyAYnlM7Uq2yUNVyhM3oDZJ4br9o4IShJdE4kn +BxB6lz7jC2pUY1WOxyTmwIO/NBKA13JN7kqNBVNrUYs= +-----END SIGNATURE----- diff --git a/tests/data/.sbws/tor/state b/tests/data/.sbws/tor/state new file mode 100644 index 00000000..de976dce --- /dev/null +++ b/tests/data/.sbws/tor/state @@ -0,0 +1,17 @@ +# Tor state file last generated on 2019-05-11 06:32:31 local time +# Other times below are in UTC +# You *do not* need to edit this file. + +Guard in=default rsa_id=270A861ABED22EC2B625198BCCD7B2B9DBFFC93C nickname=exit1 sampled_on=2019-05-01T10:58:03 sampled_by=0.3.5.8 listed=1 +Guard in=default rsa_id=117A456C911114076BEB4E757AC48B16CC0CCC5F nickname=relay1mbyteMAB sampled_on=2019-05-04T00:54:05 sampled_by=0.3.5.8 listed=1 +Guard in=default rsa_id=693F73187624BE760AAD2A12C5ED89DB1DE044F5 nickname=relay3 sampled_on=2019-05-03T16:41:51 sampled_by=0.3.5.8 listed=1 +Guard in=default rsa_id=E894C65997F8EC96558B554176EEEA39C6A43EF6 nickname=relay7 sampled_on=2019-05-09T10:13:49 sampled_by=0.3.5.8 listed=1 +Guard in=default rsa_id=4D664E247E530CA5CD5176B8C1A6DABC9531F0B0 nickname=relay4 sampled_on=2019-05-05T14:37:27 sampled_by=0.3.5.8 listed=1 +Guard in=default rsa_id=AA45C13025C037F056E734169891878ED0880231 nickname=auth1 sampled_on=2019-05-07T20:41:43 sampled_by=0.3.5.8 listed=1 +Guard in=default rsa_id=C0606B414423F9A2BBA2679B440056E3B07FEC85 nickname=exit2 sampled_on=2019-05-10T02:02:09 sampled_by=0.3.5.8 listed=1 +Guard in=default rsa_id=FC264325EA99D597FF94DA88379DABB64304DD9D nickname=exit3 sampled_on=2019-05-03T08:12:26 sampled_by=0.3.5.8 listed=1 +Guard in=default rsa_id=8E687E91DCAB967F6E4EE8E46E66F6AD05C7C625 nickname=relay2 sampled_on=2019-05-06T13:53:54 sampled_by=0.3.5.8 listed=1 +Guard in=default rsa_id=35E3B8BB71C81355649AEC5862ECB7ED7EFDBC5C nickname=auth3 sampled_on=2019-05-04T02:49:33 sampled_by=0.3.5.8 listed=1 +Guard in=default rsa_id=C7C5094677013F5BC124183C71A482D0156CDCFE nickname=relay6 sampled_on=2019-05-10T15:04:58 sampled_by=0.3.5.8 listed=1 +TorVersion Tor 0.3.5.8 +LastWritten 2019-05-11 06:32:31 diff --git a/tests/data/.sbws/v3bw/20190511_063232.v3bw b/tests/data/.sbws/v3bw/20190511_063232.v3bw new file mode 100644 index 00000000..b24ccbbe --- /dev/null +++ b/tests/data/.sbws/v3bw/20190511_063232.v3bw @@ -0,0 +1,40 @@ +1557556118 +version=1.4.0 +destinations_countries=ZZ +earliest_bandwidth=2019-05-11T06:26:41 +file_created=2019-05-11T06:32:32 +generator_started=2019-05-11T06:26:28 +latest_bandwidth=2019-05-11T06:28:38 +minimum_number_eligible_relays=9 +minimum_percent_eligible_relays=60 +number_consensus_relays=15 +number_eligible_relays=0 +percent_eligible_relays=0 +recent_consensus_count=1 +recent_measurement_attempt_count=15 +recent_measurement_failure_count=0 +recent_measurements_excluded_error_count=11 +recent_measurements_excluded_few_count=4 +recent_measurements_excluded_near_count=0 +recent_measurements_excluded_old_count=0 +recent_priority_list_count=1 +recent_priority_relay_count=15 +scanner_country=ZZ +software=sbws +software_version=1.1.1-dev0 +===== +bw=1 error_circ=0 error_destination=0 error_misc=0 error_second_relay=0 error_stream=1 master_key_ed25519=/PrTbpen3BrKNgiNRhAa93JQtnrT3LJX3Ka1+jvbWj4 nick=relay3 node_id=$693F73187624BE760AAD2A12C5ED89DB1DE044F5 relay_in_recent_consensus_count=1 relay_recent_measurement_attempt_count=1 relay_recent_measurements_excluded_error_count=1 relay_recent_priority_list_count=1 success=0 time=2019-05-11T06:28:18 under_min_report=1 unmeasured=1 vote=0 +bw=1 error_circ=0 error_destination=0 error_misc=0 error_second_relay=0 error_stream=1 master_key_ed25519=uPz8ZZNm2Gcra7BauJP5PH+7uANRraYpCj7NFtp1KdM nick=auth2 node_id=$E7B3C9A0040D628DAC88B0251AE6334D28E8F531 relay_in_recent_consensus_count=1 relay_recent_measurement_attempt_count=1 relay_recent_measurements_excluded_error_count=1 relay_recent_priority_list_count=1 success=0 time=2019-05-11T06:27:44 under_min_report=1 unmeasured=1 vote=0 +bw=1 error_circ=0 error_destination=0 error_misc=0 error_second_relay=0 error_stream=1 master_key_ed25519=zOZX+yyD7EN1W/Y2wgjuvObpFOOWK+LZIWlKW0AOcIE nick=relay1 node_id=$2ABFBACE61167A1019A56CB35B2E3362B97D8570 relay_in_recent_consensus_count=1 relay_recent_measurement_attempt_count=1 relay_recent_measurements_excluded_error_count=1 relay_recent_priority_list_count=1 success=0 time=2019-05-11T06:26:51 under_min_report=1 unmeasured=1 vote=0 +bw=1 error_circ=0 error_destination=0 error_misc=0 error_second_relay=0 error_stream=0 master_key_ed25519=wLglSEw9/DHfpNrlrqjVRSnGLVWfnm0vYxkryH4aT6Q nick=auth1 node_id=$AA45C13025C037F056E734169891878ED0880231 relay_in_recent_consensus_count=1 relay_recent_measurement_attempt_count=1 relay_recent_measurements_excluded_few_count=1 relay_recent_priority_list_count=1 success=1 time=2019-05-11T06:28:08 under_min_report=1 unmeasured=1 vote=0 +bw=1 error_circ=0 error_destination=0 error_misc=0 error_second_relay=0 error_stream=0 master_key_ed25519=2fXiF4T993i+vVwZZEQ4fYee+N8OThzCGeacvnVaNbo nick=relay1mbyteMAB node_id=$117A456C911114076BEB4E757AC48B16CC0CCC5F relay_in_recent_consensus_count=1 relay_recent_measurement_attempt_count=1 relay_recent_measurements_excluded_few_count=1 relay_recent_priority_list_count=1 success=1 time=2019-05-11T06:27:03 under_min_report=1 unmeasured=1 vote=0 +bw=1 error_circ=0 error_destination=0 error_misc=0 error_second_relay=0 error_stream=1 master_key_ed25519=NQspfAK/xkywFHV5LsQ5lLi2BmTKx2Imu4jacJ0d9os nick=relay4 node_id=$4D664E247E530CA5CD5176B8C1A6DABC9531F0B0 relay_in_recent_consensus_count=1 relay_recent_measurement_attempt_count=1 relay_recent_measurements_excluded_error_count=1 relay_recent_priority_list_count=1 success=0 time=2019-05-11T06:27:01 under_min_report=1 unmeasured=1 vote=0 +bw=1 error_circ=0 error_destination=0 error_misc=0 error_second_relay=0 error_stream=0 master_key_ed25519=Pymu1Z1eZRWgkE42xzDCYFLVKNtY743GKZzt6Im0OUw nick=relay2 node_id=$8E687E91DCAB967F6E4EE8E46E66F6AD05C7C625 relay_in_recent_consensus_count=1 relay_recent_measurement_attempt_count=1 relay_recent_measurements_excluded_few_count=1 relay_recent_priority_list_count=1 success=1 time=2019-05-11T06:27:34 under_min_report=1 unmeasured=1 vote=0 +bw=1 error_circ=0 error_destination=0 error_misc=0 error_second_relay=0 error_stream=1 master_key_ed25519=cj7V+PYPJvSANsvBOjZSiCvXuGHXFrpSnPqC46I6DgU nick=relay7 node_id=$E894C65997F8EC96558B554176EEEA39C6A43EF6 relay_in_recent_consensus_count=1 relay_recent_measurement_attempt_count=1 relay_recent_measurements_excluded_error_count=1 relay_recent_priority_list_count=1 success=0 time=2019-05-11T06:27:13 under_min_report=1 unmeasured=1 vote=0 +bw=1 error_circ=0 error_destination=0 error_misc=0 error_second_relay=0 error_stream=1 master_key_ed25519=Wiv9uOemlcFzRY1gZfBOpbQ+aJn6NG0Z/IArZFCdSdk nick=exit3 node_id=$FC264325EA99D597FF94DA88379DABB64304DD9D relay_in_recent_consensus_count=1 relay_recent_measurement_attempt_count=1 relay_recent_measurements_excluded_error_count=1 relay_recent_priority_list_count=1 success=0 time=2019-05-11T06:27:23 under_min_report=1 unmeasured=1 vote=0 +bw=1 error_circ=0 error_destination=0 error_misc=0 error_second_relay=0 error_stream=1 master_key_ed25519=r29RWlFMIdU5GUvKsWGdhQIKjIpUzqgw0yNx/7IpPFM nick=auth3 node_id=$35E3B8BB71C81355649AEC5862ECB7ED7EFDBC5C relay_in_recent_consensus_count=1 relay_recent_measurement_attempt_count=1 relay_recent_measurements_excluded_error_count=1 relay_recent_priority_list_count=1 success=0 time=2019-05-11T06:27:54 under_min_report=1 unmeasured=1 vote=0 +bw=1 error_circ=0 error_destination=0 error_misc=0 error_second_relay=0 error_stream=1 master_key_ed25519=gnREYdkUN0uYI3zF8oB+Wwm4MhM6ETPX1hEyt2m2a+Y nick=relay6 node_id=$C7C5094677013F5BC124183C71A482D0156CDCFE relay_in_recent_consensus_count=1 relay_recent_measurement_attempt_count=1 relay_recent_measurements_excluded_error_count=1 relay_recent_priority_list_count=1 success=0 time=2019-05-11T06:26:41 under_min_report=1 unmeasured=1 vote=0 +bw=1 error_circ=0 error_destination=0 error_misc=0 error_second_relay=0 error_stream=1 master_key_ed25519=C506YEdasDDQqidu4G2VLMFOwaqMX28BYkuxr1+wI9o nick=exit1 node_id=$270A861ABED22EC2B625198BCCD7B2B9DBFFC93C relay_in_recent_consensus_count=1 relay_recent_measurement_attempt_count=1 relay_recent_measurements_excluded_error_count=1 relay_recent_priority_list_count=1 success=0 time=2019-05-11T06:28:05 under_min_report=1 unmeasured=1 vote=0 +bw=1 error_circ=0 error_destination=0 error_misc=0 error_second_relay=0 error_stream=0 master_key_ed25519=5O+uCpFoBzsey33+Zzlgyy18/McmV1mpPJZ+DrZMKRc nick=relay5 node_id=$32B7178F7201F76411A99D3552F340D3597D5629 relay_in_recent_consensus_count=1 relay_recent_measurement_attempt_count=1 relay_recent_measurements_excluded_few_count=1 relay_recent_priority_list_count=1 success=1 time=2019-05-11T06:28:38 under_min_report=1 unmeasured=1 vote=0 +bw=1 error_circ=0 error_destination=0 error_misc=0 error_second_relay=0 error_stream=1 master_key_ed25519=iriD8sIKS25WGc6mLesQ2okT1Tcn81AuqnEbItJeuvY nick=exit2 node_id=$C0606B414423F9A2BBA2679B440056E3B07FEC85 relay_in_recent_consensus_count=1 relay_recent_measurement_attempt_count=1 relay_recent_measurements_excluded_error_count=1 relay_recent_priority_list_count=1 success=0 time=2019-05-11T06:28:28 under_min_report=1 unmeasured=1 vote=0 +bw=1 error_circ=0 error_destination=0 error_misc=0 error_second_relay=0 error_stream=1 master_key_ed25519=eR0HnYlzpOEGwxuFjZkGJZ6pu2eV1i6fd9lhF4UOMno nick=relay1mbyteRBR node_id=$934E06F38A391CB71DF83ECDE05DFF5CDE3AC49D relay_in_recent_consensus_count=1 relay_recent_measurement_attempt_count=1 relay_recent_measurements_excluded_error_count=1 relay_recent_priority_list_count=1 success=0 time=2019-05-11T06:27:33 under_min_report=1 unmeasured=1 vote=0 diff --git a/tests/data/.sbws/v3bw/latest.v3bw b/tests/data/.sbws/v3bw/latest.v3bw new file mode 120000 index 00000000..2acaccfa --- /dev/null +++ b/tests/data/.sbws/v3bw/latest.v3bw @@ -0,0 +1 @@ +20190511_063232.v3bw \ No newline at end of file From c3769b96b5b05cff631f6a5844b66fde5da9d99b Mon Sep 17 00:00:00 2001 From: juga0 Date: Sun, 26 May 2019 16:07:12 +0000 Subject: [PATCH 6/6] fixup! new: tests: Add test network data and v3bwfile test --- sbws/lib/v3bwfile.py | 2 ++ tests/conftest.py | 1 - tests/unit/lib/test_v3bwfile.py | 35 +++++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 46516167..66dcabb3 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -1017,6 +1017,8 @@ def is_max_bw_diff_perc_reached(bw_lines, sum_bw = sum([l.bw for l in bw_lines if getattr(l, 'consensus_bandwidth', None) and getattr(l, 'unmeasured', 0) == 0]) * 1000 + # For tests without consensus bandwidth, avoid division by 0 + sum_bw = sum_bw or 1 # Percentage difference diff_perc = ( abs(sum_consensus_bw - sum_bw) diff --git a/tests/conftest.py b/tests/conftest.py index 3c000e39..4a2caaad 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -77,7 +77,6 @@ def measurements(mock_time, sbws_conf): # Because load_recent_results_in_datadir will use time.time() # to decide which results are recent. mock_time.return_value = FAKE_TIME - print("measurements", sbws_conf.getpath('paths', 'datadir')) measurements = resultdump \ .load_recent_results_in_datadir( sbws_conf.getint('general', 'data_period'), diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 720cd4b9..81b575f3 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -7,8 +7,11 @@ from unittest import mock from sbws import __version__ as version +from sbws import settings +# XXX: refactor, these constants can be used from settings. from sbws.globals import (SPEC_VERSION, SBWS_SCALING, TORFLOW_SCALING, MIN_REPORT, TORFLOW_ROUND_DIG, PROP276_ROUND_DIG) +from sbws.lib import destination from sbws.lib.resultdump import Result, load_result_file, ResultSuccess from sbws.lib.v3bwfile import ( V3BWHeader, V3BWLine, TERMINATOR, LINE_SEP, @@ -509,3 +512,35 @@ def test_set_under_min_report(mock_consensus, conf, datadir): assert bwl.vote == 0 assert bwl.under_min_report == 1 assert bwl.bw != 1 + + +def test_header_from_results_parse(sbws_conf, measurements, + bandwidth_file_headers): + """Generate a bandwidth file from the measurements' files + generated with the test network and ensure it's the same as the + bandwidth file also generated with the test network. + + 1. generate bw file from measurements files + 2. parse data bw file + 3. compare generated bw files with the parsed data bw files + """ + scanner_country = sbws_conf['scanner'].get('country') + destinations_countries = destination \ + .parse_destinations_countries(sbws_conf) + + bw_file = V3BWFile.from_results( + measurements, scanner_country, destinations_countries, + sbws_conf.getpath('paths', 'state_fname'), + settings.SBWS_SCALE_CONSTANT, settings.TORFLOW_SCALING, + secs_away=settings.DAY_SECS, + min_num=settings.NUM_MIN_RESULTS, + consensus_path=os.path.join( + sbws_conf.getpath('tor', 'datadir'), "cached-consensus") + ) + # Replace timestamps. + [setattr(bw_file.header, k, v) for k, v in bandwidth_file_headers.items()] + + expected_bw_file = V3BWFile(None, None)\ + .from_v1_fpath(sbws_conf.getpath('paths', 'v3bw_fname')) + + assert str(bw_file.header) == str(expected_bw_file.header)