diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b62c7b0c8..4c44fc2ac 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,7 @@ stages: - pip install --upgrade requests>=2.22.0 - pip install --upgrade codecov coveralls flake8 mock pytest==4.6.3 pytest-cov selenium - pip install --upgrade -r requirements.txt + - for PLUGIN in $(ls plugins/[^disabled-]*/requirements.txt); do pip install --upgrade -r ${PLUGIN}; done script: - pip list - openssl version -a diff --git a/.travis.yml b/.travis.yml index d88210013..c9c1efa4d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ before_install: # - docker run -d -v $PWD:/root/data -p 15441:15441 -p 127.0.0.1:43110:43110 zeronet install: - pip install --upgrade -r requirements.txt + - for PLUGIN in $(ls plugins/[^disabled-]*/requirements.txt); do pip install --upgrade -r ${PLUGIN}; done - pip list before_script: - openssl version -a diff --git a/Dockerfile b/Dockerfile index b85d44f15..bf9d74190 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,28 +1,26 @@ FROM alpine:3.8 -#Base settings +# Base settings ENV HOME /root +WORKDIR /root -COPY requirements.txt /root/requirements.txt +# Add ZeroNet source +COPY . /root +VOLUME /root/data -#Install ZeroNet +# Install dependencies RUN apk --no-cache --no-progress add python3 python3-dev gcc libffi-dev musl-dev make tor openssl \ - && pip3 install -r /root/requirements.txt \ + && pip3 install -r requirements.txt \ + && for PLUGIN in $(ls plugins/[^disabled-]*/requirements.txt); do pip3 install -r ${PLUGIN}; done \ && apk del python3-dev gcc libffi-dev musl-dev make \ && echo "ControlPort 9051" >> /etc/tor/torrc \ && echo "CookieAuthentication 1" >> /etc/tor/torrc -#Add Zeronet source -COPY . /root -VOLUME /root/data - -#Control if Tor proxy is started +# Control if Tor proxy is started ENV ENABLE_TOR false -WORKDIR /root - -#Set upstart command +# Set upstart command CMD (! ${ENABLE_TOR} || tor&) && python3 zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26552 -#Expose ports +# Expose ports EXPOSE 43110 26552 diff --git a/Vagrantfile b/Vagrantfile index 24fe0c45f..eb8ff3dd9 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -5,41 +5,43 @@ VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - #Set box - config.vm.box = "ubuntu/trusty64" + # Set box + config.vm.box = "ubuntu/bionic64" - #Do not check fo updates + # Do not check fo updates config.vm.box_check_update = false - #Add private network + # Add private network config.vm.network "private_network", type: "dhcp" - #Redirect ports + # Redirect ports config.vm.network "forwarded_port", guest: 43110, host: 43110 config.vm.network "forwarded_port", guest: 15441, host: 15441 - #Sync folder using NFS if not windows + # Sync folder using NFS if not windows config.vm.synced_folder ".", "/vagrant", :nfs => !Vagrant::Util::Platform.windows? - #Virtal Box settings + # Virtal Box settings config.vm.provider "virtualbox" do |vb| # Don't boot with headless mode - #vb.gui = true + vb.gui = false # Set VM settings vb.customize ["modifyvm", :id, "--memory", "512"] vb.customize ["modifyvm", :id, "--cpus", 1] end - #Update system + # Update system config.vm.provision "shell", inline: "sudo apt-get update -y && sudo apt-get upgrade -y" - #Install deps + # Install dependencies config.vm.provision "shell", - inline: "sudo apt-get install msgpack-python python-gevent python-pip python-dev -y" + inline: "sudo apt-get install python3 python3-pip python3-dev gcc libffi-dev musl-dev make -y" config.vm.provision "shell", - inline: "sudo pip install msgpack --upgrade" + inline: "sudo pip3 install -r /vagrant/requirements.txt" + config.vm.provision "shell", + inline: "for PLUGIN in $(ls /vagrant/plugins/[^disabled-]*/requirements.txt); do sudo pip3 install -r ${PLUGIN}; done" end diff --git a/plugins/ENS/ConfigPlugin.py b/plugins/ENS/ConfigPlugin.py new file mode 100644 index 000000000..6e6bb1622 --- /dev/null +++ b/plugins/ENS/ConfigPlugin.py @@ -0,0 +1,26 @@ +from Plugin import PluginManager + +@PluginManager.registerTo('ConfigPlugin') +class ConfigPlugin: + def createArguments(self): + localProviders = ['web3py://default-ipc-providers', 'ws://127.0.0.1:8546', 'http://127.0.0.1:8545'] + mainnetProviders = ['https://cloudflare-eth.com', 'wss://mainnet.infura.io/ws', 'https://mainnet.infura.io'] + ropstenProviders = ['wss://ropsten.infura.io/ws', 'https://ropsten.infura.io'] + rinkebyProviders = ['wss://rinkeby.infura.io/ws', 'https://rinkeby.infura.io'] + goerliProviders = ['wss://goerli.infura.io/ws', 'https://goerli.infura.io'] + + group = self.parser.add_argument_group('ENS plugin') + + group.add_argument('--ens_local_providers', help='Ethereum local providers for ENS plugin', default=localProviders, metavar='protocol://address', nargs='*') + group.add_argument('--ens_mainnet_providers', help='Ethereum mainnet providers for ENS plugin', default=mainnetProviders, metavar='protocol://address', nargs='*') + group.add_argument('--ens_ropsten_providers', help='Ethereum ropsten providers for ENS plugin', default=ropstenProviders, metavar='protocol://address', nargs='*') + group.add_argument('--ens_rinkeby_providers', help='Ethereum rinkeby providers for ENS plugin', default=rinkebyProviders, metavar='protocol://address', nargs='*') + group.add_argument('--ens_goerli_providers', help='Ethereum goerli providers for ENS plugin', default=goerliProviders, metavar='protocol://address', nargs='*') + + group.add_argument('--ens_use_local', help='Use local providers for ENS plugin', action='store_true', default=True) + group.add_argument('--ens_use_mainnet', help='Use mainnet providers for ENS plugin', action='store_true', default=True) + group.add_argument('--ens_use_ropsten', help='Use ropsten providers for ENS plugin', action='store_true', default=True) + group.add_argument('--ens_use_rinkeby', help='Use rinkeby providers for ENS plugin', action='store_true', default=True) + group.add_argument('--ens_use_goerli', help='Use goerli providers for ENS plugin', action='store_true', default=True) + + return super(ConfigPlugin, self).createArguments() diff --git a/plugins/ENS/ENSResolver.py b/plugins/ENS/ENSResolver.py new file mode 100644 index 000000000..270c60580 --- /dev/null +++ b/plugins/ENS/ENSResolver.py @@ -0,0 +1,164 @@ +from Config import config + +from web3 import HTTPProvider, WebsocketProvider, IPCProvider +from ens import ENS +import gevent + +from urllib.parse import urlparse +import logging +import time +import json +import re +import os + +FILE_SCHEMES = {'file'} +HTTP_SCHEMES = {'http', 'https'} +WS_SCHEMES = {'ws', 'wss'} + +log = logging.getLogger('ENSPlugin') + +class ENSResolver: + loaded = False + cache = {} + + def __init__(self, site_manager, networks): + self.site_manager = site_manager + self.networks = networks + + def load(self): + if not self.loaded: + self.loadCache() + + greenlets = [] + for network in self.networks: + greenlets.append(gevent.spawn(self.loadNetwork, network)) + gevent.joinall(greenlets) + + self.loaded = True + + def loadNetwork(self, network): + if not network['enabled']: + return + + for provider_uri in network['providers']: + provider_scheme = urlparse(provider_uri).scheme + provider_path = urlparse(provider_uri).path + + if provider_uri == 'web3py://default-ipc-providers': + provider = IPCProvider() + + elif provider_scheme in FILE_SCHEMES: + provider = IPCProvider(provider_path) + + elif provider_scheme in HTTP_SCHEMES: + provider = HTTPProvider(provider_uri) + + elif provider_scheme in WS_SCHEMES: + provider = WebsocketProvider(provider_uri) + + try: + if provider.isConnected(): + network['instance'] = ENS(provider) + return + except: + pass + + network['enabled'] = False + + def loadCache(self, path=os.path.join(config.data_dir, 'ens_cache.json')): + if os.path.isfile(path): + try: + self.cache = json.load(open(path)) + except json.decoder.JSONDecodeError: + pass + + def saveCache(self, path=os.path.join(config.data_dir, 'ens_cache.json')): + with open(path, 'w') as file: + json.dump(self.cache, file, indent=2) + + def isDomain(self, address): + return re.match(r'(.*?)([A-Za-z0-9_-]+\.eth)$', address) + + def resolveDomain(self, domain): + if not self.loaded: + self.load() + + domain = domain.lower() + + cache_entry = self.lookupCache(domain) + if cache_entry and time.time() < cache_entry['timeout']: + log.info('cache: %s -> %s', domain, cache_entry['address']) + return cache_entry['address'] + + provider_entry = None + provider_error = None + + try: + provider_entry = self.lookupProviders(domain) + except Exception as err: + provider_error = err + + if provider_entry and not provider_error: + log.info('provider: %s -> %s', domain, provider_entry['address']) + self.saveInCache(provider_entry) + return provider_entry['address'] + + if cache_entry and provider_error: + log.info('fallback: %s -> %s', domain, cache_entry['address']) + self.extendInCache(cache_entry) + return cache_entry['address'] + + def lookupCache(self, domain): + entry = self.cache.get(domain) + + if not entry: + return None + + for network in self.networks: + if entry['network'] == network['name'] and not network['enabled']: + return None + + return entry + + def lookupProviders(self, domain): + error = None + + greenlets = [] + for network in self.networks: + greenlets.append(gevent.spawn(self.lookupProvider, domain, network)) + gevent.joinall(greenlets) + + for greenlet in greenlets: + try: + entry = greenlet.get() + except Exception as err: + error = err + + if entry and entry['address']: + return entry + + if error: + raise error + + def lookupProvider(self, domain, network): + if not network['enabled']: + return None + + name = network['name'] + instance = network['instance'] + + content = instance.content(domain) + + if content and content['type'] == 'zeronet': + return {'network': name, 'domain': domain, 'address': content['hash']} + + def saveInCache(self, entry): + entry['timeout'] = time.time() + 60 * 60 + self.cache[entry['domain']] = entry + + self.saveCache() + + def extendInCache(self, entry): + self.cache[entry['domain']]['timeout'] = time.time() + 60 * 15 + + self.saveCache() diff --git a/plugins/ENS/SiteManagerPlugin.py b/plugins/ENS/SiteManagerPlugin.py new file mode 100644 index 000000000..54611f9b3 --- /dev/null +++ b/plugins/ENS/SiteManagerPlugin.py @@ -0,0 +1,73 @@ +from Config import config +from Plugin import PluginManager + +from .ENSResolver import ENSResolver + +allow_reload = False + +@PluginManager.registerTo('SiteManager') +class SiteManagerPlugin: + _ens_resolver = None + + @property + def ens_resolver(self): + if not self._ens_resolver: + local = { + 'name': 'local', + 'providers': config.ens_local_providers, + 'enabled': config.ens_use_local, + 'instance': None + } + + mainnet = { + 'name': 'mainnet', + 'providers': config.ens_mainnet_providers, + 'enabled': config.ens_use_mainnet, + 'instance': None + } + + ropsten = { + 'name': 'ropsten', + 'providers': config.ens_ropsten_providers, + 'enabled': config.ens_use_ropsten, + 'instance': None + } + + rinkeby = { + 'name': 'rinkeby', + 'providers': config.ens_rinkeby_providers, + 'enabled': config.ens_use_rinkeby, + 'instance': None + } + + goerli = { + 'name': 'goerli', + 'providers': config.ens_goerli_providers, + 'enabled': config.ens_use_goerli, + 'instance': None + } + + networks = [ + local, + mainnet, + ropsten, + rinkeby, + goerli + ] + + self._ens_resolver = ENSResolver( + site_manager=self, + networks=networks + ) + + return self._ens_resolver + + def load(self, *args, **kwargs): + super(SiteManagerPlugin, self).load(*args, **kwargs) + self.ens_resolver.load() + + def isDomain(self, address): + return self.ens_resolver.isDomain(address) or super(SiteManagerPlugin, self).isDomain(address) + + def resolveDomain(self, domain): + return self.ens_resolver.resolveDomain(domain) or super(SiteManagerPlugin, self).resolveDomain(domain) diff --git a/plugins/ENS/__init__.py b/plugins/ENS/__init__.py new file mode 100644 index 000000000..473988653 --- /dev/null +++ b/plugins/ENS/__init__.py @@ -0,0 +1,2 @@ +from . import ConfigPlugin +from . import SiteManagerPlugin diff --git a/plugins/ENS/plugin_info.json b/plugins/ENS/plugin_info.json new file mode 100644 index 000000000..cb6c8d031 --- /dev/null +++ b/plugins/ENS/plugin_info.json @@ -0,0 +1,5 @@ +{ + "name": "ENS", + "description": "Support Ethereum Name Service as domain system.", + "default": "enabled" +} diff --git a/plugins/ENS/requirements.txt b/plugins/ENS/requirements.txt new file mode 100644 index 000000000..b7c62127b --- /dev/null +++ b/plugins/ENS/requirements.txt @@ -0,0 +1 @@ +web3 @ git+https://github.com/filips123/web3.py@abb32516a3898ace515f2162374be387b2733403