From d2a571e8c75621e996dbd8f6b234cabab26ef249 Mon Sep 17 00:00:00 2001 From: Noah Martin Date: Thu, 11 Dec 2014 10:29:05 -0800 Subject: [PATCH 01/10] Use powers of 2 for file size --- progress.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/progress.py b/progress.py index 17305a0..06f9d9a 100644 --- a/progress.py +++ b/progress.py @@ -157,10 +157,10 @@ def updateCallback(self): self.timeDiff = time.time() - self.lastUpdated self.lastUpdated = time.time() - def printableUnits(self, value): + def printableUnits(self, value, base=1000): i = 0 for i in range(1, len(self.units)+1): - if value // pow(1000, i) == 0: + if value // pow(base, i) == 0: break i-=1 return i @@ -172,10 +172,11 @@ def __str__(self): if self.value[2] and self.progress == self.expected_size: self.fill_char = "V" - i = self.printableUnits(self.expected_size) - j = self.printableUnits(self.progress) + baseDiskSize = pow(2, 10) + i = self.printableUnits(self.expected_size, baseDiskSize) + j = self.printableUnits(self.progress, baseDiskSize) - progress = "{0:.3f}".format(self.progress/pow(1000, j)) + self.units[j] + "/" + "{0:.3f}".format(self.expected_size/pow(1000, i)) + self.units[i] + progress = "{0:.3f}".format(self.progress/pow(baseDiskSize, j)) + self.units[j] + "/" + "{0:.3f}".format(self.expected_size/pow(baseDiskSize, i)) + self.units[i] speed = 0 if self.timeDiff != 0: speed = (self.lastProgress[1] - self.lastProgress[0])/self.timeDiff From 9a69628060e82481eefb3e723c6e28adc5968de9 Mon Sep 17 00:00:00 2001 From: Noah Martin Date: Thu, 11 Dec 2014 10:50:35 -0800 Subject: [PATCH 02/10] Verbose mode doesn't enter fullscreen --- progress.py | 4 ++++ warp.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/progress.py b/progress.py index 06f9d9a..8cfffa2 100644 --- a/progress.py +++ b/progress.py @@ -30,6 +30,9 @@ def redraw(self): def exit(self): self.screen.exit() + def fullscreen(self): + self.screen.fullscreen() + class Screen(object): def __init__(self): @@ -39,6 +42,7 @@ def __init__(self): self.next_line_top = 0 self.next_line_bottom = 1 + def fullscreen(self): print self.term.enter_fullscreen() def redraw(self): diff --git a/warp.py b/warp.py index 723171b..da1193f 100755 --- a/warp.py +++ b/warp.py @@ -32,6 +32,8 @@ def main(remote_host, recursive, file_src, file_dest, tcp_mode, disable_verify, logger.setLevel(logging.DEBUG) global gui gui = mock.Mock() + else: + gui.fullscreen() startTime = time.time() From 2db4b50934191ed9c1b23bdd069c45d2baecdd50 Mon Sep 17 00:00:00 2001 From: Aviv Kiss Date: Thu, 11 Dec 2014 10:54:58 -0800 Subject: [PATCH 03/10] attempted fix for guy weirdness --- progress.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/progress.py b/progress.py index 8cfffa2..cd22dbe 100644 --- a/progress.py +++ b/progress.py @@ -77,6 +77,10 @@ def add_component(self, component, to_bottom=False): self.next_line_bottom += 1 def exit(self): + for line in (self.bottom_lines + self.top_lines): + for componenet in line.componenet: + componenet.active = False + print self.term.clear() print self.term.exit_fullscreen() From ab53a97ab91e14377abb07587eba09cd37f1b2d8 Mon Sep 17 00:00:00 2001 From: Aviv Kiss Date: Thu, 11 Dec 2014 11:16:38 -0800 Subject: [PATCH 04/10] No longer terminating server explicitly --- client_transfer_controller.py | 3 ++- progress.py | 12 +++++++++--- warp.py | 4 ++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/client_transfer_controller.py b/client_transfer_controller.py index 166bebc..af39416 100644 --- a/client_transfer_controller.py +++ b/client_transfer_controller.py @@ -118,4 +118,5 @@ def is_transfer_success(self): return reduce(lambda y, x: 0 + y if x.transfer_success is True else 1 + y, self.transfer_agents, 0) == 0 def close(self): - self.server_channel.root.get_transfer_manager().finish() + # self.server_channel.root.get_transfer_manager().finish() + pass diff --git a/progress.py b/progress.py index cd22dbe..c30991c 100644 --- a/progress.py +++ b/progress.py @@ -77,9 +77,11 @@ def add_component(self, component, to_bottom=False): self.next_line_bottom += 1 def exit(self): - for line in (self.bottom_lines + self.top_lines): - for componenet in line.componenet: - componenet.active = False + for line in self.bottom_lines.itervalues(): + line.exit() + + for line in self.top_lines.itervalues(): + line.exit() print self.term.clear() print self.term.exit_fullscreen() @@ -95,6 +97,10 @@ def __init__(self, comp=None): def add_component(self, component): self.components.append(component) + def exit(self): + for componenet in self.components: + componenet.active = False + def __iter__(self): for each in self.components: yield each diff --git a/warp.py b/warp.py index da1193f..ccfc94e 100755 --- a/warp.py +++ b/warp.py @@ -76,9 +76,13 @@ def main(remote_host, recursive, file_src, file_dest, tcp_mode, disable_verify, else: logger.warn("Failed to send file.") + logger.debug("Redrawing GUI.") gui.redraw() + logger.debug("Closing Controller.") controller.close() + logger.debug("Closing Connection.") connection.close() + logger.debug("Closing Channel.") channel.close() logger.debug("Closed connections.") From c68477bc3e5113084ffe194e5c17da12a685a067 Mon Sep 17 00:00:00 2001 From: Philip Papadopoulos Date: Tue, 16 Dec 2014 16:11:13 -0800 Subject: [PATCH 05/10] Make a warp rpm and wrapper scripts --- Makefile | 41 +++++++++++++++++++++++++++++++++++++++++ server.sh | 8 ++++++++ warp.sh | 8 ++++++++ warp.spec | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+) create mode 100644 Makefile create mode 100755 server.sh create mode 100755 warp.sh create mode 100644 warp.spec diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..9b7eca4 --- /dev/null +++ b/Makefile @@ -0,0 +1,41 @@ +## warp +NAME = warp +VERSION = 0.1 +TARFILE = $(NAME)-$(VERSION).tar.gz +PYSOURCES = client_transfer_controller.py progress.py \ + client_udt_manager.py server.py \ + common_tools.py server_transfer_controller.py \ + config.py server_udt_manager.py \ + connection.py transfer_manager.py \ + file_transfer_agent.py warp.py \ + forward.py +SCRIPTS = warp.sh server.sh + +## Generic +RPMDIRS = SOURCES BUILD SRPMS RPMS BUILDROOT + +## Help +HELPTEXT = "\ + make help -- This message \n\ + make rpm -- makes warp rpm \n\ + make clean -- cleans everything \n\ + " + +help: + @ echo -e $(HELPTEXT) + +$(RPMDIRS): + mkdir $@ + +SOURCES/$(TARFILE): $(RPMDIRS) + - /bin/rm -rf $(NAME)-$(VERSION) + mkdir $(NAME)-$(VERSION) + cp $(PYSOURCES) $(SCRIPTS) $(NAME)-$(VERSION) + tar czf $@ $(NAME)-$(VERSION) + +rpm: SOURCES/$(TARFILE) + rpmbuild -ba --buildroot=$(PWD)/BUILDROOT $(NAME).spec + + +clean: + /bin/rm -rf $(NAME)-$(VERSION) $(RPMDIRS) diff --git a/server.sh b/server.sh new file mode 100755 index 0000000..dcfb7a9 --- /dev/null +++ b/server.sh @@ -0,0 +1,8 @@ +#!/bin/bash +WARPHOME=/opt/warp +MODNAME=opt-python +PRELOADED=$(echo ${LOADEDMODULES} | grep $MODNAME) +if [ -z $PRELOADED ]; then module load $MODNAME; fi +PYTHONPATH=${WARPHOME}:${PYTHONPATH} ${WARPHOME}/server.py $* +if [ -z $PRELOADED ]; then module unload $MODNAME; fi + diff --git a/warp.sh b/warp.sh new file mode 100755 index 0000000..f439e88 --- /dev/null +++ b/warp.sh @@ -0,0 +1,8 @@ +#!/bin/bash +WARPHOME=/opt/warp +MODNAME=opt-python +PRELOADED=$(echo ${LOADEDMODULES} | grep $MODNAME) +if [ -z $PRELOADED ]; then module load $MODNAME; fi +PYTHONPATH=${WARPHOME}:${PYTHONPATH} ${WARPHOME}/warp.py $* +if [ -z $PRELOADED ]; then module unload $MODNAME; fi + diff --git a/warp.spec b/warp.spec new file mode 100644 index 0000000..17e78f0 --- /dev/null +++ b/warp.spec @@ -0,0 +1,36 @@ +Summary: warp UDT-based file transfer +Name: warp +Version: 0.1 +Release: 2 +License: GPL +Vendor: University of California +Group: System Environment/Base +Source: %{name}-%{version}.tar.gz +%define _topdir %(echo $PWD)/ +%define pkgroot /opt/%{name} + +%description +UDT is a reliable UDP based application level data transport protocol for +distributed data intensive applications over wide area high-speed networks. +Warp is a nice command-line client, written in Python, around UDT +%prep +%setup +%build +printf "\n\n\n### build ###\n\n\n" +%install +printf "\n\n\n### install ###\n\n\n" + +ROOT=$RPM_BUILD_ROOT +INSTALL=/usr/bin/install + +mkdir -p $ROOT/%{pkgroot} +mkdir -p $ROOT/usr/bin + +$INSTALL *.py $ROOT/%{pkgroot} +$INSTALL -m 755 warp.sh $ROOT/usr/bin/warp +$INSTALL -m 755 server.sh $ROOT/usr/bin/warp-server + +%files +%{pkgroot} +/usr/bin/* + From 2d6aa52a9ac21bf68864e55cad1779ab8e2ed606 Mon Sep 17 00:00:00 2001 From: Aviv Kiss Date: Tue, 30 Dec 2014 08:28:33 -0800 Subject: [PATCH 06/10] User specific log files meant to address Phil issue #1 --- .gitignore | 10 ++++++++++ config.py | 6 +++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index fc2abb4..9450068 100644 --- a/.gitignore +++ b/.gitignore @@ -80,3 +80,13 @@ test/foo test/IMG_2252.JPG *.pdf + +foo/bar/foo1 + +foo/bar/foo2 + +foo/bar2 + +foo/bar1 + +test diff --git a/config.py b/config.py index e709ac4..a978bb1 100644 --- a/config.py +++ b/config.py @@ -6,6 +6,7 @@ import logging import __main__ as main +import os, pwd LOG_LEVEL = logging.INFO @@ -20,7 +21,10 @@ NONCE_SIZE = 32 PORT = 29977 -def get_file_logger(logger_name, filepath="/var/tmp/warp.log"): +# Creates user specific log files. +DEFAULT_LOG_FILE_NAME = "/var/tmp/" + "warp-" + pwd.getpwuid(os.getuid())[0] + ".log" + +def get_file_logger(logger_name, filepath=DEFAULT_LOG_FILE_NAME): """ Returns a formatted logger that logs to a file and the console. Takes the logger name as a parameters and optional filepath. From f139b6fbd667525871e28d7958f910d0ac65e0ac Mon Sep 17 00:00:00 2001 From: Aviv Kiss Date: Tue, 30 Dec 2014 08:53:33 -0800 Subject: [PATCH 07/10] Server now terminates Previously the server process would stick around, it now terminates after handling one connection. --- connection.py | 4 ++-- server.py | 4 ++-- transfer_manager.py | 1 - warp.py | 2 -- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/connection.py b/connection.py index df08d11..37b2490 100644 --- a/connection.py +++ b/connection.py @@ -55,8 +55,8 @@ def connect(self): return self.channel def close(self): - pass - # self.forward_thread.exit() + logger.debug("Closing Channel.") + self.channel.close() @staticmethod def unpack_remote_host(remote_host): diff --git a/server.py b/server.py index 61f66fb..6189920 100755 --- a/server.py +++ b/server.py @@ -7,7 +7,7 @@ """ from config import * -from rpyc.utils.server import ThreadedServer +from rpyc.utils.server import OneShotServer from common_tools import * import plac from server_transfer_controller import ServerTransferController @@ -19,7 +19,7 @@ def main(): os.chdir(expanduser("~")) - server = ThreadedServer(ServerTransferController, hostname='localhost', port=0, protocol_config={"allow_public_attrs": True}) + server = OneShotServer(ServerTransferController, hostname='localhost', port=0, protocol_config={"allow_public_attrs": True}) sys.stderr.write(str(server.port)) sys.stderr.write(' ') server.start() diff --git a/transfer_manager.py b/transfer_manager.py index dadd625..3caa044 100644 --- a/transfer_manager.py +++ b/transfer_manager.py @@ -2,7 +2,6 @@ from config import * from common_tools import * - class TransferManager: def __init__(self): pass diff --git a/warp.py b/warp.py index ccfc94e..202170a 100755 --- a/warp.py +++ b/warp.py @@ -82,8 +82,6 @@ def main(remote_host, recursive, file_src, file_dest, tcp_mode, disable_verify, controller.close() logger.debug("Closing Connection.") connection.close() - logger.debug("Closing Channel.") - channel.close() logger.debug("Closed connections.") gui.exit() From d71254ff6a3048ef1a30b5accbf9441aac88034f Mon Sep 17 00:00:00 2001 From: Aviv Kiss Date: Tue, 30 Dec 2014 09:08:41 -0800 Subject: [PATCH 08/10] Fix for Phil issue #3 Fix for when the source is a file and the destination is a folder, should now put the file in the destination folder. --- client_transfer_controller.py | 4 ++++ transfer_manager.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/client_transfer_controller.py b/client_transfer_controller.py index af39416..eaff037 100644 --- a/client_transfer_controller.py +++ b/client_transfer_controller.py @@ -43,6 +43,10 @@ def _start(self): transfer_manager = self.server_channel.root.get_transfer_manager() if not self.recursive: try: + if not transfer_manager.isfile(self.file_dest): + file_name = os.path.split(self.file_src)[1] + self.file_dest = os.path.join(self.file_dest, file_name) + transfer_agent = FileTransferAgent(ClientUDTManager(self.server_channel, self.hostname, self.tcp_mode), transfer_manager, self.file_src, self.file_dest, self.verify, False, self.stat) self.files_processed += 1 except EOFError: diff --git a/transfer_manager.py b/transfer_manager.py index 3caa044..3a699da 100644 --- a/transfer_manager.py +++ b/transfer_manager.py @@ -9,7 +9,7 @@ def __init__(self): def __del__(self): pass - def is_file(self, filepath): + def isfile(self, filepath): return os.path.isfile(filepath) def get_file_hash(self, filepath, block_count=0): From 43e0027dc9856b202cdf1200c4e86c4383521dd9 Mon Sep 17 00:00:00 2001 From: Noah Martin Date: Tue, 27 Jan 2015 12:34:39 -0800 Subject: [PATCH 09/10] Bug fixes --- client_transfer_controller.py | 2 +- transfer_manager.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/client_transfer_controller.py b/client_transfer_controller.py index eaff037..59a48bf 100644 --- a/client_transfer_controller.py +++ b/client_transfer_controller.py @@ -43,7 +43,7 @@ def _start(self): transfer_manager = self.server_channel.root.get_transfer_manager() if not self.recursive: try: - if not transfer_manager.isfile(self.file_dest): + if transfer_manager.isdir(self.file_dest): file_name = os.path.split(self.file_src)[1] self.file_dest = os.path.join(self.file_dest, file_name) diff --git a/transfer_manager.py b/transfer_manager.py index 3a699da..6c9d785 100644 --- a/transfer_manager.py +++ b/transfer_manager.py @@ -12,6 +12,9 @@ def __del__(self): def isfile(self, filepath): return os.path.isfile(filepath) + def isdir(self, filepath): + return os.path.isdir(filepath) + def get_file_hash(self, filepath, block_count=0): return getHash(filepath, block_count) From 6049f49d1d9a71a95095b8afdca89787c2cbf5d6 Mon Sep 17 00:00:00 2001 From: Noah Martin Date: Tue, 27 Jan 2015 17:35:42 -0800 Subject: [PATCH 10/10] Catch error when trying to use a closed connection --- forward.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/forward.py b/forward.py index 3cd2d80..effad18 100644 --- a/forward.py +++ b/forward.py @@ -79,11 +79,13 @@ def handle(self): break self.request.send(data) - peername = self.request.getpeername() - chan.close() - self.request.close() - verbose('Tunnel closed from %r' % (peername,)) - + try: + peername = self.request.getpeername() + chan.close() + self.request.close() + verbose('Tunnel closed from %r' % (peername,)) + except: + pass def forward_tunnel(local_port, remote_host, remote_port, transport): # this is a little convoluted, but lets me configure things for the Handler