From 16a9a335efe76e51b0cacffd9047fd3f50e309e6 Mon Sep 17 00:00:00 2001 From: lling Date: Wed, 15 Nov 2017 22:42:42 -0800 Subject: [PATCH 01/24] python shebang --- asl_sdr_hackfest/postmaster.py | 2 ++ 1 file changed, 2 insertions(+) mode change 100644 => 100755 asl_sdr_hackfest/postmaster.py diff --git a/asl_sdr_hackfest/postmaster.py b/asl_sdr_hackfest/postmaster.py old mode 100644 new mode 100755 index c957984..5ddfaed --- a/asl_sdr_hackfest/postmaster.py +++ b/asl_sdr_hackfest/postmaster.py @@ -1,3 +1,5 @@ +#!/usr/bin/python + import threading from asl_sdr_hackfest.service import Service From be286ee68faab082fdc8cf8448980e41c05b3599 Mon Sep 17 00:00:00 2001 From: lling Date: Thu, 16 Nov 2017 01:37:29 -0800 Subject: [PATCH 02/24] services test program --- asl_sdr_hackfest/serviceLoopback.py | 51 +++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100755 asl_sdr_hackfest/serviceLoopback.py diff --git a/asl_sdr_hackfest/serviceLoopback.py b/asl_sdr_hackfest/serviceLoopback.py new file mode 100755 index 0000000..e9888b6 --- /dev/null +++ b/asl_sdr_hackfest/serviceLoopback.py @@ -0,0 +1,51 @@ +#!/usr/bin/python + +from asl_sdr_hackfest.service import * +import sys + +class ServiceLoopback: + def __init__(self, portA = 5786, portB = 6875): + try: + self.services = [Service(portA, portB), Service(portB, portA)] + except: + raise + + def run (self): + while True: + try: + t = threading.Timer(0.25, self.services[0].outputData, {"data": "Hello"}) + t.start() + t.join() + # self.services[0].outputData ("Hello") + print self.services[1].readData() + except KeyboardInterrupt: + for service in self.services: + service.stop() + break + except: + print "No messages." + continue + + +if __name__ == "__main__": + slb = ServiceLoopback() + slb.run() + + # BUG: This code works, but the above doesn't + # s1 = Service(5786, 6875) + # s2 = Service(6875, 5786) + # + # s1.start() + # s2.start() + # + # while True: + # s1.outputData("Hello") + # try: + # print s2.readData() + # except KeyboardInterrupt: + # s1.stop() + # s2.stop() + # break + # except: + # print "exception!" + # continue From f111304c0291b51914c973fa5b0a98e70f50e81d Mon Sep 17 00:00:00 2001 From: lling Date: Wed, 15 Nov 2017 18:14:33 -0800 Subject: [PATCH 03/24] added chaos layer and utilities --- chaos_layer.py | 132 ++++++++++++++++++++++++++++++++++++++++++ chaos_layer_tester.py | 65 +++++++++++++++++++++ entropy_engine.py | 28 +++++++++ 3 files changed, 225 insertions(+) create mode 100644 chaos_layer.py create mode 100755 chaos_layer_tester.py create mode 100644 entropy_engine.py diff --git a/chaos_layer.py b/chaos_layer.py new file mode 100644 index 0000000..59b8e65 --- /dev/null +++ b/chaos_layer.py @@ -0,0 +1,132 @@ +#! /usr/bin/python + +from asl_sdr_hackfest.protocols.network_layer import Frame + +# from random import SystemRandom as sysran +import queue +import sys +import bitarray +import threading + +from entropy_engine import * + +def to_bitarray(bytearray): + bits = '' + for byte in bytearray: + # Remove '0b' prefix and pad up to 8 zeros per byte + bits += bin(byte).split('0b')[1].zfill(8) + return bitarray.bitarray(bits) + +def to_bytearray(bits): + # Note: python does implicit conversions + if isinstance(bits, bitarray.bitarray): + return bytearray(bits) + elif isinstance(bits, str): + return bytearray(bits, 'utf8') + else: + raise + +class Chaos_layer(threading.Thread, object): + def __init__ (self, chance_to_run = 0.5): + self.bits = bitarray.bitarray(Frame.headerlength*8) + + self.chance_for_fun = 0 + if (chance_to_run <= 1): + self.chance_for_fun *= 100 + else: + self.chance_for_fun = chance_to_run + + self.d6 = Dice() + self.d20 = D20() + self.d100 = D100() + + def run (self, bytearr_queue, bytearr): + roll100 = self.d100.roll() + roll6 = self.d6.roll() + + chaos_tool = None + + if (roll100 < self.chance_for_fun and roll6 == 1): + chaos_tool = self.bitFlips + elif (roll100 < self.chance_for_fun and roll6 == 2): + chaos_tool = self.zeroField + elif (roll100 < self.chance_for_fun and roll6 == 3): + chaos_tool = self.genetic + elif (roll100 < self.chance_for_fun and roll6 == 4): + chaos_tool = self.delayed_enqueue + elif (roll100 < self.chance_for_fun and roll6 == 5): + chaos_tool = self.random_delayed_enqueue + elif (roll100 < self.chance_for_fun and roll6 == 6): + chaos_tool = None + else: + chaos_tool = None + # raise + + try: + chaos_tool (bytearr_queue, bytearr) + except: + bytearr_queue.put(bytearr) + + def bitFlips (self, bytearr_queue, bytearr): + f_bits = to_bitarray(bytearr) + f_bits.invert() + + f_bytes = to_bytearray(f_bits) + bytearr_queue.put(f_bytes) + + # TODO: write test? + def zeroField (self, bytearr_queue, bytearr): + frame = Frame.unpack_frame(bytearr) + field = self.d6.roll() + if field == 0: + frame.addr = None + elif field == 1: + frame.qos = None + elif field == 2: + frame.MF = None + elif field == 3: + frame.NACK = None + elif field == 4: + frame.length = None + elif field == 5: + frame.fragment = None + else: + frame.packetid = None + frame.checksum = None + frame.payload = None + + bytearr_queue.put(Frame.pack_frame(frame)) + + def genetic (self, bytearr_queue, bytearr): + f_bits = to_bitarray(bytearr) + + begin = self.d20.roll() + end = len(f_bits) - self.d20.roll() + + if begin > end: + begin, end = end, begin + + self.bits[begin:end], f_bits[begin:end] = f_bits[begin:end], self.bits[begin:end] + + # TODO: For mutation, 100 % 64 != 0 (not even) + roll6 = self.d6.roll() + roll100 = self.d100.roll() + for n in range(roll6): + if (roll100 > self.chance_for_fun): + f_bits[roll100%len(f_bits)] = not f_bits[roll100%len(f_bits)] + + bytearr_queue.put(to_bytearray(f_bits)) + + def delayed_enqueue (self, bytearr_queue, bytearr, secs = 1): + # Note: intentionally not really thread-safe per put_nowait + if (not bytearr_queue.full()): + print ("delay enqueue by %d seconds" % secs) + t = threading.Timer(secs, bytearr_queue.put_nowait, [bytearr]) + t.start() + return t + else: + raise + + def random_delayed_enqueue (self, bytearr_queue, bytearr): + secs = self.d6.roll() + return self.delayed_enqueue(bytearr_queue, bytearr, secs) diff --git a/chaos_layer_tester.py b/chaos_layer_tester.py new file mode 100755 index 0000000..2f7ea1f --- /dev/null +++ b/chaos_layer_tester.py @@ -0,0 +1,65 @@ +#! /usr/bin/python + +from chaos_layer import * + +class Chaos_layer_tester(object): + def __init__(self, chaos_layer = Chaos_layer(), max_queue_size = 32): + self.cl = chaos_layer + self.bq = queue.Queue (max_queue_size) + + def bitFlips_test (self): + print "bitFlips" + ba = bitarray.bitarray(Frame.headerlength*8) + ba.setall(0) + Ba = to_bytearray(ba) + + self.cl.bitFlips(self.bq, Ba) + Ba = self.bq.get() + + ba = to_bitarray(Ba) + ## Note: This should print + ## '1111111111001111111111111111111111111111111111111111111111111111' + ## because Frame.qos field masks 0x0F + print ba + + def genetic_test (self, passes = 42): + print "genetics" + ba = bitarray.bitarray(Frame.headerlength*8) + print ba + print self.cl.bits + for n in range(passes): + Ba = to_bytearray(ba) + self.cl.genetic(self.bq, Ba) + Ba = self.bq.get() + print to_bitarray(Ba) + + def delayed_enqueue_test (self, delayed_enqueued_func, queue_max_size = 32): + print "delayed enqueue" + data_queue = self.bq + Ba = bytearray("helloworld", "utf-8") + threads = [] + for n in range(queue_max_size): + threads.append(delayed_enqueued_func(data_queue, Ba)) + for thread in threads: + thread.join() + print data_queue.qsize() + for n in range(data_queue.qsize()): + data = data_queue.get_nowait() + print ("%d: %s" % (n, data)) + + def uniform_delayed_enqueue_test (self, queue_max_size = 32): + self.delayed_enqueue_test(self.cl.delayed_enqueue, queue_max_size) + + def random_delayed_enqueue_test (self, queue_max_size = 32): + self.delayed_enqueue_test(self.cl.random_delayed_enqueue, queue_max_size) + + def burst (self, queue_max_size = 32) : + self.delayed_enqueue_test(self.cl.random_delayed_enqueue, queue_max_size) + +if __name__ == "__main__": + clt = Chaos_layer_tester() + clt.bitFlips_test() + clt.genetic_test() + clt.burst() + + clt.cl.run(clt.bq, bytearray("helloworld", "utf-8")) diff --git a/entropy_engine.py b/entropy_engine.py new file mode 100644 index 0000000..ede5238 --- /dev/null +++ b/entropy_engine.py @@ -0,0 +1,28 @@ +#! /usr/bin/python + +import random + +class Randomizer(object): + def __init__(self): + random.seed() + +class Dice(Randomizer): + def __init__(self, min = 1, max = 6): + self._min = min + self._max = max + self = super(Dice, self).__init__() + + def roll(self): + return random.randint(self._min, self._max) + +class D16(Dice): + def __init__(self): + self = super(D16, self).__init__(1, 16) + +class D20(Dice): + def __init__(self): + self = super(D20, self).__init__(1, 20) + +class D100(Dice): + def __init(self): + self = super(D100, self).__init__(1, 100) From df4fe078911bfe3e8e135788514a0a3c57fd5948 Mon Sep 17 00:00:00 2001 From: lling Date: Thu, 16 Nov 2017 02:02:09 -0800 Subject: [PATCH 04/24] stop sending cats --- apps/netCat.py | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/apps/netCat.py b/apps/netCat.py index 39c98c7..5d6126a 100755 --- a/apps/netCat.py +++ b/apps/netCat.py @@ -24,15 +24,12 @@ def __init__(self, img_fp, ip_address='127.0.0.1', port='6127'): self.pub.bind('tcp://%s:%s' % (self._ip, self._port)) def send(self): - try: - img = Image.open(self._img_file) - img_pickled = pickle.dumps(img) + img = Image.open(self._img_file) + img_pickled = pickle.dumps(img) - print("sending {fp}".format(fp=self._img_file)) - self.pub.send(img_pickled) - print("done sending {fp}".format(fp=self._img_file)) - except: - print("failed to send") + print("sending {fp}".format(fp=self._img_file)) + self.pub.send(img_pickled) + print("done sending {fp}".format(fp=self._img_file)) class netCatReceiver(netCat): def __init__(self, ip_address='127.0.0.1', port='6127'): @@ -76,22 +73,26 @@ def __init__(self): else: print("receive: ./netCat.py \n transmit: ./netCat.py cat_image") - def run(self): - if len(sys.argv) > 1: - self.sender.send() - else: - self.receiver.recv() + while True: + try: + if len(sys.argv) > 1: + self.sender.send() + else: + self.receiver.recv() + except KeyboardInterrupt: + # BUG: sender will not call sit() on SIGINT + nc.sit() + break + except: + if len(sys.argv) > 1: + print "failed to send" + else: + print "failed to receive" def sit(self): print("good kitty!") if __name__ == "__main__": nc = netCatLauncher() - while True: - try: - nc.run() - except KeyboardInterrupt: - # BUG: sender will not call sit() on SIGINT - nc.sit() - break + nc.run() From f8f7b92f908e6a01a72c4fe22c6d71331df99d4b Mon Sep 17 00:00:00 2001 From: Zachary Kolarich Date: Thu, 16 Nov 2017 12:21:05 -0500 Subject: [PATCH 05/24] removed addr from frame header --- asl_sdr_hackfest/protocols/network_layer.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/asl_sdr_hackfest/protocols/network_layer.py b/asl_sdr_hackfest/protocols/network_layer.py index 759279c..0aad845 100644 --- a/asl_sdr_hackfest/protocols/network_layer.py +++ b/asl_sdr_hackfest/protocols/network_layer.py @@ -7,17 +7,17 @@ class Frame: - headerlength = 8 + headerlength = 7 # addr, qos, len, len, id, fragment, check, check - addr_pos = 0 - qos_pos = 1 - len_pos = 2 - packetid_pos = 4 - fragment_pos = 5 - checksum_pos = 6 + # addr_pos = 0 + qos_pos = 0 + len_pos = 1 + packetid_pos = 3 + fragment_pos = 4 + checksum_pos = 5 def __init__(self): - self.addr = None + self.addr = None # None self.qos = None self.MF = None self.NACK = None @@ -31,9 +31,10 @@ def __init__(self): def unpack_frame(bytearr): f = Frame() - (f.addr, qosbyte, f.length, f.packetid, f.fragment, f.checksum) = struct.unpack('!BBHBBH', + (qosbyte, f.length, f.packetid, f.fragment, f.checksum) = struct.unpack('!BHBBH', bytearr[:Frame.headerlength]) + f.addr = 0 f.qos = qosbyte & 0x0F f.MF = qosbyte & 128 != 0 f.NACK = qosbyte & 64 != 0 @@ -46,7 +47,7 @@ def pack_frame(f): # everything should be in network order (big endian) qosbyte = (f.qos & 0x0F) | (128 if f.MF else 0) | (64 if f.NACK else 0) - strut = struct.pack('!BBHBBH', f.addr, qosbyte, f.length, f.packetid, f.fragment, f.checksum) + strut = struct.pack('!BHBBH', qosbyte, f.length, f.packetid, f.fragment, f.checksum) return strut + f.payload From dced6089a79e8a7ba11868c8fb80b2f83ef7a49b Mon Sep 17 00:00:00 2001 From: Brandon Haines Date: Thu, 16 Nov 2017 16:12:21 -0500 Subject: [PATCH 06/24] Adding fragment culling at egress saturation. --- .../protocols/network_layer_handler.py | 50 +++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/asl_sdr_hackfest/protocols/network_layer_handler.py b/asl_sdr_hackfest/protocols/network_layer_handler.py index f6a8d70..d256ac2 100644 --- a/asl_sdr_hackfest/protocols/network_layer_handler.py +++ b/asl_sdr_hackfest/protocols/network_layer_handler.py @@ -1,8 +1,14 @@ import threading import queue +import time from asl_sdr_hackfest.protocols.qos import QoS -from asl_sdr_hackfest.protocols.network_layer import NetworkLayerReceive, NetworkLayerTransmit +from asl_sdr_hackfest.protocols.network_layer import NetworkLayerReceive, NetworkLayerTransmit, Frame + + + +EGRESS_WINDOW_LOW = 8 +EGRESS_WINDOW_HIGH = 12 @@ -45,9 +51,11 @@ def run(self): class NetworkLayerTransmitHandler(threading.Thread, object): - def __init__(self, output_data_func): + def __init__(self, output_data_func, max_frames_per_second = 20): self.running = False + self.xmit_delay = float(1)/float(max_frames_per_second) + self.out_queue = queue.Queue() self.in_queues = [] for i in range(0,16): @@ -74,7 +82,43 @@ def run(self): self.running = True while (self.running is True): - self.tx_class.do_transmit(mtu = 255) + self.tx_class.do_transmit(mtu = 96 - Frame.headerlength) + time.sleep(self.xmit_delay) if self.out_queue.empty() is False: + if self.out_queue.qsize() > EGRESS_WINDOW_HIGH: + self.egress_cull() data = self.out_queue.get() self.output_data_func(data) + + + def egress_cull(self): + init_size = self.out_queue.qsize() + + tmp_egress = [] + while self.out_queue.empty() is False: + tmp_egress.append(self.out_queue.get()) + + while len(tmp_egress) > EGRESS_WINDOW_LOW: + max_cos = 0 + for pkt in tmp_egress: + cos = pkt[Frame.qos_pos] & 0x0f + if cos > max_cos: + max_cos = cos + + candidates = [] + for i in range(0, len(tmp_egress)): + cos = tmp_egress[i][Frame.qos_pos] & 0x0f + if cos == max_cos: + candidates.append(i) + + candidates.reverse() + while len(tmp_egress) > EGRESS_WINDOW_LOW and len(candidates) > 0: + target = candidates.pop() + del(tmp_egress[target]) + + tmp_egress.reverse() + while len(tmp_egress) > 0: + self.out_queue.put(tmp_egress.pop()) + + culled_size = self.out_queue.qsize() + print('Egress queue culled from {} to {}.'.format(init_size, culled_size)) From 96768c96d37a7e154aadc837dbb83d5267c8e7c5 Mon Sep 17 00:00:00 2001 From: Brandon Haines Date: Thu, 16 Nov 2017 16:13:18 -0500 Subject: [PATCH 07/24] Adding PDU strip/add to ZMQ interfaces. --- README.md | 3 --- apps/run_all.py | 2 +- apps/run_one.py | 2 +- asl_sdr_hackfest/zmq_pub.py | 7 ++++++- asl_sdr_hackfest/zmq_sub.py | 8 +++++++- setup.py | 3 +-- 6 files changed, 16 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 3f2a5b7..899e1bb 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,6 @@ Adversarial Science Lab's repository for the DARPA SDR Hackfest ## Installation -### python3 -pip3 install . - ### python2 sudo pip install . diff --git a/apps/run_all.py b/apps/run_all.py index f9fdc78..294f4cf 100755 --- a/apps/run_all.py +++ b/apps/run_all.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python import signal diff --git a/apps/run_one.py b/apps/run_one.py index 78c9c29..172fa3f 100755 --- a/apps/run_one.py +++ b/apps/run_one.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python import signal diff --git a/asl_sdr_hackfest/zmq_pub.py b/asl_sdr_hackfest/zmq_pub.py index ac152ce..f9d7f79 100644 --- a/asl_sdr_hackfest/zmq_pub.py +++ b/asl_sdr_hackfest/zmq_pub.py @@ -1,4 +1,5 @@ import zmq +import numpy, pmt @@ -13,4 +14,8 @@ def __init__(self, portOut): def send(self, data): - self._socketOut.send(data) + car = pmt.make_dict() + data = bytes(data) + cdr = pmt.to_pmt(data) + pdu = pmt.cons(car, cdr) + self._socketOut.send(pmt.serialize_str(pdu)) diff --git a/asl_sdr_hackfest/zmq_sub.py b/asl_sdr_hackfest/zmq_sub.py index 720e396..7696b76 100644 --- a/asl_sdr_hackfest/zmq_sub.py +++ b/asl_sdr_hackfest/zmq_sub.py @@ -1,6 +1,8 @@ import zmq from zmq import Again as ZMQ_sub_timeout +import numpy, pmt +import binascii class ZMQ_sub(object): def __init__(self, portIn, timeout = 100): @@ -18,4 +20,8 @@ def __init__(self, portIn, timeout = 100): def recv(self): - return self._socketIn.recv() + msg = self._socketIn.recv() + pdu = pmt.deserialize_str(msg) + cdr = pmt.to_python(pmt.cdr(pdu)) + cdr = numpy.getbuffer(cdr) + return cdr diff --git a/setup.py b/setup.py index 0679f87..2b007a9 100755 --- a/setup.py +++ b/setup.py @@ -11,11 +11,10 @@ description = ("Adversarial Science Lab's code for DARPA's SDR Hackfest."), license = "LGPL v2.1", url = "https://github.com/haineb/ASLHackfest", -# package_dir = {'asl_sdr_hackfest': 'src'}, -# package_dir = {'asl_sdr_hackfest': 'src', 'asl_sdr_hackfest.protocols': 'src/protocols'}, packages = find_packages(), install_requires=[ 'bitstring', 'zmq', + 'numpy', ], ) From e59ff548dfdf2084d5347ce8f648a77f529a733a Mon Sep 17 00:00:00 2001 From: Brandon Haines Date: Thu, 16 Nov 2017 16:30:52 -0500 Subject: [PATCH 08/24] Fixing PDU encapsulation type bug. --- asl_sdr_hackfest/zmq_pub.py | 1 + 1 file changed, 1 insertion(+) diff --git a/asl_sdr_hackfest/zmq_pub.py b/asl_sdr_hackfest/zmq_pub.py index f9d7f79..3df57d8 100644 --- a/asl_sdr_hackfest/zmq_pub.py +++ b/asl_sdr_hackfest/zmq_pub.py @@ -16,6 +16,7 @@ def __init__(self, portOut): def send(self, data): car = pmt.make_dict() data = bytes(data) + data = numpy.frombuffer(data, dtype=numpy.uint8) cdr = pmt.to_pmt(data) pdu = pmt.cons(car, cdr) self._socketOut.send(pmt.serialize_str(pdu)) From e2fa81930a6bb8d31a7b9179b072dfc5bdfc3d86 Mon Sep 17 00:00:00 2001 From: Brandon Haines Date: Thu, 16 Nov 2017 18:08:53 -0500 Subject: [PATCH 09/24] Surpressing requires for setup. --- setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 2b007a9..25ca4df 100755 --- a/setup.py +++ b/setup.py @@ -12,9 +12,9 @@ license = "LGPL v2.1", url = "https://github.com/haineb/ASLHackfest", packages = find_packages(), - install_requires=[ - 'bitstring', - 'zmq', - 'numpy', - ], +# install_requires=[ +# 'bitstring', +# 'zmq', +# 'numpy', +# ], ) From 934056ee78d28c06b6b1dd10a6bac4a95ba33a1f Mon Sep 17 00:00:00 2001 From: Brandon Haines Date: Fri, 17 Nov 2017 01:39:51 -0500 Subject: [PATCH 10/24] fixing culling bug. --- asl_sdr_hackfest/protocols/network_layer_handler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/asl_sdr_hackfest/protocols/network_layer_handler.py b/asl_sdr_hackfest/protocols/network_layer_handler.py index d256ac2..53ad590 100644 --- a/asl_sdr_hackfest/protocols/network_layer_handler.py +++ b/asl_sdr_hackfest/protocols/network_layer_handler.py @@ -111,10 +111,11 @@ def egress_cull(self): if cos == max_cos: candidates.append(i) - candidates.reverse() + tmp_egress.reverse() while len(tmp_egress) > EGRESS_WINDOW_LOW and len(candidates) > 0: target = candidates.pop() del(tmp_egress[target]) + tmp_egress.reverse() tmp_egress.reverse() while len(tmp_egress) > 0: From 2ed4ccc46ea6b5e9a66b71badbda46053469dae6 Mon Sep 17 00:00:00 2001 From: lling Date: Fri, 17 Nov 2017 00:02:41 -0800 Subject: [PATCH 11/24] conflict resolution --- .../protocols/network_layer_handler.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/asl_sdr_hackfest/protocols/network_layer_handler.py b/asl_sdr_hackfest/protocols/network_layer_handler.py index 53ad590..3afd1c4 100644 --- a/asl_sdr_hackfest/protocols/network_layer_handler.py +++ b/asl_sdr_hackfest/protocols/network_layer_handler.py @@ -111,15 +111,19 @@ def egress_cull(self): if cos == max_cos: candidates.append(i) - tmp_egress.reverse() - while len(tmp_egress) > EGRESS_WINDOW_LOW and len(candidates) > 0: + to_drop = len(tmp_egress) - EGRESS_WINDOW_LOW + + candidates.reverse() + while to_drop > 0 and len(candidates) > 0: + to_drop -= 1 target = candidates.pop() - del(tmp_egress[target]) - tmp_egress.reverse() + tmp_egress[target] = None tmp_egress.reverse() while len(tmp_egress) > 0: - self.out_queue.put(tmp_egress.pop()) + val = tmp_egress.pop() + if val is not None: + self.out_queue.put() culled_size = self.out_queue.qsize() print('Egress queue culled from {} to {}.'.format(init_size, culled_size)) From 9d0f3557d6661cd2c72c77a1df4ef768bad0b2cc Mon Sep 17 00:00:00 2001 From: lling Date: Thu, 16 Nov 2017 23:58:17 -0800 Subject: [PATCH 12/24] decode and dump mavlink message types --- asl_sdr_hackfest/pmt_mav_dump.py | 27 +++++++++++++++++++++++++++ asl_sdr_hackfest/zmq_sub.py | 8 ++++++++ 2 files changed, 35 insertions(+) create mode 100644 asl_sdr_hackfest/pmt_mav_dump.py diff --git a/asl_sdr_hackfest/pmt_mav_dump.py b/asl_sdr_hackfest/pmt_mav_dump.py new file mode 100644 index 0000000..f42d19c --- /dev/null +++ b/asl_sdr_hackfest/pmt_mav_dump.py @@ -0,0 +1,27 @@ +from pymavlink import mavutil +import pmt +import numpy + +class PMT_MAV_dump(object): + # NOTE: default arguments derived from MAVProxy's --master argument + def __init__(self, ip = "127.0.0.1", remote_port = 5760, protocol = 'tcp'): + print ('connecting to %s:%s:%d' % (protocol, ip, remote_port)) + self.mc = mavutil.mavlink_connection(device='%s:%s:%d' % (protocol, ip, remote_port), retries = 9000) + + # NOTE: have this method invert ZMQ_pub modules' send method + def method(self, pmt_cons): + s = pmt.deserialize_str(pmt_cons) + + if pmt.is_pair(s): + car = pmt.car(s) + cdr = pmt.to_python(pmt.cdr(s)) + cdr = numpy.getbuffer(cdr) + + meta = car + data = self.mc.mav.decode(bytearray(cdr)) + + # NOTE: to fully parse fields of message use this for reference: + # https://www.samba.org/tridge/UAV/pymavlink/apidocs/classIndex.html + return \ + "gr metadata: %s \ + message type: %s" % (meta, data.get_type()) diff --git a/asl_sdr_hackfest/zmq_sub.py b/asl_sdr_hackfest/zmq_sub.py index 7696b76..7cf024a 100644 --- a/asl_sdr_hackfest/zmq_sub.py +++ b/asl_sdr_hackfest/zmq_sub.py @@ -4,6 +4,8 @@ import binascii +import pmt_mav_dump as pmd + class ZMQ_sub(object): def __init__(self, portIn, timeout = 100): self._ipAddress = '127.0.0.1' @@ -13,6 +15,9 @@ def __init__(self, portIn, timeout = 100): self._socketIn = self._zmqContext.socket(zmq.SUB) self._socketIn.RCVTIMEO = timeout self._socketIn.connect('tcp://%s:%s' % (self._ipAddress,self._portIn)) + + self.decode = pmd.PMT_MAV_dump() + try: self._socketIn.setsockopt(zmq.SUBSCRIBE, '') # python2 except TypeError: @@ -24,4 +29,7 @@ def recv(self): pdu = pmt.deserialize_str(msg) cdr = pmt.to_python(pmt.cdr(pdu)) cdr = numpy.getbuffer(cdr) + + print self.decode.method(msg) + return cdr From 51103702d48d074e5e3c832eb412e16c5a4cbcce Mon Sep 17 00:00:00 2001 From: lling Date: Fri, 17 Nov 2017 10:11:01 -0800 Subject: [PATCH 13/24] cl args passed correctly --- apps/netCat.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/apps/netCat.py b/apps/netCat.py index 5d6126a..dd03ebf 100755 --- a/apps/netCat.py +++ b/apps/netCat.py @@ -60,15 +60,22 @@ def __init__(self): self.receiver = netCatReceiver() # usage: netCat.py (image_file) elif len(sys.argv) == 2: - self.sender = netCatSender(sys.argv[1]) + self.sender = netCatSender(img_fp=sys.argv[1]) self.receiver = None # usage: netCat.py (ip, port) elif len(sys.argv) == 3: self.sender = None - self.receiver = netCatReceiver() + self.receiver = netCatReceiver(\ + ip_address=sys.argv[1], \ + port=sys.argv[2] \ + ) # usage: netCat.py (image_file, ip, port) elif len(sys.argv) == 4: - self.sender = netCatSender(sys.argv[1]) + self.sender = netCatSender(\ + img_fp=sys.argv[1], \ + ip_address=sys.argv[2],\ + port=sys.argv[3] \ + ) self.receiver = None else: print("receive: ./netCat.py \n transmit: ./netCat.py cat_image") @@ -76,18 +83,17 @@ def __init__(self): def run(self): while True: try: - if len(sys.argv) > 1: + if self.sender is not None: self.sender.send() - else: + if self.receiver is not None: self.receiver.recv() except KeyboardInterrupt: - # BUG: sender will not call sit() on SIGINT nc.sit() break except: - if len(sys.argv) > 1: + if self.sender is not None: print "failed to send" - else: + if self.receive is not None: print "failed to receive" def sit(self): From ae0a8da737dba0c53169921658969504f18c1a67 Mon Sep 17 00:00:00 2001 From: Brandon Haines Date: Thu, 16 Nov 2017 04:28:00 -0500 Subject: [PATCH 14/24] Re-enabling RTP. Breaks SITL integration. --- asl_sdr_hackfest/postmaster.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/asl_sdr_hackfest/postmaster.py b/asl_sdr_hackfest/postmaster.py index ace17b0..8815083 100755 --- a/asl_sdr_hackfest/postmaster.py +++ b/asl_sdr_hackfest/postmaster.py @@ -77,8 +77,8 @@ def run(self): if srv['type'] == 'radio': self.frame_rx.ingest_data(data) elif srv['type'] == 'client': -# rtp_header = self.rtp_handler_tx.tx(srv['config']['ssrc'], 'gsm') # This is a byte array, not a header class -# data = rtp_header + data + rtp_header = self.rtp_handler_tx.tx(srv['config']['ssrc'], 'gsm') # This is a byte array, not a header class + data = rtp_header + data qos_header = QoS.header_calculate(data) # This is a header class, not a byte array qos_header.set_priority_code(srv['config']['qos']) data = qos_header.to_bytearray() + data @@ -88,15 +88,15 @@ def run(self): def frameDeframed(self, data): cls, data = QoS.header_consume(data) -# rtp_header, data = self.rtp_handler_rx.header_consume(data) -# ssrc = rtp_header.get_ssrc() + rtp_header, data = self.rtp_handler_rx.header_consume(data) + ssrc = rtp_header.get_ssrc() for srv in self.services: srv = self.services[srv] if srv['type'] == 'client': -# conf = srv['config'] -# if conf is not None and ssrc == conf['ssrc']: - srv['service'].outputData(data) + conf = srv['config'] + if conf is not None and ssrc == conf['ssrc']: + srv['service'].outputData(data) def stop(self): From 17790f83fc48f583d175d599cc187d64be655018 Mon Sep 17 00:00:00 2001 From: Dario Date: Thu, 16 Nov 2017 10:01:29 -0800 Subject: [PATCH 15/24] Fixed ssrc numbers and timestamp type issues. Also removed upper bound on incoming rtp timestamps. --- asl_sdr_hackfest/protocols/rtp_handler.py | 67 +++++++++++++++++------ 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/asl_sdr_hackfest/protocols/rtp_handler.py b/asl_sdr_hackfest/protocols/rtp_handler.py index 51dff98..85140c8 100644 --- a/asl_sdr_hackfest/protocols/rtp_handler.py +++ b/asl_sdr_hackfest/protocols/rtp_handler.py @@ -24,10 +24,12 @@ def tx(self, ssrc, p_type): # Fields the handler will fill if ssrc not in self.streams.keys(): - r.ssrc = self.new_tx_stream(p_type) - else: - r.ssrc = ssrc - + r.ssrc = self.new_tx_stream(p_type, ssrc) + #else: + # r.ssrc = ssrc + r.ssrc = ssrc + print('ssrc = ', ssrc) + print('r.ssrc = ', r.ssrc) r.seq_num = self.streams[r.ssrc].get_seq_num() r.payload_type = self.streams[r.ssrc].payload_type r.timestamp = self.streams[r.ssrc].create_timestamp32() @@ -45,13 +47,14 @@ def rx(self, rtp_bytearray): return r - def new_tx_stream(self, p_type): - new_ssrc = random.randint(1, self.INT32_MAX) - if new_ssrc in self.streams.keys(): - self.new_tx_stream(p_type) - else: - self.streams[new_ssrc] = Stream(new_ssrc, p_type, tx=True) - return new_ssrc + def new_tx_stream(self, p_type, ssrc): + self.streams[ssrc] = Stream(ssrc, p_type, tx=True) + #new_ssrc = random.randint(1, self.INT32_MAX) + #if new_ssrc in self.streams.keys(): + # self.new_tx_stream(p_type) + #else: + # self.streams[new_ssrc] = Stream(new_ssrc, p_type, tx=True) + #return new_ssrc def new_rx_stream(self, ssrc, p_type): self.streams[ssrc] = Stream(ssrc, p_type, tx=False) @@ -92,6 +95,26 @@ def create_timestamp32(self): self.last_timestamp = struct.pack('!L', t) return self.last_timestamp + def unpack_timestamp32(self, ts): + new_ts = 0 + try: + new_ts = ts.uint + return new_ts + except AttributeError: + print("timestamp was not a BitArray") + pass + try: + new_ts = struct.unpack('!L', ts) + return new_ts + except stuct.error: + print("timestamp was not a valid struct") + pass + if isinstance(ts, int): + print("timestamp was always an int") + return ts + print("Weird timestamp. Returning zero") + return 0 + ''' For rx side. Updates the last_seq_num and last_timestamp for the stream @@ -137,20 +160,28 @@ def check_seq_num_window(self, seq_num): return True def check_timestamp_window(self, ts): - time_top = self.last_timestamp + self.time_win_top + print("Check timestamp window") + print("Last timestamp = ", self.unpack_timestamp32(self.last_timestamp)) + print("Incoming timestamp = ", self.unpack_timestamp32(ts)) + print("UINT32_MAX = ", self.UINT32_MAX) + time_top = self.unpack_timestamp32(self.last_timestamp) + self.time_win_top + ts_in = self.unpack_timestamp32(ts) + print("Top of window = ", time_top) if (time_top < self.UINT32_MAX): - if ( ts > (time_top)): + if ( ts_in > (time_top)): print("Timestamp is too big") - return False + return True + #return False #if ( ts < (struct.unpack('L', struct.pack('L', (self.last_timestamp - self.time_win_btm)) ))): - if ( ts < (self.last_timestamp - self.time_win_btm)): + if ( ts_in < (self.unpack_timestamp32(self.last_timestamp) - self.time_win_btm)): print("Timestamp is too small") return False else: - if ( ts > (time_top - self.UINT32_MAX)): + if ( ts_in > (time_top - self.UINT32_MAX)): print("Timestamp is too big") - return False - if ( ts < (self.last_timestamp - self.time_win_btm)): + return True + #return False + if ( ts_in < (self.unpack_timestamp32(self.last_timestamp) - self.time_win_btm)): print("Timestamp is too small") return False return True From 3effddb2fcc4d1cf229ea838af2ad5f39d23dcee Mon Sep 17 00:00:00 2001 From: Dario Date: Thu, 16 Nov 2017 10:41:46 -0800 Subject: [PATCH 16/24] Cleaned up some prints --- asl_sdr_hackfest/protocols/protocol.py | 2 +- asl_sdr_hackfest/protocols/rtp.py | 6 ++++++ asl_sdr_hackfest/protocols/rtp_handler.py | 9 ++------- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/asl_sdr_hackfest/protocols/protocol.py b/asl_sdr_hackfest/protocols/protocol.py index 86f0c3d..76ecbaf 100644 --- a/asl_sdr_hackfest/protocols/protocol.py +++ b/asl_sdr_hackfest/protocols/protocol.py @@ -24,7 +24,7 @@ def bin_formater(self, value, length): return "0b" + f #if (isinstance(value, str) and (length == 32)): elif isinstance(value, str): - print("Packed str = ", ("0b" + ''.join('{0:08b}'.format(x, 'b') for x in bytearray(value)))) + #print("Packed str = ", ("0b" + ''.join('{0:08b}'.format(x, 'b') for x in bytearray(value)))) return ("0b" + ''.join('{0:08b}'.format(x, 'b') for x in bytearray(value))) else: try: diff --git a/asl_sdr_hackfest/protocols/rtp.py b/asl_sdr_hackfest/protocols/rtp.py index 2ebd118..dcc4aab 100644 --- a/asl_sdr_hackfest/protocols/rtp.py +++ b/asl_sdr_hackfest/protocols/rtp.py @@ -50,12 +50,18 @@ def to_bitarray(self): ba = BitArray(self.TWO2) ba.append(self.TRUE) if self.padding else ba.append(self.FALSE) ba.append(self.TRUE) if self.extension else ba.append(self.FALSE) + #print("Format csrc_count") ba.append(self.bin_formater(self.csrc_count, 4)) ba.append(self.TRUE) if self.marker else ba.append(self.FALSE) + #print("Format payload_type") ba.append(self.bin_formater(self.payload_type, 7)) + #print("Format seq_num") ba.append(self.bin_formater(self.seq_num, 16)) + #print("Format timestamp") ba.append(self.bin_formater(self.timestamp, 32)) + #print("Format ssrc") ba.append(self.bin_formater(self.ssrc, 32)) + #print("Format csrc") ba.append(self.bin_formater(self.csrc, 32)) return ba diff --git a/asl_sdr_hackfest/protocols/rtp_handler.py b/asl_sdr_hackfest/protocols/rtp_handler.py index 85140c8..7cf5048 100644 --- a/asl_sdr_hackfest/protocols/rtp_handler.py +++ b/asl_sdr_hackfest/protocols/rtp_handler.py @@ -28,8 +28,6 @@ def tx(self, ssrc, p_type): #else: # r.ssrc = ssrc r.ssrc = ssrc - print('ssrc = ', ssrc) - print('r.ssrc = ', r.ssrc) r.seq_num = self.streams[r.ssrc].get_seq_num() r.payload_type = self.streams[r.ssrc].payload_type r.timestamp = self.streams[r.ssrc].create_timestamp32() @@ -128,6 +126,7 @@ def update(self, seq_num, ts): self.last_seq_num = seq_num else: print("Received invalid sequence number for this stream") + print("May want to drop it.") #return if self.check_timestamp_window(ts): self.last_timestamp = ts @@ -138,6 +137,7 @@ def update(self, seq_num, ts): else: # Outside timestamp window print("Received invalid timestamp for this stream") + print("May want to drop it.") #return def check_seq_num_window(self, seq_num): @@ -160,13 +160,8 @@ def check_seq_num_window(self, seq_num): return True def check_timestamp_window(self, ts): - print("Check timestamp window") - print("Last timestamp = ", self.unpack_timestamp32(self.last_timestamp)) - print("Incoming timestamp = ", self.unpack_timestamp32(ts)) - print("UINT32_MAX = ", self.UINT32_MAX) time_top = self.unpack_timestamp32(self.last_timestamp) + self.time_win_top ts_in = self.unpack_timestamp32(ts) - print("Top of window = ", time_top) if (time_top < self.UINT32_MAX): if ( ts_in > (time_top)): print("Timestamp is too big") From 393566f594b2dc4cf0dd0c3f2c81326389878d95 Mon Sep 17 00:00:00 2001 From: Dario Date: Thu, 16 Nov 2017 12:03:54 -0800 Subject: [PATCH 17/24] Resolved rtp handler stripping too much data. Successfully ran control, sitl, flow, and run_all. Stack seems good. All that is left with rtp is take advantage of seq numbers and timestamps. --- asl_sdr_hackfest/postmaster.py | 1 - asl_sdr_hackfest/protocols/rtp.py | 3 +++ asl_sdr_hackfest/protocols/rtp_handler.py | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/asl_sdr_hackfest/postmaster.py b/asl_sdr_hackfest/postmaster.py index 8815083..cab684a 100755 --- a/asl_sdr_hackfest/postmaster.py +++ b/asl_sdr_hackfest/postmaster.py @@ -87,7 +87,6 @@ def run(self): def frameDeframed(self, data): cls, data = QoS.header_consume(data) - rtp_header, data = self.rtp_handler_rx.header_consume(data) ssrc = rtp_header.get_ssrc() diff --git a/asl_sdr_hackfest/protocols/rtp.py b/asl_sdr_hackfest/protocols/rtp.py index dcc4aab..b2037ac 100644 --- a/asl_sdr_hackfest/protocols/rtp.py +++ b/asl_sdr_hackfest/protocols/rtp.py @@ -16,7 +16,10 @@ class RTP(Protocol): #34 : 'h263' } + # HEADER_SIZE in bits HEADER_SIZE = 128 + # HEADER_LENGTH in bytes + HEADER_LENGTH = 16 def __init__(self, *args, **kwargs): self.version = kwargs.get('version') diff --git a/asl_sdr_hackfest/protocols/rtp_handler.py b/asl_sdr_hackfest/protocols/rtp_handler.py index 7cf5048..401126f 100644 --- a/asl_sdr_hackfest/protocols/rtp_handler.py +++ b/asl_sdr_hackfest/protocols/rtp_handler.py @@ -58,8 +58,8 @@ def new_rx_stream(self, ssrc, p_type): self.streams[ssrc] = Stream(ssrc, p_type, tx=False) def header_consume(self, data): - rtp_header = self.rx(data[:RTP.HEADER_SIZE]) - return (rtp_header, data[RTP.HEADER_SIZE:]) + rtp_header = self.rx(data[:RTP.HEADER_LENGTH]) + return (rtp_header, data[RTP.HEADER_LENGTH:]) class Stream(object): def __init__(self, ssrc, p_type, tx=False): From abea80f6d8adc71a1d38a8787036aff4a83a3034 Mon Sep 17 00:00:00 2001 From: Dario Date: Thu, 16 Nov 2017 13:26:34 -0800 Subject: [PATCH 18/24] Postmaster will now drop packets ir RTP seq_num is not valid. --- asl_sdr_hackfest/postmaster.py | 21 +++++++++++++-------- asl_sdr_hackfest/protocols/rtp_handler.py | 20 +++++++++++--------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/asl_sdr_hackfest/postmaster.py b/asl_sdr_hackfest/postmaster.py index cab684a..6fc64e0 100755 --- a/asl_sdr_hackfest/postmaster.py +++ b/asl_sdr_hackfest/postmaster.py @@ -87,15 +87,20 @@ def run(self): def frameDeframed(self, data): cls, data = QoS.header_consume(data) - rtp_header, data = self.rtp_handler_rx.header_consume(data) - ssrc = rtp_header.get_ssrc() + #rtp_header is an instance of RTP + rtp_header_and_validity, data = self.rtp_handler_rx.header_consume(data) + rtp_header = rtp_header_and_validity[0] + validity = rtp_header_and_validity[1] + # Seq number is valid, process data + if validity: + ssrc = rtp_header.get_ssrc() + for srv in self.services: + srv = self.services[srv] + if srv['type'] == 'client': + conf = srv['config'] + if conf is not None and ssrc == conf['ssrc']: + srv['service'].outputData(data) - for srv in self.services: - srv = self.services[srv] - if srv['type'] == 'client': - conf = srv['config'] - if conf is not None and ssrc == conf['ssrc']: - srv['service'].outputData(data) def stop(self): diff --git a/asl_sdr_hackfest/protocols/rtp_handler.py b/asl_sdr_hackfest/protocols/rtp_handler.py index 401126f..306fad1 100644 --- a/asl_sdr_hackfest/protocols/rtp_handler.py +++ b/asl_sdr_hackfest/protocols/rtp_handler.py @@ -41,8 +41,8 @@ def rx(self, rtp_bytearray): r.from_bytearray(rtp_bytearray) if r.ssrc not in self.streams.keys(): self.new_rx_stream(r.ssrc, r.payload_type) - self.streams[r.ssrc].update(r.seq_num, r.timestamp) - return r + valid = self.streams[r.ssrc].update(r.seq_num, r.timestamp) + return (r, valid) def new_tx_stream(self, p_type, ssrc): @@ -58,8 +58,8 @@ def new_rx_stream(self, ssrc, p_type): self.streams[ssrc] = Stream(ssrc, p_type, tx=False) def header_consume(self, data): - rtp_header = self.rx(data[:RTP.HEADER_LENGTH]) - return (rtp_header, data[RTP.HEADER_LENGTH:]) + rtp_header_and_validity = self.rx(data[:RTP.HEADER_LENGTH]) + return (rtp_header_and_validity, data[RTP.HEADER_LENGTH:]) class Stream(object): def __init__(self, ssrc, p_type, tx=False): @@ -121,24 +121,26 @@ def update(self, seq_num, ts): if ((self.last_seq_num == 0) and (self.last_timestamp == 10000)): self.last_seq_num = seq_num self.last_timestamp = ts - return + return True if self.check_seq_num_window(seq_num): self.last_seq_num = seq_num else: print("Received invalid sequence number for this stream") print("May want to drop it.") - #return + return False if self.check_timestamp_window(ts): self.last_timestamp = ts + return True elif (self.last_timestamp == 10000): print("Reinitialize stream ", self.ssrc) self.last_seq_num = seq_num self.last_timestamp = ts + return True else: # Outside timestamp window print("Received invalid timestamp for this stream") print("May want to drop it.") - #return + return True def check_seq_num_window(self, seq_num): win_top = self.last_seq_num + self.window_top @@ -164,7 +166,7 @@ def check_timestamp_window(self, ts): ts_in = self.unpack_timestamp32(ts) if (time_top < self.UINT32_MAX): if ( ts_in > (time_top)): - print("Timestamp is too big") + #print("Timestamp is too big") return True #return False #if ( ts < (struct.unpack('L', struct.pack('L', (self.last_timestamp - self.time_win_btm)) ))): @@ -173,7 +175,7 @@ def check_timestamp_window(self, ts): return False else: if ( ts_in > (time_top - self.UINT32_MAX)): - print("Timestamp is too big") + #print("Timestamp is too big") return True #return False if ( ts_in < (self.unpack_timestamp32(self.last_timestamp) - self.time_win_btm)): From 41a268f3432c8c204c30b46f9354a0b56af2aa93 Mon Sep 17 00:00:00 2001 From: Dario Date: Thu, 16 Nov 2017 22:32:14 -0800 Subject: [PATCH 19/24] Access point scanner and visualization stuff. --- python-iwlist/iwlist.py | 39 +++++ python-iwlist/readme.md | 56 +++++++ python-iwlist/sample.py | 72 +++++++++ python-iwlist/scanner.py | 99 ++++++++++++ python-iwlist/test.py | 63 ++++++++ python-iwlist/test/basic/scan.txt | 52 +++++++ python-iwlist/test/basic/vectors.json | 28 ++++ python-iwlist/test/issue_5/scan.txt | 192 ++++++++++++++++++++++++ python-iwlist/test/issue_5/vectors.json | 93 ++++++++++++ python-iwlist/update_demo.py | 36 +++++ python-iwlist/viz.py | 26 ++++ 11 files changed, 756 insertions(+) create mode 100644 python-iwlist/iwlist.py create mode 100644 python-iwlist/readme.md create mode 100644 python-iwlist/sample.py create mode 100644 python-iwlist/scanner.py create mode 100644 python-iwlist/test.py create mode 100644 python-iwlist/test/basic/scan.txt create mode 100644 python-iwlist/test/basic/vectors.json create mode 100644 python-iwlist/test/issue_5/scan.txt create mode 100644 python-iwlist/test/issue_5/vectors.json create mode 100644 python-iwlist/update_demo.py create mode 100644 python-iwlist/viz.py diff --git a/python-iwlist/iwlist.py b/python-iwlist/iwlist.py new file mode 100644 index 0000000..7f56dd7 --- /dev/null +++ b/python-iwlist/iwlist.py @@ -0,0 +1,39 @@ +import re +import subprocess + +cellNumberRe = re.compile(r"^Cell\s+(?P.+)\s+-\s+Address:\s(?P.+)$") +regexps = [ + re.compile(r"^ESSID:\"(?P.*)\"$"), + re.compile(r"^Protocol:(?P.+)$"), + re.compile(r"^Mode:(?P.+)$"), + re.compile(r"^Frequency:(?P[\d.]+) (?P.+) \(Channel (?P\d+)\)$"), + re.compile(r"^Encryption key:(?P.+)$"), + re.compile(r"^Quality=(?P\d+)/(?P\d+)\s+Signal level=(?P.+) d.+$"), + re.compile(r"^Signal level=(?P\d+)/(?P\d+).*$"), +] + +# Runs the comnmand to scan the list of networks. +# Must run as super user. +# Does not specify a particular device, so will scan all network devices. +def scan(interface='wlan0'): + cmd = ["iwlist", interface, "scan"] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + points = proc.stdout.read().decode('utf-8') + return points + +# Parses the response from the command "iwlist scan" +def parse(content): + cells = [] + lines = content.split('\n') + for line in lines: + line = line.strip() + cellNumber = cellNumberRe.search(line) + if cellNumber is not None: + cells.append(cellNumber.groupdict()) + continue + for expression in regexps: + result = expression.search(line) + if result is not None: + cells[-1].update(result.groupdict()) + continue + return cells diff --git a/python-iwlist/readme.md b/python-iwlist/readme.md new file mode 100644 index 0000000..27b1594 --- /dev/null +++ b/python-iwlist/readme.md @@ -0,0 +1,56 @@ +# python-iwlist + +Python scanner and parser for wireless networks + +## Usage + +``` +>>> import iwlist +>>> content = iwlist.scan(interface='wlan0') +>>> cells = iwlist.parse(content) +``` + +## Output + +``` +>>> print cells + +[ + { + "cellnumber": "01", + "mac": "00:11:22:33:44:55", + "essid": "Wireless Network 1", + "mode": "Master", + "frequency": "2.437", + "frequency_units": "GHz", + "channel": "6", + "encryption": "on", + "signal_level": "32", + "signal_total": "70", + "db": "-78" + }, + { + "cellnumber": "02", + "mac": "FE:DC:BA:98:76:54", + "essid": "AB-Guest-Wifi", + "protocol": "IEEE 802.11bgn", + "mode": "Master", + "frequency": "2.462", + "frequency_units": "GHz", + "channel": "11", + "encryption": "on", + "signal_level": "43", + "signal_total": "100" + } +] +``` + +## Notes + +Any script calling `iwlist.scan()` should run as superuser to get live values. + +## Tests + +``` +$ python test.py +``` diff --git a/python-iwlist/sample.py b/python-iwlist/sample.py new file mode 100644 index 0000000..7d8c979 --- /dev/null +++ b/python-iwlist/sample.py @@ -0,0 +1,72 @@ +#!/usr/bin/python + +import matplotlib.pyplot as plt +import numpy as np +import time + +class access_point(object): + def __init__(self, mac, essid, channel, encryption, sig_level): + self.mac = mac + self.essid = essid + self.channel = channel + self.encryption = encryption + self.sig_level = sig_level + + +access_points = () +ap1 = access_point('abc', 'nasawifi', 11, True, 10) +ap2 = access_point('abc', 'nasawifi', 11, True, 33) +ap3 = access_point('abc', 'nasawifi', 11, True, 25) +ap4 = access_point('abc', 'nasawifi', 11, True, 24) +ap5 = access_point('abc', 'nasawifi', 11, True, 40) +access_points = access_points + (ap1,) +access_points = access_points + (ap2,) +access_points = access_points + (ap3,) +access_points = access_points + (ap4,) +access_points = access_points + (ap5,) +print("Access_points length = " + str(len(access_points))) + +N=5 +mens_means = (20,35, 30, 35, 27) +men_std = (2,3,4,1,2) + +ind = np.arange(N) # the x locations for the groups +width = 0.35 + +fig, ax = plt.subplots() +rects1 = ax.bar(ind, mens_means, width, color='r', yerr=men_std) + +women_means = (25, 32, 34, 20, 25) +women_std = (3, 5, 2, 3, 3) + + +ap_means = (20,35, 30, 35, 27) +#other_means = (32, 61, 20, 25, 12) + +rects2 = ax.bar(ind + width, women_means, width, color='y', yerr=women_std) + +rects3 = ax.bar(ind + width + width, ap_means, width, color='b', yerr=women_std) + +ax.set_ylabel('dBs') +ax.set_title('Scores by group and gender') +ax.set_xticks(ind + width / 2) +ax.set_xticklabels(('1','2', '3', '4', '5', '6', '7', '8', '9', '10', '11')) + +ax.legend((rects1[0], rects2[0]), ('Men', 'Women')) + +def autolabel(rects): + for rect in rects: + height = rect.get_height() + ax.text(rect.get_x() + rect.get_width()/2., 1.05*height, + '%d' % int(height), + ha='center', va='bottom') + +autolabel(rects1) +autolabel(rects2) +autolabel(rects3) + +plt.show() + +time.sleep(3) +women_means = (30, 37, 34, 29, 37) + diff --git a/python-iwlist/scanner.py b/python-iwlist/scanner.py new file mode 100644 index 0000000..8245af7 --- /dev/null +++ b/python-iwlist/scanner.py @@ -0,0 +1,99 @@ +#!/usr/bin/python + + +import iwlist +import time +import psutil +import struct + + +def pack_ap(ap): + #s = struct.Struct() + s = struct.pack('!b', int(ap['signal_level_dBm'])) + if ap['encryption'] is 'yes': + s += struct.pack('!b', ((int(ap['channel']) & 0x0F) | 128)) + else: + s += struct.pack('!b', ((int(ap['channel']) & 0x0F) | 0)) + s += struct.pack('!b', int(ap['signal_level_dBm'])) + s += struct.pack('!s', "".join(ap['mac'].split())) + s += struct.pack('!s', ap['essid']) + return s + + +INTERFACE='wlp2s0' + +access_points = dict() +keys= ['essid', 'encryption', 'signal_level_dBm', 'channel'] +keys_short = ['essid', 'encryption', 'channel'] + +new_stuff = dict() +#content = iwlist.scan(interface=INTERFACE) +#cells = iwlist.parse(content) + +#while True: +#for a in range (1,5): +while True: + new_stuff = list() + content = iwlist.scan(interface=INTERFACE) + cells = iwlist.parse(content) + for c in cells: + if c['mac'] not in access_points.keys(): + ap = dict() + for k in keys: + try: + ap[k] = str(c[k]) + print(k, c[k]) + except KeyError: + pass + ap['last_time'] = time.time() + ap['mac'] = str(c['mac']) + print("Mac = '" + ap['mac'] + "'") + access_points[c['mac']] = ap + new_stuff.append(pack_ap(ap)) + print + else: + #print(c['mac'] + " is in access_points.keys ") + if (time.time()-access_points[c['mac']]['last_time']) > 30: + print("Haven't heard from " + str(access_points[c['mac']])) + print("in 30 seconds. Removing.") + del(access_points[c['mac']]) + continue + for k in keys: #keys_short: + access_points[c['mac']]['last_time'] = time.time() + if access_points[c['mac']][k] != c[k]: #'mac']:#[k]: + access_points[c['mac']][k] = c[k] + new_stuff.append(pack_ap(access_points[c['mac']])) + #new_stuff[c['mac']] = access_points[c['mac']] + #print(c['mac'] + ' ' + k + 'changed') + #print('Old = ' + access_points[c['mac']][k]) + #print('New = ' + c[k])#'mac'][k]) + #print + print("CYCLE COMPLETE") + + print("CPU use percentage = " + str(psutil.cpu_percent(interval=None))) + print("Memory usage = " + str(psutil.virtual_memory().percent) + "%") + print("Battery = " + str(psutil.sensors_battery().percent) + "%") + print("Length of access_points as bytes = " + str(len(bytes(access_points)))) + print("Length of new_stuff as bytes = " + str(len(bytes(new_stuff)))) + time.sleep(1) + +print +print("Number of access points= " + str(len(access_points.keys()))) + +print("Access_points keys:") +print(access_points.keys()) +for k in access_points.keys(): #.keys(): + #print(type(k)) + print k + print access_points[k]['signal_level_dBm'] + print access_points[k] + +print +print("All access points:") +print(access_points) +#for ap in access_points: +# print(ap['signal_level_dBm']) #'signal_level_dBm']) +# continue + + + diff --git a/python-iwlist/test.py b/python-iwlist/test.py new file mode 100644 index 0000000..397fd89 --- /dev/null +++ b/python-iwlist/test.py @@ -0,0 +1,63 @@ +import json +import os +import iwlist +import unittest + +def fileContent(filename): + f = open(filename) + content = f.read() + f.close() + return content + +class TestParse(unittest.TestCase): + + def setUp(self): + dirs = os.listdir("test") + self.cases = [] + for d in dirs: + scanFile = os.path.join("test", d, "scan.txt") + vectorsFile = os.path.join("test", d, "vectors.json") + case = { + "name": d, + "parsed": iwlist.parse(fileContent(scanFile)), + "expected": json.loads(fileContent(vectorsFile)), + } + self.cases.append(case) + + def tearDown(self): + self.cases = [] + + def test_parse_length(self): + for case in self.cases: + self.assertEqual(len(case["expected"]), len(case["parsed"])) + + def test_cells_have_all_expected_keys(self): + for case in self.cases: + for i in range(len(case["expected"])): + e = case["expected"][i] + p = case["parsed"][i] + for key in e: + msg = "\nkey is in expected but missing in parsed:\nkey: %s\ntestdir: %s\ncellnumber: %s" % (key, case["name"], e["cellnumber"]) + self.assertTrue(key in p, msg) + + def test_cells_dont_have_extra_keys(self): + for case in self.cases: + for i in range(len(case["expected"])): + e = case["expected"][i] + p = case["parsed"][i] + for key in p: + msg = "\nkey was parsed but missing in expected:\nkey: %s\ntestdir: %s\ncellnumber: %s" % (key, case["name"], e["cellnumber"]) + self.assertTrue(key in e, msg) + + def test_cells_have_expected_values(self): + for case in self.cases: + for i in range(len(case["expected"])): + e = case["expected"][i] + for k in e: + ev = e[k] + pv = case["parsed"][i][k] + msg = "\nwrong value for field:\ntestdir: %s\nfield: %s\ncellnumber: %s\nexpected: %s\nactual: %s" % (case["name"], k, e["cellnumber"], ev, pv) + self.assertTrue(ev == pv, msg) + +if __name__ == '__main__': + unittest.main() diff --git a/python-iwlist/test/basic/scan.txt b/python-iwlist/test/basic/scan.txt new file mode 100644 index 0000000..022e8c7 --- /dev/null +++ b/python-iwlist/test/basic/scan.txt @@ -0,0 +1,52 @@ +wlan0 Scan completed : + Cell 01 - Address: 00:11:22:33:44:55 + Channel:6 + Frequency:2.437 GHz (Channel 6) + Quality=32/70 Signal level=-78 dBm + Encryption key:on + ESSID:"Wireless Network 1" + Bit Rates:1 Mb/s; 2 Mb/s; 5.5 Mb/s; 11 Mb/s; 9 Mb/s + 18 Mb/s; 36 Mb/s; 54 Mb/s + Bit Rates:6 Mb/s; 12 Mb/s; 24 Mb/s; 48 Mb/s + Mode:Master + Extra:tsf=0000048ab37999f9 + Extra: Last beacon: 276ms ago + IE: Unknown: 0006466F6E7A6965 + IE: Unknown: 010882848B961224486C + IE: Unknown: 030106 + IE: Unknown: 2A0102 + IE: Unknown: 32040C183060 + IE: Unknown: 2D1A6E1017FFFFFF0001000000000000000000000000000000000000 + IE: Unknown: 3D1606070700000000000000000000000000000000000000 + IE: Unknown: 3E0100 + IE: IEEE 802.11i/WPA2 Version 1 + Group Cipher : CCMP + Pairwise Ciphers (1) : CCMP + Authentication Suites (1) : PSK + IE: Unknown: DD180050F2020101000003A4000027A4000042435E0062322F00 + IE: Unknown: 0B05000D467A12 + IE: Unknown: 7F0101 + IE: Unknown: DD07000C4300000000 + IE: Unknown: DD1E00904C336E1017FFFFFF0001000000000000000000000000000000000000 + IE: Unknown: DD1A00904C3406070700000000000000000000000000000000000000 + IE: Unknown: DD9D0050F204104A0001101044000101103B00010310470010BC329E001DD811B28601001DAAB28FF81021001852616C696E6B20546563686E6F6C6F67792C20436F72702E1023001C52616C696E6B20576972656C6573732041636365737320506F696E74102400065254323836301042000831323334353637381054000800060050F20400011011000952616C696E6B415053100800020084103C000100 + Cell 02 - Address: FE:DC:BA:98:76:54 + ESSID:"AB-Wireless-Guest" + Protocol:IEEE 802.11bgn + Mode:Master + Frequency:2.462 GHz (Channel 11) + Encryption key:on + Bit Rates:1 Mb/s; 2 Mb/s; 5.5 Mb/s; 11 Mb/s; 18 Mb/s + 24 Mb/s; 36 Mb/s; 54 Mb/s; 6 Mb/s; 9 Mb/s + 12 Mb/s; 48 Mb/s + Extra:wpa_ie=dd1c0050f20101000050f20202000050f2040050f20201000050f2020c00 + IE: WPA Version 1 + Group Cipher : TKIP + Pairwise Ciphers (2) : CCMP TKIP + Authentication Suites (1) : PSK + Extra:rsn_ie=30180100000fac020200000fac04000fac020100000fac020c00 + IE: IEEE 802.11i/WPA2 Version 1 + Group Cipher : TKIP + Pairwise Ciphers (2) : CCMP TKIP + Authentication Suites (1) : PSK + Signal level=43/100 diff --git a/python-iwlist/test/basic/vectors.json b/python-iwlist/test/basic/vectors.json new file mode 100644 index 0000000..4eb6acc --- /dev/null +++ b/python-iwlist/test/basic/vectors.json @@ -0,0 +1,28 @@ +[ + { + "cellnumber": "01", + "mac": "00:11:22:33:44:55", + "essid": "Wireless Network 1", + "mode": "Master", + "frequency": "2.437", + "frequency_units": "GHz", + "channel": "6", + "encryption": "on", + "signal_quality": "32", + "signal_total": "70", + "signal_level_dBm": "-78" + }, + { + "cellnumber": "02", + "mac": "FE:DC:BA:98:76:54", + "essid": "AB-Wireless-Guest", + "protocol": "IEEE 802.11bgn", + "mode": "Master", + "frequency": "2.462", + "frequency_units": "GHz", + "channel": "11", + "encryption": "on", + "signal_quality": "43", + "signal_total": "100" + } +] diff --git a/python-iwlist/test/issue_5/scan.txt b/python-iwlist/test/issue_5/scan.txt new file mode 100644 index 0000000..fc3da5e --- /dev/null +++ b/python-iwlist/test/issue_5/scan.txt @@ -0,0 +1,192 @@ + Cell 10 - Address: A0:21:B7:6A:13:64 + Channel:6 + Frequency:2.437 GHz (Channel 6) + Quality=22/70 Signal level=-88 dBm + Encryption key:on + ESSID:"Frank Kwok" + Bit Rates:1 Mb/s; 2 Mb/s; 5.5 Mb/s; 11 Mb/s; 18 Mb/s + 24 Mb/s; 36 Mb/s; 54 Mb/s + Bit Rates:6 Mb/s; 9 Mb/s; 12 Mb/s; 48 Mb/s + Mode:Master + Extra:tsf=0000006dc301c102 + Extra: Last beacon: 14340ms ago + IE: Unknown: 000A4672616E6B204B776F6B + IE: Unknown: 010882840B162430486C + IE: Unknown: 030106 + IE: Unknown: 2A0100 + IE: Unknown: 2F0100 + IE: IEEE 802.11i/WPA2 Version 1 + Group Cipher : CCMP + Pairwise Ciphers (1) : CCMP + Authentication Suites (1) : PSK + IE: Unknown: 32040C121860 + IE: Unknown: 2D1A6C181BFF00000000000000000000000000000000000000000000 + IE: Unknown: 3D1606081600000000000000000000000000000000000000 + IE: Unknown: 4A0E14000A002C01C800140005001900 + IE: Unknown: 7F0101 + IE: Unknown: DD7F0050F204104A00011010440001021041000100103B00010310470010B4A0E34ADC6D662B05F4894A88D5EB791021000D4E4554474541522C20496E632E10230009574E5231303030763310240009574E523130303076331042000538333235381054000800060050F204000110110009574E52313030307633100800020084 + IE: Unknown: DD090010180201F0050000 + IE: Unknown: DD180050F2020101800003A4000027A4000042435E0062322F00 + IE: Unknown: DD1E00904C336C181BFF00000000000000000000000000000000000000000000 + IE: Unknown: DD1A00904C3406081600000000000000000000000000000000000000 + Cell 11 - Address: B8:08:D7:F7:B8:24 + Channel:9 + Frequency:2.452 GHz (Channel 9) + Quality=61/70 Signal level=-49 dBm + Encryption key:on + ESSID:"DODO-B81E" + Bit Rates:1 Mb/s; 2 Mb/s; 5.5 Mb/s; 11 Mb/s; 18 Mb/s + 24 Mb/s; 36 Mb/s; 54 Mb/s + Bit Rates:6 Mb/s; 9 Mb/s; 12 Mb/s; 48 Mb/s + Mode:Master + Extra:tsf=0000012bfb8b5aa5 + Extra: Last beacon: 14340ms ago + IE: Unknown: 0009444F444F2D42383145 + IE: Unknown: 010882848B962430486C + IE: Unknown: 030109 + IE: Unknown: 2A0104 + IE: Unknown: 2F0104 + IE: IEEE 802.11i/WPA2 Version 1 + Group Cipher : CCMP + Pairwise Ciphers (1) : CCMP + Authentication Suites (1) : PSK + IE: Unknown: 32040C121860 + IE: Unknown: 2D1AAC181BFFFF000000000000000000000000000000000000000000 + IE: Unknown: 3D1609001700000000000000000000000000000000000000 + IE: Unknown: 4A0E14000A002C01C800140005001900 + IE: Unknown: 7F080100000000000040 + IE: Unknown: DD1000E0FC80000000230000000000000000 + IE: Unknown: DD090010180209000C0000 + IE: WPA Version 1 + Group Cipher : CCMP + Pairwise Ciphers (1) : CCMP + Authentication Suites (1) : PSK + IE: Unknown: DD180050F2020101800003A4000027A4000042435E0062322F00 + Cell 12 - Address: 00:60:64:ED:61:B4 + Channel:9 + Frequency:2.452 GHz (Channel 9) + Quality=57/70 Signal level=-53 dBm + Encryption key:on + ESSID:"Tengeriin ulaach" + Bit Rates:1 Mb/s; 2 Mb/s; 5.5 Mb/s; 11 Mb/s; 9 Mb/s + 18 Mb/s; 36 Mb/s; 54 Mb/s + Bit Rates:6 Mb/s; 12 Mb/s; 24 Mb/s; 48 Mb/s + Mode:Master + Extra:tsf=0000009dcdb77fe2 + Extra: Last beacon: 14340ms ago + IE: Unknown: 001054656E67657269696E20756C61616368 + IE: Unknown: 010882848B961224486C + IE: Unknown: 030109 + IE: Unknown: 2A0100 + IE: Unknown: 32040C183060 + IE: Unknown: 2D1AEE1117FFFF0000010000000000000000000000000C0000000000 + IE: Unknown: 3D1609000600000000000000000000000000000000000000 + IE: WPA Version 1 + Group Cipher : TKIP + Pairwise Ciphers (2) : TKIP CCMP + Authentication Suites (1) : PSK + IE: IEEE 802.11i/WPA2 Version 1 + Group Cipher : TKIP + Pairwise Ciphers (2) : TKIP CCMP + Authentication Suites (1) : PSK + IE: Unknown: DD180050F2020101000003A4000027A4000042435E0062322F00 + IE: Unknown: 0B0506003A127A + IE: Unknown: 4A0E14000A002C01C800140005001900 + IE: Unknown: DD07000C4307000000 + IE: Unknown: 0706415520010D10 + IE: Unknown: DD7F0050F204104A0001101044000102103B00010310470010BC329E001DD811B28601006064ED61B410210004416D697410230009334720526F75746572102400085742522D303030311042000831323334353637381054000800060050F20400011011001054656E67657269696E20756C6161636810080002008C103C000101 + Cell 13 - Address: FA:8F:CA:39:0A:E8 + Channel:9 + Frequency:2.452 GHz (Channel 9) + Quality=53/70 Signal level=-57 dBm + Encryption key:off + ESSID:"" + Bit Rates:1 Mb/s; 2 Mb/s; 5.5 Mb/s; 11 Mb/s; 6 Mb/s + 9 Mb/s; 12 Mb/s; 18 Mb/s + Bit Rates:24 Mb/s; 36 Mb/s; 48 Mb/s; 54 Mb/s + Mode:Master + Extra:tsf=0000009dcdb7a142 + Extra: Last beacon: 15028ms ago + IE: Unknown: 0000 + IE: Unknown: 010882848B960C121824 + IE: Unknown: 030109 + IE: Unknown: 050401020000 + IE: Unknown: 2A0100 + IE: Unknown: 2D1A2C1103FF00000000000000000000000000000000000000000000 + IE: Unknown: 32043048606C + IE: Unknown: 3D1609000100000000000000000000000000000000000000 + IE: Unknown: DD180050F2020101800003A4000027A4000042435E0062322F00 + Cell 14 - Address: 30:91:8F:99:2E:A1 + Channel:11 + Frequency:2.462 GHz (Channel 11) + Quality=60/70 Signal level=-50 dBm + Encryption key:on + ESSID:"Telstra992EA1" + Bit Rates:1 Mb/s; 2 Mb/s; 5.5 Mb/s; 11 Mb/s; 18 Mb/s + 24 Mb/s; 36 Mb/s; 54 Mb/s + Bit Rates:6 Mb/s; 9 Mb/s; 12 Mb/s; 48 Mb/s + Mode:Master + Extra:tsf=00000010a8f55018 + Extra: Last beacon: 14340ms ago + IE: Unknown: 000D54656C73747261393932454131 + IE: Unknown: 010882848B9624B0486C + IE: Unknown: 03010B + IE: Unknown: 2A0100 + IE: Unknown: 32048C129860 + IE: IEEE 802.11i/WPA2 Version 1 + Group Cipher : CCMP + Pairwise Ciphers (1) : CCMP + Authentication Suites (1) : PSK + IE: Unknown: 46053208010000 + IE: Unknown: 2D1A1C081BFFFF000000000000000000000000000000000000000000 + IE: Unknown: 3D160B081500000000000000000000000000000000000000 + IE: Unknown: 7F080400080000000040 + IE: Unknown: DD9A0050F204104A00011010440001021057000101103B00010310470010C3479693A0735041BE93CB8E34C365E71021000B546563686E69636F6C6F721023000E546563686E69636F6C6F72205447102400073739376E207633104200093135303354414739391054000800060050F204000110110015546563686E69636F6C6F722054473739376E20763310080002268C1049000600372A000120 + IE: Unknown: DD090010180203000C0000 + IE: Unknown: DD180050F2020101040003A4000027A4000042435E0062322F00 + Cell 15 - Address: 32:91:8F:99:2E:A4 + Channel:11 + Frequency:2.462 GHz (Channel 11) + Quality=58/70 Signal level=-52 dBm + Encryption key:off + ESSID:"Fon WiFi" + Bit Rates:1 Mb/s; 2 Mb/s; 5.5 Mb/s; 11 Mb/s; 18 Mb/s + 24 Mb/s; 36 Mb/s; 54 Mb/s + Bit Rates:6 Mb/s; 9 Mb/s; 12 Mb/s; 48 Mb/s + Mode:Master + Extra:tsf=00000010a8f5d5fe + Extra: Last beacon: 14340ms ago + IE: Unknown: 0008466F6E2057694669 + IE: Unknown: 010882848B9624B0486C + IE: Unknown: 03010B + IE: Unknown: 2A0100 + IE: Unknown: 32048C129860 + IE: Unknown: 46053208010000 + IE: Unknown: 2D1A1C081BFFFF000000000000000000000000000000000000000000 + IE: Unknown: 3D160B081500000000000000000000000000000000000000 + IE: Unknown: 7F03040008 + IE: Unknown: DD090010180200000C0000 + IE: Unknown: DD180050F2020101800003A4000027A4000042435E0062322F00 + Cell 16 - Address: 32:91:8F:99:2E:A3 + Channel:11 + Frequency:2.462 GHz (Channel 11) + Quality=57/70 Signal level=-53 dBm + Encryption key:off + ESSID:"Telstra Air" + Bit Rates:1 Mb/s; 2 Mb/s; 5.5 Mb/s; 11 Mb/s; 18 Mb/s + 24 Mb/s; 36 Mb/s; 54 Mb/s + Bit Rates:6 Mb/s; 9 Mb/s; 12 Mb/s; 48 Mb/s + Mode:Master + Extra:tsf=00000010a8f574c1 + Extra: Last beacon: 14340ms ago + IE: Unknown: 000B54656C7374726120416972 + IE: Unknown: 010882848B9624B0486C + IE: Unknown: 03010B + IE: Unknown: 2A0100 + IE: Unknown: 32048C129860 + IE: Unknown: 46053208010000 + IE: Unknown: 2D1A1C081BFFFF000000000000000000000000000000000000000000 + IE: Unknown: 3D160B081500000000000000000000000000000000000000 + IE: Unknown: 7F03040008 + IE: Unknown: DD090010180200000C0000 + IE: Unknown: DD180050F2020101800003A4000027A4000042435E0062322F00 diff --git a/python-iwlist/test/issue_5/vectors.json b/python-iwlist/test/issue_5/vectors.json new file mode 100644 index 0000000..c500818 --- /dev/null +++ b/python-iwlist/test/issue_5/vectors.json @@ -0,0 +1,93 @@ +[ + { + "cellnumber": "10", + "mac": "A0:21:B7:6A:13:64", + "essid": "Frank Kwok", + "mode": "Master", + "frequency": "2.437", + "frequency_units": "GHz", + "channel": "6", + "encryption": "on", + "signal_quality": "22", + "signal_total": "70", + "signal_level_dBm": "-88" + }, + { + "cellnumber": "11", + "mac": "B8:08:D7:F7:B8:24", + "essid": "DODO-B81E", + "mode": "Master", + "frequency": "2.452", + "frequency_units": "GHz", + "channel": "9", + "encryption": "on", + "signal_quality": "61", + "signal_total": "70", + "signal_level_dBm": "-49" + }, + { + "cellnumber": "12", + "mac": "00:60:64:ED:61:B4", + "essid": "Tengeriin ulaach", + "mode": "Master", + "frequency": "2.452", + "frequency_units": "GHz", + "channel": "9", + "encryption": "on", + "signal_quality": "57", + "signal_total": "70", + "signal_level_dBm": "-53" + }, + { + "cellnumber": "13", + "mac": "FA:8F:CA:39:0A:E8", + "essid": "", + "mode": "Master", + "frequency": "2.452", + "frequency_units": "GHz", + "channel": "9", + "encryption": "off", + "signal_quality": "53", + "signal_total": "70", + "signal_level_dBm": "-57" + }, + { + "cellnumber": "14", + "mac": "30:91:8F:99:2E:A1", + "essid": "Telstra992EA1", + "mode": "Master", + "frequency": "2.462", + "frequency_units": "GHz", + "channel": "11", + "encryption": "on", + "signal_quality": "60", + "signal_total": "70", + "signal_level_dBm": "-50" + }, + { + "cellnumber": "15", + "mac": "32:91:8F:99:2E:A4", + "essid": "Fon WiFi", + "mode": "Master", + "frequency": "2.462", + "frequency_units": "GHz", + "channel": "11", + "encryption": "off", + "signal_quality": "58", + "signal_total": "70", + "signal_level_dBm": "-52" + }, + { + "cellnumber": "16", + "mac": "32:91:8F:99:2E:A3", + "essid": "Telstra Air", + "mode": "Master", + "frequency": "2.462", + "frequency_units": "GHz", + "channel": "11", + "encryption": "off", + "signal_quality": "57", + "signal_total": "70", + "signal_level_dBm": "-53" + } +] diff --git a/python-iwlist/update_demo.py b/python-iwlist/update_demo.py new file mode 100644 index 0000000..c0959dd --- /dev/null +++ b/python-iwlist/update_demo.py @@ -0,0 +1,36 @@ +import matplotlib.pyplot as plt +import matplotlib.animation as anim + +def plot_cont(fun, xmax): + y = [] + fig = plt.figure() + ax = fig.add_subplot(1,1,1) + + def update(i): + yi = fun() + y.append(yi) + x = range(len(y)) + ax.clear() + ax.plot(x, y) + print i, ': ', yi + + a = anim.FuncAnimation(fig, update, frames=xmax, repeat=False) + plt.show() +''' +import matplotlib.pyplot as plt +import numpy as np + +x = np.linspace(0, 6*np.pi, 100) +y = np.sin(x) + +# You probably won't need this if you're embedding things in a tkinter plot... +plt.ion() + +fig = plt.figure() +ax = fig.add_subplot(111) +line1, = ax.plot(x, y, 'r-') # Returns a tuple of line objects, thus the comma + +for phase in np.linspace(0, 10*np.pi, 500): + line1.set_ydata(np.sin(x + phase)) + fig.canvas.draw() +''' diff --git a/python-iwlist/viz.py b/python-iwlist/viz.py new file mode 100644 index 0000000..c5f7e08 --- /dev/null +++ b/python-iwlist/viz.py @@ -0,0 +1,26 @@ +#!/usr/bin/python + +import matplotlib.pyplot as plt +import numpy as np +import time + + +class Access_Point(object): + def __init__(self, mac, essid, channel, encryption, sig_level): + self.mac = mac + self.essid = essid + self.channel = channel + self.encryption = encryption + self.sig_level = sig_level + +class Viz(object): + def __init__(self): + self.access_points = dict() + + + def update_ap(self, ap): + self.access_points[ap.mac] = ap + + + def run(self): + N = len(access_points) From 6d23ddf49486d9d9adebbfddf562db7f70b01392 Mon Sep 17 00:00:00 2001 From: Dario Date: Fri, 17 Nov 2017 00:15:59 -0800 Subject: [PATCH 20/24] Kept playing with matplotlib. --- python-iwlist/sample.py | 17 ++++++++++++++++- python-iwlist/viz.py | 2 ++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/python-iwlist/sample.py b/python-iwlist/sample.py index 7d8c979..0198fd9 100644 --- a/python-iwlist/sample.py +++ b/python-iwlist/sample.py @@ -39,8 +39,9 @@ def __init__(self, mac, essid, channel, encryption, sig_level): women_means = (25, 32, 34, 20, 25) women_std = (3, 5, 2, 3, 3) +ap_sig_levels = (20,35, 30, 35, 27) +for a in access_points: -ap_means = (20,35, 30, 35, 27) #other_means = (32, 61, 20, 25, 12) rects2 = ax.bar(ind + width, women_means, width, color='y', yerr=women_std) @@ -70,3 +71,17 @@ def autolabel(rects): time.sleep(3) women_means = (30, 37, 34, 29, 37) +''' +This is for formating the ticks on the x axis +https://stackoverflow.com/questions/17158382/centering-x-tick-labels-between-tick-marks-in-matplotlib +''' +import matplotlib.ticker as ticker + +# a is an axes object, e.g. from figure.get_axes() + +# Hide major tick labels +a.xaxis.set_major_formatter(ticker.NullFormatter()) + +# Customize minor tick labels +a.xaxis.set_minor_locator(ticker.FixedLocator([1.5,2.5,3.5,4.5,5.5])) +a.xaxis.set_minor_formatter(ticker.FixedFormatter(['1','2','3','4','5'])) diff --git a/python-iwlist/viz.py b/python-iwlist/viz.py index c5f7e08..9d51a53 100644 --- a/python-iwlist/viz.py +++ b/python-iwlist/viz.py @@ -24,3 +24,5 @@ def update_ap(self, ap): def run(self): N = len(access_points) + fig, ax = plt.subplots(ncols=11) + plt.bar() From 474f42d2527b4f0b75185bff7b7306c3e82bad01 Mon Sep 17 00:00:00 2001 From: Brandon Haines Date: Thu, 16 Nov 2017 17:45:07 -0500 Subject: [PATCH 21/24] Adding frame level debugging. --- asl_sdr_hackfest/postmaster.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/asl_sdr_hackfest/postmaster.py b/asl_sdr_hackfest/postmaster.py index 6fc64e0..26871dd 100755 --- a/asl_sdr_hackfest/postmaster.py +++ b/asl_sdr_hackfest/postmaster.py @@ -1,6 +1,7 @@ #!/usr/bin/python import threading +import binascii from asl_sdr_hackfest.service import Service @@ -51,7 +52,7 @@ def __init__(self, services_config): ''' self.frame_rx = NetworkLayerReceiveHandler(output_data_func = self.frameDeframed) - self.frame_tx = NetworkLayerTransmitHandler(output_data_func = self.services['radio']['service'].outputData) + self.frame_tx = NetworkLayerTransmitHandler(output_data_func = self.debugTxCallback) self.rtp_handler_rx = RTP_Handler() self.rtp_handler_tx = RTP_Handler() @@ -59,6 +60,12 @@ def __init__(self, services_config): threading.Thread.__init__(self) + def debugTxCallback(self, data): + print('Transmitting frame:') + print(binascii.hexlify(data)) + self.services['radio']['service'].outputData(data) + + def run(self): self.running = True @@ -75,6 +82,8 @@ def run(self): if data is None: continue if srv['type'] == 'radio': + print('Receiving frame:') + print(binascii.hexlify(data)) self.frame_rx.ingest_data(data) elif srv['type'] == 'client': rtp_header = self.rtp_handler_tx.tx(srv['config']['ssrc'], 'gsm') # This is a byte array, not a header class From 69c17a366fe725fd8c0bdaaf8dbb4d6bfb1a711f Mon Sep 17 00:00:00 2001 From: Zachary Kolarich Date: Fri, 17 Nov 2017 14:21:37 -0500 Subject: [PATCH 22/24] added zmq --- python-iwlist/scanner.py | 26 ++++++++++++++++++++++++-- python-iwlist/viz.py | 24 ++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/python-iwlist/scanner.py b/python-iwlist/scanner.py index 8245af7..8800340 100644 --- a/python-iwlist/scanner.py +++ b/python-iwlist/scanner.py @@ -4,7 +4,24 @@ import iwlist import time import psutil +import pmt import struct +import zmq + +ZMQ_IP = '127.0.0.1' +ZMQ_PORT = 5058 + +zmq_context = zmq.Context() +socketOut = zmq_context.socket(zmq.PUB) +socketOut.bind('tcp://%s:%s' % (ZMQ_IP, ZMQ_PORT)) + + +def scanner_send(data): + car = pmt.make_dict() + data = bytes(data) + cdr = pmt.to_pmt(data) + pdu = pmt.cons(car, cdr) + socketOut.send(pmt.serialize_str(pdu)) def pack_ap(ap): @@ -20,13 +37,12 @@ def pack_ap(ap): return s -INTERFACE='wlp2s0' +INTERFACE='wlp3s0b1'#'wlp2s0' access_points = dict() keys= ['essid', 'encryption', 'signal_level_dBm', 'channel'] keys_short = ['essid', 'encryption', 'channel'] -new_stuff = dict() #content = iwlist.scan(interface=INTERFACE) #cells = iwlist.parse(content) @@ -68,6 +84,12 @@ def pack_ap(ap): #print('Old = ' + access_points[c['mac']][k]) #print('New = ' + c[k])#'mac'][k]) #print + + bytearr = bytearray() + for val in new_stuff: + bytearr += val + + scanner_send(bytearr) print("CYCLE COMPLETE") print("CPU use percentage = " + str(psutil.cpu_percent(interval=None))) diff --git a/python-iwlist/viz.py b/python-iwlist/viz.py index 9d51a53..04c1b99 100644 --- a/python-iwlist/viz.py +++ b/python-iwlist/viz.py @@ -1,6 +1,10 @@ #!/usr/bin/python import matplotlib.pyplot as plt +import numpy +import pmt +import zmq + import numpy as np import time @@ -14,15 +18,31 @@ def __init__(self, mac, essid, channel, encryption, sig_level): self.sig_level = sig_level class Viz(object): - def __init__(self): + def __init__(self, portIn, timeout = 100): self.access_points = dict() + self._zmqContext = zmq.Context() + self._socketIn = self._zmqContext.socket(zmq.SUB) + self._socketIn.RCVTIMEO = timeout + self._socketIn.connect('tcp://%s:%s' % (self._ipAddress,self._portIn)) + try: + self._socketIn.setsockopt(zmq.SUBSCRIBE, '') # python2 + except TypeError: + self._socketIn.setsockopt_string(zmq.SUBSCRIBE, '') # python3, if this throws an exception... give up... + + def recv(self): + msg = self._socketIn.recv() + pdu = pmt.deserialize_str(msg) + cdr = pmt.to_python(pmt.cdr(pdu)) + cdr = numpy.getbuffer(cdr) + return cdr + def update_ap(self, ap): self.access_points[ap.mac] = ap def run(self): - N = len(access_points) + N = len(self.access_points) fig, ax = plt.subplots(ncols=11) plt.bar() From 194af2b85205df9c4da1a70e02e932c24bdae258 Mon Sep 17 00:00:00 2001 From: lling Date: Fri, 17 Nov 2017 14:11:26 -0800 Subject: [PATCH 23/24] label services for debugging --- apps/run_all.py | 10 +++++----- asl_sdr_hackfest/service.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/run_all.py b/apps/run_all.py index 294f4cf..db2fa11 100755 --- a/apps/run_all.py +++ b/apps/run_all.py @@ -15,7 +15,7 @@ def run(self): services = {} radio = { - 'service': Service(portIn = 6126, portOut = 6127), + 'service': Service(portIn = 6126, portOut = 6127, name = 'radio'), 'type': 'radio', 'config': None, } @@ -26,7 +26,7 @@ def run(self): 'ssrc': 0, } mavlink = { - 'service': Service(portIn = 6128, portOut = 6131), + 'service': Service(portIn = 6128, portOut = 6131, name = 'gcs?'), 'type': 'client', 'config': mavlink_config, } @@ -37,7 +37,7 @@ def run(self): 'ssrc': 1, } cats = { - 'service': Service(portIn = 5058, portOut = 5059), + 'service': Service(portIn = 5058, portOut = 5059, name = 'cats'), 'type': 'client', 'config': cats_config, } @@ -57,7 +57,7 @@ def run(self): 'ssrc': 0, } mavlink = { - 'service': Service(portIn = 6130, portOut = 6129), + 'service': Service(portIn = 6130, portOut = 6129, name = 'uav?'), 'type': 'client', 'config': mavlink_config, } @@ -68,7 +68,7 @@ def run(self): 'ssrc': 1, } cats = { - 'service': Service(portIn = 5158, portOut = 5159), + 'service': Service(portIn = 5158, portOut = 5159, name = 'cats?'), 'type': 'client', 'config': cats_config, } diff --git a/asl_sdr_hackfest/service.py b/asl_sdr_hackfest/service.py index 47625bf..8345d4e 100644 --- a/asl_sdr_hackfest/service.py +++ b/asl_sdr_hackfest/service.py @@ -7,7 +7,7 @@ class Service(threading.Thread, object): - def __init__(self, portIn, portOut): + def __init__(self, portIn, portOut, name=None): self.running = False self._ZMQ_in = ZMQ_sub(portIn = portIn) From d4d87a0dc91e6eabfed89dea849de26ae31270a4 Mon Sep 17 00:00:00 2001 From: lling Date: Fri, 17 Nov 2017 14:11:26 -0800 Subject: [PATCH 24/24] label services for debugging --- apps/run_all.py | 10 +++++----- asl_sdr_hackfest/service.py | 4 +++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/apps/run_all.py b/apps/run_all.py index 294f4cf..db2fa11 100755 --- a/apps/run_all.py +++ b/apps/run_all.py @@ -15,7 +15,7 @@ def run(self): services = {} radio = { - 'service': Service(portIn = 6126, portOut = 6127), + 'service': Service(portIn = 6126, portOut = 6127, name = 'radio'), 'type': 'radio', 'config': None, } @@ -26,7 +26,7 @@ def run(self): 'ssrc': 0, } mavlink = { - 'service': Service(portIn = 6128, portOut = 6131), + 'service': Service(portIn = 6128, portOut = 6131, name = 'gcs?'), 'type': 'client', 'config': mavlink_config, } @@ -37,7 +37,7 @@ def run(self): 'ssrc': 1, } cats = { - 'service': Service(portIn = 5058, portOut = 5059), + 'service': Service(portIn = 5058, portOut = 5059, name = 'cats'), 'type': 'client', 'config': cats_config, } @@ -57,7 +57,7 @@ def run(self): 'ssrc': 0, } mavlink = { - 'service': Service(portIn = 6130, portOut = 6129), + 'service': Service(portIn = 6130, portOut = 6129, name = 'uav?'), 'type': 'client', 'config': mavlink_config, } @@ -68,7 +68,7 @@ def run(self): 'ssrc': 1, } cats = { - 'service': Service(portIn = 5158, portOut = 5159), + 'service': Service(portIn = 5158, portOut = 5159, name = 'cats?'), 'type': 'client', 'config': cats_config, } diff --git a/asl_sdr_hackfest/service.py b/asl_sdr_hackfest/service.py index 47625bf..8b696fa 100644 --- a/asl_sdr_hackfest/service.py +++ b/asl_sdr_hackfest/service.py @@ -7,7 +7,7 @@ class Service(threading.Thread, object): - def __init__(self, portIn, portOut): + def __init__(self, portIn, portOut, name=None): self.running = False self._ZMQ_in = ZMQ_sub(portIn = portIn) @@ -16,6 +16,8 @@ def __init__(self, portIn, portOut): self._input_queue = queue.Queue() threading.Thread.__init__(self) + self.setName(name) + def run(self):