From f72154d7440a09c8d428427d4d975744ddae458e Mon Sep 17 00:00:00 2001 From: Tyler Kobil Date: Sun, 31 Jan 2021 18:18:14 -0500 Subject: [PATCH 01/14] adding json dump - still need to fix commas --- pyx12/jsonwriter.py | 147 ++++++++++++++++++++++++++++++++++++++++ pyx12/x12json_simple.py | 98 +++++++++++++++++++++++++++ pyx12/x12n_document.py | 17 ++++- 3 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 pyx12/jsonwriter.py create mode 100644 pyx12/x12json_simple.py diff --git a/pyx12/jsonwriter.py b/pyx12/jsonwriter.py new file mode 100644 index 00000000..a2010fcf --- /dev/null +++ b/pyx12/jsonwriter.py @@ -0,0 +1,147 @@ +# A simple JSON Generator Based off of the XML Writer in pyx12.xmlwriter +import sys + +""" +Tyler's Notes +loop: [ + seg: { + ele: ..., + ele: ... + }, + seg: { + ele: ..., + ele: ... + }, + loop: [ + + ], + ... +] + +Todo: Handle commas correctly. Otherwise, almost there! +""" +class JSONriter(object): + """ + Doctest: + >>>from jsonwriter import JSONriter + >>>writer = JSONriter() + >>>#Notice: there is no error checking to ensure that the root element + >>>#specified in the doctype matches the top-level element generated + >>>writer.push(u"xsa") + >>>#Another element with child elements + >>>writer.push(u"vendor") + >>>#Element with simple text (#PCDATA) content + >>>writer.elem(u"name", u"Centigrade systems") + >>>writer.elem(u"email", u"info@centigrade.bogus") + >>>writer.elem(u"vendor", u"Centigrade systems") + >>>#Close currently open element ("vendor) + >>>writer.pop() + >>>#Element with an attribute + >>>writer.push(u"product", {u"id": u"100\\u00B0"}) + >>>writer.elem(u"name", u"100\\u00B0 Server") + >>>writer.elem(u"version", u"1.0") + >>>writer.elem(u"last-release", u"20030401") + >>>#Empty element + >>>writer.empty(u"changes") + >>>writer.pop() + >>>writer.pop() + startElement + endElement + emptyElement + text, data + endDocument + attribute + indentation + close + flush + """ + + def __init__(self, out=sys.stdout, encoding="utf-8", indent=" "): + """ + out - a stream for the output + encoding - an encoding used to wrap the output for unicode + indent - white space used for indentation + """ + self.encoding = encoding + self.out = out + self.stack = [] + self.indent = indent + # self._write("{") + + def push(self, elem, attrs={}): + """ + Create an element which will have child elements + """ + if elem == "comp": + # self.stack.append(elem) + return + elif elem == "subele": + import pdb;pdb.set_trace() + self._indent() + for (_, v) in list(attrs.items()): + if elem == "loop": + self._write("""{"%s": [\n""" % self._escape_attr(v)) + elif elem == "seg": + self._write("""{"%s": {\n""" % self._escape_attr(v)) + else: + import pdb;pdb.set_trace() + self.stack.append(elem) + + def elem(self, elem, content, attrs={}, last=False): + """ + Create an element with text content only + """ + self._indent() + for (_, v) in list(attrs.items()): + if last: + self._write(""""%s": "%s"\n""" % (self._escape_attr(v), self._escape_cont(content))) + else: + self._write(""""%s": "%s",\n""" % (self._escape_attr(v), self._escape_cont(content))) # Fix Commas + + def pop(self, last=False): + """ + Close an element started with the push() method + """ + if last: + if len(self.stack) > 0: + elem = self.stack[-1] + del self.stack[-1] + + if elem == "seg": + self._indent() + self._write("}}\n") + elif elem == "loop": + self._indent() + self._write("]}\n") + else: + if len(self.stack) > 0: + elem = self.stack[-1] + del self.stack[-1] + if elem == "seg": + self._indent() + self._write("}},\n") + elif elem == "loop": + self._indent() + self._write("]},\n") + + def __len__(self): + return len(self.stack) + + def _indent(self): + self._write(self.indent * (len(self.stack) * 2)) + + def _escape_cont(self, text): + if text is None: + return None + return text.replace("&", "&")\ + .replace("<", "<").replace(">", ">") + + def _escape_attr(self, text): + if text is None: + return None + return text.replace("&", "&") \ + .replace("'", "'").replace("<", "<")\ + .replace(">", ">").replace('\n', '') + + def _write(self, strval): + self.out.write(strval) \ No newline at end of file diff --git a/pyx12/x12json_simple.py b/pyx12/x12json_simple.py new file mode 100644 index 00000000..554fbe69 --- /dev/null +++ b/pyx12/x12json_simple.py @@ -0,0 +1,98 @@ +from os.path import commonprefix +import logging + +from .x12xml_simple import x12xml_simple +from .jsonwriter import JSONriter +from .errors import EngineError +from .map_walker import pop_to_parent_loop + +logger = logging.getLogger('pyx12.x12json.simple') + +class X12JsonSimple(x12xml_simple): + def __init__(self, fd): + self.writer = JSONriter(fd) + self.last_path = [] + + def __del__(self): + while len(self.writer) > 0: + self.writer.pop() + self.writer._write('}') + + def seg(self, seg_node, seg_data, last_seg=False): + """ + Generate XML for the segment data and matching map node + + @param seg_node: Map Node + @type seg_node: L{node} + @param seg_data: Segment object + @type seg_data: L{segment} + """ + if not seg_node.is_segment(): + raise EngineError('Node must be a segment') + parent = pop_to_parent_loop(seg_node) # Get enclosing loop + # check path for new loops to be added + cur_path = self._path_list(parent.get_path()) + # if seg_node.id == 'GS_LOOP': + # import pdb; pdb.set_trace() + if self.last_path == cur_path and seg_node.is_first_seg_in_loop(): + # loop repeat + self.writer.pop() + (xname, attrib) = self._get_loop_info(cur_path[-1]) + self.writer.push(xname, attrib) + else: + last_path = self.last_path + match_idx = self._get_path_match_idx(last_path, cur_path) + root_path = self._path_list(commonprefix(['/'.join(cur_path), '/'.join(last_path)])) + if seg_node.is_first_seg_in_loop() and root_path == cur_path: + match_idx -= 1 + loop_struct = range(len(last_path) - 1, match_idx - 1, -1) + for i in loop_struct: + if i == loop_struct[-1]: + self.writer.pop() + else: + self.writer.pop() + for i in range(match_idx, len(cur_path)): + (xname, attrib) = self._get_loop_info(cur_path[i]) + # if len(cur_path) > 3: + if seg_node.is_first_seg_in_loop(): + self.writer.push(xname, attrib) + else: + self.writer.push(xname, attrib) + seg_node_id = self._get_node_id(seg_node, parent, seg_data) + (xname, attrib) = self._get_seg_info(seg_node_id) + # (seg_node.name, attrib) = self._get_seg_info(seg_node_id) + if seg_node.is_first_seg_in_loop: + # import pdb;pdb.set_trace() + self.writer.push(xname, attrib) + else: + self.writer.push(xname, attrib) + loop_struct = range(len(seg_data)) + for i in loop_struct: + if i == loop_struct[-1]: + last = True + else: + last = False + child_node = seg_node.get_child_node_by_idx(i) + if child_node.usage == 'N' or seg_data.get('%02i' % (i + 1)).is_empty(): + pass # Do not try to ouput for invalid or empty elements + elif child_node.is_composite(): + (xname, attrib) = self._get_comp_info(seg_node_id) + self.writer.push(xname, attrib) + comp_data = seg_data.get('%02i' % (i + 1)) + for j in range(len(comp_data)): + # import pdb;pdb.set_trace() + subele_node = child_node.get_child_node_by_idx(j) + (xname, attrib) = self._get_subele_info(subele_node.id) + self.writer.elem(xname, comp_data[j].get_value(), attrib, last) + # self.writer.pop() # end composite + elif child_node.is_element(): + if seg_data.get_value('%02i' % (i + 1)) == '': + pass + #self.writer.empty(u"ele", attrs={u'id': child_node.id}) + else: + (xname, attrib) = self._get_ele_info(child_node.id) + self.writer.elem(xname, seg_data.get_value('%02i' % (i + 1)), attrib, last) + else: + raise EngineError('Node must be a either an element or a composite') + self.writer.pop(last=False) # end segment + self.last_path = cur_path \ No newline at end of file diff --git a/pyx12/x12n_document.py b/pyx12/x12n_document.py index 3a83b729..dfcd3426 100644 --- a/pyx12/x12n_document.py +++ b/pyx12/x12n_document.py @@ -26,6 +26,7 @@ import pyx12.x12file from pyx12.map_walker import walk_tree import pyx12.x12xml_simple +import pyx12.x12json_simple def _reset_counter_to_isa_counts(walker): @@ -47,7 +48,7 @@ def _reset_counter_to_gs_counts(walker): def x12n_document(param, src_file, fd_997, fd_html, - fd_xmldoc=None, xslt_files=None, map_path=None, + fd_xmldoc=None, fd_jsondoc=None, xslt_files=None, map_path=None, callback=None): """ Primary X12 validation function @@ -89,12 +90,15 @@ def x12n_document(param, src_file, fd_997, fd_html, err_iter = pyx12.error_handler.err_iter(errh) if fd_xmldoc: xmldoc = pyx12.x12xml_simple.x12xml_simple(fd_xmldoc, param.get('simple_dtd')) + if fd_jsondoc: + fd_jsondoc = pyx12.x12json_simple.X12JsonSimple(fd_jsondoc) #basedir = os.path.dirname(src_file) #erx = errh_xml.err_handler(basedir=basedir) valid = True for seg in src: + # import pdb;pdb.set_trace() #find node orig_node = node @@ -216,11 +220,15 @@ def x12n_document(param, src_file, fd_997, fd_html, if fd_xmldoc: xmldoc.seg(node, seg) + + if fd_jsondoc: + # if seg == src[-1] + fd_jsondoc.seg(node, seg) if False: print('\n\n') #erx.Write(src.cur_line) - + #erx.handleErrors(src.pop_errors()) src.cleanup() # Catch any skipped loop trailers errh.handle_errors(src.pop_errors()) @@ -232,7 +240,12 @@ def x12n_document(param, src_file, fd_997, fd_html, del html if fd_xmldoc: + # fd_xmldoc.writer._write("tlyer") del xmldoc + + if fd_jsondoc: + # fd_jsondoc.writer._write("tlyer") + del fd_jsondoc #visit_debug = pyx12.error_debug.error_debug_visitor(sys.stdout) #errh.accept(visit_debug) From 91b475384d0b795efaa29f9aa5685157253c7164 Mon Sep 17 00:00:00 2001 From: Tyler Kobil Date: Mon, 1 Feb 2021 11:49:35 -0500 Subject: [PATCH 02/14] comma support --- pyx12/jsonwriter.py | 24 +++++++++++++++--------- pyx12/x12json_simple.py | 32 ++++++++++++++++++-------------- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/pyx12/jsonwriter.py b/pyx12/jsonwriter.py index a2010fcf..efe81f16 100644 --- a/pyx12/jsonwriter.py +++ b/pyx12/jsonwriter.py @@ -68,7 +68,7 @@ def __init__(self, out=sys.stdout, encoding="utf-8", indent=" "): self.indent = indent # self._write("{") - def push(self, elem, attrs={}): + def push(self, elem, attrs={}, first=False): """ Create an element which will have child elements """ @@ -78,13 +78,19 @@ def push(self, elem, attrs={}): elif elem == "subele": import pdb;pdb.set_trace() self._indent() - for (_, v) in list(attrs.items()): - if elem == "loop": - self._write("""{"%s": [\n""" % self._escape_attr(v)) - elif elem == "seg": - self._write("""{"%s": {\n""" % self._escape_attr(v)) - else: - import pdb;pdb.set_trace() + + if first: + for (_, v) in list(attrs.items()): + if elem == "loop": + self._write("""{"%s": [\n""" % self._escape_attr(v)) + elif elem == "seg": + self._write("""{"%s": {\n""" % self._escape_attr(v)) + else: + for (_, v) in list(attrs.items()): + if elem == "loop": + self._write(""",{"%s": [\n""" % self._escape_attr(v)) + elif elem == "seg": + self._write(""",{"%s": {\n""" % self._escape_attr(v)) self.stack.append(elem) def elem(self, elem, content, attrs={}, last=False): @@ -98,7 +104,7 @@ def elem(self, elem, content, attrs={}, last=False): else: self._write(""""%s": "%s",\n""" % (self._escape_attr(v), self._escape_cont(content))) # Fix Commas - def pop(self, last=False): + def pop(self, last=True): """ Close an element started with the push() method """ diff --git a/pyx12/x12json_simple.py b/pyx12/x12json_simple.py index 554fbe69..263c0fb8 100644 --- a/pyx12/x12json_simple.py +++ b/pyx12/x12json_simple.py @@ -16,7 +16,7 @@ def __init__(self, fd): def __del__(self): while len(self.writer) > 0: self.writer.pop() - self.writer._write('}') + # self.writer._write('}') def seg(self, seg_node, seg_data, last_seg=False): """ @@ -32,13 +32,13 @@ def seg(self, seg_node, seg_data, last_seg=False): parent = pop_to_parent_loop(seg_node) # Get enclosing loop # check path for new loops to be added cur_path = self._path_list(parent.get_path()) - # if seg_node.id == 'GS_LOOP': - # import pdb; pdb.set_trace() if self.last_path == cur_path and seg_node.is_first_seg_in_loop(): # loop repeat self.writer.pop() (xname, attrib) = self._get_loop_info(cur_path[-1]) - self.writer.push(xname, attrib) + # if attrib['id'] == '2110': + # import pdb;pdb.set_trace() + self.writer.push(xname, attrib, first=False) else: last_path = self.last_path match_idx = self._get_path_match_idx(last_path, cur_path) @@ -53,19 +53,20 @@ def seg(self, seg_node, seg_data, last_seg=False): self.writer.pop() for i in range(match_idx, len(cur_path)): (xname, attrib) = self._get_loop_info(cur_path[i]) - # if len(cur_path) > 3: - if seg_node.is_first_seg_in_loop(): - self.writer.push(xname, attrib) + # Write a Loop + # if attrib['id'] == '2110': + # import pdb;pdb.set_trace() + if attrib['id'] == 'ISA_LOOP': + self.writer.push(xname, attrib, first=True) else: - self.writer.push(xname, attrib) + self.writer.push(xname, attrib, first=False) seg_node_id = self._get_node_id(seg_node, parent, seg_data) (xname, attrib) = self._get_seg_info(seg_node_id) - # (seg_node.name, attrib) = self._get_seg_info(seg_node_id) - if seg_node.is_first_seg_in_loop: + if seg_node.is_first_seg_in_loop(): # import pdb;pdb.set_trace() - self.writer.push(xname, attrib) + self.writer.push(xname, attrib, first=True) else: - self.writer.push(xname, attrib) + self.writer.push(xname, attrib, first=False) loop_struct = range(len(seg_data)) for i in loop_struct: if i == loop_struct[-1]: @@ -77,7 +78,10 @@ def seg(self, seg_node, seg_data, last_seg=False): pass # Do not try to ouput for invalid or empty elements elif child_node.is_composite(): (xname, attrib) = self._get_comp_info(seg_node_id) - self.writer.push(xname, attrib) + if i == loop_struct[0]: + self.writer.push(xname, attrib, first=True) + else: + self.writer.push(xname, attrib, first=False) comp_data = seg_data.get('%02i' % (i + 1)) for j in range(len(comp_data)): # import pdb;pdb.set_trace() @@ -94,5 +98,5 @@ def seg(self, seg_node, seg_data, last_seg=False): self.writer.elem(xname, seg_data.get_value('%02i' % (i + 1)), attrib, last) else: raise EngineError('Node must be a either an element or a composite') - self.writer.pop(last=False) # end segment + self.writer.pop() # end segment self.last_path = cur_path \ No newline at end of file From 279f5964668ed7fa8c3fdd418eea16c543e8a7f6 Mon Sep 17 00:00:00 2001 From: Tyler Kobil Date: Mon, 1 Feb 2021 13:10:14 -0500 Subject: [PATCH 03/14] single line output for spark reading --- pyx12/jsonwriter.py | 78 +++++++++++------------------------------ pyx12/x12json_simple.py | 25 ++++++++----- pyx12/x12n_document.py | 1 - 3 files changed, 37 insertions(+), 67 deletions(-) diff --git a/pyx12/jsonwriter.py b/pyx12/jsonwriter.py index efe81f16..1f5f2c81 100644 --- a/pyx12/jsonwriter.py +++ b/pyx12/jsonwriter.py @@ -1,32 +1,12 @@ # A simple JSON Generator Based off of the XML Writer in pyx12.xmlwriter import sys -""" -Tyler's Notes -loop: [ - seg: { - ele: ..., - ele: ... - }, - seg: { - ele: ..., - ele: ... - }, - loop: [ - ], - ... -] - -Todo: Handle commas correctly. Otherwise, almost there! -""" class JSONriter(object): """ Doctest: >>>from jsonwriter import JSONriter >>>writer = JSONriter() - >>>#Notice: there is no error checking to ensure that the root element - >>>#specified in the doctype matches the top-level element generated >>>writer.push(u"xsa") >>>#Another element with child elements >>>writer.push(u"vendor") @@ -41,22 +21,9 @@ class JSONriter(object): >>>writer.elem(u"name", u"100\\u00B0 Server") >>>writer.elem(u"version", u"1.0") >>>writer.elem(u"last-release", u"20030401") - >>>#Empty element - >>>writer.empty(u"changes") - >>>writer.pop() - >>>writer.pop() - startElement - endElement - emptyElement - text, data - endDocument - attribute - indentation - close - flush """ - def __init__(self, out=sys.stdout, encoding="utf-8", indent=" "): + def __init__(self, out=sys.stdout, encoding="utf-8", indent="\t"): """ out - a stream for the output encoding - an encoding used to wrap the output for unicode @@ -66,31 +33,29 @@ def __init__(self, out=sys.stdout, encoding="utf-8", indent=" "): self.out = out self.stack = [] self.indent = indent - # self._write("{") def push(self, elem, attrs={}, first=False): """ Create an element which will have child elements """ if elem == "comp": - # self.stack.append(elem) return - elif elem == "subele": - import pdb;pdb.set_trace() + # elif elem == "subele": + # import pdb;pdb.set_trace() self._indent() if first: for (_, v) in list(attrs.items()): if elem == "loop": - self._write("""{"%s": [\n""" % self._escape_attr(v)) + self._write("""{"%s": [""" % self._escape_attr(v)) #newline elif elem == "seg": - self._write("""{"%s": {\n""" % self._escape_attr(v)) + self._write("""{"%s": {""" % self._escape_attr(v)) #newline else: for (_, v) in list(attrs.items()): if elem == "loop": - self._write(""",{"%s": [\n""" % self._escape_attr(v)) + self._write(""",{"%s": [""" % self._escape_attr(v)) #newline elif elem == "seg": - self._write(""",{"%s": {\n""" % self._escape_attr(v)) + self._write(""",{"%s": {""" % self._escape_attr(v)) #newline self.stack.append(elem) def elem(self, elem, content, attrs={}, last=False): @@ -100,40 +65,39 @@ def elem(self, elem, content, attrs={}, last=False): self._indent() for (_, v) in list(attrs.items()): if last: - self._write(""""%s": "%s"\n""" % (self._escape_attr(v), self._escape_cont(content))) + self._write('''"%s": "%s"''' % (self._escape_attr(v), self._escape_cont(content))) #Newline else: - self._write(""""%s": "%s",\n""" % (self._escape_attr(v), self._escape_cont(content))) # Fix Commas + self._write(""""%s": "%s",""" % (self._escape_attr(v), self._escape_cont(content))) # Newline def pop(self, last=True): """ Close an element started with the push() method """ - if last: - if len(self.stack) > 0: - elem = self.stack[-1] - del self.stack[-1] - + if len(self.stack) > 0: + elem = self.stack[-1] + del self.stack[-1] + if last: if elem == "seg": self._indent() - self._write("}}\n") + self._write("}}") #newline elif elem == "loop": self._indent() - self._write("]}\n") - else: - if len(self.stack) > 0: - elem = self.stack[-1] - del self.stack[-1] + self._write("]}") #newline + else: if elem == "seg": self._indent() - self._write("}},\n") + self._write("}},") #newline elif elem == "loop": self._indent() - self._write("]},\n") + self._write("]},") #newline def __len__(self): return len(self.stack) def _indent(self): + return + # Todo : enable multi line json output + # This gets tricky with formatting commas and indents! self._write(self.indent * (len(self.stack) * 2)) def _escape_cont(self, text): diff --git a/pyx12/x12json_simple.py b/pyx12/x12json_simple.py index 263c0fb8..3a523114 100644 --- a/pyx12/x12json_simple.py +++ b/pyx12/x12json_simple.py @@ -12,11 +12,11 @@ class X12JsonSimple(x12xml_simple): def __init__(self, fd): self.writer = JSONriter(fd) self.last_path = [] + self.visited = [] def __del__(self): while len(self.writer) > 0: self.writer.pop() - # self.writer._write('}') def seg(self, seg_node, seg_data, last_seg=False): """ @@ -36,8 +36,6 @@ def seg(self, seg_node, seg_data, last_seg=False): # loop repeat self.writer.pop() (xname, attrib) = self._get_loop_info(cur_path[-1]) - # if attrib['id'] == '2110': - # import pdb;pdb.set_trace() self.writer.push(xname, attrib, first=False) else: last_path = self.last_path @@ -54,16 +52,15 @@ def seg(self, seg_node, seg_data, last_seg=False): for i in range(match_idx, len(cur_path)): (xname, attrib) = self._get_loop_info(cur_path[i]) # Write a Loop - # if attrib['id'] == '2110': - # import pdb;pdb.set_trace() - if attrib['id'] == 'ISA_LOOP': + parent_loop = cur_path[i-1] + if parent_loop not in self.visited: + self.visited.append(parent_loop) self.writer.push(xname, attrib, first=True) else: self.writer.push(xname, attrib, first=False) seg_node_id = self._get_node_id(seg_node, parent, seg_data) (xname, attrib) = self._get_seg_info(seg_node_id) if seg_node.is_first_seg_in_loop(): - # import pdb;pdb.set_trace() self.writer.push(xname, attrib, first=True) else: self.writer.push(xname, attrib, first=False) @@ -72,7 +69,17 @@ def seg(self, seg_node, seg_data, last_seg=False): if i == loop_struct[-1]: last = True else: - last = False + # Check to see if any of the next children exist. + # If no, then we are on last node + try: + next_children = [seg_node.get_child_node_by_idx(index) for index in loop_struct[i+1:]] + except IndexError: + next_children = [] + next_node_exists = [not(child_node.usage == 'N' or seg_data.get('%02i' % (i + 1)).is_empty()) for child_node in next_children] + if any(next_node_exists): + last = False + else: + last = True child_node = seg_node.get_child_node_by_idx(i) if child_node.usage == 'N' or seg_data.get('%02i' % (i + 1)).is_empty(): pass # Do not try to ouput for invalid or empty elements @@ -84,7 +91,6 @@ def seg(self, seg_node, seg_data, last_seg=False): self.writer.push(xname, attrib, first=False) comp_data = seg_data.get('%02i' % (i + 1)) for j in range(len(comp_data)): - # import pdb;pdb.set_trace() subele_node = child_node.get_child_node_by_idx(j) (xname, attrib) = self._get_subele_info(subele_node.id) self.writer.elem(xname, comp_data[j].get_value(), attrib, last) @@ -99,4 +105,5 @@ def seg(self, seg_node, seg_data, last_seg=False): else: raise EngineError('Node must be a either an element or a composite') self.writer.pop() # end segment + self.visited.append(parent.id) self.last_path = cur_path \ No newline at end of file diff --git a/pyx12/x12n_document.py b/pyx12/x12n_document.py index dfcd3426..92d9e89e 100644 --- a/pyx12/x12n_document.py +++ b/pyx12/x12n_document.py @@ -222,7 +222,6 @@ def x12n_document(param, src_file, fd_997, fd_html, xmldoc.seg(node, seg) if fd_jsondoc: - # if seg == src[-1] fd_jsondoc.seg(node, seg) if False: From ee7153e364bb77ee08f040709b276614e0ed4dbd Mon Sep 17 00:00:00 2001 From: Tyler Kobil Date: Mon, 1 Feb 2021 13:16:10 -0500 Subject: [PATCH 04/14] docstrings --- pyx12/x12json_simple.py | 5 +++-- pyx12/x12n_document.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pyx12/x12json_simple.py b/pyx12/x12json_simple.py index 3a523114..68a5db0e 100644 --- a/pyx12/x12json_simple.py +++ b/pyx12/x12json_simple.py @@ -20,7 +20,7 @@ def __del__(self): def seg(self, seg_node, seg_data, last_seg=False): """ - Generate XML for the segment data and matching map node + Generate JSON for the segment data and matching map node @param seg_node: Map Node @type seg_node: L{node} @@ -105,5 +105,6 @@ def seg(self, seg_node, seg_data, last_seg=False): else: raise EngineError('Node must be a either an element or a composite') self.writer.pop() # end segment - self.visited.append(parent.id) + if parent.id not in self.visited: + self.visited.append(parent.id) self.last_path = cur_path \ No newline at end of file diff --git a/pyx12/x12n_document.py b/pyx12/x12n_document.py index 92d9e89e..ea715ee7 100644 --- a/pyx12/x12n_document.py +++ b/pyx12/x12n_document.py @@ -61,6 +61,8 @@ def x12n_document(param, src_file, fd_997, fd_html, @type fd_html: file descriptor @param fd_xmldoc: XML output document @type fd_xmldoc: file descriptor + @param fd_jsondoc: JSON output document (outputs to single line) + @type fd_jsondoc: file descriptor @rtype: boolean """ logger = logging.getLogger('pyx12') @@ -239,11 +241,9 @@ def x12n_document(param, src_file, fd_997, fd_html, del html if fd_xmldoc: - # fd_xmldoc.writer._write("tlyer") del xmldoc if fd_jsondoc: - # fd_jsondoc.writer._write("tlyer") del fd_jsondoc #visit_debug = pyx12.error_debug.error_debug_visitor(sys.stdout) From 0eba38913f6f37d47d73ebe1044285866a907ba2 Mon Sep 17 00:00:00 2001 From: Tyler Kobil Date: Mon, 1 Feb 2021 17:10:49 -0500 Subject: [PATCH 05/14] adding support for dumping fields rather than codes --- pyx12/jsonwriter.py | 10 +-- pyx12/x12json_simple.py | 136 ++++++++++++++++++++++++++++++++++++++-- pyx12/x12n_document.py | 1 - 3 files changed, 138 insertions(+), 9 deletions(-) diff --git a/pyx12/jsonwriter.py b/pyx12/jsonwriter.py index 1f5f2c81..55db4c0d 100644 --- a/pyx12/jsonwriter.py +++ b/pyx12/jsonwriter.py @@ -23,16 +23,18 @@ class JSONriter(object): >>>writer.elem(u"last-release", u"20030401") """ - def __init__(self, out=sys.stdout, encoding="utf-8", indent="\t"): + def __init__(self, out=sys.stdout, encoding="utf-8", indent="\t", words_mode=True): """ out - a stream for the output encoding - an encoding used to wrap the output for unicode indent - white space used for indentation + words_mode - boolean for using string fields rather than codes in output """ self.encoding = encoding self.out = out self.stack = [] self.indent = indent + self.words_mode = words_mode def push(self, elem, attrs={}, first=False): """ @@ -40,8 +42,6 @@ def push(self, elem, attrs={}, first=False): """ if elem == "comp": return - # elif elem == "subele": - # import pdb;pdb.set_trace() self._indent() if first: @@ -107,11 +107,13 @@ def _escape_cont(self, text): .replace("<", "<").replace(">", ">") def _escape_attr(self, text): + if self.words_mode: + return text if text is None: return None return text.replace("&", "&") \ .replace("'", "'").replace("<", "<")\ - .replace(">", ">").replace('\n', '') + .replace(">", ">") def _write(self, strval): self.out.write(strval) \ No newline at end of file diff --git a/pyx12/x12json_simple.py b/pyx12/x12json_simple.py index 68a5db0e..63867b72 100644 --- a/pyx12/x12json_simple.py +++ b/pyx12/x12json_simple.py @@ -9,24 +9,152 @@ logger = logging.getLogger('pyx12.x12json.simple') class X12JsonSimple(x12xml_simple): - def __init__(self, fd): - self.writer = JSONriter(fd) + def __init__(self, fd, words_mode=True): + """ + @param fd: File stream for output + @param words_mode: Dump JSON using string names for fields rather than codes. + """ + self.writer = JSONriter(fd, words_mode=words_mode) self.last_path = [] self.visited = [] + self.words_mode = words_mode def __del__(self): while len(self.writer) > 0: self.writer.pop() - def seg(self, seg_node, seg_data, last_seg=False): + @staticmethod + def get_parents_in_path(node): + # Todo: Ensure all x12 files start with Interchange Control Header + def safe_get_parent(node): + try: + if node.parent.id == 'ISA_LOOP': + return [] + else: + return [str(node.parent.name)] + except AttributeError: + return [] + parent_nodes_in_path = [] + parents = safe_get_parent(node) + while len(parents) > 0: + parent_nodes_in_path += parents + node = node.parent + parents = safe_get_parent(node) + # if parents[0] == "Application Sender's Code": + # import pdb;pdb.set_trace() + # print("hello") + return ['Interchange Control Header'] + list(reversed(parent_nodes_in_path)) + + def seg_with_names(self, seg_node, seg_data): """ - Generate JSON for the segment data and matching map node + Generate JSON for the segment data and matching map node. + Essentially the same as "seg", however this will write + String field names, rather than codes. @param seg_node: Map Node @type seg_node: L{node} @param seg_data: Segment object @type seg_data: L{segment} """ + if not seg_node.is_segment(): + raise EngineError('Node must be a segment') + parent = pop_to_parent_loop(seg_node) # Get enclosing loop + # check path for new loops to be added + cur_path = self._path_list(parent.get_path()) + if self.last_path == cur_path and seg_node.is_first_seg_in_loop(): + # loop repeat + self.writer.pop() + (xname, attrib) = self._get_loop_info(cur_path[-1]) + attrib['id'] = parent.name + self.writer.push(xname, attrib, first=False) + else: + last_path = self.last_path + match_idx = self._get_path_match_idx(last_path, cur_path) + root_path = self._path_list(commonprefix(['/'.join(cur_path), '/'.join(last_path)])) + if seg_node.is_first_seg_in_loop() and root_path == cur_path: + match_idx -= 1 + loop_struct = range(len(last_path) - 1, match_idx - 1, -1) + for i in loop_struct: + if i == loop_struct[-1]: + self.writer.pop() + else: + self.writer.pop() + for i in range(match_idx, len(cur_path)): + (xname, attrib) = self._get_loop_info(cur_path[i]) + # Write a Loop + parent_path_nodes = self.get_parents_in_path(seg_node) + attrib['id'] = parent_path_nodes[i] + parent_loop = cur_path[i-1] + if parent_loop not in self.visited: + self.visited.append(parent_loop) + self.writer.push(xname, attrib, first=True) + else: + self.writer.push(xname, attrib, first=False) + seg_node_id = self._get_node_id(seg_node, parent, seg_data) + (xname, attrib) = self._get_seg_info(seg_node_id) + attrib['id'] = seg_node.name + if seg_node.is_first_seg_in_loop(): + self.writer.push(xname, attrib, first=True) + else: + self.writer.push(xname, attrib, first=False) + loop_struct = range(len(seg_data)) + for i in loop_struct: + if i == loop_struct[-1]: + last = True + else: + # Check to see if any of the next children exist. + # If no, then we are on last node + try: + next_children = [seg_node.get_child_node_by_idx(index) for index in loop_struct[i+1:]] + except IndexError: + next_children = [] + next_node_exists = [not(child_node.usage == 'N' or seg_data.get('%02i' % (i + 1)).is_empty()) for child_node in next_children] + if any(next_node_exists): + last = False + else: + last = True + child_node = seg_node.get_child_node_by_idx(i) + if child_node.usage == 'N' or seg_data.get('%02i' % (i + 1)).is_empty(): + pass # Do not try to ouput for invalid or empty elements + elif child_node.is_composite(): + (xname, attrib) = self._get_comp_info(seg_node_id) + attrib['id'] = child_node.name + if i == loop_struct[0]: + self.writer.push(xname, attrib, first=True) + else: + self.writer.push(xname, attrib, first=False) + comp_data = seg_data.get('%02i' % (i + 1)) + for j in range(len(comp_data)): + subele_node = child_node.get_child_node_by_idx(j) + (xname, attrib) = self._get_subele_info(subele_node.id) + attrib['id'] = subele_node.name + self.writer.elem(xname, comp_data[j].get_value(), attrib, last) + elif child_node.is_element(): + if seg_data.get_value('%02i' % (i + 1)) == '': + pass + else: + attrib['id'] = child_node.name + self.writer.elem(xname, seg_data.get_value('%02i' % (i + 1)), attrib, last) + else: + raise EngineError('Node must be a either an element or a composite') + self.writer.pop() # end segment + if parent.id not in self.visited: + self.visited.append(parent.id) + self.last_path = cur_path + + + def seg(self, seg_node, seg_data): + """ + Generate JSON for the segment data and matching map node + @param seg_node: Map Node + @type seg_node: L{node} + @param seg_data: Segment object + @type seg_data: L{segment} + """ + if self.words_mode: + self.seg_with_names(seg_node, seg_data) + return + if not seg_node.is_segment(): raise EngineError('Node must be a segment') parent = pop_to_parent_loop(seg_node) # Get enclosing loop diff --git a/pyx12/x12n_document.py b/pyx12/x12n_document.py index ea715ee7..2d331415 100644 --- a/pyx12/x12n_document.py +++ b/pyx12/x12n_document.py @@ -100,7 +100,6 @@ def x12n_document(param, src_file, fd_997, fd_html, valid = True for seg in src: - # import pdb;pdb.set_trace() #find node orig_node = node From e99edeff937a7e89859723605d0a3b3c30185ee2 Mon Sep 17 00:00:00 2001 From: Tyler Kobil Date: Tue, 2 Feb 2021 14:39:56 -0500 Subject: [PATCH 06/14] error handling --- pyx12/error_handler.py | 31 ++++++++++++++++++++++++------- pyx12/jsonwriter.py | 6 ++++-- pyx12/x12json_simple.py | 1 - pyx12/x12n_document.py | 11 ++++++++--- 4 files changed, 36 insertions(+), 13 deletions(-) diff --git a/pyx12/error_handler.py b/pyx12/error_handler.py index 6763b354..69d2355b 100644 --- a/pyx12/error_handler.py +++ b/pyx12/error_handler.py @@ -23,7 +23,6 @@ class err_iter(object): """ Iterate over the error tree - Implements an odd iterator??? """ @@ -78,7 +77,7 @@ class err_handler(object): """ The interface to the error handling structures. """ - def __init__(self): + def __init__(self, errhandler): """ """ @@ -93,6 +92,16 @@ def __init__(self): self.seg_node_added = False self.cur_ele_node = None self.cur_line = 0 + self.errhandler = errhandler + self.err_cnt = 0 + + def add_summary(self, element_count): + """ + Params: element_count - reference to how many elements written in json output + """ + sout = "Data Elements Written: {}".format(element_count) + sout += "Number of Errors Found: {}".format(self.err_cnt) + self.errhandler.write(sout) def accept(self, visitor): """ @@ -228,6 +237,8 @@ def isa_error(self, err_cde, err_str): sout = '' sout += 'Line:%i ' % (self.cur_isa_node.get_cur_line()) sout += 'ISA:%s - %s' % (err_cde, err_str) + self.errhandler.write(sout) + self.err_cnt += 1 logger.error(sout) self.cur_isa_node.add_error(err_cde, err_str) @@ -241,6 +252,8 @@ def gs_error(self, err_cde, err_str): sout = '' sout += 'Line:%i ' % (self.cur_gs_node.get_cur_line()) sout += 'GS:%s - %s' % (err_cde, err_str) + self.errhandler.write(sout) + self.err_cnt += 1 logger.error(sout) self.cur_gs_node.add_error(err_cde, err_str) @@ -254,6 +267,8 @@ def st_error(self, err_cde, err_str): sout = '' sout += 'Line:%i ' % (self.cur_st_node.get_cur_line()) sout += 'ST:%s - %s' % (err_cde, err_str) + self.errhandler.write(sout) + self.err_cnt += 1 logger.error(sout) self.cur_st_node.add_error(err_cde, err_str) @@ -278,6 +293,9 @@ def seg_error(self, err_cde, err_str, err_value=None, src_line=None): sout += 'SEG:%s - %s' % (err_cde, err_str) if err_value: sout += ' (%s)' % err_value + sout += '' + self.errhandler.write(sout) + self.err_cnt += 1 logger.error(sout) def ele_error(self, err_cde, err_str, bad_value, refdes=None): @@ -295,6 +313,9 @@ def ele_error(self, err_cde, err_str, bad_value, refdes=None): sout += 'ELE:%s - %s' % (err_cde, err_str) if bad_value: sout += ' (%s)' % (bad_value) + sout += '' + self.errhandler.write(sout) + self.err_cnt += 1 logger.error(sout) #print self.cur_ele_node.errors @@ -587,7 +608,6 @@ def __init__(self, parent, seg_data, src): @type seg_data: L{segment} @param src: X12file source @type src: L{X12file} - """ self.seg_data = seg_data self.isa_id = src.get_isa_id() @@ -720,7 +740,6 @@ def __repr__(self): class err_st(err_node): """ ST loops - Needs: 1. Transaction set id code (837, 834) 2. Transaction set control number @@ -771,7 +790,6 @@ def add_error(self, err_cde, err_str): def close(self, node, seg_data, src): """ Close ST loop - @param node: SE node @type node: L{node} @param seg_data: Segment object @@ -935,7 +953,6 @@ class err_ele(err_node): """ Element Errors - Holds and generates output for element and composite/sub-element errors - Each element with an error creates a new err_ele instance. """ def __init__(self, parent, map_node): @@ -1307,4 +1324,4 @@ def is_closed(self): def __repr__(self): """ """ - return '%i: %s' % (-1, self.id) + return '%i: %s' % (-1, self.id) \ No newline at end of file diff --git a/pyx12/jsonwriter.py b/pyx12/jsonwriter.py index 55db4c0d..cbadc729 100644 --- a/pyx12/jsonwriter.py +++ b/pyx12/jsonwriter.py @@ -35,6 +35,7 @@ def __init__(self, out=sys.stdout, encoding="utf-8", indent="\t", words_mode=Tru self.stack = [] self.indent = indent self.words_mode = words_mode + self.element_count = 0 def push(self, elem, attrs={}, first=False): """ @@ -64,6 +65,7 @@ def elem(self, elem, content, attrs={}, last=False): """ self._indent() for (_, v) in list(attrs.items()): + self.element_count += 1 if last: self._write('''"%s": "%s"''' % (self._escape_attr(v), self._escape_cont(content))) #Newline else: @@ -104,11 +106,11 @@ def _escape_cont(self, text): if text is None: return None return text.replace("&", "&")\ - .replace("<", "<").replace(">", ">") + .replace("<", "<").replace(">", ">").replace("\t", "") def _escape_attr(self, text): if self.words_mode: - return text + return text.replace(' ', '_').replace(',', '') if text is None: return None return text.replace("&", "&") \ diff --git a/pyx12/x12json_simple.py b/pyx12/x12json_simple.py index 63867b72..1bbd3b83 100644 --- a/pyx12/x12json_simple.py +++ b/pyx12/x12json_simple.py @@ -50,7 +50,6 @@ def seg_with_names(self, seg_node, seg_data): Generate JSON for the segment data and matching map node. Essentially the same as "seg", however this will write String field names, rather than codes. - @param seg_node: Map Node @type seg_node: L{node} @param seg_data: Segment object diff --git a/pyx12/x12n_document.py b/pyx12/x12n_document.py index 2d331415..dc33c990 100644 --- a/pyx12/x12n_document.py +++ b/pyx12/x12n_document.py @@ -14,6 +14,7 @@ """ import logging +import sys # Intrapackage imports import pyx12.error_handler @@ -49,7 +50,7 @@ def _reset_counter_to_gs_counts(walker): def x12n_document(param, src_file, fd_997, fd_html, fd_xmldoc=None, fd_jsondoc=None, xslt_files=None, map_path=None, - callback=None): + callback=None, errhandler=sys.stderr): """ Primary X12 validation function @param param: pyx12.param instance @@ -64,9 +65,11 @@ def x12n_document(param, src_file, fd_997, fd_html, @param fd_jsondoc: JSON output document (outputs to single line) @type fd_jsondoc: file descriptor @rtype: boolean + @param errhandler: Error Output Document + @type errhandler: file descriptor """ logger = logging.getLogger('pyx12') - errh = pyx12.error_handler.err_handler() + errh = pyx12.error_handler.err_handler(errhandler=errhandler) # Get X12 DATA file try: @@ -243,6 +246,8 @@ def x12n_document(param, src_file, fd_997, fd_html, del xmldoc if fd_jsondoc: + element_cnt = fd_jsondoc.writer.element_count + errh.add_summary(element_cnt) del fd_jsondoc #visit_debug = pyx12.error_debug.error_debug_visitor(sys.stdout) @@ -278,4 +283,4 @@ def x12n_document(param, src_file, fd_997, fd_html, return True except Exception: print(errh) - return False + return False \ No newline at end of file From ef2f76d94ac1a13317b1a37f69bea58a5d6ec874 Mon Sep 17 00:00:00 2001 From: Tyler Kobil Date: Thu, 4 Feb 2021 08:50:30 -0500 Subject: [PATCH 07/14] enabling return of json --- pyx12/jsonwriter.py | 15 +++++++++++++-- pyx12/x12json_simple.py | 18 +++++++++++++++--- pyx12/x12n_document.py | 21 ++++++++++++++------- 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/pyx12/jsonwriter.py b/pyx12/jsonwriter.py index cbadc729..8c12c533 100644 --- a/pyx12/jsonwriter.py +++ b/pyx12/jsonwriter.py @@ -42,7 +42,8 @@ def push(self, elem, attrs={}, first=False): Create an element which will have child elements """ if elem == "comp": - return + # import pdb;pdb.set_trace() + pass # To-Do: Ensure composites are being written correctly! self._indent() if first: @@ -51,12 +52,16 @@ def push(self, elem, attrs={}, first=False): self._write("""{"%s": [""" % self._escape_attr(v)) #newline elif elem == "seg": self._write("""{"%s": {""" % self._escape_attr(v)) #newline + elif elem == "comp": + self._write(""""%s": {""" % self._escape_attr(v)) #newline else: for (_, v) in list(attrs.items()): if elem == "loop": self._write(""",{"%s": [""" % self._escape_attr(v)) #newline elif elem == "seg": self._write(""",{"%s": {""" % self._escape_attr(v)) #newline + elif elem == "comp": + self._write(""","%s": {""" % self._escape_attr(v)) #newline self.stack.append(elem) def elem(self, elem, content, attrs={}, last=False): @@ -85,6 +90,9 @@ def pop(self, last=True): elif elem == "loop": self._indent() self._write("]}") #newline + elif elem == "comp": + self._indent() + self._write("}") else: if elem == "seg": self._indent() @@ -92,6 +100,9 @@ def pop(self, last=True): elif elem == "loop": self._indent() self._write("]},") #newline + elif elem == "comp": + self._indent() + self._write("},") def __len__(self): return len(self.stack) @@ -110,7 +121,7 @@ def _escape_cont(self, text): def _escape_attr(self, text): if self.words_mode: - return text.replace(' ', '_').replace(',', '') + return text.replace(' ', '_').replace(',', '').replace('(', '[').replace(')', ']') if text is None: return None return text.replace("&", "&") \ diff --git a/pyx12/x12json_simple.py b/pyx12/x12json_simple.py index 1bbd3b83..c6623450 100644 --- a/pyx12/x12json_simple.py +++ b/pyx12/x12json_simple.py @@ -20,6 +20,9 @@ def __init__(self, fd, words_mode=True): self.words_mode = words_mode def __del__(self): + self.finalize() + + def finalize(self): while len(self.writer) > 0: self.writer.pop() @@ -124,10 +127,15 @@ def seg_with_names(self, seg_node, seg_data): self.writer.push(xname, attrib, first=False) comp_data = seg_data.get('%02i' % (i + 1)) for j in range(len(comp_data)): + if j == range(len(comp_data))[-1]: + elem_last = True + else: + elem_last = False subele_node = child_node.get_child_node_by_idx(j) (xname, attrib) = self._get_subele_info(subele_node.id) attrib['id'] = subele_node.name - self.writer.elem(xname, comp_data[j].get_value(), attrib, last) + self.writer.elem(xname, comp_data[j].get_value(), attrib, elem_last) + self.writer.pop(last=last) # end composite elif child_node.is_element(): if seg_data.get_value('%02i' % (i + 1)) == '': pass @@ -218,10 +226,14 @@ def seg(self, seg_node, seg_data): self.writer.push(xname, attrib, first=False) comp_data = seg_data.get('%02i' % (i + 1)) for j in range(len(comp_data)): + if j == range(len(comp_data))[-1]: + elem_last = True + else: + elem_last = False subele_node = child_node.get_child_node_by_idx(j) (xname, attrib) = self._get_subele_info(subele_node.id) - self.writer.elem(xname, comp_data[j].get_value(), attrib, last) - # self.writer.pop() # end composite + self.writer.elem(xname, comp_data[j].get_value(), attrib, elem_last) + self.writer.pop(last=last) # end composite elif child_node.is_element(): if seg_data.get_value('%02i' % (i + 1)) == '': pass diff --git a/pyx12/x12n_document.py b/pyx12/x12n_document.py index dc33c990..841b49a8 100644 --- a/pyx12/x12n_document.py +++ b/pyx12/x12n_document.py @@ -49,7 +49,7 @@ def _reset_counter_to_gs_counts(walker): def x12n_document(param, src_file, fd_997, fd_html, - fd_xmldoc=None, fd_jsondoc=None, xslt_files=None, map_path=None, + fd_xmldoc=None, fd_jsondoc=None, return_jsonstring=False, dump_json_to_words=True,xslt_files=None, map_path=None, callback=None, errhandler=sys.stderr): """ Primary X12 validation function @@ -96,7 +96,7 @@ def x12n_document(param, src_file, fd_997, fd_html, if fd_xmldoc: xmldoc = pyx12.x12xml_simple.x12xml_simple(fd_xmldoc, param.get('simple_dtd')) if fd_jsondoc: - fd_jsondoc = pyx12.x12json_simple.X12JsonSimple(fd_jsondoc) + fd_jsondoc = pyx12.x12json_simple.X12JsonSimple(fd_jsondoc, words_mode=dump_json_to_words) #basedir = os.path.dirname(src_file) #erx = errh_xml.err_handler(basedir=basedir) @@ -248,7 +248,6 @@ def x12n_document(param, src_file, fd_997, fd_html, if fd_jsondoc: element_cnt = fd_jsondoc.writer.element_count errh.add_summary(element_cnt) - del fd_jsondoc #visit_debug = pyx12.error_debug.error_debug_visitor(sys.stdout) #errh.accept(visit_debug) @@ -278,9 +277,17 @@ def x12n_document(param, src_file, fd_997, fd_html, pass try: if not valid or errh.get_error_count() > 0: - return False + success = False else: - return True + success = True except Exception: - print(errh) - return False \ No newline at end of file + print(errh) #todo: change to logging + success = False + + if fd_jsondoc: + fd_jsondoc.finalize() + if return_jsonstring: + return fd_jsondoc.writer.out.content, success + + return "", success + \ No newline at end of file From 75901adfd9b1a542eb79de538da7bca79cf6fa7d Mon Sep 17 00:00:00 2001 From: Tyler Kobil Date: Thu, 4 Feb 2021 08:52:44 -0500 Subject: [PATCH 08/14] removing comment --- pyx12/jsonwriter.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyx12/jsonwriter.py b/pyx12/jsonwriter.py index 8c12c533..937407b1 100644 --- a/pyx12/jsonwriter.py +++ b/pyx12/jsonwriter.py @@ -41,9 +41,6 @@ def push(self, elem, attrs={}, first=False): """ Create an element which will have child elements """ - if elem == "comp": - # import pdb;pdb.set_trace() - pass # To-Do: Ensure composites are being written correctly! self._indent() if first: From 490709757d3321dcdbe1fde092f4ee0d685c0de9 Mon Sep 17 00:00:00 2001 From: Tyler Kobil Date: Thu, 4 Feb 2021 13:40:29 -0500 Subject: [PATCH 09/14] jsonwriter test --- pyx12/test/test_jsonwriter.py | 86 +++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 pyx12/test/test_jsonwriter.py diff --git a/pyx12/test/test_jsonwriter.py b/pyx12/test/test_jsonwriter.py new file mode 100644 index 00000000..93732081 --- /dev/null +++ b/pyx12/test/test_jsonwriter.py @@ -0,0 +1,86 @@ +import os.path +import sys +import os +import unittest + +try: + from StringIO import StringIO +except: + from io import StringIO + +import tempfile + +from pyx12.jsonwriter import JSONriter + +class TestJsonWriter(unittest.TestCase): + """ + """ + + def test_write_loop_first(self): + # With StringIO Object + try: + fd = StringIO(encoding='ascii') + except: + fd = StringIO() + + writer = JSONriter(fd) + attrs = {'id': 'TestLoopFirst'} + writer.push('loop', attrs, first=True) + while len(writer) > 0: + writer.pop() + writer.pop() + expected = """{"TestLoopFirst": []}""" + self.assertEqual(fd.getvalue(), expected) + fd.close() + + # With Temp File + _, filename = tempfile.mkstemp('.json', 'pyx12_') + with open(filename, 'w') as fd: + writer = JSONriter(fd) + attrs = {'id': 'TestLoopFirst'} + writer.push('loop', attrs, first=True) + while len(writer) > 0: + writer.pop() + + with open(filename, 'r') as fd: + self.assertEqual(fd.read(), expected) + + try: + os.remove(filename) + except: + pass + + def test_write_loop(self): + # With StringIO Object + try: + fd = StringIO(encoding='ascii') + except: + fd = StringIO() + + writer = JSONriter(fd) + attrs = {'id': 'TestLoopFirst'} + writer.push('loop', attrs) + writer.pop() + expected = """,{"TestLoopFirst": []}""" + self.assertEqual(fd.getvalue(), expected) + fd.close() + + # With Temp File + _, filename = tempfile.mkstemp('.json', 'pyx12_') + with open(filename, 'w') as fd: + writer = JSONriter(fd) + attrs = {'id': 'TestLoopFirst'} + writer.push('loop', attrs) + while len(writer) > 0: + writer.pop() + + with open(filename, 'r') as fd: + self.assertEqual(fd.read(), expected) + + try: + os.remove(filename) + except: + pass + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From f221821fc333a955d6db0e6ce29d290ead2c16b4 Mon Sep 17 00:00:00 2001 From: Tyler Kobil Date: Thu, 4 Feb 2021 16:42:06 -0500 Subject: [PATCH 10/14] removing return_jsonstr option --- pyx12/x12n_document.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pyx12/x12n_document.py b/pyx12/x12n_document.py index 841b49a8..7579c133 100644 --- a/pyx12/x12n_document.py +++ b/pyx12/x12n_document.py @@ -49,7 +49,7 @@ def _reset_counter_to_gs_counts(walker): def x12n_document(param, src_file, fd_997, fd_html, - fd_xmldoc=None, fd_jsondoc=None, return_jsonstring=False, dump_json_to_words=True,xslt_files=None, map_path=None, + fd_xmldoc=None, fd_jsondoc=None, dump_json_to_words=True,xslt_files=None, map_path=None, callback=None, errhandler=sys.stderr): """ Primary X12 validation function @@ -286,8 +286,6 @@ def x12n_document(param, src_file, fd_997, fd_html, if fd_jsondoc: fd_jsondoc.finalize() - if return_jsonstring: - return fd_jsondoc.writer.out.content, success - return "", success + return success \ No newline at end of file From af582907e93ee9f93a1bc78f604ba7e1b21d26a2 Mon Sep 17 00:00:00 2001 From: Tyler Kobil Date: Fri, 5 Feb 2021 16:07:38 -0500 Subject: [PATCH 11/14] 837 bug --- pyx12/jsonwriter.py | 12 ++++++------ pyx12/x12json_simple.py | 4 +++- pyx12/x12n_document.py | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/pyx12/jsonwriter.py b/pyx12/jsonwriter.py index 937407b1..33b4d901 100644 --- a/pyx12/jsonwriter.py +++ b/pyx12/jsonwriter.py @@ -46,19 +46,19 @@ def push(self, elem, attrs={}, first=False): if first: for (_, v) in list(attrs.items()): if elem == "loop": - self._write("""{"%s": [""" % self._escape_attr(v)) #newline + self._write("""{"%s": [""" % self._escape_attr(v)) elif elem == "seg": - self._write("""{"%s": {""" % self._escape_attr(v)) #newline + self._write("""{"%s": {""" % self._escape_attr(v)) elif elem == "comp": - self._write(""""%s": {""" % self._escape_attr(v)) #newline + self._write(""""%s": {""" % self._escape_attr(v)) else: for (_, v) in list(attrs.items()): if elem == "loop": - self._write(""",{"%s": [""" % self._escape_attr(v)) #newline + self._write(""",{"%s": [""" % self._escape_attr(v)) elif elem == "seg": - self._write(""",{"%s": {""" % self._escape_attr(v)) #newline + self._write(""",{"%s": {""" % self._escape_attr(v)) elif elem == "comp": - self._write(""","%s": {""" % self._escape_attr(v)) #newline + self._write(""""%s": {""" % self._escape_attr(v)) #component is essentially an element self.stack.append(elem) def elem(self, elem, content, attrs={}, last=False): diff --git a/pyx12/x12json_simple.py b/pyx12/x12json_simple.py index c6623450..12d41535 100644 --- a/pyx12/x12json_simple.py +++ b/pyx12/x12json_simple.py @@ -219,7 +219,9 @@ def seg(self, seg_node, seg_data): if child_node.usage == 'N' or seg_data.get('%02i' % (i + 1)).is_empty(): pass # Do not try to ouput for invalid or empty elements elif child_node.is_composite(): - (xname, attrib) = self._get_comp_info(seg_node_id) + (xname, attrib) = self._get_comp_info(seg_node_id) #consider child_node.id for child node as key, not parent! + # if attrib['id'] == 'CLM': + # import pdb;pdb.set_trace() if i == loop_struct[0]: self.writer.push(xname, attrib, first=True) else: diff --git a/pyx12/x12n_document.py b/pyx12/x12n_document.py index 7579c133..9dbee3f1 100644 --- a/pyx12/x12n_document.py +++ b/pyx12/x12n_document.py @@ -49,7 +49,7 @@ def _reset_counter_to_gs_counts(walker): def x12n_document(param, src_file, fd_997, fd_html, - fd_xmldoc=None, fd_jsondoc=None, dump_json_to_words=True,xslt_files=None, map_path=None, + fd_xmldoc=None, fd_jsondoc=None, dump_json_to_words=False, xslt_files=None, map_path=None, callback=None, errhandler=sys.stderr): """ Primary X12 validation function From 82a880fcab9df512f764170bd32c65b360bfa874 Mon Sep 17 00:00:00 2001 From: Tyler Kobil Date: Fri, 5 Feb 2021 16:09:12 -0500 Subject: [PATCH 12/14] remove comments --- pyx12/x12json_simple.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyx12/x12json_simple.py b/pyx12/x12json_simple.py index 12d41535..b7dab528 100644 --- a/pyx12/x12json_simple.py +++ b/pyx12/x12json_simple.py @@ -220,8 +220,6 @@ def seg(self, seg_node, seg_data): pass # Do not try to ouput for invalid or empty elements elif child_node.is_composite(): (xname, attrib) = self._get_comp_info(seg_node_id) #consider child_node.id for child node as key, not parent! - # if attrib['id'] == 'CLM': - # import pdb;pdb.set_trace() if i == loop_struct[0]: self.writer.push(xname, attrib, first=True) else: From aff2b40125866c6bfcbb9ed83842e946d0ad4ac1 Mon Sep 17 00:00:00 2001 From: Tyler Kobil Date: Mon, 15 Feb 2021 18:29:05 -0500 Subject: [PATCH 13/14] component node names --- pyx12/x12json_simple.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyx12/x12json_simple.py b/pyx12/x12json_simple.py index b7dab528..d0011388 100644 --- a/pyx12/x12json_simple.py +++ b/pyx12/x12json_simple.py @@ -119,7 +119,7 @@ def seg_with_names(self, seg_node, seg_data): if child_node.usage == 'N' or seg_data.get('%02i' % (i + 1)).is_empty(): pass # Do not try to ouput for invalid or empty elements elif child_node.is_composite(): - (xname, attrib) = self._get_comp_info(seg_node_id) + (xname, attrib) = self._get_comp_info(child_node.id) attrib['id'] = child_node.name if i == loop_struct[0]: self.writer.push(xname, attrib, first=True) @@ -219,8 +219,7 @@ def seg(self, seg_node, seg_data): if child_node.usage == 'N' or seg_data.get('%02i' % (i + 1)).is_empty(): pass # Do not try to ouput for invalid or empty elements elif child_node.is_composite(): - (xname, attrib) = self._get_comp_info(seg_node_id) #consider child_node.id for child node as key, not parent! - if i == loop_struct[0]: + (xname, attrib) = self._get_comp_info(child_node.id) # formerly seg_node_id self.writer.push(xname, attrib, first=True) else: self.writer.push(xname, attrib, first=False) From c4eda813f79ff946818bd1d77207691a095eb737 Mon Sep 17 00:00:00 2001 From: Tyler Kobil Date: Mon, 15 Feb 2021 18:33:04 -0500 Subject: [PATCH 14/14] editing mistake --- pyx12/x12json_simple.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyx12/x12json_simple.py b/pyx12/x12json_simple.py index d0011388..06dd3c47 100644 --- a/pyx12/x12json_simple.py +++ b/pyx12/x12json_simple.py @@ -220,6 +220,7 @@ def seg(self, seg_node, seg_data): pass # Do not try to ouput for invalid or empty elements elif child_node.is_composite(): (xname, attrib) = self._get_comp_info(child_node.id) # formerly seg_node_id + if i == loop_struct[0]: self.writer.push(xname, attrib, first=True) else: self.writer.push(xname, attrib, first=False)