Diff between old NW and NW used by MTC

README.md

@@ -1,3 +1,11 @@
+Current Branch Changes:
+=======================
+This branch's purpose is to be used for TM1.5.1.10 model runs.
+It was created from the 9 Feb 2023 commit e95e5a3.
+
+please contact [email protected] with questions
+
+
 NetworkWrangler
 ===============
 
@@ -8,23 +16,42 @@ set of projects to that base network.
 
 The base network and resulting network are in the Citilabs Cube format (http://www.citilabs.com/software/cube/)
 
+See also: [next generation Network Wrangler](https://github.com/wsp-sag/network_wrangler)
+
 Contributors
 =======
 NetworkWrangler is the brainchild of Billy Charlton, who was the Deputy Director for Technology Services at SFCTA through 2011.
 Contributors include:
 * Elizabeth Sall, 2010-2014
-* Lisa Zorn, 2010-2014
+* Lisa Zorn, 2010-2014, 2018-Present (MTC fork)
 * Dan Tischler, 2011-Present
 * Drew Cooper, 2013-Present
 * Bhargava Sana, 2015-Present
 
-Usage
+Install
 =======
+NetworkWrangler recently was updated to Python3. Recommand using conda to install and manage NetworkWrangler environment. For example, run the following commands in Anaconda Prompt:
+```
+conda env create -f M:\Software\Anaconda\NetworkWrangler_py3.yml
+conda activate NetworkWrangler-py3
+cd \NetworkWrangler
+pip install -e .
+```
 
-Build a network by running the `build_network.py` script  in the `/scripts` folder.
-
-   python build_network.py [-c configword] [-m test] network_specification.py
+Usage
+=======
+If needed, set the PATHs before calling the build_network.py script, for example:
+```
+set PATH=%PATH%;C:\Users\mtcpb\.conda\envs\NetworkWrangler-py3;C:\Users\mtcpb\.conda\envs\NetworkWrangler-py3\Scripts
+set PATH=%PATH%;C:\Program Files\Citilabs\CubeVoyager;
+set PYTHONPATH=%PYTHONPATH%;C:\Users\mtcpb\Documents\GitHub\NetworkWrangler
+set PYTHONPATH=%PYTHONPATH%;C:\Users\mtcpb\Documents\GitHub\NetworkWrangler\_static
+```
 
+With PATHs set, build a network by running the `build_network.py` script  in the `/scripts` folder.
+```
+python build_network.py [-c configword] [-m test] network_specification.py
+```
 
 This will build a network using the specifications in `network_specification.py`, which should define the variables listed below (in this script)
 
@@ -34,4 +61,4 @@ If test mode is specified (with -m test), then the following happen:
   * TAG is not used for TEST_PROJECTS
 
 The [-c configword] is if you want an optional word for your network_specification.py
-  (e.g. to have multiple scenarios in one file).  Access it via CONFIG_WORD.
+  (e.g. to have multiple scenarios in one file).  Access it via CONFIG_WORD.

Wrangler/HighwayNetwork.py

@@ -86,6 +86,19 @@ def applyBasenetwork(self, parentdir, networkdir, gitdir, tierNetworkName):
         # done
         self.applyingBasenetwork = False
 
+    def saveNetworkFiles(self, suffix, to_suffix):
+        """
+        Since roadway networks are not stored in memory but in files, this is useful
+        for when the network builder is doing something tricky.
+        """
+        for filename in ["FREEFLOW.BLD", "turnsam.pen", "turnspm.pen", "turnsop.pen", "tolls.csv"]:
+            if to_suffix:
+                shutil.copy2(src=filename, dst="{}{}".format(filename, suffix))
+                WranglerLogger.debug("Copying {:20} to {}{}".format(filename, filename, suffix))
+            else:
+                shutil.copy2(src="{}{}".format(filename, suffix), dst=filename)
+                WranglerLogger.debug("Copying {:20} to {}".format(filename+suffix, filename))
+
     def applyProject(self, parentdir, networkdir, gitdir, projectsubdir=None, **kwargs):
         """
         Applies a roadway project by calling ``runtpp`` on the ``apply.s`` script.
@@ -145,6 +158,7 @@ def applyProject(self, parentdir, networkdir, gitdir, projectsubdir=None, **kwar
 
             nodemerge = re.compile("NODEMERGE: \d+")
             linkmerge = re.compile("LINKMERGE: \d+-\d+")
+            cube_success = re.compile("\s*(VOYAGER)\s+(ReturnCode)\s*=\s*([01])\s+")
             license_error = False
             for line in cubeStdout:
                 line = line.rstrip()
@@ -165,9 +179,18 @@ def applyProject(self, parentdir, networkdir, gitdir, projectsubdir=None, **kwar
                 time.sleep(1)
                 continue
 
+            if cuberet != 0 and cuberet != 1 and len(cubeStdout) > 1:
+
+                # cuberet may be wrong -- check last stdout
+                WranglerLogger.debug("checking cubeStdout[-1]: {}".format(cubeStdout[-1]))
+                # WranglerLogger.debug("match: {}".format(re.match(cube_success,cubeStdout[-1])))
+                if re.match(cube_success,cubeStdout[-1]):
+                    WranglerLogger.debug("Overriding cuberet {} with 0 due to last cubeStdout line".format(cuberet))
+                    cuberet = 0
 
             if cuberet != 0 and cuberet != 1:
-                WranglerLogger.fatal("FAIL! Project: "+applyScript)
+                WranglerLogger.debug("cubeStdout: {}".format(cubeStdout))
+                WranglerLogger.fatal("FAIL! Project: {}  cuberet={}".format(applyScript, cuberet))
                 raise NetworkException("HighwayNetwork applyProject failed; see log file")
 
             else:
@@ -212,7 +235,7 @@ def mergeTolls(self, tollsfile, newtollsfile):
         """
         Merge the given tolls file with the existing.
         """
-        WranglerLogger.debug("TODO: merging tolls")     
+        WranglerLogger.debug("mergeTolls({},{}) called".format(tollsfile, newtollsfile))     
 
         # read the original file -- fac_index is the key
         tolls_config = collections.OrderedDict()
@@ -222,6 +245,7 @@ def mergeTolls(self, tollsfile, newtollsfile):
         # not using csv.DictReader because in python2, it doesn't read an ordered dict :(
         for row in tolls_reader:
             row_dict = collections.OrderedDict(zip(fieldnames, row))
+            WranglerLogger.debug("row_dict: {}".format(row_dict))
             tolls_config[row_dict["fac_index"]] = row_dict
         tolls.close()
 
@@ -245,7 +269,7 @@ def mergeTolls(self, tollsfile, newtollsfile):
         newtolls.close()
 
         # write it out
-        tolls = open(tollsfile, mode='wb')
+        tolls = open(tollsfile, mode='w', newline='')  # newline arg passed because of https://docs.python.org/3/library/csv.html#id3
         tolls_writer = csv.writer(tolls)
         tolls_writer.writerow(fieldnames)
         for fac_index, row in tolls_config.items():

Wrangler/Network.py

@@ -53,22 +53,31 @@ def _runAndLog(self, cmd, run_dir=".", logStdoutAndStderr=False, env=None):
         if env:
             myenv = copy.deepcopy(os.environ)
             myenv.update(env)
-            # ranglerLogger.debug("Using environment {}".format(myenv))
-
-        proc = subprocess.Popen( cmd, cwd = run_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, env=myenv )
-        retStdout = []
-        for line in proc.stdout:
-            line = line.strip(b'\r\n')
-            if logStdoutAndStderr: WranglerLogger.debug("stdout: " + line)
-            retStdout.append(line)
-
-        retStderr = []
-        for line in proc.stderr:
-            line = line.strip(b'\r\n')
-            if logStdoutAndStderr: WranglerLogger.debug("stderr: " + line)
-            retStderr.append(line)
-        retcode  = proc.wait()
-        WranglerLogger.debug("Received %d from [%s] run in [%s]" % (retcode, cmd, run_dir))
+            # WranglerLogger.debug("Using environment {}".format(myenv))
+
+        try:
+            proc = subprocess.Popen( cmd, cwd = run_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, env=myenv )
+            retStdout = []
+            for line in proc.stdout:
+                if type(line)==bytes: line = line.decode()  # convert to string, not byetes
+                line = line.strip('\r\n')
+                if logStdoutAndStderr: WranglerLogger.debug("stdout: " + line)
+                retStdout.append(line)
+
+            retStderr = []
+            for line in proc.stderr:
+                if type(line)==bytes: line = line.decode() # convert to string, not byetes
+                line = line.strip('\r\n')
+                if logStdoutAndStderr: WranglerLogger.debug("stderr: " + line)
+                retStderr.append(line)
+            retcode  = proc.wait()
+            WranglerLogger.debug("Received {} from [{}] run in [{}]".format(retcode, cmd, run_dir))
+
+        except Exception as inst:
+            WranglerLogger.error('Exception caught')
+            WranglerLogger.error(type(inst))    # the exception instance
+            WranglerLogger.error(inst.args)     # arguments stored in .args
+            WranglerLogger.error(inst)          # __str__ allows args to be printed directly,
         return (retcode, retStdout, retStderr)
 
     def getReqs(self, networkdir, projectsubdir=None, tag=None, projtype=None, tempdir=None):
@@ -419,12 +428,17 @@ def getCommit(self, gitdir):
         Figures out the SHA1 hash commit string for the given gitdir (so gitdir is a git dir).
         (e.g. a 40-character hex string)
         """
+        # WranglerLogger.debug("getCommit({}) called".format(gitdir))
+        # SET_CAPCLASS is special and setup by NetworkWrangler
+        if gitdir.endswith("set_capclass"):
+            return "set_capclass_N/A"
+
         cmd = r"git log -1"
         (retcode, retstdout, retstderr) = self._runAndLog(cmd, gitdir)
         if len(retstdout)<3:
             raise NetworkException("Git log failed; see log file")
 
-        m = re.match(git_commit_pattern, retstdout[0].decode("utf-8"))
+        m = re.match(git_commit_pattern, retstdout[0])
         if not m:
             raise NetworkException("Didn't understand git log output: [" + retstdout[0] + "]")
 

Wrangler/Node.py

@@ -1,4 +1,5 @@
 import os,sys
+from .Logger import WranglerLogger
 
 __all__ = ['Node']
 
@@ -46,6 +47,19 @@ def getNum(self):
         """
         return abs(int(self.num))
 
+    def __eq__(self, other):
+        """
+        For finding nodes in a list of Node objects
+        """
+        if isinstance(other, int):
+            return int(self.num) == other
+        elif isinstance(other, Node):
+            return int(self.num) == int(other.num)
+        elif isinstance(other, str):
+            return int(self.num) == int(other)
+        else:
+            WranglerLogger.error("Node.__eq__ called with other type {}".format(type(other)))
+
     def replaceNum(self, num):
         """
         Replace the node number with the given num,

Wrangler/Supplink.py

@@ -19,7 +19,7 @@ class Supplink(dict):
     def __init__(self):
         dict.__init__(self)
 
-        MODES_INV = dict((v,k) for k,v in Supplink.MODES.iteritems())
+        MODES_INV = dict((v,k) for k,v in Supplink.MODES.items())
 
         self.id=''  # string, e.g. "1-7719"
         self.comment=None

Wrangler/TransitAssignmentData.py

@@ -173,10 +173,9 @@ def initializeFields(self, headerRow=None):
                                 "OWNER",
                                 "AB_VOL","AB_BRDA","AB_XITA","AB_BRDB","AB_XITB",
                                 "BA_VOL","BA_BRDA","BA_XITA","BA_BRDB","BA_XITB"]
-        # print("csvColnames = {}".format(self.csvColnames))
-
-        self.colnameToCsvIndex = dict((self.csvColnames[idx],idx) for idx in range(len(self.csvColnames)))
 
+        self.colnameToCsvIndex = dict((self.csvColnames[idx],idx) for idx in range(len(self.csvColnames)))
+
         # copy these directly
         self.trnAsgnFields = {"A":      'u4',
                               "B":      'u4',
@@ -189,7 +188,7 @@ def initializeFields(self, headerRow=None):
                               "NAME":   'a13',
                               "OWNER":  'a10',
                               }
-        self.trnAsgnCopyFields = self.trnAsgnFields.keys()
+        self.trnAsgnCopyFields = list(self.trnAsgnFields.keys())
         # these are in the dbf not the csv (grrrr)
         self.trnAsgnFields["FREQ"]      = 'f4'
         self.trnAsgnFields["SEQ"]       = 'u1'
@@ -233,7 +232,6 @@ def initializeFields(self, headerRow=None):
         for field in self.trnAsgnAdditiveFields:
             self.aggregateFields[field]='f4'
 
-
     def readTransitAssignmentCsvs(self):
         """
         Read the transit assignment dbfs, the direct output of Cube's transit assignment.
@@ -264,7 +262,7 @@ def readTransitAssignmentCsvs(self):
                 numrecs = 0
                 totalrows = 0
 
-                filereader = csv.reader(open(filename, 'rb'), delimiter=',', quoting=csv.QUOTE_NONE)
+                filereader = csv.reader(open(filename, 'r'), delimiter=',', quoting=csv.QUOTE_NONE)
                 for row in filereader:
 
                     # header row?
@@ -294,16 +292,15 @@ def readTransitAssignmentCsvs(self):
 
                 self.trnAsgnTable = DataTable(numRecords=numrecs,
                                               fieldNames=self.trnAsgnFields.keys(),
-                                              numpyFieldTypes=self.trnAsgnFields.values())
+                                              numpyFieldTypes=list(self.trnAsgnFields.values()))
                 ABNameSeqSet = set()
                 WranglerLogger.debug("Created dataTable")
 
             # Go through the records
             newrownum = 0  # row number in the trnAsgnTable,ABNameSeq_List -- rows we're keeping
             oldrownum = 0  # row number in the csv,dbf -- all input rows
 
-            filereader = csv.reader(open(filename, 'rb'), delimiter=',', quoting=csv.QUOTE_NONE)
-
+            filereader = csv.reader(open(filename, 'r'), delimiter=',', quoting=csv.QUOTE_NONE)
             # for the first csv only, also read the dbf for the freq and seq fields
             if mode == self.MODES[0]:
                 if self.modelType == Network.MODEL_TYPE_CHAMP:
@@ -343,7 +340,6 @@ def readTransitAssignmentCsvs(self):
                     for field in self.trnAsgnCopyFields:
 
                         try:
-
                             # integer fields
                             if self.trnAsgnFields[field][0] in ['u','b']:
                                 if row[self.colnameToCsvIndex[field]]=="":
@@ -403,6 +399,7 @@ def readTransitAssignmentCsvs(self):
                         ABNameSeq = tryABNameSeq
                     self.trnAsgnTable[newrownum]["ABNAMESEQ"] = ABNameSeq
                     ABNameSeqSet.add(ABNameSeq)
+                    # WranglerLogger.debug("AABNAMESEQ={} type={}".format(ABNameSeq, type(ABNameSeq)))
 
                     ABNameSeq_List.append((int(row[self.colnameToCsvIndex["A"]]),
                                            int(row[self.colnameToCsvIndex["B"]]),
@@ -425,12 +422,12 @@ def readTransitAssignmentCsvs(self):
                        self.trnAsgnTable[newrownum]["PERIODCAP"] = 0
 
                     # if we still don't have a system, warn
-                    if self.trnAsgnTable[newrownum]["SYSTEM"] == "" and not warnline.has_key(linename):
+                    if self.trnAsgnTable[newrownum]["SYSTEM"] == "" and linename not in warnline:
                         WranglerLogger.warning("No default system: " + linename)
                         warnline[linename] =1
 
                     #---------add in any grouping that may want to use
-                    if self.lineToGroup.has_key(linename):
+                    if linename in self.lineToGroup:
                         self.trnAsgnTable[newrownum]["GROUP"] = self.lineToGroup[linename]
                     else:
                         self.trnAsgnTable[newrownum]["GROUP"] = ""
@@ -446,7 +443,7 @@ def readTransitAssignmentCsvs(self):
 
                 # Add in the subsequent assignment files
                 else:
-                     
+
                     # print oldrownum, newrownum, ABNameSeq_List[newrownum]
                     # print row[self.colnameToCsvIndex["NAME"]], ABNameSeq_List[oldrownum][2]
                     if ((int(row[self.colnameToCsvIndex["A"]]) != ABNameSeq_List[newrownum][0]) or
@@ -462,7 +459,9 @@ def readTransitAssignmentCsvs(self):
                                 row[self.colnameToCsvIndex["B"]] + " " + \
                                 row[self.colnameToCsvIndex["NAME"]].rstrip()
                     if ABNameSeq_List[newrownum][3]>0:
-                        ABNameSeq += " " + str(ABNameSeq_List[newrownum][3])                    
+                        ABNameSeq += " " + str(ABNameSeq_List[newrownum][3])
+                    # convert to bytes - the index in the dataTable is a byte string
+                    ABNameSeq = ABNameSeq.encode('utf-8')
                     for field in self.trnAsgnAdditiveFields:
                         if row[self.colnameToCsvIndex[field]] !="":
                             self.trnAsgnTable[ABNameSeq][field] += float(row[self.colnameToCsvIndex[field]])
@@ -512,8 +511,8 @@ def buildAggregateTable(self):
             ABSet.add(row["AB"])
 
         self.aggregateTable = DataTable(numRecords=len(ABSet),
-                                        fieldNames=self.aggregateFields.keys(),
-                                        numpyFieldTypes=self.aggregateFields.values())
+                                        fieldNames=list(self.aggregateFields.keys()),
+                                        numpyFieldTypes=list(self.aggregateFields.values()))
         ABtoRowIndex = {}
         rowsUsed = 0
         for row in self.trnAsgnTable:

Wrangler/TransitCapacity.py

@@ -154,7 +154,7 @@ def getSystemAndVehicleType(self, linename, timeperiod):
         Convenience function.  Returns tuple: best guess of (system, vehicletype)
         """
         linenameU = linename.upper()
-        if self.linenameToAttributes.has_key(linenameU):
+        if linenameU in self.linenameToAttributes:
             return (self.linenameToAttributes[linenameU][TransitCapacity.ATTR_SYSTEM], 
                     self.linenameToAttributes[linenameU][TransitCapacity.TIMEPERIOD_TO_VEHTYPIDX[timeperiod]])
 
@@ -183,7 +183,7 @@ def getFullname(self, linename, timeperiod):
         Returns best guess of fullname, or empty string if unknown
         """
         linenameU = linename.upper()
-        if self.linenameToAttributes.has_key(linenameU):
+        if linenameU in self.linenameToAttributes:
             return self.linenameToAttributes[linenameU][TransitCapacity.ATTR_FULLNAME]
         else:
             return ""