Merge pull request #38 from jpn--/repair

Repair for Multiprocessing
psrc · Mar 23, 2023 · 0e4a04d · 0e4a04d
2 parents daa073e + 1365514
commit 0e4a04d
Showing 14 changed files with 294 additions and 87 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,2 +1,4 @@
 data
 output
+**/__pycache__
+**/.ipynb_checkpoints
diff --git a/configs/logging.yaml b/configs/logging.yaml
@@ -24,19 +24,32 @@ logging:
       handlers: [console, logfile]
       propagate: false
 
+    filelock:
+      level: WARN
+
+    sharrow:
+      level: INFO
+
+    blib2to3:
+      level: WARN
+
+    black:
+      level: WARN
+
   handlers:
 
     logfile:
       class: logging.FileHandler
-      filename: !!python/object/apply:activitysim.core.config.log_file_path ['activitysim.log']
+      filename:
+        get_log_file_path: 'activitysim.log'
       mode: w
       formatter: fileFormatter
       level: NOTSET
 
     console:
       class: logging.StreamHandler
       stream: ext://sys.stdout
-      formatter: simpleFormatter
+      formatter: elapsedFormatter
       level: NOTSET
 
   formatters:
@@ -52,3 +65,7 @@ logging:
       format: '%(asctime)s - %(levelname)s - %(name)s - %(message)s'
       datefmt: '%d/%m/%Y %H:%M:%S'
 
+    elapsedFormatter:
+      (): activitysim.core.tracing.ElapsedTimeFormatter
+      format: '[{elapsedTime}] {levelname:s}: {message:s}'
+      style: '{'
diff --git a/configs/network_los.yaml b/configs/network_los.yaml
@@ -8,8 +8,9 @@ write_skim_cache: True
 zone_system: 2
 
 # glob 'skims*.omx' will match one or more files: skims.omx, skims1.omx, skims2.omx...
-taz_skims: skims*.omx
-
+taz_skims:
+  omx: skims*.omx
+  zarr: skims.zarr
 
 maz: maz.csv
 

diff --git a/configs_dev/logging.yaml b/configs_dev/logging.yaml
@@ -24,19 +24,32 @@ logging:
       handlers: [console, logfile]
       propagate: false
 
+    filelock:
+      level: WARN
+
+    sharrow:
+      level: INFO
+
+    blib2to3:
+      level: WARN
+
+    black:
+      level: WARN
+
   handlers:
 
     logfile:
       class: logging.FileHandler
-      filename: !!python/object/apply:activitysim.core.config.log_file_path ['activitysim.log']
+      filename:
+        get_log_file_path: 'activitysim.log'
       mode: w
       formatter: fileFormatter
       level: NOTSET
 
     console:
       class: logging.StreamHandler
       stream: ext://sys.stdout
-      formatter: simpleFormatter
+      formatter: elapsedFormatter
       level: NOTSET
 
   formatters:
@@ -52,3 +65,7 @@ logging:
       format: '%(asctime)s - %(levelname)s - %(name)s - %(message)s'
       datefmt: '%d/%m/%Y %H:%M:%S'
 
+    elapsedFormatter:
+      (): activitysim.core.tracing.ElapsedTimeFormatter
+      format: '[{elapsedTime}] {levelname:s}: {message:s}'
+      style: '{'
diff --git a/configs_dev/network_los.yaml b/configs_dev/network_los.yaml
@@ -10,7 +10,7 @@ zone_system: 2
 # glob 'skims*.omx' will match one or more files: skims.omx, skims1.omx, skims2.omx...
 taz_skims: 
   omx: skims*.omx
-
+  zarr: skims.zarr
 
 
 maz: maz.csv

diff --git a/data/skims.omx b/data/skims.omx
diff --git a/test/configs/settings.yaml b/test/configs/settings.yaml
@@ -37,3 +37,6 @@ output_tables:
   sort: True
   tables:
     - trips
+    - tours
+    - households
+    - persons
diff --git a/test/configs_mp/settings_mp.yaml b/test/configs_mp/settings_mp.yaml
@@ -0,0 +1,50 @@
+inherit_settings: settings.yaml
+
+MKL_NUM_THREADS: 1
+#activitysim run -c configs_mp -s settings_mp.yaml -c configs -d data/full -o output_test
+
+# turn shadow_pricing on and off for all models (e.g. school and work)
+# shadow pricing is deprecated for less than full samples
+# see shadow_pricing.yaml for additional settings
+use_shadow_pricing: False
+
+
+# raise error if any sub-process fails without waiting for others to complete
+fail_fast: True
+
+multiprocess: True
+num_processes: 2
+
+multiprocess_steps:
+  - name: mp_initialize
+    begin: initialize_landuse
+  - name: mp_accessibility
+    begin: compute_accessibility
+    slice:
+      tables:
+        - accessibility
+      exclude: True
+  - name: mp_initialize_hh
+    begin: initialize_households
+  - name: mp_households
+    begin: school_location
+    slice:
+      tables:
+        - households
+        - persons
+  - name: mp_summarize
+    begin: write_data_dictionary
+
+output_tables:
+  h5_store: False
+  action: include
+  prefix: final_
+  sort: True
+  tables:
+    - checkpoints
+    - accessibility
+    - land_use
+    - households
+    - persons
+
+
diff --git a/test/regress/final_trips.csv b/test/regress/final_trips.csv
diff --git a/test/test_psrc.py b/test/test_psrc.py
@@ -1,48 +1,65 @@
 # ActivitySim
 # See full license in LICENSE.txt.
-import os
-import subprocess
-import pkg_resources
+from pathlib import Path
 
-import pandas as pd
-import pandas.testing as pdt
+from activitysim.core import workflow
 
-from activitysim.core import inject
+test_dir = Path(__file__).parent
+top_dir = test_dir.parent
 
 
-def teardown_function(func):
-    inject.clear_cache()
-    inject.reinject_decorated_tables()
+def _test_psrc(tmp_path, dataframe_regression, mp=False, use_sharrow=True):
+    import activitysim.abm  # noqa: F401 # register steps
 
+    if mp:
+        configs_dir = (
+            test_dir.joinpath("configs_mp"),
+            test_dir.joinpath("configs"),
+            top_dir.joinpath("configs_dev"),
+        )
+    else:
+        configs_dir = (
+            test_dir.joinpath("configs"),
+            top_dir.joinpath("configs_dev"),
+        )
 
-def test_psrc():
+    settings = {}
+    if use_sharrow:
+        settings["sharrow"] = "test"
 
-    def example_path(dirname):
-        resource = os.path.join('examples', 'example_psrc', dirname)
-        return pkg_resources.resource_filename('activitysim', resource)
+    state = workflow.State.make_default(
+        working_dir=top_dir,
+        configs_dir=configs_dir,
+        data_dir=top_dir.joinpath("data"),
+        output_dir=tmp_path,  # test_dir.joinpath("output"),
+        settings=settings,
+    )
+    state.import_extensions("extensions")
 
-    def test_path(dirname):
-        return os.path.join(os.path.dirname(__file__), dirname)
+    # persist sharrow cache in local output
+    sharrow_cache_dir = test_dir.joinpath("output", "cache")
+    sharrow_cache_dir.mkdir(parents=True, exist_ok=True)
+    state.filesystem.sharrow_cache_dir = sharrow_cache_dir
 
-    def regress():
-        regress_trips_df = pd.read_csv(test_path('regress/final_trips.csv'))
-        final_trips_df = pd.read_csv(test_path('output/final_trips.csv'))
+    state.run.all()
 
-        # person_id,household_id,tour_id,primary_purpose,trip_num,outbound,trip_count,purpose,
-        # destination,origin,destination_logsum,depart,trip_mode,mode_choice_logsum
-        # compare_cols = []
-        pdt.assert_frame_equal(final_trips_df, regress_trips_df)
+    for t in ("trips", "tours", "persons", "households"):
+        df = state.get_dataframe(t)
+        # check that in-memory result table is as expected
+        dataframe_regression.check(df, basename=t)
 
-    file_path = os.path.join(os.path.dirname(__file__), 'simulation.py')
 
-    subprocess.run(['coverage', 'run', '-a', file_path,
-                    '-c', test_path('configs'), '-c', example_path('configs'),
-                    '-d', example_path('data'),
-                    '-o', test_path('output')], check=True)
+def test_psrc(tmp_path, dataframe_regression):
+    return _test_psrc(tmp_path, dataframe_regression, mp=False, use_sharrow=False)
 
-    regress()
 
+def test_psrc_sh(tmp_path, dataframe_regression):
+    return _test_psrc(tmp_path, dataframe_regression, mp=False, use_sharrow=True)
 
-if __name__ == '__main__':
 
-    test_psrc()
+def test_psrc_mp(tmp_path, dataframe_regression):
+    return _test_psrc(tmp_path, dataframe_regression, mp=True, use_sharrow=False)
+
+
+def test_psrc_sh_mp(tmp_path, dataframe_regression):
+    return _test_psrc(tmp_path, dataframe_regression, mp=True, use_sharrow=True)
diff --git a/test/test_psrc/households.csv b/test/test_psrc/households.csv
@@ -0,0 +1,11 @@
+household_id,home_zone_id,income,hhsize,HHT,auto_ownership,num_workers,sample_rate,income_in_thousands,income_segment,num_non_workers,num_drivers,num_adults,num_children,num_young_children,num_children_5_to_15,num_children_16_to_17,num_college_age,num_young_adults,non_family,family,home_is_urban,home_is_rural,hh_work_auto_savings_ratio,num_under16_not_at_school,num_travel_active,num_travel_active_adults,num_travel_active_preschoolers,num_travel_active_children,num_travel_active_non_preschoolers,participates_in_jtf_model,joint_tour_frequency,num_hh_joint_tours
+277269,453,148500,1,1,1,1,0.001,148.5,4,0,1,1,0,0,0,0,0,0,False,True,False,False,0.32268959283828735,0,1,1,0,0,1,False,0_tours,0
+252257,164,71500,2,1,0,1,0.001,71.5,3,1,2,2,0,0,0,0,0,0,False,True,False,False,0.28730747103691101,0,2,2,0,0,2,True,0_tours,0
+306161,435,82500,2,1,2,2,0.001,82.5,3,0,2,2,0,0,0,0,0,2,False,True,False,False,0.51287847757339478,0,2,2,0,0,2,True,0_tours,0
+271530,538,82000,2,1,2,2,0.001,82,3,0,2,2,0,0,0,0,0,2,False,True,True,False,0.49424159526824951,0,2,2,0,0,2,True,0_tours,0
+290576,509,8500,2,6,0,2,0.001,8.5,1,0,2,2,0,0,0,0,2,0,True,False,False,False,0.11518096923828125,0,2,2,0,0,2,True,0_tours,0
+244914,40,80000,2,1,2,0,0.001,80,3,2,2,2,0,0,0,0,0,0,False,True,False,False,0,0,2,2,0,0,2,True,1_Eat,1
+291838,252,467000,1,1,1,1,0.001,467,4,0,1,1,0,0,0,0,0,1,False,True,True,False,0.1512654721736908,0,1,1,0,0,1,False,0_tours,0
+244270,379,18000,1,1,1,0,0.001,18,1,1,1,1,0,0,0,0,0,0,False,True,False,False,0,0,1,1,0,0,1,False,0_tours,0
+250839,370,39400,6,1,1,0,0.001,39.399999999999999,2,6,6,6,0,0,0,0,1,2,False,True,False,False,0,0,3,3,0,0,3,True,0_tours,0
+244148,151,290000,4,1,2,2,0.001,290,4,2,2,2,2,1,2,0,0,0,False,True,False,False,0.94347560405731201,0,3,1,0,2,3,True,0_tours,0
diff --git a/test/test_psrc/persons.csv b/test/test_psrc/persons.csv
@@ -0,0 +1,24 @@
+person_id,household_id,age,PNUM,sex,pemploy,pstudent,ptype,age_16_to_19,age_16_p,adult,male,female,has_non_worker,has_retiree,has_preschool_kid,has_driving_kid,has_school_kid,has_full_time,has_part_time,has_university,student_is_employed,nonstudent_to_school,is_student,is_gradeschool,is_highschool,is_university,school_segment,is_worker,home_zone_id,time_factor_work,time_factor_nonwork,school_zone_id,school_location_logsum,distance_to_school,roundtrip_auto_time_to_school,workplace_zone_id,workplace_location_logsum,distance_to_work,workplace_in_cbd,work_zone_area_type,roundtrip_auto_time_to_work,work_auto_savings,work_auto_savings_ratio,free_parking_at_work,cdap_activity,travel_active,under16_not_at_school,has_preschool_kid_at_home,has_school_kid_at_home,mandatory_tour_frequency,work_and_school_and_worker,work_and_school_and_student,num_mand,num_work_tours,num_joint_tours,non_mandatory_tour_frequency,num_non_mand,num_escort_tours,num_eatout_tours,num_shop_tours,num_maint_tours,num_discr_tours,num_social_tours,num_non_escort_tours
+557351,244148,44,1,2,1,3,1,False,True,True,False,True,False,False,False,False,True,True,False,False,False,False,False,False,False,False,0,True,151,0.7891534497345577,0.39007043700087773,-1,,,0,276,10.298988619370965,1.5642563104629517,True,1,18.356212615966797,41.662990570068359,0.34719160199165344,False,H,False,False,False,False,,False,False,0,0,0,0,0,0,0,0,0,0,0,0
+557352,244148,42,2,1,1,3,1,False,True,True,True,False,False,False,False,False,True,True,False,False,False,False,False,False,False,False,0,True,151,1.673739405224558,0.52558214756766519,-1,,,0,126,10.097032132995134,2.547050952911377,True,1,21.674797058105469,71.554084777832031,0.59628403186798096,True,M,True,False,False,False,work1,False,False,1,1,0,16,2,0,0,2,0,0,0,2
+557353,244148,7,3,1,4,1,7,False,False,False,True,False,False,False,False,False,True,True,False,False,False,False,True,True,False,False,1,False,151,0.51270632404314798,0.73730950677203055,329,8.5349513812198943,1.6893030405044556,18.889913558959961,-1,,,False,,0,0,0,False,M,True,False,False,False,school1,False,False,1,0,0,0,0,0,0,0,0,0,0,0
+557354,244148,5,4,1,4,1,7,False,False,False,True,False,False,False,False,False,True,True,False,False,False,False,True,True,False,False,1,False,151,1.490604485554442,1.3942289070617651,414,8.4660777101373199,0.80245286226272583,14.048561096191406,-1,,,False,,0,0,0,False,M,True,False,False,False,school1,False,False,1,0,0,4,1,0,1,0,0,0,0,1
+557688,244270,57,1,2,3,3,4,False,True,True,False,True,False,False,False,False,False,False,False,False,False,False,False,False,False,False,0,False,379,1.2969643388620089,1.417022467714607,-1,,,0,-1,,,False,,0,0,0,False,N,True,False,False,False,,False,False,0,0,0,16,2,0,0,2,0,0,0,2
+559273,244914,74,1,1,3,3,5,False,True,True,True,False,False,True,False,False,False,False,False,False,False,False,False,False,False,False,0,False,40,0.44913834445626905,0.3718556450440218,-1,,,0,-1,,,False,,0,0,0,False,N,True,False,False,False,,False,False,0,0,1,0,0,0,1,0,0,0,0,0
+559274,244914,74,2,2,3,3,5,False,True,True,False,True,False,True,False,False,False,False,False,False,False,False,False,False,False,False,0,False,40,1.6405246825841262,0.52504238300680561,-1,,,0,-1,,,False,,0,0,0,False,N,True,False,False,False,,False,False,0,0,1,9,2,0,0,0,1,1,0,2
+571771,250839,83,1,2,3,3,5,False,True,True,False,True,True,False,False,False,False,False,False,False,False,False,False,False,False,False,0,False,370,0.50347953829156655,0.75880106006753478,-1,,,0,-1,,,False,,0,0,0,False,H,False,False,False,False,,False,False,0,0,0,0,0,0,0,0,0,0,0,0
+571772,250839,58,2,2,3,3,4,False,True,True,False,True,True,True,False,False,False,False,False,False,False,False,False,False,False,False,0,False,370,0.3502994618184655,1.3326368623494023,-1,,,0,-1,,,False,,0,0,0,False,H,False,False,False,False,,False,False,0,0,0,0,0,0,0,0,0,0,0,0
+571773,250839,30,3,1,3,3,4,False,True,True,True,False,True,True,False,False,False,False,False,False,False,False,False,False,False,False,0,False,370,0.84107771915910434,2.1641100898339389,-1,,,0,-1,,,False,,0,0,0,False,H,False,False,False,False,,False,False,0,0,0,0,0,0,0,0,0,0,0,0
+571774,250839,27,4,1,3,3,4,False,True,True,True,False,True,True,False,False,False,False,False,False,False,False,False,False,False,False,0,False,370,2.1622365777704258,1.568605899398249,-1,,,0,-1,,,False,,0,0,0,False,N,True,False,False,False,,False,False,0,0,0,2,1,0,0,0,0,0,1,1
+571775,250839,24,5,2,3,3,4,False,True,True,False,True,True,True,False,False,False,False,False,False,False,False,False,False,False,False,0,False,370,1.2631936273263977,0.80653680151807494,-1,,,0,-1,,,False,,0,0,0,False,N,True,False,False,False,,False,False,0,0,0,33,2,1,0,0,0,1,0,1
+571776,250839,60,6,1,3,3,4,False,True,True,True,False,True,True,False,False,False,False,False,False,False,False,False,False,False,False,0,False,370,0.64324623098237033,0.76063570842218875,-1,,,0,-1,,,False,,0,0,0,False,N,True,False,False,False,,False,False,0,0,0,3,2,0,0,0,0,1,1,2
+574668,252257,70,1,1,2,3,2,False,True,True,True,False,True,False,False,False,False,False,False,False,False,False,False,False,False,False,0,True,164,0.61260571703767752,0.47857501430399835,-1,,,0,31,10.390741991677825,1.277260422706604,False,5,15.233522415161133,34.476898193359375,0.28730747103691101,False,M,True,False,False,False,work1,False,False,1,1,0,1,1,0,0,0,0,1,0,1
+574669,252257,59,2,2,3,3,4,False,True,True,False,True,False,False,False,False,False,False,True,False,False,False,False,False,False,False,0,False,164,0.50040040559634336,1.3605164491645823,-1,,,0,-1,,,False,,0,0,0,False,N,True,False,False,False,,False,False,0,0,0,1,1,0,0,0,0,1,0,1
+606169,271530,31,1,2,1,3,1,False,True,True,False,True,False,False,False,False,False,False,True,False,False,False,False,False,False,False,0,True,538,0.3023598034620133,1.1620136066731588,-1,,,0,67,10.604493232717626,2.0605380535125732,False,5,21.2562255859375,31.878585815429688,0.26565489172935486,True,M,True,False,False,False,work1,False,False,1,1,0,1,1,0,0,0,0,1,0,1
+606170,271530,31,2,1,2,3,2,False,True,True,True,False,False,False,False,False,False,True,False,False,False,False,False,False,False,False,0,True,538,0.47406744249975963,1.3290210338027793,-1,,,0,197,10.535276132084466,1.3679345846176147,False,3,18.252376556396484,27.430404663085938,0.22858670353889465,False,N,True,False,False,False,,False,False,0,0,0,1,1,0,0,0,0,1,0,1
+615465,277269,66,1,2,1,3,1,False,True,True,False,True,False,False,False,False,False,False,False,False,False,False,False,False,False,False,0,True,453,0.72017069166054271,2.1508159257616226,-1,,,0,93,10.466816360214549,2.3995010852813721,True,1,22.372966766357422,38.722751617431641,0.32268959283828735,False,M,True,False,False,False,work1,False,False,1,1,0,0,0,0,0,0,0,0,0,0
+637915,290576,21,1,2,2,2,3,False,True,True,False,True,False,False,False,False,False,False,True,True,True,False,True,False,False,True,3,True,509,0.4893017380052852,0.46749563255176207,540,10.459504282268009,0.47725531458854675,10.02202033996582,515,10.840799310941202,0.48858052492141724,True,0,15.693637847900391,3.7695827484130859,0.031413190066814423,False,M,True,False,False,False,work1,False,False,1,1,0,0,0,0,0,0,0,0,0,0
+637916,290576,21,2,2,2,2,3,False,True,True,False,True,False,False,False,False,False,False,True,True,True,False,True,False,False,True,3,True,509,0.88835892156032059,2.7678305988574117,540,10.260792542615654,0.47725531458854675,10.02202033996582,106,10.766475719726619,0.70290935039520264,True,1,18.064239501953125,10.052133560180664,0.083767779171466827,False,M,True,False,False,False,work1,False,False,1,1,0,0,0,0,0,0,0,0,0,0
+640993,291838,26,1,2,1,3,1,False,True,True,False,True,False,False,False,False,False,False,False,False,False,False,False,False,False,False,0,True,252,1.0860263848187248,0.50615285845704905,-1,,,0,455,10.654487949861633,0.77209359407424927,True,0,12.73188591003418,18.151857376098633,0.1512654721736908,False,M,True,False,False,False,work1,False,False,1,1,0,0,0,0,0,0,0,0,0,0
+664907,306161,26,1,1,1,3,1,False,True,True,True,False,False,False,False,False,False,True,False,False,False,False,False,False,False,False,0,True,435,1.6121447161129019,1.0180287301985604,-1,,,0,116,10.331668842670334,1.484452486038208,False,2,20.634883880615234,38.406398773193359,0.32005330920219421,False,M,True,False,False,False,work1,False,False,1,1,0,0,0,0,0,0,0,0,0,0
+664908,306161,30,2,1,1,3,1,False,True,True,True,False,False,False,False,False,False,True,False,False,False,False,False,False,False,False,0,True,435,0.44351858564866897,0.93295504937307205,-1,,,0,429,10.623193652741008,0.92926663160324097,False,3,14.051645278930664,23.139020919799805,0.19282516837120056,False,M,True,False,False,False,work1,False,False,1,1,0,0,0,0,0,0,0,0,0,0
diff --git a/test/test_psrc/tours.csv b/test/test_psrc/tours.csv
@@ -0,0 +1,33 @@
+tour_id,person_id,tour_type,tour_type_count,tour_type_num,tour_num,tour_count,tour_category,number_of_participants,destination,origin,household_id,tdd,start,end,duration,composition,destination_logsum,tour_mode,mode_choice_logsum,atwork_subtour_frequency,parent_tour_id,stop_frequency,primary_purpose
+22851471,557352,work,1,1,1,1,mandatory,1,126,151,244148,47,7,17,10,,,DRIVEALONEFREE,0.056365019756161247,business1,,0out_0in,work
+22851504,557353,school,1,1,1,1,mandatory,1,329,151,244148,40,7,10,3,,,WALK_LR,1.6826546008346579,,,0out_0in,school
+22851545,557354,school,1,1,1,1,mandatory,1,414,151,244148,45,7,15,8,,,SHARED2FREE,1.5881138499629117,,,0out_0in,school
+23561427,574668,work,1,1,1,1,mandatory,1,31,164,252257,28,6,15,9,,,WALK_LR,-3.6694498524520189,eat,,0out_0in,work
+24852968,606169,work,1,1,1,1,mandatory,1,67,538,271530,47,7,17,10,,,DRIVEALONEFREE,1.4890603702212339,no_subtours,,0out_0in,work
+25234104,615465,work,1,1,1,1,mandatory,1,93,453,277269,64,8,18,10,,,DRIVEALONEFREE,1.0532550640822764,no_subtours,,0out_0in,work
+26154554,637915,work,1,1,1,1,mandatory,1,515,509,290576,121,12,21,9,,,WALK_LOC,-3.3053649697951526,no_subtours,,0out_1in,work
+26154595,637916,work,1,1,1,1,mandatory,1,106,509,290576,79,9,18,9,,,WALK,-3.7326636538978128,no_subtours,,1out_2in,work
+26280752,640993,work,1,1,1,1,mandatory,1,455,252,291838,65,8,19,11,,,WALK_LOC,1.008344431102048,eat,,0out_0in,work
+27261226,664907,work,1,1,1,1,mandatory,1,116,435,306161,48,7,18,11,,,WALK_COM,0.49420625122664724,business1,,1out_1in,work
+27261267,664908,work,1,1,1,1,mandatory,1,429,435,306161,64,8,18,10,,,WALK,1.4131240351018572,no_subtours,,0out_0in,work
+22930206,559273,eatout,1,1,1,1,joint,2,30,40,244914,135,14,14,0,adults,10.042944310292039,WALK,7.7156230615813994,,,0out_0in,eatout
+22851465,557352,shopping,2,1,1,2,non_mandatory,1,5,151,244148,172,18,21,3,,9.700740488552837,WALK,0.89351936973119339,,,0out_3in,shopping
+22851466,557352,shopping,2,2,2,2,non_mandatory,1,30,151,244148,184,21,21,0,,9.6319143325843335,WALK,1.0398708789393183,,,0out_0in,shopping
+22851520,557354,eatout,1,1,1,1,non_mandatory,1,373,151,244148,156,16,18,2,,10.130222623908336,SHARED2FREE,1.624422650933226,,,0out_0in,eatout
+22865241,557688,shopping,2,1,1,2,non_mandatory,1,30,379,244270,9,5,14,9,,9.4766223738598985,DRIVEALONEFREE,0.43129539941014872,,,0out_0in,shopping
+22865242,557688,shopping,2,2,2,2,non_mandatory,1,30,379,244270,162,17,17,0,,9.4499362039302266,WALK,0.42158309816587025,,,0out_0in,shopping
+22930262,559274,othmaint,1,1,1,2,non_mandatory,1,103,40,244914,124,13,13,0,,9.775376160042061,DRIVEALONEFREE,1.0164227986514049,,,0out_0in,othmaint
+22930259,559274,othdiscr,1,1,2,2,non_mandatory,1,8,40,244914,154,16,16,0,,11.868774013496134,WALK,1.3773194765158845,,,0out_0in,othdiscr
+23442770,571774,social,1,1,1,1,non_mandatory,1,126,370,250839,77,9,16,7,,9.624390879863812,SHARED2FREE,0.37545931482299572,,,0out_2in,social
+23442784,571775,escort,1,1,1,2,non_mandatory,1,308,370,250839,162,17,17,0,,29.944677954123843,SHARED3FREE,28.030549720067121,,,0out_0in,escort
+23442800,571775,othdiscr,1,1,2,2,non_mandatory,1,515,370,250839,171,18,20,2,,12.428919773731662,BIKE,1.650564258981768,,,0out_0in,othdiscr
+23442841,571776,othdiscr,1,1,1,2,non_mandatory,1,194,370,250839,73,9,12,3,,12.708183072258528,SHARED3FREE,1.105148813412655,,,0out_0in,othdiscr
+23442852,571776,social,1,1,2,2,non_mandatory,1,171,370,250839,163,17,18,1,,10.136916742814289,WALK,1.0342801871257519,,,0out_0in,social
+23561413,574668,othdiscr,1,1,1,1,non_mandatory,1,106,164,252257,171,18,20,2,,9.1548675442603109,WALK_LR,-3.3182804246636857,,,0out_0in,othdiscr
+23561454,574669,othdiscr,1,1,1,1,non_mandatory,1,286,164,252257,58,8,12,4,,9.2297380535581635,WALK_LOC,-4.2118218358588999,,,0out_3in,othdiscr
+24852954,606169,othdiscr,1,1,1,1,non_mandatory,1,154,538,271530,164,17,19,2,,13.089387082111553,BIKE,1.4584546609929969,,,0out_0in,othdiscr
+24852995,606170,othdiscr,1,1,1,1,non_mandatory,1,522,538,271530,112,12,12,0,,12.888647509444096,SHARED2FREE,0.94415977040101362,,,0out_1in,othdiscr
+22851432,557352,business,1,1,1,1,atwork,1,114,126,244148,113,12,13,1,,17.394810483629929,WALK,0.91717980360319962,,22851471,0out_0in,atwork
+23561392,574668,eat,1,1,1,1,atwork,1,126,31,252257,85,10,10,0,,11.996399812790559,TNC_SINGLE,3.7422410462649389,,23561427,0out_0in,atwork
+26280717,640993,eat,1,1,1,1,atwork,1,126,455,291838,113,12,13,1,,16.851302738955258,WALK,0.10900120796030888,,26280752,0out_1in,atwork
+27261187,664907,business,1,1,1,1,atwork,1,126,116,306161,118,12,18,6,,17.074010791604088,WALK,0.46171736206450786,,27261226,0out_1in,atwork
diff --git a/test/test_psrc/trips.csv b/test/test_psrc/trips.csv
@@ -0,0 +1,82 @@
+trip_id,person_id,household_id,primary_purpose,trip_num,outbound,trip_count,destination,origin,tour_id,purpose,destination_logsum,depart,trip_mode,mode_choice_logsum
+182811457,557352,244148,atwork,1,True,1,114,126,22851432,atwork,,12,WALK,0.024405698037469916
+182811461,557352,244148,atwork,1,False,1,126,114,22851432,work,,13,WALK,0.024405698037469916
+182811721,557352,244148,shopping,1,True,1,5,151,22851465,shopping,,18,WALK,-1.0422942043308638
+182811725,557352,244148,shopping,1,False,4,414,5,22851465,escort,10.138248301125826,21,WALK,1.3553830621140546
+182811726,557352,244148,shopping,2,False,4,373,414,22851465,shopping,9.0922266744376365,21,WALK,0.37100991269626599
+182811727,557352,244148,shopping,3,False,4,99,373,22851465,shopping,10.331511449130069,21,WALK,0.60891842223096837
+182811728,557352,244148,shopping,4,False,4,151,99,22851465,home,,21,WALK,1.6054755164353953
+182811729,557352,244148,shopping,1,True,1,30,151,22851466,shopping,,21,WALK,-0.35102217784889356
+182811733,557352,244148,shopping,1,False,1,151,30,22851466,home,,21,WALK,-0.35102217784889356
+182811769,557352,244148,work,1,True,1,126,151,22851471,work,,7,DRIVEALONEFREE,-0.57519104139122235
+182811773,557352,244148,work,1,False,1,151,126,22851471,home,,17,DRIVEALONEFREE,-0.56186255125999951
+182812033,557353,244148,school,1,True,1,329,151,22851504,school,,7,WALK_LOC,-0.41883364900540948
+182812037,557353,244148,school,1,False,1,151,329,22851504,home,,10,WALK_LOC,-0.68579660252170849
+182812161,557354,244148,eatout,1,True,1,373,151,22851520,eatout,,16,SHARED2FREE,0.66119773153929984
+182812165,557354,244148,eatout,1,False,1,151,373,22851520,home,,18,SHARED2FREE,0.66125029523103651
+182812361,557354,244148,school,1,True,1,414,151,22851545,school,,7,SHARED2FREE,-1.4604903756024996
+182812365,557354,244148,school,1,False,1,151,414,22851545,home,,15,WALK,-1.4604980526169284
+182921929,557688,244270,shopping,1,True,1,30,379,22865241,shopping,,5,DRIVEALONEFREE,-0.25932849289186205
+182921933,557688,244270,shopping,1,False,1,379,30,22865241,home,,14,DRIVEALONEFREE,-0.28072053692545346
+182921937,557688,244270,shopping,1,True,1,30,379,22865242,shopping,,17,WALK,-0.83497766343984448
+182921941,557688,244270,shopping,1,False,1,379,30,22865242,home,,17,WALK,-1.0150336990211082
+183441649,559273,244914,eatout,1,True,1,30,40,22930206,eatout,,14,WALK,-1.0488171076963082
+183441653,559273,244914,eatout,1,False,1,40,30,22930206,home,,14,WALK,-1.0488171076963082
+183442073,559274,244914,othdiscr,1,True,1,8,40,22930259,othdiscr,,16,WALK,-0.35396088421807165
+183442077,559274,244914,othdiscr,1,False,1,40,8,22930259,home,,16,WALK,-0.35396088421807165
+183442097,559274,244914,othmaint,1,True,1,103,40,22930262,othmaint,,13,DRIVEALONEFREE,0.043093566726020271
+183442101,559274,244914,othmaint,1,False,1,40,103,22930262,home,,13,DRIVEALONEFREE,0.043093566726020271
+187542161,571774,250839,social,1,True,1,126,370,23442770,social,,9,SHARED2FREE,0.1675592734356082
+187542165,571774,250839,social,1,False,3,207,126,23442770,shopping,11.401315391634027,16,SHARED2FREE,0.30482486580864609
+187542166,571774,250839,social,2,False,3,542,207,23442770,escort,11.19300797280245,16,SHARED2FREE,0.18042144537149354
+187542167,571774,250839,social,3,False,3,370,542,23442770,home,,16,SHARED2FREE,0.14976710579843056
+187542273,571775,250839,escort,1,True,1,308,370,23442784,escort,,17,SHARED3FREE,0.73180062438394466
+187542277,571775,250839,escort,1,False,1,370,308,23442784,home,,17,SHARED3FREE,0.73127476639003341
+187542401,571775,250839,othdiscr,1,True,1,515,370,23442800,othdiscr,,18,BIKE,2.8615829563964126
+187542405,571775,250839,othdiscr,1,False,1,370,515,23442800,home,,20,BIKE,2.8615829563964126
+187542729,571776,250839,othdiscr,1,True,1,194,370,23442841,othdiscr,,9,SHARED3FREE,0.66106154049911836
+187542733,571776,250839,othdiscr,1,False,1,370,194,23442841,home,,12,DRIVEALONEFREE,0.68087610948896449
+187542817,571776,250839,social,1,True,1,171,370,23442852,social,,17,WALK,-0.42527721892918696
+187542821,571776,250839,social,1,False,1,370,171,23442852,home,,18,WALK,-0.42527721892918696
+188491137,574668,252257,atwork,1,True,1,126,31,23561392,atwork,,10,TNC_SINGLE,-0.7148410636105611
+188491141,574668,252257,atwork,1,False,1,31,126,23561392,work,,10,TNC_SINGLE,-0.70854023337175531
+188491305,574668,252257,othdiscr,1,True,1,106,164,23561413,othdiscr,,18,WALK_LOC,1.5051115869574876
+188491309,574668,252257,othdiscr,1,False,1,164,106,23561413,home,,20,WALK_LOC,1.4392768704758816
+188491417,574668,252257,work,1,True,1,31,164,23561427,work,,6,WALK_LOC,0.037776304293340078
+188491421,574668,252257,work,1,False,1,164,31,23561427,home,,15,WALK,0.11116627385271838
+188491633,574669,252257,othdiscr,1,True,1,286,164,23561454,othdiscr,,8,WALK_LOC,-0.66486076841711983
+188491637,574669,252257,othdiscr,1,False,4,106,286,23561454,eatout,7.7963141634215258,12,WALK,-0.78722779756849337
+188491638,574669,252257,othdiscr,2,False,4,552,106,23561454,shopping,11.265685052032758,12,WALK_LOC,1.6207620003702197
+188491639,574669,252257,othdiscr,3,False,4,5,552,23561454,eatout,9.6325686378355737,12,WALK,0.72924455718509273
+188491640,574669,252257,othdiscr,4,False,4,164,5,23561454,home,,12,WALK_LOC,-0.014214490292795368
+198823633,606169,271530,othdiscr,1,True,1,154,538,24852954,othdiscr,,17,BIKE,-0.52350367109113771
+198823637,606169,271530,othdiscr,1,False,1,538,154,24852954,home,,19,BIKE,-0.52350367109113771
+198823745,606169,271530,work,1,True,1,67,538,24852968,work,,7,DRIVEALONEFREE,-0.0098719387823825288
+198823749,606169,271530,work,1,False,1,538,67,24852968,home,,17,DRIVEALONEFREE,-0.008140538035364275
+198823961,606170,271530,othdiscr,1,True,1,522,538,24852995,othdiscr,,12,SHARED2FREE,0.43257459904149015
+198823965,606170,271530,othdiscr,1,False,2,505,522,24852995,othdiscr,12.976215385796159,12,DRIVEALONEFREE,0.14867490919295304
+198823966,606170,271530,othdiscr,2,False,2,538,505,24852995,home,,12,SHARED2FREE,0.56645258725908731
+201872833,615465,277269,work,1,True,1,93,453,25234104,work,,8,DRIVEALONEFREE,-0.27432440136532665
+201872837,615465,277269,work,1,False,1,453,93,25234104,home,,18,DRIVEALONEFREE,-0.27616373517752191
+209236433,637915,290576,work,1,True,1,515,509,26154554,work,,12,WALK,3.3314527907312899
+209236437,637915,290576,work,1,False,2,289,515,26154554,shopping,18.786966638559221,21,WALK_LOC,3.3318666407313828
+209236438,637915,290576,work,2,False,2,509,289,26154554,home,,21,WALK,2.2244112509352827
+209236761,637916,290576,work,1,True,2,540,509,26154595,univ,10.702100614969668,9,WALK,-0.31612303683759235
+209236762,637916,290576,work,2,True,2,106,540,26154595,work,,10,WALK,0.25956881262090326
+209236765,637916,290576,work,1,False,3,552,106,26154595,shopping,9.8487129659045003,17,WALK,0.93332848089306197
+209236766,637916,290576,work,2,False,3,455,552,26154595,eatout,11.213864641973714,18,WALK,0.42965506632965039
+209236767,637916,290576,work,3,False,3,509,455,26154595,home,,18,WALK,1.7080170712527258
+210245737,640993,291838,atwork,1,True,1,126,455,26280717,atwork,,12,WALK,-0.0029626359254870116
+210245741,640993,291838,atwork,1,False,2,407,126,26280717,shopping,11.489693252281262,13,WALK,0.29576828087122692
+210245742,640993,291838,atwork,2,False,2,455,407,26280717,work,,13,WALK,2.3692972750439063
+210246017,640993,291838,work,1,True,1,455,252,26280752,work,,8,WALK_LOC,2.1710686457584636
+210246021,640993,291838,work,1,False,1,252,455,26280752,home,,19,WALK_LOC,2.1273642877834806
+218089497,664907,306161,atwork,1,True,1,126,116,27261187,atwork,,12,WALK,0.26310847588828384
+218089501,664907,306161,atwork,1,False,2,407,126,27261187,shopping,12.151304170410677,18,WALK,0.11899506653640328
+218089502,664907,306161,atwork,2,False,2,116,407,27261187,work,,18,WALK,3.0811944062121963
+218089809,664907,306161,work,1,True,2,515,435,27261226,work,14.216378821036573,7,WALK_LOC,2.5984081016950502
+218089810,664907,306161,work,2,True,2,116,515,27261226,work,,9,WALK,0.13267940262222438
+218089813,664907,306161,work,1,False,2,515,116,27261226,othdiscr,13.117470465750708,17,WALK,0.0472275541936664
+218089814,664907,306161,work,2,False,2,435,515,27261226,home,,18,WALK,2.6237074521139445
+218090137,664908,306161,work,1,True,1,429,435,27261267,work,,8,WALK,0.049800199429109425
+218090141,664908,306161,work,1,False,1,435,429,27261267,home,,18,WALK,0.04940989414267883
-Original file line number
+Diff line change
@@ @@ -37,3 +37,6 @@ output_tables: @@
       sort: True
       tables:
         - trips
+        - tours
+        - households
+        - persons