Add an end to end diff test (#553)

* Add an end to end diff test * Use `RandomState` for consistent random numbers across numpy versions. --------- Co-authored-by: drewoldag <[email protected]>
dirac-institute · Aug 28, 2023 · 1aed598 · 1aed598
1 parent 0bb8677
commit 1aed598
Showing 5 changed files with 245 additions and 28 deletions.
diff --git a/demo/goldens/out_end2end.csv b/demo/goldens/out_end2end.csv
@@ -0,0 +1,120 @@
+ObjID,FieldMJD,fieldRA,fieldDec,AstRA(deg),AstDec(deg),AstrometricSigma(deg),optFilter,observedPSFMag,observedTrailedSourceMag,PhotometricSigmaPSF(mag),PhotometricSigmaTrailedSource(mag),fiveSigmaDepth,fiveSigmaDepthAtSource
+632,60289.31975,142.7072408,8.25046,143.2909363,8.4698029,1.18e-05,r,23.111,23.055,0.085,0.085,23.986,23.986
+632,60293.34228,144.6014516,8.8158938,143.2660347,8.4266695,2.11e-05,i,23.016,22.807,0.135,0.135,23.34,23.307
+632,60295.34863,141.989729,8.6301382,143.2396462,8.4103275,1.51e-05,i,22.874,22.715,0.119,0.119,23.459,23.436
+632,60303.34152,143.0411449,9.2770025,143.0453212,8.3797758,7e-06,r,22.87,22.867,0.061,0.061,24.264,24.259
+632,60303.35323,143.0411449,9.2770025,143.0449122,8.3797833,1.11e-05,i,22.611,22.651,0.099,0.099,23.59,23.584
+632,60315.2441,141.4554595,8.1858813,142.5089099,8.4349784,1.36e-05,r,22.856,22.878,0.084,0.084,23.783,23.771
+632,60315.26793,141.4554595,8.1858813,142.5075204,8.4352444,1.17e-05,i,22.59,22.656,0.09,0.09,23.595,23.583
+632,60322.248,141.0466609,9.4406351,142.071369,8.5214773,7.9e-06,g,23.178,22.972,0.06,0.06,24.591,24.558
+632,60322.2717,141.0466609,9.4406351,142.0696963,8.5218368,5.9e-06,r,22.642,22.626,0.051,0.051,24.315,24.282
+632,60328.19755,141.6678165,7.1548011,141.6420702,8.6235672,1.79e-05,z,22.549,22.445,0.139,0.139,22.962,22.918
+632,60328.25587,140.9158928,9.8725584,141.6375268,8.6246818,1.03e-05,i,22.513,22.417,0.08,0.079,23.619,23.579
+632,60328.27875,140.9158928,9.8725584,141.6357209,8.6251056,1.76e-05,z,22.41,22.431,0.136,0.136,22.982,22.943
+632,60328.30071,141.6678165,7.1548011,141.6340086,8.625544,1.71e-05,z,22.534,22.351,0.134,0.133,23.006,22.962
+632,60329.25405,142.8361496,7.6203923,141.5610477,8.6441767,9.4e-06,g,23.168,23.043,0.065,0.065,24.462,24.39
+632,60329.27614,140.7371655,9.7972372,141.5592827,8.6446433,1.15e-05,r,22.559,22.566,0.072,0.072,23.83,23.794
+632,60340.20215,140.7268621,9.0201761,140.6613886,8.8967183,1.04e-05,i,22.181,22.332,0.089,0.089,23.291,23.291
+632,60340.22599,140.7268621,9.0201761,140.6593032,8.8973594,1.37e-05,z,22.443,22.166,0.119,0.119,22.942,22.942
+632,60344.1987,140.326146,7.7906532,140.3140553,9.0038758,3.22e-05,g,23.056,22.912,0.111,0.111,23.583,23.563
+632,60344.22335,140.326146,7.7906532,140.3118933,9.0045462,9.6e-06,r,22.319,22.335,0.061,0.061,23.802,23.782
+632,60344.27754,140.326146,7.7906532,140.3069869,9.0060328,2.74e-05,i,22.564,22.106,0.121,0.121,22.894,22.875
+632,60344.30119,140.326146,7.7906532,140.304925,9.0067237,2.52e-05,z,22.254,22.068,0.152,0.152,22.616,22.596
+632,60345.18907,140.2280447,8.5837329,140.227295,9.0314038,1.23e-05,r,22.405,22.374,0.067,0.067,23.668,23.668
+632,60346.16835,141.2076529,9.0398667,140.1412747,9.0590229,1.68e-05,r,22.437,22.429,0.079,0.079,23.477,23.466
+632,60348.25973,140.0293005,8.965671,139.9570284,9.1190459,6.9e-06,r,22.271,22.358,0.049,0.049,24.011,24.011
+632,60351.19357,139.8492054,8.6320566,139.6992301,9.2054812,7.3e-06,g,22.842,22.891,0.05,0.05,24.472,24.472
+632,60351.21734,139.8492054,8.6320566,139.6971011,9.206185,5.7e-06,r,22.354,22.405,0.043,0.043,24.163,24.163
+632,60377.1174,135.9939995,10.2602506,137.6751627,9.9984771,2.69e-05,r,22.63,22.623,0.115,0.115,23.411,23.329
+632,60377.14141,135.9939995,10.2602506,137.673626,9.9991875,8.7e-06,i,22.584,22.648,0.076,0.076,23.784,23.703
+632,60384.12561,137.578367,10.4153852,137.2810763,10.1951456,9e-06,r,22.64,22.72,0.067,0.067,24.041,24.041
+632,60394.06222,137.3391149,10.3333232,136.8862044,10.4401648,4.18e-05,z,22.05,22.312,0.278,0.278,22.32,22.32
+632,60399.0693,137.5007909,9.6835542,136.7666029,10.5451419,2.45e-05,r,22.959,23.119,0.138,0.138,23.341,23.327
+632,60399.09351,137.5007909,9.6835542,136.7661364,10.5456194,1.57e-05,i,22.814,22.709,0.124,0.124,23.356,23.343
+632,60404.02967,136.6652419,10.7150041,136.7036138,10.635401,1.48e-05,r,22.915,22.835,0.093,0.093,23.834,23.834
+632,60407.01166,137.9003026,10.4601787,136.6929055,10.6825785,1.01e-05,r,22.922,23.015,0.078,0.078,24.083,24.064
+632,60407.03549,137.9003026,10.4601787,136.6928544,10.682932,1.08e-05,i,22.917,22.91,0.094,0.094,23.755,23.735
+632,60426.96955,136.7340669,12.2432215,137.1521929,10.8475201,2.27e-05,r,23.196,23.155,0.133,0.133,23.621,23.579
+632,60426.97045,137.6919103,9.2401203,137.1521697,10.8474387,2.4e-05,r,23.163,23.167,0.136,0.136,23.638,23.552
+632,60426.98127,137.6919103,9.2401203,137.1526394,10.8474706,2.59e-05,i,22.88,22.634,0.158,0.158,23.348,23.263
+632,60432.96527,138.1759203,10.3996733,137.4653236,10.8432737,1.47e-05,r,23.04,23.141,0.108,0.108,23.861,23.858
+632,60432.97609,138.1759203,10.3996733,137.4659164,10.8432341,1.12e-05,i,23.159,23.28,0.104,0.104,23.8,23.797
+632,60435.95804,136.7133685,10.5177289,137.6505195,10.8316753,1.94e-05,r,23.223,22.804,0.137,0.137,23.608,23.6
+632,60435.96879,136.7133685,10.5177289,137.6511953,10.831653,1.61e-05,i,22.962,23.214,0.127,0.127,23.584,23.576
+632,60437.02296,138.9314847,10.5647336,137.7208526,10.8259796,1.88e-05,r,23.013,22.973,0.12,0.12,23.777,23.757
+632,60437.04671,138.9314847,10.5647336,137.7224211,10.8258789,2.98e-05,i,23.108,22.904,0.177,0.177,23.207,23.187
+39265,60347.2776,196.15222,-30.2579928,196.9513628,-31.5022453,2.8e-06,r,18.417,18.414,0.003,0.003,24.037,24.0
+39265,60347.3022,196.15222,-30.2579928,196.9491641,-31.5041886,2.8e-06,i,18.221,18.224,0.003,0.003,23.744,23.707
+39265,60352.23648,196.3355205,-32.3333,196.4659108,-31.8646381,2.8e-06,r,18.382,18.371,0.002,0.002,24.407,24.407
+39265,60352.26156,196.3355205,-32.3333,196.4630528,-31.8663676,2.8e-06,i,18.188,18.178,0.003,0.003,23.939,23.939
+39265,60355.27522,196.7281447,-30.4743488,196.1072699,-32.0612428,2.8e-06,g,19.004,18.992,0.003,0.003,24.853,24.771
+39265,60355.30005,196.7281447,-30.4743488,196.1040567,-32.0627817,2.8e-06,r,18.353,18.343,0.002,0.002,24.432,24.35
+39265,60357.22379,196.3576384,-32.7841511,195.8536052,-32.176277,2.8e-06,i,18.141,18.139,0.003,0.003,23.658,23.657
+39265,60370.38399,192.6418095,-32.5378881,193.6751007,-32.7017701,2.8e-06,i,18.021,18.014,0.004,0.004,23.121,23.116
+39265,60370.39696,192.6418095,-32.5378881,193.6725322,-32.7020423,2.9e-06,z,18.071,18.061,0.006,0.006,22.52,22.515
+39265,60371.23081,195.2225231,-33.5546893,193.509971,-32.7191964,2.8e-06,z,18.057,18.052,0.005,0.005,22.882,22.803
+39265,60371.39142,195.2225231,-33.5546893,193.477522,-32.722214,2.8e-06,i,18.015,17.991,0.004,0.004,23.258,23.175
+39265,60372.3966,193.6312686,-33.2358203,193.2764582,-32.7395931,2.8e-06,z,18.058,18.04,0.005,0.005,22.845,22.845
+39265,60373.20776,193.6570145,-33.5014042,193.1122022,-32.7514375,2.8e-06,r,18.186,18.175,0.003,0.003,24.033,24.028
+39265,60373.23256,193.6570145,-33.5014042,193.1070153,-32.7517895,2.8e-06,i,17.999,17.982,0.003,0.003,23.627,23.622
+39265,60374.34702,192.1142191,-34.1159491,192.8756718,-32.7646344,2.8e-06,i,17.992,17.97,0.003,0.003,23.839,23.79
+39265,60374.37338,192.1142191,-34.1159491,192.8700422,-32.7648691,2.8e-06,z,18.048,18.029,0.004,0.004,23.095,23.047
+39265,60376.33574,192.885252,-33.56991,192.452844,-32.7780308,2.8e-06,g,18.818,18.796,0.002,0.002,24.822,24.817
+39265,60376.35645,192.885252,-33.56991,192.4482758,-32.7780872,2.8e-06,r,18.167,18.146,0.002,0.002,24.374,24.369
+39265,60377.36057,192.8842668,-33.5905256,192.2296059,-32.7800532,2.8e-06,r,18.148,18.134,0.003,0.003,23.995,23.987
+39265,60381.33992,192.8817522,-32.219521,191.3328222,-32.7561965,2.8e-06,r,18.124,18.102,0.002,0.002,24.428,24.391
+39265,60381.36238,192.8817522,-32.219521,191.3275563,-32.7558885,2.8e-06,i,17.939,17.908,0.002,0.002,23.902,23.865
+39265,60387.30118,189.3286377,-33.5336428,189.9155719,-32.6246595,2.8e-06,z,17.936,17.908,0.003,0.003,23.337,23.327
+39265,60387.32523,189.3286377,-33.5336428,189.9096074,-32.6238638,2.8e-06,i,17.865,17.859,0.003,0.003,23.48,23.47
+39265,60397.34108,186.1196704,-31.8560311,187.4164037,-32.1469478,2.8e-06,z,17.873,17.845,0.005,0.005,22.586,22.572
+39265,60398.32359,186.4026034,-30.9279057,187.1696013,-32.083637,3.2e-06,z,17.859,17.844,0.007,0.007,21.96,21.932
+39265,60402.27963,187.5349786,-32.4817212,186.1800149,-31.800601,2.8e-06,z,17.866,17.829,0.003,0.003,23.313,23.283
+39265,60403.2865,187.334917,-30.7222495,185.9300234,-31.721604,2.8e-06,i,17.8,17.779,0.003,0.003,23.679,23.619
+39265,60406.0597,185.645687,-30.2346361,185.248943,-31.4902746,2.8e-06,r,17.998,17.967,0.002,0.002,24.317,24.29
+39265,60407.2205,185.4166348,-30.6409676,184.9661836,-31.3877483,2.8e-06,g,18.645,18.62,0.002,0.002,24.81,24.807
+39265,60407.24474,185.4166348,-30.6409676,184.9602154,-31.3855556,2.8e-06,r,17.992,17.967,0.002,0.002,24.258,24.255
+39265,60407.27341,185.4166348,-30.6409676,184.9531721,-31.3829478,2.8e-06,r,17.999,17.967,0.002,0.002,24.335,24.331
+39265,60407.29728,185.4166348,-30.6409676,184.9473234,-31.380766,2.8e-06,i,17.808,17.78,0.002,0.002,23.744,23.74
+39265,60408.11825,183.5202571,-31.2328893,184.7501526,-31.3062074,2.8e-06,r,17.989,17.974,0.002,0.002,24.147,24.136
+39265,60411.22259,183.7822018,-31.9683561,184.0138918,-31.0101173,2.8e-06,g,18.666,18.628,0.002,0.002,24.927,24.919
+39265,60411.24676,183.7822018,-31.9683561,184.0081515,-31.007703,2.8e-06,r,18.007,17.98,0.002,0.002,24.312,24.304
+39265,60413.102,182.1016585,-29.847999,183.5800785,-30.8210609,2.8e-06,r,18.019,17.99,0.002,0.002,24.397,24.329
+39265,60414.22197,181.6621325,-31.2405931,183.3252087,-30.7051507,2.9e-06,r,17.999,17.994,0.003,0.003,23.376,23.323
+39265,60414.24588,181.6621325,-31.2405931,183.3197191,-30.702622,3e-06,i,17.808,17.803,0.004,0.004,22.756,22.703
+39265,60424.21651,182.4994442,-29.2493868,181.2334028,-29.5863099,3e-06,i,17.892,17.893,0.007,0.007,22.084,22.069
+39265,60424.24032,182.4994442,-29.2493868,181.2287725,-29.5834642,3.1e-06,z,17.959,17.942,0.008,0.008,21.931,21.916
+39265,60425.04489,181.3900564,-28.9727969,181.0769728,-29.4884137,2.8e-06,z,17.978,17.956,0.005,0.005,22.651,22.651
+39265,60427.22296,181.6882319,-29.3941104,180.6761456,-29.228328,2.9e-06,z,17.991,17.985,0.006,0.006,22.427,22.422
+39265,60431.00232,181.5037149,-28.7645167,180.0295546,-28.77078,2.9e-06,g,18.825,18.817,0.003,0.003,24.149,24.123
+39265,60431.04718,181.5037149,-28.7645167,180.0220556,-28.7653573,2.8e-06,r,18.17,18.169,0.003,0.003,23.848,23.822
+39265,60436.11444,178.1673749,-27.4232305,179.2526988,-28.1462425,2.8e-06,r,18.243,18.228,0.002,0.002,24.397,24.379
+39265,60436.13866,178.1673749,-27.4232305,179.2492123,-28.1432662,2.8e-06,i,18.053,18.039,0.003,0.002,23.946,23.928
+39265,60452.12429,176.4481133,-25.0336079,177.6061288,-26.2636453,2.9e-06,z,18.303,18.292,0.007,0.007,22.517,22.447
+39265,60461.07326,178.4695629,-25.9864001,177.2011126,-25.3316939,2.8e-06,i,18.338,18.349,0.004,0.004,23.445,23.418
+39265,60461.0975,178.4695629,-25.9864001,177.2004201,-25.3293106,2.9e-06,z,18.407,18.399,0.006,0.006,22.856,22.828
+39265,60490.0355,178.5100284,-24.1976256,178.1202807,-23.252555,2.9e-06,z,18.697,18.695,0.006,0.006,23.068,23.059
+39265,60490.05966,178.5100284,-24.1976256,178.1222231,-23.2514557,2.8e-06,i,18.641,18.64,0.004,0.004,23.491,23.481
+39265,60490.99441,177.6348378,-22.2110496,178.2009389,-23.2106304,2.8e-06,r,18.834,18.83,0.003,0.003,24.192,24.178
+39265,60509.9941,181.8585333,-23.5909222,180.3263581,-22.730266,5.8e-06,z,18.816,18.808,0.018,0.018,21.779,21.701
+387449,60359.35523,208.4228825,3.2913902,209.5785639,2.9265148,1.37e-05,r,23.873,23.967,0.132,0.131,24.413,24.394
+387449,60377.29371,208.0727629,5.2920259,209.1115464,4.6019526,1.74e-05,r,23.68,23.975,0.134,0.133,24.138,24.116
+387449,60378.30503,208.2165207,4.1568996,209.0294502,4.7083379,1.5e-05,g,24.038,24.131,0.123,0.122,24.697,24.689
+387449,60378.32888,208.2165207,4.1568996,209.0273167,4.7108532,1.57e-05,r,23.484,23.72,0.126,0.125,24.184,24.176
+387449,60378.34833,208.2165207,4.1568996,209.0255611,4.7129269,1.54e-05,r,23.514,23.835,0.125,0.124,24.192,24.184
+387449,60378.36867,208.2165207,4.1568996,209.0237374,4.7150442,2.37e-05,i,23.596,23.311,0.181,0.18,23.643,23.635
+387449,60383.37916,208.6704424,4.8872052,208.533715,5.2513581,1.79e-05,r,23.658,23.564,0.134,0.133,24.036,24.036
+387449,60400.23204,206.6164126,7.128562,206.0540422,7.0228025,2.43e-05,i,23.272,23.431,0.189,0.186,23.324,23.324
+387449,60400.25582,206.6164126,7.128562,206.0496755,7.0250977,3.21e-05,z,22.998,22.776,0.256,0.251,22.951,22.951
+387449,60402.14827,205.0758992,5.9832784,205.7120449,7.2065104,1.71e-05,g,23.941,23.722,0.123,0.121,24.434,24.401
+387449,60402.17159,206.1880626,8.5516011,205.7076504,7.2086773,2.01e-05,r,23.468,23.354,0.138,0.136,23.817,23.779
+387449,60402.17204,205.0758992,5.9832784,205.7075834,7.2087628,1.88e-05,r,23.383,23.439,0.133,0.131,23.863,23.83
+387449,60402.23067,206.1880626,8.5516011,205.6965748,7.2142826,1.45e-05,i,23.112,23.338,0.136,0.133,23.756,23.718
+387449,60406.2427,203.697998,7.108249,204.9545748,7.577384,2.93e-05,g,23.936,23.926,0.152,0.151,24.14,24.111
+387449,60410.26785,204.6843632,8.5256584,204.1886307,7.907977,1.13e-05,r,23.441,23.37,0.097,0.095,24.198,24.195
+387449,60410.29163,204.6843632,8.5256584,204.1839187,7.909841,1.5e-05,i,23.089,23.057,0.134,0.131,23.72,23.718
+387449,60412.21618,202.1961898,7.8606109,203.8148801,8.054131,1.28e-05,r,23.486,23.324,0.106,0.104,24.172,24.102
+387449,60412.21842,205.1318972,8.585858,203.8144455,8.0542924,1.21e-05,r,23.285,23.365,0.102,0.1,24.191,24.154
+387449,60412.24244,205.1318972,8.585858,203.8096577,8.0560284,2.19e-05,i,23.082,23.093,0.158,0.156,23.546,23.51
+387449,60430.12461,200.5041267,10.2788879,200.5626698,8.8892973,7.73e-05,g,23.556,24.489,0.305,0.305,23.583,23.548
+387449,60434.11074,200.8476207,8.3288768,199.9454482,8.9398527,3.25e-05,r,23.771,23.395,0.184,0.183,23.778,23.767
+387449,60434.17878,200.8476207,8.3288768,199.9349207,8.9402984,3.02e-05,r,23.647,24.035,0.179,0.179,23.813,23.801
diff --git a/src/sorcha/utilities/diffTestUtils.py b/src/sorcha/utilities/diffTestUtils.py
@@ -0,0 +1,75 @@
+import numpy as np
+import os
+import tempfile
+
+from shutil import copyfile
+
+from sorcha.sorcha import runLSSTPostProcessing
+from sorcha.utilities.dataUtilitiesForTests import get_demo_filepath, get_data_out_filepath
+from sorcha.utilities.sorchaArguments import sorchaArguments
+
+
+def compare_result_files(test_output, golden_output):
+    """Compare the results in test_output to those in golden_output.
+
+    Parameters
+    ----------
+    test_output (str): The path and file name of the test results.
+
+    golden_output (str): The path and file name of the golden set results.
+
+    Returns
+    -------
+    bool : Indicates whether the results are the same.
+    """
+    test_data = np.genfromtxt(test_output, delimiter=",", dtype=str, skip_header=1)
+    golden_data = np.genfromtxt(golden_output, delimiter=",", dtype=str, skip_header=1)
+    if np.shape(test_data) != np.shape(golden_data):
+        return False
+    print(np.shape(test_data))
+    (num_rows, num_cols) = np.shape(test_data)
+
+    # Check each column, casting to float and using 'close' matches when possible
+    # and exact string matches otherwise.
+    for c in range(num_cols):
+        if np.can_cast(test_data[0][c], float):
+            if not np.allclose(test_data[:, c].astype(float), golden_data[:, c].astype(float)):
+                return False
+        else:
+            if not np.all(test_data[:, c] == golden_data[:, c]):
+                return False
+    return True
+
+
+def override_seed_and_run(outpath):
+    """Run the full Rubin sim on the demo data and a fixed seed.
+
+    WARNING: Never use a fixed seed for scientific analysis. This is
+    for testing purposes only.
+
+    Parameters
+    ----------
+    outpath (str): The path for the output files.
+    """
+
+    cmd_args_dict = {
+        "paramsinput": get_demo_filepath("sspp_testset_colours.txt"),
+        "orbinfile": get_demo_filepath("sspp_testset_orbits.des"),
+        "oifoutput": get_demo_filepath("example_oif_output.txt"),
+        "configfile": get_demo_filepath("PPConfig_test.ini"),
+        "pointing_database": get_demo_filepath("baseline_v2.0_1yr.db"),
+        "outpath": outpath,
+        "makeTemporaryEphemerisDatabase": False,
+        "readTemporaryEphemerisDatabase": False,
+        "deleteTemporaryEphemerisDatabase": False,
+        "surveyname": "LSST",
+        "outfilestem": f"out_end2end",
+        "verbose": False,
+    }
+    args = sorchaArguments(cmd_args_dict)
+
+    # Override the random number generator seed.
+    # WARNING: This is only accceptable in a test and should never be used for
+    # science results.
+    args._rng = np.random.RandomState(2023)
+    runLSSTPostProcessing(args)
diff --git a/src/sorcha/utilities/generateGoldens.py b/src/sorcha/utilities/generateGoldens.py
@@ -0,0 +1,23 @@
+import os
+import tempfile
+
+from shutil import copyfile
+
+from sorcha.utilities.diffTestUtils import override_seed_and_run
+from sorcha.utilities.dataUtilitiesForTests import get_demo_filepath
+
+if __name__ == "__main__":
+    # Create a goldens directory if it does not exist.
+    golden_dir = get_demo_filepath("goldens")
+    if not os.path.exists(golden_dir):
+        os.mkdir(golden_dir)
+
+    # Use a temporary directory to dump logs and results.
+    with tempfile.TemporaryDirectory() as dir_name:
+        override_seed_and_run(dir_name)
+
+        res_name = os.path.join(dir_name, "out_end2end.csv")
+        if not os.path.exists(res_name):
+            sys.exit(f"ERROR: Unable to find output file {res_name}")
+
+        copyfile(res_name, os.path.join(golden_dir, "out_end2end.csv"))
diff --git a/tests/sorcha/test_demo_end2end.py b/tests/sorcha/test_demo_end2end.py
@@ -0,0 +1,27 @@
+import os
+import tempfile
+
+from sorcha.utilities.dataUtilitiesForTests import get_demo_filepath
+from sorcha.utilities.diffTestUtils import compare_result_files, override_seed_and_run
+
+
+def test_demo_end2end():
+    """run the full rubin sim to ensure there are no errors."""
+    golden_dir = get_demo_filepath("goldens")
+    golden_fn = os.path.join(golden_dir, "out_end2end.csv")
+    print(f"Golden File: {golden_fn}")
+    if not os.path.isdir(golden_dir) or not os.path.isfile(golden_fn):
+        print("ERROR: No goldens generated. You must first run:")
+        print("  python src/sorcha/utilities/generateGoldens.py")
+        assert False
+
+    with tempfile.TemporaryDirectory() as dir_name:
+        override_seed_and_run(dir_name)
+        res_file = os.path.join(dir_name, "out_end2end.csv")
+        assert os.path.isfile(res_file)
+        print(f"Res File: {res_file}")
+
+        if not compare_result_files(res_file, golden_fn):
+            print("Result files do not match. There may be an error in the code, or you may need")
+            print("to regenerate the goldens with 'python src/sorcha/utilities/generateGoldens.py'")
+            assert False
diff --git a/tests/sorcha/test_integration.py b/tests/sorcha/test_integration.py