-
Notifications
You must be signed in to change notification settings - Fork 12
/
Copy pathForecastingWrapper.py
153 lines (141 loc) · 7.67 KB
/
ForecastingWrapper.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
######################################################
# Import modules
import resources.application as app
import os, pickle, datetime, time, warnings, sys, pandas
warnings.filterwarnings("ignore")
from PyQt5 import QtWidgets
from resources.modules.Miscellaneous.generateModel import Model
if __name__ == '__main__':
######################################################
# Wrapper script inputs and outputs
pathString = r"C:\Users\jrocha\Desktop\_TLWRK_STUFF\Forecasting\06JUN2023"
wyInput = 2023
outList = []
outList.append('FILENAME,RUNMESSAGE,PREDICTION,PREDICTIONRANGE[P10],PREDICTIONRANGE[P25],PREDICTIONRANGE[P50],PREDICTIONRANGE[P75],PREDICTIONRANGE[P90],MODELEQUATION,MODELVARIABLES[n-Predictors],XVALUES[n-Predictors]')
######################################################
# Start PyForecast in the background
print('#########################################')
print('Starting dummy PyForecast instance in the background...')
# Begin loading the application
dummyApp = QtWidgets.QApplication(sys.argv)
pyCast = app.mainWindow()
pyCast.hide()
print('----- OK!')
print(' ')
time.sleep(5)
######################################################
# Loop through all the forecast files in the input directory
forecastDirectory = os.path.normpath(pathString)
forecastExtension = ['fcst']
forecastFiles = [fn for fn in os.listdir(forecastDirectory) if any(fn.endswith(ext) for ext in forecastExtension)]
for file in forecastFiles:
fname = file
fname = forecastDirectory + "\\" + fname
outListEntry = file + ","
######################################################
# Load data and populate all the PyForecast tables
print(fname + '#########################################')
print('Loading forecast file...')
with open(fname, 'rb') as readfile:
try:
pyCast.datasetTable = pickle.load(readfile)
except:
print('WARNING: No datasetTable in saved forecast file...')
try:
pyCast.dataTable = pickle.load(readfile)
except:
print('WARNING: No dataTable in saved forecast file...')
try:
pyCast.datasetOperationsTable = pickle.load(readfile)
except:
print('WARNING: No datasetOperationsTable in saved forecast file...')
try:
pyCast.modelRunsTable = pickle.load(readfile)
except:
print('WARNING: No modelRunsTable in saved forecast file...')
try:
pyCast.forecastEquationsTable = pickle.load(readfile)
except:
print('WARNING: No forecastEquationsTable in saved forecast file...')
try:
pyCast.savedForecastEquationsTable = pickle.load(readfile)
except:
print('WARNING: No savedForecastEquationsTable in saved forecast file...')
try:
pyCast.forecastsTable = pickle.load(readfile)
except:
print('WARNING: No forecastsTable in saved forecast file...')
# with open('resources/temp/user_options.txt', 'w') as writefile:
# writefile.write(pickle.load(readfile))
print('----- OK!')
print(' ')
######################################################
# Download and update data to the user defined WY
print(fname + '#########################################')
print('Downloading and updating data...')
endYear = datetime.date.today().year
if datetime.date.today().month > 9:
endYear = endYear + 1
endYear = wyInput
startYear = pyCast.dataTable.index.max()[0].year
if startYear == endYear:
startYear = endYear - 1
pyCast.dataTab.startYearInput.setValue(startYear)
pyCast.dataTab.endYearInput.setValue(endYear)
pyCast.dataTab.freshDownloadOption.setChecked(False)
pyCast.dataTab.updateOption.setChecked(True)
#pyCast.dataTab.freshDownloadOption.setChecked(True)
#pyCast.dataTab.updateOption.setChecked(False)
pyCast.downloadData()
while pyCast.threadPool.activeThreadCount() > 0:
print(' still downloading data - active threads=' + str(pyCast.threadPool.activeThreadCount()))
time.sleep(5)
time.sleep(5)
print('----- OK!')
print(' ')
#print(pyCast.dataTable.dropna().tail(n=25))
######################################################
# Loop through the saved forecast models and generate forecasts
print(fname + '#########################################')
print('Generating forecasts...')
for modelIdx in range(len(pyCast.savedForecastEquationsTable)):
ithModelEntry = pyCast.savedForecastEquationsTable.iloc[modelIdx]
ithModel = Model(parent=pyCast,forecastEquationTableEntry=ithModelEntry)
try:
# JR - BREAKPOINT GOES ON THE NEXT LINE
ithModel.generate() #Regenerate Model
if endYear not in ithModel.x.index:
print('Forecast cannot be issued -- missing data...')
ithOutListEntry = outListEntry + 'MISSING DATA ModelId=' + str(ithModelEntry.name) + ',nan,nan,nan,nan,nan,nan'
print('----- FAIL!')
else:
try:
# JR - BREAKPOINT GOES ON THE NEXT LINE
ithModel.predict(ithModel.x.loc[endYear]) #Make a prediction
# Build model output strings
equationString = ithModel.modelReport[ithModel.modelReport.index("----- MODEL EQUATION -----") + 1]
equationVariables = ithModel.modelReport[(ithModel.modelReport.index("----- MODEL VARIABLES -----") + 1):(ithModel.modelReport.index("----- MODEL EQUATION -----") - 1)]
equationVariables = [w.replace(',', ';') for w in equationVariables]
equationVariablesString = ','.join([' '.join(ithString.split()) for ithString in equationVariables])
xValues = pandas.concat([ithModel.x.loc[endYear], pyCast.datasetTable], axis=1, join="inner").iloc[:, [4, 2, 5, 3, 0]]
xStrings = xValues.to_string(header=False, index=False, index_names=False).split('\n')
xString = ','.join([' '.join(ithString.split()) for ithString in xStrings])
# Build model output row
ithOutListEntry = outListEntry + 'OK ModelId=' + str(ithModelEntry.name) + ',' + str(ithModel.prediction) + ',' + str(','.join(map(str, ithModel.predictionRange.values[[10,25,50,75,90],0]))) + ',' + equationString + ',' + equationVariablesString + ',' + xString
print('----- OK!')
except:
ithOutListEntry = outListEntry + 'FAILED ModelId=' + str(ithModelEntry.name) + ',nan,nan,nan,nan,nan,nan'
print('----- FAIL!')
except:
ithOutListEntry = outListEntry + 'FAILED regenerating model - potential model and/or data errors...'
print('----- FAIL!')
# Add output row to output array
outList.append(ithOutListEntry.replace("[", "").replace("]", ""))
print(' ')
######################################################
# Write output file
date_time = datetime.datetime.now()
tsString = date_time.strftime("%d%b%YT%H%M%S")
with open(forecastDirectory + r'\ForecastWrapperOutput-' + tsString.upper() + '.csv', 'w') as f:
for item in outList:
f.write("%s\n" % item)