-
Notifications
You must be signed in to change notification settings - Fork 4
/
AstroBinUpload.py
292 lines (253 loc) · 11.3 KB
/
AstroBinUpload.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
#!/usr/bin/env python3
import os
from utils import initialise_logging,summarize_session, Headers,Processing,Sites,utils_version
import sys
version = '1.3.11'
'''
# Changes:
# v1.1.2 9th January 2024
# Allows initialisation of the script without a config.ini file and no directory paths
# Explicitly prevents single files being passed as arguments
# Added utils_version check
# v1.1.3 10th January 2024
# Deal with WBPP
# v1.1.3 13th January 2024
# Redesign of how code parses data files
# v1.1.4 14th January 2024
# New utils.py script v1.1.4
# 22nd January 2024
# Saved as version 1.2.0 this is now a candidate release version
# New utils.py script v1.2.0
# 28th January 2024
# Saved as version 1.2.1
# New utils.py script v1.2.1
# 29th January 2024
# Saved as version 1.2.2
# New utils.py script v1.2.2
# 30th January 2024
# Saved as version 1.2.3
# New utils.py script v1.2.3
# 6th February 2024
# Saved as version 1.2.4
# New utils.py script v1.2.4
# 7th February 2024
# Saved as version 1.2.5
# New utils.py script v1.2.5
# 7th February 2024
# Saved as version 1.2.6
# New utils.py script v1.2.6
# Saved as version 1.2.7
#added logging of data frames
# New utils.py script v1.2.8
# Saved as version 1.2.8
# added logging of data frames
# added logging of basic header data frames
# New utils.py script v1.2.8
# 8th February 2024
# New utils.py script v1.2.9
# Saved as version 1.2.9
# added DEBUG switch to save dataframes to csv files
# use --debug in command line to save dataframes to csv files
# 9th February 2024
# New utils.py script v1.2.10
# Saved as version 1.2.10
# 12th February 2024
# Release version 1.3.0
# 26th February 2024
# Release version 1.3.1
# Modified debugging file dumps, so they occour after the data has been processed not all at the the end
# New utils.py script v1.3.1
# Release version 1.3.2
# New utils.py script v1.3.2
# 4th March 2024
# Release version 1.3.3
# New utils.py script v1.3.3
# 5th March 2024
# Release version 1.3.4
# New utils.py script v1.3.4
# 5th May 2024
# Release version 1.3.5
# New utils.py script v1.3.5
# Added the ability to call the script with no arguments and it will process the current directory
# All debug, txt and csv output files are saved to a subdirectory of the directory being processed
# 16 June 2024
# Release version 1.3.6
# New utils.py script v1.3.6
# Added the ability to call the script with --debug flag to save dataframes to csv files
# Corrected error caused by site coordinates and date format of some images
# modified script to save all output files to a subdirectory of the directory being processed
# Release version 1.3.7
# allows the scipt to be called from an image directory and process the images in that directory but allows for calibration directories to be passed as arguments
# eg AstroBinUploadv1_3_7.py "." /home/user/images/calibration
# version 1.3.8
# 28th September 2024
# Allows the processing of LIGHTFRAMES as well as LIGHT frames
# Modification in the process headers function to allow the processing of LIGHTFRAMES as well as LIGHT frames
# version 1.3.9
# 28th September 2024
# Allows the processing of LIGHTFRAMES and Light Frames as well as LIGHT frames
# Modification in the process headers function to allow the processing of LIGHTFRAMES Light Frames as well as LIGHT frames
# version 1.3.10
# 29th September 2024
# Deals with the case where a fractional part of a second is present in some date-obs keyword but not in others
# Deals with the case where the filter names in the light frames have trailing white spaces.
# version 1.3.11
# 16th October 2024
# Fixes bug where running the script for the first time from the installation directory would fail
# Deals with the case where the filter names in the light frames have trailing white spaces.
# Modifed script to take a new default parameter for the config file
# The parameter is USEOBSDATE and is set to True if the actual date of the observation session is to be used when aggregating data
# for the astrobin upload .csv output.
# If this prameter is set to False then the date the observation session was started is used.
# Deals with the case where the filter names in the light frames have trailing white spaces.
'''
#see utils.py for change details
# Determine the script's directory
script_dir = os.path.dirname(os.path.abspath(__file__))
# CONFIGFILENAME should only exist in the directory where the script is located
CONFIGFILENAME = os.path.join(script_dir, 'config.ini')
PRECISION = 4
DEBUG = False
def main() -> None:
"""
Main function to process directories containing FITS files, aggregate parameters,
get site data, summarize the session, and export the summary and AstroBin data.
"""
#Check for debug flag in arguments and set debug flag if found
DEBUG = '--debug' in sys.argv
#remove the debug flag from the arguments
if DEBUG:
sys.argv.remove('--debug')
# Validate directory paths
# bug fix for running the script for the first time from the installation directory here
if len(sys.argv) < 2: # and os.path.isfile(CONFIGFILENAME):
err = "No directory path provided. Please provide a directory path as an argument."
#logger.error(err)
print(err)
sys.exit(1)
elif len(sys.argv) >= 2 and sys.argv[1] == ".":
directory_paths = [os.getcwd()]+ sys.argv[2:]
else:
directory_paths = sys.argv[1:]
#Output directory is the first argument
output_dir = sys.argv[1]
# Ensure the output directory path is absolute
output_dir = os.path.abspath(output_dir)
# Construct the path for the new directory
output_dir_path = os.path.join(output_dir, 'AstroBinUploadInfo')
# Create the output directory called 'AstroBinUploadInfo' if it does not exist in output_dir
os.makedirs(output_dir_path, exist_ok=True)
print(f"Output directory: {output_dir_path}")
# Set LOGFILENAME based on the output directory
LOGFILENAME = os.path.join(output_dir_path, 'AstroBinUploader.log')
# Create the directory for the log file if it does not exist
os.makedirs(os.path.dirname(LOGFILENAME), exist_ok=True)
# Initialise logging
logger = initialise_logging(LOGFILENAME)
logger.info("Logging initialised.")
print("Logging initialised.")
#log the version of the script
logger.info(f"main version : {version}")
#log the version of utils
logger.info(f"utils version: {utils_version}")
print(f"main version : {version}")
print(f"utils version: {utils_version}")
#log the provided arguments
logger.info(f"Calling function and arguments provided: {sys.argv}")
logger.info("")
# Validate directory paths
for directory in directory_paths:
if not os.path.exists(directory):
err = f"The path '{directory}' does not exist. Exiting the program."
logger.error(err)
print(err)
sys.exit(1)
elif not os.path.isdir(directory):
err = f"The path '{directory}' is not a directory. Exiting the program."
logger.error(err)
print(err)
sys.exit(1)
logger.info(f"Processing directory: {directory}")
#check version of utils script is same version as this script
if utils_version != version:
print(f"Error: utils version is {utils_version} and must be {version}")
print("Exiting the program.")
logger.error(f"utils version is {utils_version} and must be {version}")
logger.error("Exiting the program.")
sys.exit(1)
# Create headers object
headers = Headers(CONFIGFILENAME, logger, PRECISION)
print("Headers object created.")
logger.info("Headers object created.")
# Check if a configuration file was created
if headers.change:
# inform user that a new config.ini file was created and exit
print ("A new config.ini file was created. Please edit this before re-running the script: ")
sys.exit(0)
# Create processing object
processing = Processing(headers, logger)
logger.info("Processing object created.")
# Create sites object
sites = Sites(headers, logger)
logger.info("Sites object created.")
# Extract FITS headers
logger.info("Reading FITS headers...")
print('\nReading FITS headers...\n')
headers_df,basic_headers = headers.process_directories(directory_paths)
if DEBUG:
#if debug is True save dataframes
#save basic headers to a csv file
basic_headers_csv = os.path.join(output_dir_path, os.path.basename(directory_paths[0]).replace(" ", "_") + "_basic_headers.csv")
basic_headers.to_csv(basic_headers_csv, index=False)
#save modied headers to a csv file
headers_csv = os.path.join(output_dir_path, os.path.basename(directory_paths[0]).replace(" ", "_") + "_headers.csv")
headers_df.to_csv(headers_csv, index=False)
logger.info("Debugging is True.")
logger.info(f"basic headers exported to {basic_headers_csv}")
logger.info(f"headers exported to {headers_csv}")
# Get location information
logger.info("Getting site data...")
site_modified_headers_df = sites.get_site_data(headers_df)
if DEBUG:
modified_csv = os.path.join(output_dir_path, os.path.basename(directory_paths[0]).replace(" ", "_") + "_modified.csv")
site_modified_headers_df.to_csv(modified_csv, index=False)
logger.info(f"site modified headers exported to {modified_csv}")
# Aggregate parameters
logger.info("Aggregating parameters...")
aggregated_df = processing.aggregate_parameters(site_modified_headers_df)
if DEBUG:
# Save the aggregated data to a csv file
aggregated_csv = os.path.join(output_dir_path, os.path.basename(directory_paths[0]).replace(" ", "_") + "_aggregated.csv")
aggregated_df.to_csv(aggregated_csv, index=False)
logger.info(f"aggregated headers exported to {aggregated_csv}")
# Create and print summary of observation session
logger.info("Summarizing session...")
summary_txt = summarize_session(aggregated_df, logger,headers.number_of_images_processed)
# Transform data for AstroBin output
logger.info("Creating AstroBin output...")
output_csv = os.path.basename(directory_paths[0]).replace(" ", "_") + "_acquisition.csv"
astrobin_df = processing.create_astrobin_output(aggregated_df)
summary_txt += "\n "+output_csv
summary_txt+= "\n\n "+ astrobin_df.to_string(index=False).replace('\n', '\n ') + "\n "
# Export summary to a text file
logger.info("Exporting summary to text file...")
summary_file = os.path.join(output_dir_path, os.path.basename(directory_paths[0]).replace(" ", "_") + "_session_summary.txt")
with open(summary_file, 'w') as file:
file.write(summary_txt)
summary_txt+=f"\n Processing summary exported to {summary_file}\n"
# Export final data to CSV
# Export final data to CSV
if not astrobin_df.empty:
logger.info("\nExporting AstroBin data to CSV...")
output_csv = os.path.join(output_dir_path, os.path.basename(directory_paths[0]).replace(" ", "_") + "_acquisition.csv")
astrobin_df.to_csv(output_csv, index=False)
summary_txt+=f"\n AstroBin data exported to {output_csv}\n"
else:
logger.info("No AstroBin data to export.")
print("\n No AstroBin data to export.\n")
logger.info("Processing completed.")
summary_txt+="\n Processing complete.\n"
print(summary_txt)
return
if __name__ == "__main__":
main()