Skip to content

Commit

Permalink
Do not bin profile evaluation percentages.
Browse files Browse the repository at this point in the history
  • Loading branch information
@mihaeladuta
mihaeladuta committed Feb 22, 2024
1 parent 17545d9 commit b7a9cd7
Show file tree
Hide file tree
Showing 4 changed files with 59 additions and 59 deletions.
Binary file modified data/processed/sheets/Results.parquet
View file
Binary file not shown.
80 changes: 39 additions & 41 deletions logs/Results.log
View file
Original file line number Diff line number Diff line change
@@ -1,41 +1,39 @@
2024-02-22 14:28:18,700 [INFO] Results - read sheet from 'data/raw/REF-2021-Results-All-2022-05-06.xlsx'
2024-02-22 14:28:19,089 [INFO] Results - parsed sheet: 7552 records
2024-02-22 14:28:19,091 [INFO] Results - rename 'Main panel' to 'Main panel code'
2024-02-22 14:28:19,093 [INFO] Results - replace '['/', ':']' with '_' in 'Institution name'
2024-02-22 14:28:19,096 [INFO] Results - add columns for panel names
2024-02-22 14:28:19,100 [INFO] Results - replace '-' with na in ['4 stars', '3 stars', '2 stars', '1 star', 'Unclassified']
2024-02-22 14:28:19,103 [INFO] Results - bin percentages for ['4 stars', '3 stars', '2 stars', '1 star', 'Unclassified']
2024-02-22 14:28:19,104 [INFO] Results - drop columns '['Institution code (UKPRN)', 'Institution sort order']'
2024-02-22 14:28:19,112 [INFO] Results - pivot to make wide format for ratings per profile to enable analyses
2024-02-22 14:28:19,113 [INFO] Results - make categorical ['Outputs profile - Unclassified (binned)', 'Impact profile - 4 stars (binned)', 'Outputs profile - 2 stars (binned)', 'Overall profile - 4 stars (binned)', 'Overall profile - 3 stars (binned)', 'Outputs profile - 1 star (binned)', 'Environment profile - Unclassified (binned)', 'Impact profile - 1 star (binned)', 'Environment profile - 4 stars (binned)', 'Environment profile - 2 stars (binned)', 'Outputs profile - 3 stars (binned)', 'Impact profile - 2 stars (binned)', 'Impact profile - 3 stars (binned)', 'Environment profile - 3 stars (binned)', 'Impact profile - Unclassified (binned)', 'Overall profile - 2 stars (binned)', 'Outputs profile - 4 stars (binned)', 'Overall profile - 1 star (binned)', 'Overall profile - Unclassified (binned)', 'Environment profile - 1 star (binned)']
2024-02-22 14:28:19,325 [INFO] Results - read dataset from 'data/processed/sheets/Outputs.parquet'
2024-02-22 14:28:19,340 [INFO] Results - added column 'Output submissions (added)'
2024-02-22 14:28:19,345 [INFO] Results - added column 'Output submissions - Chapter in book (added)'
2024-02-22 14:28:19,353 [INFO] Results - added column 'Output submissions - Journal article (added)'
2024-02-22 14:28:19,357 [INFO] Results - added column 'Output submissions - Authored book (added)'
2024-02-22 14:28:19,361 [INFO] Results - added column 'Output submissions - Edited book (added)'
2024-02-22 14:28:19,365 [INFO] Results - added column 'Output submissions - Exhibition (added)'
2024-02-22 14:28:19,369 [INFO] Results - added column 'Output submissions - Performance (added)'
2024-02-22 14:28:19,372 [INFO] Results - added column 'Output submissions - Digital or visual media (added)'
2024-02-22 14:28:19,376 [INFO] Results - added column 'Output submissions - Conference contribution (added)'
2024-02-22 14:28:19,380 [INFO] Results - added column 'Output submissions - Scholarly edition (added)'
2024-02-22 14:28:19,383 [INFO] Results - added column 'Output submissions - Other (added)'
2024-02-22 14:28:19,387 [INFO] Results - added column 'Output submissions - Working paper (added)'
2024-02-22 14:28:19,391 [INFO] Results - added column 'Output submissions - Patent/ published patent application (added)'
2024-02-22 14:28:19,394 [INFO] Results - added column 'Output submissions - Composition (added)'
2024-02-22 14:28:19,398 [INFO] Results - added column 'Output submissions - Website content (added)'
2024-02-22 14:28:19,402 [INFO] Results - added column 'Output submissions - Design (added)'
2024-02-22 14:28:19,405 [INFO] Results - added column 'Output submissions - Artefact (added)'
2024-02-22 14:28:19,409 [INFO] Results - added column 'Output submissions - Research report for external body (added)'
2024-02-22 14:28:19,413 [INFO] Results - added column 'Output submissions - Research data sets and databases (added)'
2024-02-22 14:28:19,416 [INFO] Results - added column 'Output submissions - Translation (added)'
2024-02-22 14:28:19,420 [INFO] Results - added column 'Output submissions - Software (added)'
2024-02-22 14:28:19,423 [INFO] Results - added column 'Output submissions - Devices and products (added)'
2024-02-22 14:28:19,541 [INFO] Results - read dataset from 'data/processed/sheets/ImpactCaseStudies.parquet'
2024-02-22 14:28:19,546 [INFO] Results - added column 'Impact case study submissions (added)'
2024-02-22 14:28:19,556 [INFO] Results - read dataset from 'data/processed/sheets/ResearchDoctoralDegreesAwarded.parquet'
2024-02-22 14:28:19,560 [INFO] Results - added columns '['Total number of doctoral degrees awarded (added)']'
2024-02-22 14:28:19,744 [INFO] Results - read dataset from '/Users/mihaela/Documents/work/ssi_work/ref-2021-analysis/data/processed/environment_statements/EnvironmentStatementsUnitLevel.parquet'
2024-02-22 14:28:19,753 [INFO] Results - merged with unit environment statements: 1888 records
2024-02-22 14:28:19,754 [INFO] Results - make categorical ['Institution name', 'Multiple submission name', 'Multiple submission letter', 'Joint submission', 'Unit of assessment name', 'Main panel name']
2024-02-22 14:28:20,196 [INFO] Results - write dataset to 'data/processed/sheets/Results.parquet'
2024-02-22 15:44:03,758 [INFO] Results - read sheet from 'data/raw/REF-2021-Results-All-2022-05-06.xlsx'
2024-02-22 15:44:04,148 [INFO] Results - parsed sheet: 7552 records
2024-02-22 15:44:04,149 [INFO] Results - rename 'Main panel' to 'Main panel code'
2024-02-22 15:44:04,152 [INFO] Results - replace '['/', ':']' with '_' in 'Institution name'
2024-02-22 15:44:04,154 [INFO] Results - add columns for panel names
2024-02-22 15:44:04,159 [INFO] Results - replace '-' with na in ['4 stars', '3 stars', '2 stars', '1 star', 'Unclassified']
2024-02-22 15:44:04,160 [INFO] Results - drop columns '['Institution sort order', 'Institution code (UKPRN)']'
2024-02-22 15:44:04,166 [INFO] Results - pivot to make wide format for ratings per profile to enable analyses
2024-02-22 15:44:04,653 [INFO] Results - read dataset from 'data/processed/sheets/Outputs.parquet'
2024-02-22 15:44:04,664 [INFO] Results - added column 'Output submissions (added)'
2024-02-22 15:44:04,668 [INFO] Results - added column 'Output submissions - Chapter in book (added)'
2024-02-22 15:44:04,675 [INFO] Results - added column 'Output submissions - Journal article (added)'
2024-02-22 15:44:04,678 [INFO] Results - added column 'Output submissions - Authored book (added)'
2024-02-22 15:44:04,681 [INFO] Results - added column 'Output submissions - Edited book (added)'
2024-02-22 15:44:04,684 [INFO] Results - added column 'Output submissions - Exhibition (added)'
2024-02-22 15:44:04,686 [INFO] Results - added column 'Output submissions - Performance (added)'
2024-02-22 15:44:04,689 [INFO] Results - added column 'Output submissions - Digital or visual media (added)'
2024-02-22 15:44:04,691 [INFO] Results - added column 'Output submissions - Conference contribution (added)'
2024-02-22 15:44:04,694 [INFO] Results - added column 'Output submissions - Scholarly edition (added)'
2024-02-22 15:44:04,696 [INFO] Results - added column 'Output submissions - Other (added)'
2024-02-22 15:44:04,699 [INFO] Results - added column 'Output submissions - Working paper (added)'
2024-02-22 15:44:04,701 [INFO] Results - added column 'Output submissions - Patent/ published patent application (added)'
2024-02-22 15:44:04,704 [INFO] Results - added column 'Output submissions - Composition (added)'
2024-02-22 15:44:04,706 [INFO] Results - added column 'Output submissions - Website content (added)'
2024-02-22 15:44:04,709 [INFO] Results - added column 'Output submissions - Design (added)'
2024-02-22 15:44:04,711 [INFO] Results - added column 'Output submissions - Artefact (added)'
2024-02-22 15:44:04,714 [INFO] Results - added column 'Output submissions - Research report for external body (added)'
2024-02-22 15:44:04,716 [INFO] Results - added column 'Output submissions - Research data sets and databases (added)'
2024-02-22 15:44:04,719 [INFO] Results - added column 'Output submissions - Translation (added)'
2024-02-22 15:44:04,721 [INFO] Results - added column 'Output submissions - Software (added)'
2024-02-22 15:44:04,723 [INFO] Results - added column 'Output submissions - Devices and products (added)'
2024-02-22 15:44:04,843 [INFO] Results - read dataset from 'data/processed/sheets/ImpactCaseStudies.parquet'
2024-02-22 15:44:04,846 [INFO] Results - added column 'Impact case study submissions (added)'
2024-02-22 15:44:04,858 [INFO] Results - read dataset from 'data/processed/sheets/ResearchDoctoralDegreesAwarded.parquet'
2024-02-22 15:44:04,860 [INFO] Results - added columns '['Total number of doctoral degrees awarded (added)']'
2024-02-22 15:44:05,034 [INFO] Results - read dataset from '/Users/mihaela/Documents/work/ssi_work/ref-2021-analysis/data/processed/environment_statements/EnvironmentStatementsUnitLevel.parquet'
2024-02-22 15:44:05,041 [INFO] Results - merged with unit environment statements: 1888 records
2024-02-22 15:44:05,041 [INFO] Results - make categorical ['Main panel name', 'Institution name', 'Multiple submission name', 'Unit of assessment name', 'Joint submission', 'Multiple submission letter']
2024-02-22 15:44:05,476 [INFO] Results - write dataset to 'data/processed/sheets/Results.parquet'
31 changes: 16 additions & 15 deletions src/REF2021_processing/process_submissions_and_results.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,22 @@ def preprocess_results(dset):
f"%s - replace '%s' with na in {cb.COLUMNS_STARS}", sname, text_to_replace
)

# bin percentages
# ---------------
# COL_RESULTS_PERC_STAFF_SUBMITTED, COL_RESULTS_TOTAL_FTE_SUBMITTED_JOINT not binned, not < 100%
for column in cb.COLUMNS_STARS:
dset = utils.bin_percentages(
dset, column, f"{column}{cb.COLUMN_NAME_BINNED_SUFFIX}"
)
logging.info("%s - bin percentages for %s", sname, cb.COLUMNS_STARS)
# do not bin these percentages as it is not actually useful
# # bin percentages
# # ---------------
# # COL_RESULTS_PERC_STAFF_SUBMITTED, COL_RESULTS_TOTAL_FTE_SUBMITTED_JOINT not binned, not < 100%
# for column in cb.COLUMNS_STARS:
# dset = utils.bin_percentages(
# dset, column, f"{column}{cb.COLUMN_NAME_BINNED_SUFFIX}"
# )
# logging.info("%s - bin percentages for %s", sname, cb.COLUMNS_STARS)

# # make all binned columns categorical
# dset = utils.make_columns_categorical(
# dset,
# [column for column in dset.columns if cb.COLUMN_NAME_BINNED_SUFFIX in column],
# sname,
# )

# drop the columns not relevant for current visualisations
# --------------------------------------------------------
Expand All @@ -50,13 +58,6 @@ def preprocess_results(dset):

dset = utils.pivot_results_by_profile(dset, sname)

# make all binned columns categorical
dset = utils.make_columns_categorical(
dset,
[column for column in dset.columns if cb.COLUMN_NAME_BINNED_SUFFIX in column],
sname,
)

# read and merge information from research groups
# -----------------------------------------------
# the submission of research groups was not compulsory and not structured
Expand Down
7 changes: 4 additions & 3 deletions src/REF2021_processing/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -388,9 +388,10 @@ def pivot_results_by_profile(dset, sname):
cb.COL_RESULTS_TOTAL_FTE_SUBMITTED_JOINT,
]
columns_values.extend(cb.COLUMNS_STARS)
columns_values.extend(
[f"{column}{cb.COLUMN_NAME_BINNED_SUFFIX}" for column in columns_values[2:]]
)
# percentages are not binned
# columns_values.extend(
# [f"{column}{cb.COLUMN_NAME_BINNED_SUFFIX}" for column in columns_values[2:]]
# )
suffix = "profile"

# columns to drop from the wide format because they are duplicates
Expand Down

0 comments on commit b7a9cd7

Please sign in to comment.