diff --git a/import-task/src/main/scala/bio/ferlab/fhir/etl/transformations/Transformations.scala b/import-task/src/main/scala/bio/ferlab/fhir/etl/transformations/Transformations.scala index cc135e9e..01e53139 100644 --- a/import-task/src/main/scala/bio/ferlab/fhir/etl/transformations/Transformations.scala +++ b/import-task/src/main/scala/bio/ferlab/fhir/etl/transformations/Transformations.scala @@ -235,7 +235,7 @@ object Transformations { .withColumn("data_category_cs", filter(col("category")(0)("coding"), col => col("system") === DOCUMENT_DATA_CATEGORY)(0) ) - .withColumn("data_category", when(isnull(col("data_category_cs")("display")), col("data_category_cs")("code")) + .withColumn("data_category", when(isnull(col("data_category_cs")("display")), col("data_category_cs")("display")) .otherwise(col("data_category_cs")("code")) ) .withColumn("content_exp", explode(col("content"))) diff --git a/prepare-index/src/main/scala/bio/ferlab/fhir/etl/PrepareIndex.scala b/prepare-index/src/main/scala/bio/ferlab/fhir/etl/PrepareIndex.scala index b6323c8f..8230f822 100644 --- a/prepare-index/src/main/scala/bio/ferlab/fhir/etl/PrepareIndex.scala +++ b/prepare-index/src/main/scala/bio/ferlab/fhir/etl/PrepareIndex.scala @@ -18,13 +18,12 @@ object PrepareIndex extends SparkApp { val studyCentric = new StudyCentric(studyList).run() + //Remove studies that are restricted val filteredStudies = studyCentric("es_index_study_centric") .where(col("security") =!= "R") .select("study_id").collect().map(r => r.getString(0)).toList - filteredStudies.foreach(println) - new SimpleParticipant(filteredStudies).run()