From 80531c9e87f378bfa633dc94c47ba097ffa995f5 Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Tue, 24 Jan 2023 11:04:41 +1100 Subject: [PATCH 01/37] updated cloud-store-sdk 1.4.6 Signed-off-by: Deepak Devadathan --- data-products/pom.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/data-products/pom.xml b/data-products/pom.xml index 2f94cf092..b918d2b80 100644 --- a/data-products/pom.xml +++ b/data-products/pom.xml @@ -28,6 +28,11 @@ Jcenter Repo https://jcenter.bintray.com/ + + cloud-store + https://oss.sonatype.org/content/repositories/orgsunbird-1144/ + + @@ -281,7 +286,7 @@ org.sunbird cloud-store-sdk_2.12 - 1.3.0 + 1.4.6 com.microsoft.azure From c1f2363f359c705a376d9bcd521f851ee5eaf246 Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Tue, 31 Jan 2023 13:34:50 +1100 Subject: [PATCH 02/37] update the storageservice parameter list Signed-off-by: Deepak Devadathan --- .../main/scala/org/sunbird/analytics/jobs/ESCloudUploader.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etl-jobs/src/main/scala/org/sunbird/analytics/jobs/ESCloudUploader.scala b/etl-jobs/src/main/scala/org/sunbird/analytics/jobs/ESCloudUploader.scala index 17e6a2bd8..37f27a228 100644 --- a/etl-jobs/src/main/scala/org/sunbird/analytics/jobs/ESCloudUploader.scala +++ b/etl-jobs/src/main/scala/org/sunbird/analytics/jobs/ESCloudUploader.scala @@ -46,7 +46,7 @@ object ESCloudUploader { .saveAsTextFile(outputFilePath) // backup the output file to cloud - val storageService = StorageServiceFactory.getStorageService(StorageConfig(config.getString("cloudStorage.provider"), config.getString("cloudStorage.accountName"), config.getString("cloudStorage.accountKey"))) + val storageService = StorageServiceFactory.getStorageService(StorageConfig(config.getString("cloudStorage.provider"), config.getString("cloudStorage.accountName"), config.getString("cloudStorage.accountKey"),Option(config.getString("cloudStorage.accountEndpoint")),Option(""))) storageService.upload(config.getString("cloudStorage.container"), outputFilePath + "/part-00000", config.getString("cloudStorage.objectKey"), isDirectory = Option(false)) println("successfully backed up file to cloud!") System.exit(0) From f7cdf6219ebedbeb4f0198fc6fe3c8e5380299e0 Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Thu, 13 Apr 2023 23:42:12 +1000 Subject: [PATCH 03/37] added commons-pool2 to poms.xml Signed-off-by: Deepak Devadathan --- data-products/pom.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/data-products/pom.xml b/data-products/pom.xml index b918d2b80..3509b7704 100644 --- a/data-products/pom.xml +++ b/data-products/pom.xml @@ -350,7 +350,12 @@ 0.7.1 test - + + + org.apache.commons + commons-pool2 + + src/main/scala src/test/scala From 03164433a8e9ed55d4a52272ec4c2d9b9f53d8f3 Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Thu, 13 Apr 2023 23:44:23 +1000 Subject: [PATCH 04/37] updated version for common-pool2 Signed-off-by: Deepak Devadathan --- data-products/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/data-products/pom.xml b/data-products/pom.xml index 3509b7704..92ae98386 100644 --- a/data-products/pom.xml +++ b/data-products/pom.xml @@ -354,6 +354,7 @@ org.apache.commons commons-pool2 + 2.0 From faf2980766a236a1af9e3deb06f792b052dcc026 Mon Sep 17 00:00:00 2001 From: subhash_chandra_budde Date: Thu, 13 Apr 2023 21:23:23 +0530 Subject: [PATCH 05/37] Updated cassandra connector version --- data-products/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-products/pom.xml b/data-products/pom.xml index 92ae98386..0c1f4f5de 100644 --- a/data-products/pom.xml +++ b/data-products/pom.xml @@ -13,7 +13,7 @@ 2.12 2.12.10 3.0 - 3.1.0 + 3.2.1 @@ -178,7 +178,7 @@ com.datastax.spark spark-cassandra-connector_${scala.maj.version} - 3.1.0 + 3.2.0 From 23eb0db5e53ab0c128de1d2396bf0cb9ce4d93fc Mon Sep 17 00:00:00 2001 From: subhash_chandra_budde Date: Thu, 13 Apr 2023 21:50:11 +0530 Subject: [PATCH 06/37] Reverting jar version changes --- data-products/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-products/pom.xml b/data-products/pom.xml index 0c1f4f5de..d8a7ff87c 100644 --- a/data-products/pom.xml +++ b/data-products/pom.xml @@ -13,7 +13,7 @@ 2.12 2.12.10 3.0 - 3.2.1 + 3.1.3 @@ -178,7 +178,7 @@ com.datastax.spark spark-cassandra-connector_${scala.maj.version} - 3.2.0 + 3.1.0 From 981679ad0272e7c0c31a5bcc0c63cafdb21dbc3a Mon Sep 17 00:00:00 2001 From: subhash_chandra_budde Date: Thu, 13 Apr 2023 22:58:20 +0530 Subject: [PATCH 07/37] Updated the jar versions --- data-products/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-products/pom.xml b/data-products/pom.xml index d8a7ff87c..0c1f4f5de 100644 --- a/data-products/pom.xml +++ b/data-products/pom.xml @@ -13,7 +13,7 @@ 2.12 2.12.10 3.0 - 3.1.3 + 3.2.1 @@ -178,7 +178,7 @@ com.datastax.spark spark-cassandra-connector_${scala.maj.version} - 3.1.0 + 3.2.0 From 2d99b8fb89b57235f7fbc3682058607b8874e237 Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Fri, 14 Apr 2023 19:23:30 +1000 Subject: [PATCH 08/37] added spark-cassandra-connector-assembly Signed-off-by: Deepak Devadathan --- data-products/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-products/pom.xml b/data-products/pom.xml index 0c1f4f5de..c86647b53 100644 --- a/data-products/pom.xml +++ b/data-products/pom.xml @@ -177,7 +177,7 @@ --> com.datastax.spark - spark-cassandra-connector_${scala.maj.version} + spark-cassandra-connector-assembly_${scala.maj.version} 3.2.0 From 35e0b497b0af50d971b1203587e77e9a991ddc7f Mon Sep 17 00:00:00 2001 From: subhash_chandra_budde Date: Fri, 14 Apr 2023 19:50:46 +0530 Subject: [PATCH 09/37] updated spark version --- data-products/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-products/pom.xml b/data-products/pom.xml index c86647b53..34c1aa94c 100644 --- a/data-products/pom.xml +++ b/data-products/pom.xml @@ -12,7 +12,7 @@ 1.4.11 2.12 2.12.10 - 3.0 + 3.2 3.2.1 From 47843b7249ae3c91effd9f6b204be49bb5c04518 Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Sun, 16 Apr 2023 02:46:58 +1000 Subject: [PATCH 10/37] added debug line Signed-off-by: Deepak Devadathan --- .../scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala index da5a0b3cb..d140a1f81 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala @@ -187,6 +187,7 @@ trait OnDemandExhaustJob { storageConfig.fileName } val objKey = url.replace(filePrefix, ""); + JobLogger.log("***DEBUG***", Some(Map("URL" -> url, "FILEPREFIX" -> filePrefix, "OBJKEY" -> objKey, "LOCALPATH" -> localPath)), INFO) if (storageConfig.store.equals("local")) { fc.getHadoopFileUtil().copy(filePrefix, localPath, conf) } From fce049477fc1e74d5ad60a2eb7ab99dc6991c17e Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Sun, 16 Apr 2023 04:27:30 +1000 Subject: [PATCH 11/37] debug lines added Signed-off-by: Deepak Devadathan --- .../org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala index d140a1f81..86e567286 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala @@ -187,7 +187,9 @@ trait OnDemandExhaustJob { storageConfig.fileName } val objKey = url.replace(filePrefix, ""); - JobLogger.log("***DEBUG***", Some(Map("URL" -> url, "FILEPREFIX" -> filePrefix, "OBJKEY" -> objKey, "LOCALPATH" -> localPath)), INFO) + JobLogger.log("Request is zipAndEncrypt", Some(Map("requestId" -> request.request_id, "url" -> url)), INFO) + JobLogger.log("Request is zipAndEncrypt", Some(Map("requestId" -> request.request_id, "filePrefix" -> filePrefix)), INFO) + JobLogger.log("Request is zipAndEncrypt", Some(Map("requestId" -> request.request_id, "localPath" -> localPath)), INFO) if (storageConfig.store.equals("local")) { fc.getHadoopFileUtil().copy(filePrefix, localPath, conf) } From c00f2855440e1c83a493f9b8289fa66a929b387b Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Sun, 16 Apr 2023 18:03:02 +1000 Subject: [PATCH 12/37] added debug Signed-off-by: Deepak Devadathan --- .../scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala index 86e567286..dad33a852 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala @@ -190,6 +190,7 @@ trait OnDemandExhaustJob { JobLogger.log("Request is zipAndEncrypt", Some(Map("requestId" -> request.request_id, "url" -> url)), INFO) JobLogger.log("Request is zipAndEncrypt", Some(Map("requestId" -> request.request_id, "filePrefix" -> filePrefix)), INFO) JobLogger.log("Request is zipAndEncrypt", Some(Map("requestId" -> request.request_id, "localPath" -> localPath)), INFO) + JobLogger.log("Request is zipAndEncrypt", Some(Map("requestId" -> request.request_id, "objKey" -> objKey)), INFO) if (storageConfig.store.equals("local")) { fc.getHadoopFileUtil().copy(filePrefix, localPath, conf) } From eebb7d596912587c147986fa2f71a4e94d3f42ba Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Sun, 16 Apr 2023 18:37:11 +1000 Subject: [PATCH 13/37] changed debug message Signed-off-by: Deepak Devadathan --- .../sunbird/analytics/exhaust/OnDemandExhaustJob.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala index dad33a852..29b22ed47 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala @@ -187,10 +187,10 @@ trait OnDemandExhaustJob { storageConfig.fileName } val objKey = url.replace(filePrefix, ""); - JobLogger.log("Request is zipAndEncrypt", Some(Map("requestId" -> request.request_id, "url" -> url)), INFO) - JobLogger.log("Request is zipAndEncrypt", Some(Map("requestId" -> request.request_id, "filePrefix" -> filePrefix)), INFO) - JobLogger.log("Request is zipAndEncrypt", Some(Map("requestId" -> request.request_id, "localPath" -> localPath)), INFO) - JobLogger.log("Request is zipAndEncrypt", Some(Map("requestId" -> request.request_id, "objKey" -> objKey)), INFO) + JobLogger.log("Request is zipAndEncrypt1", Some(Map("requestId" -> request.request_id, "url" -> url)), INFO) + JobLogger.log("Request is zipAndEncrypt1", Some(Map("requestId" -> request.request_id, "filePrefix" -> filePrefix)), INFO) + JobLogger.log("Request is zipAndEncrypt1", Some(Map("requestId" -> request.request_id, "localPath" -> localPath)), INFO) + JobLogger.log("Request is zipAndEncrypt1", Some(Map("requestId" -> request.request_id, "objKey" -> objKey)), INFO) if (storageConfig.store.equals("local")) { fc.getHadoopFileUtil().copy(filePrefix, localPath, conf) } From 259bc56b3f621e3a4ef044ab71a5897b62122310 Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Sun, 16 Apr 2023 23:49:35 +1000 Subject: [PATCH 14/37] expicitly setting s3 hadoop configuration Signed-off-by: Deepak Devadathan --- .../scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala index 45434a051..3eb46c88b 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala @@ -52,8 +52,8 @@ trait BaseReportsJob { val storageSecret = modelParams.getOrElse("storageSecretConfig", "reports_storage_secret").asInstanceOf[String]; store.toLowerCase() match { case "s3" => - spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", AppConf.getConfig(storageKey)); - spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", AppConf.getConfig(storageSecret)); + spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", storageKey); + spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", storageSecret); case "azure" => val storageKeyValue = AppConf.getConfig(storageKey); spark.sparkContext.hadoopConfiguration.set("fs.azure", "org.apache.hadoop.fs.azure.NativeAzureFileSystem") From d88e33caa2059dfea681c40e4d78649fb435cf3b Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Mon, 17 Apr 2023 00:03:09 +1000 Subject: [PATCH 15/37] reverted s3 spark cconfiguration Signed-off-by: Deepak Devadathan --- .../scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala index 3eb46c88b..45434a051 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala @@ -52,8 +52,8 @@ trait BaseReportsJob { val storageSecret = modelParams.getOrElse("storageSecretConfig", "reports_storage_secret").asInstanceOf[String]; store.toLowerCase() match { case "s3" => - spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", storageKey); - spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", storageSecret); + spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", AppConf.getConfig(storageKey)); + spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", AppConf.getConfig(storageSecret)); case "azure" => val storageKeyValue = AppConf.getConfig(storageKey); spark.sparkContext.hadoopConfiguration.set("fs.azure", "org.apache.hadoop.fs.azure.NativeAzureFileSystem") From 2c8ffb87c2cee29f8d1361fcc6da570215bf0258 Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Mon, 17 Apr 2023 06:41:45 +1000 Subject: [PATCH 16/37] added oci specific hadoop conf Signed-off-by: Deepak Devadathan --- .../scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala | 5 +++++ .../org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala | 2 ++ 2 files changed, 7 insertions(+) diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala index 45434a051..c34e1a3c9 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala @@ -50,10 +50,15 @@ trait BaseReportsJob { val store = modelParams.getOrElse("store", "local").asInstanceOf[String]; val storageKey = modelParams.getOrElse("storageKeyConfig", "reports_storage_key").asInstanceOf[String]; val storageSecret = modelParams.getOrElse("storageSecretConfig", "reports_storage_secret").asInstanceOf[String]; + val storageEndpoint = modelParams.getOrElse("storageEndpoint", "reports_storage_endpoint").asInstanceOf[String]; store.toLowerCase() match { case "s3" => spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", AppConf.getConfig(storageKey)); spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", AppConf.getConfig(storageSecret)); + case "oci" => + spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", AppConf.getConfig(storageKey)); + spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", AppConf.getConfig(storageSecret)); + spark.sparkContext.hadoopConfiguration.set("fs.s3n.endpoint", AppConf.getConfig(storageEndpoint)); case "azure" => val storageKeyValue = AppConf.getConfig(storageKey); spark.sparkContext.hadoopConfiguration.set("fs.azure", "org.apache.hadoop.fs.azure.NativeAzureFileSystem") diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala index 29b22ed47..1ea9c0520 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala @@ -180,6 +180,8 @@ trait OnDemandExhaustJob { // $COVERAGE-OFF$ Disabling scoverage case "s3" => CommonUtil.getS3File(storageConfig.container, ""); + case "oci" => + CommonUtil.getS3File(storageConfig.container, ""); case "azure" => CommonUtil.getAzureFile(storageConfig.container, "", storageConfig.accountKey.getOrElse("azure_storage_key")) // $COVERAGE-ON$ for case: local From 32e00df48355a06389908f0958959fdb09f9a37b Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Mon, 17 Apr 2023 07:38:59 +1000 Subject: [PATCH 17/37] updated hadoop conf for oci Signed-off-by: Deepak Devadathan --- .../org/sunbird/analytics/exhaust/BaseReportsJob.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala index c34e1a3c9..a64fb39b0 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala @@ -56,9 +56,12 @@ trait BaseReportsJob { spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", AppConf.getConfig(storageKey)); spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", AppConf.getConfig(storageSecret)); case "oci" => - spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", AppConf.getConfig(storageKey)); - spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", AppConf.getConfig(storageSecret)); - spark.sparkContext.hadoopConfiguration.set("fs.s3n.endpoint", AppConf.getConfig(storageEndpoint)); + // spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", AppConf.getConfig(storageKey)); + // spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", AppConf.getConfig(storageSecret)); + // spark.sparkContext.hadoopConfiguration.set("fs.s3n.endpoint", AppConf.getConfig(storageEndpoint)); + spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", storageKey); + spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", storageSecret); + spark.sparkContext.hadoopConfiguration.set("fs.s3n.endpoint", storageEndpoint); case "azure" => val storageKeyValue = AppConf.getConfig(storageKey); spark.sparkContext.hadoopConfiguration.set("fs.azure", "org.apache.hadoop.fs.azure.NativeAzureFileSystem") From 615694eb025c71628367b89ed9e8f321f9eb05c0 Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Mon, 17 Apr 2023 07:50:49 +1000 Subject: [PATCH 18/37] added loggers Signed-off-by: Deepak Devadathan --- .../org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala index 1ea9c0520..c37afa031 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala @@ -172,6 +172,8 @@ trait OnDemandExhaustJob { private def zipAndEncrypt(url: String, storageConfig: StorageConfig, request: JobRequest)(implicit conf: Configuration, fc: FrameworkContext): String = { val path = Paths.get(url); + JobLogger.log("Request is zipAndEncrypt1", Some(Map("requestId" -> request.request_id, "key" -> storageConfig.accountKey.getOrElse(""))), INFO) + JobLogger.log("Request is zipAndEncrypt1", Some(Map("requestId" -> request.request_id, "secret" -> storageConfig.secretKey.getOrElse(""))), INFO) val storageService = fc.getStorageService(storageConfig.store, storageConfig.accountKey.getOrElse(""), storageConfig.secretKey.getOrElse("")); val tempDir = AppConf.getConfig("spark_output_temp_dir") + request.request_id + "/" val localPath = tempDir + path.getFileName; From 099d5a0ab6afa40568c662db5eae73757a3c4a2e Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Mon, 17 Apr 2023 21:32:13 +1000 Subject: [PATCH 19/37] updated etb metrics Signed-off-by: Deepak Devadathan --- .../org/sunbird/analytics/model/report/ETBMetricsModel.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/data-products/src/main/scala/org/sunbird/analytics/model/report/ETBMetricsModel.scala b/data-products/src/main/scala/org/sunbird/analytics/model/report/ETBMetricsModel.scala index a1ca4a742..eba016e87 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/model/report/ETBMetricsModel.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/model/report/ETBMetricsModel.scala @@ -227,6 +227,10 @@ object ETBMetricsModel extends IBatchModelTemplate[Empty,Empty,FinalOutput,Final val key = AppConf.getConfig("azure_storage_key") val file = conf("file") s"wasb://$bucket@$key.blob.core.windows.net/$file" + case "oci" => + val bucket = conf("bucket") + val file = conf("file") + s"s3n://$bucket/$file" } val scansCount = sparkSession.read From 7461fda7e537077deaa94a81be50d48db36ea076 Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Tue, 18 Apr 2023 13:22:50 +1000 Subject: [PATCH 20/37] updated spark hd configuration Signed-off-by: Deepak Devadathan --- .../scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala index a64fb39b0..14f884ff8 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala @@ -61,7 +61,7 @@ trait BaseReportsJob { // spark.sparkContext.hadoopConfiguration.set("fs.s3n.endpoint", AppConf.getConfig(storageEndpoint)); spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", storageKey); spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", storageSecret); - spark.sparkContext.hadoopConfiguration.set("fs.s3n.endpoint", storageEndpoint); + // spark.sparkContext.hadoopConfiguration.set("fs.s3n.endpoint", storageEndpoint); case "azure" => val storageKeyValue = AppConf.getConfig(storageKey); spark.sparkContext.hadoopConfiguration.set("fs.azure", "org.apache.hadoop.fs.azure.NativeAzureFileSystem") From a9f549aabf19d0e8ace5c851ca7d8a47a0639079 Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Tue, 18 Apr 2023 15:04:41 +1000 Subject: [PATCH 21/37] updated the spark hdp configuration Signed-off-by: Deepak Devadathan --- .../org/sunbird/analytics/exhaust/BaseReportsJob.scala | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala index 14f884ff8..d0ad5d54f 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala @@ -56,12 +56,8 @@ trait BaseReportsJob { spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", AppConf.getConfig(storageKey)); spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", AppConf.getConfig(storageSecret)); case "oci" => - // spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", AppConf.getConfig(storageKey)); - // spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", AppConf.getConfig(storageSecret)); - // spark.sparkContext.hadoopConfiguration.set("fs.s3n.endpoint", AppConf.getConfig(storageEndpoint)); - spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", storageKey); - spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", storageSecret); - // spark.sparkContext.hadoopConfiguration.set("fs.s3n.endpoint", storageEndpoint); + spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", AppConf.getConfig(storageKey)); + spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", AppConf.getConfig(storageSecret)); case "azure" => val storageKeyValue = AppConf.getConfig(storageKey); spark.sparkContext.hadoopConfiguration.set("fs.azure", "org.apache.hadoop.fs.azure.NativeAzureFileSystem") From ac08a4e2bca59c64e386cd70e98b25a7478142ee Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Tue, 18 Apr 2023 18:40:59 +1000 Subject: [PATCH 22/37] updated jets3t Signed-off-by: Deepak Devadathan --- adhoc-scripts/pom.xml | 6 +++--- data-products/pom.xml | 6 +++--- etl-jobs/pom.xml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/adhoc-scripts/pom.xml b/adhoc-scripts/pom.xml index 62372e31d..cecd81577 100644 --- a/adhoc-scripts/pom.xml +++ b/adhoc-scripts/pom.xml @@ -43,7 +43,7 @@ jets3t - net.java.dev.jets3t + org.jets3t org.slf4j @@ -81,9 +81,9 @@ provided - net.java.dev.jets3t + org.jets3t jets3t - 0.9.4 + 0.9.7 provided diff --git a/data-products/pom.xml b/data-products/pom.xml index 34c1aa94c..7df163a48 100644 --- a/data-products/pom.xml +++ b/data-products/pom.xml @@ -95,7 +95,7 @@ jets3t - net.java.dev.jets3t + org.jets3t org.slf4j @@ -140,9 +140,9 @@ provided - net.java.dev.jets3t + org.jets3t jets3t - 0.9.4 + 0.9.7 provided diff --git a/etl-jobs/pom.xml b/etl-jobs/pom.xml index 6633d8803..339702d03 100644 --- a/etl-jobs/pom.xml +++ b/etl-jobs/pom.xml @@ -24,7 +24,7 @@ jets3t - net.java.dev.jets3t + org.jets3t org.apache.xbean From e8ba034ea3af2333437cad433d58971da787cd9f Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Thu, 20 Apr 2023 01:25:08 +1000 Subject: [PATCH 23/37] changed the table name Signed-off-by: Deepak Devadathan --- .../analytics/exhaust/collection/BaseCollectionExhaustJob.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/BaseCollectionExhaustJob.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/BaseCollectionExhaustJob.scala index 7d82a19a8..a4b2559d5 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/BaseCollectionExhaustJob.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/BaseCollectionExhaustJob.scala @@ -47,7 +47,7 @@ trait BaseCollectionExhaustJob extends BaseReportsJob with IJob with OnDemandExh private val userConsentDBSettings = Map("table" -> "user_consent", "keyspace" -> AppConf.getConfig("sunbird.user.keyspace"), "cluster" -> "UserCluster"); private val collectionBatchDBSettings = Map("table" -> "course_batch", "keyspace" -> AppConf.getConfig("sunbird.courses.keyspace"), "cluster" -> "LMSCluster"); private val systemDBSettings = Map("table" -> "system_settings", "keyspace" -> AppConf.getConfig("sunbird.user.keyspace"), "cluster" -> "UserCluster"); - private val userEnrolmentDBSettings = Map("table" -> "user_enrolments", "keyspace" -> AppConf.getConfig("sunbird.user.report.keyspace"), "cluster" -> "ReportCluster"); + private val userEnrolmentDBSettings = Map("table" -> "report_user_enrolments", "keyspace" -> AppConf.getConfig("sunbird.user.report.keyspace"), "cluster" -> "ReportCluster"); private val redisFormat = "org.apache.spark.sql.redis"; val cassandraFormat = "org.apache.spark.sql.cassandra"; From 79dad41d3ed3f6b99f44086289d90d5439db6a1a Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Wed, 26 Apr 2023 13:06:33 +1000 Subject: [PATCH 24/37] added debug Signed-off-by: Deepak Devadathan --- .../scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala index c37afa031..17227b40b 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala @@ -195,6 +195,7 @@ trait OnDemandExhaustJob { JobLogger.log("Request is zipAndEncrypt1", Some(Map("requestId" -> request.request_id, "filePrefix" -> filePrefix)), INFO) JobLogger.log("Request is zipAndEncrypt1", Some(Map("requestId" -> request.request_id, "localPath" -> localPath)), INFO) JobLogger.log("Request is zipAndEncrypt1", Some(Map("requestId" -> request.request_id, "objKey" -> objKey)), INFO) + JobLogger.log("Request is zipAndEncrypt1", Some(Map("requestId" -> request.request_id, "objKey" -> objKey)), INFO) if (storageConfig.store.equals("local")) { fc.getHadoopFileUtil().copy(filePrefix, localPath, conf) } From b56476ce3816ebfe14c13bfe574099252c03fc32 Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Fri, 28 Apr 2023 16:17:29 +1000 Subject: [PATCH 25/37] reverted reporting cass table name Signed-off-by: Deepak Devadathan --- .../analytics/exhaust/collection/BaseCollectionExhaustJob.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/BaseCollectionExhaustJob.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/BaseCollectionExhaustJob.scala index a4b2559d5..7d82a19a8 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/BaseCollectionExhaustJob.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/collection/BaseCollectionExhaustJob.scala @@ -47,7 +47,7 @@ trait BaseCollectionExhaustJob extends BaseReportsJob with IJob with OnDemandExh private val userConsentDBSettings = Map("table" -> "user_consent", "keyspace" -> AppConf.getConfig("sunbird.user.keyspace"), "cluster" -> "UserCluster"); private val collectionBatchDBSettings = Map("table" -> "course_batch", "keyspace" -> AppConf.getConfig("sunbird.courses.keyspace"), "cluster" -> "LMSCluster"); private val systemDBSettings = Map("table" -> "system_settings", "keyspace" -> AppConf.getConfig("sunbird.user.keyspace"), "cluster" -> "UserCluster"); - private val userEnrolmentDBSettings = Map("table" -> "report_user_enrolments", "keyspace" -> AppConf.getConfig("sunbird.user.report.keyspace"), "cluster" -> "ReportCluster"); + private val userEnrolmentDBSettings = Map("table" -> "user_enrolments", "keyspace" -> AppConf.getConfig("sunbird.user.report.keyspace"), "cluster" -> "ReportCluster"); private val redisFormat = "org.apache.spark.sql.redis"; val cassandraFormat = "org.apache.spark.sql.cassandra"; From e9210001b86af992c73e9c79d8b1d1f16bf67001 Mon Sep 17 00:00:00 2001 From: subhash_chandra_budde Date: Tue, 2 May 2023 23:25:57 +0530 Subject: [PATCH 26/37] Added debug information --- .../sunbird/analytics/job/report/StateAdminGeoReportJob.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data-products/src/main/scala/org/sunbird/analytics/job/report/StateAdminGeoReportJob.scala b/data-products/src/main/scala/org/sunbird/analytics/job/report/StateAdminGeoReportJob.scala index 6f0e595df..fd69842b3 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/job/report/StateAdminGeoReportJob.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/job/report/StateAdminGeoReportJob.scala @@ -45,7 +45,8 @@ object StateAdminGeoReportJob extends IJob with StateAdminReportHelper { val container = AppConf.getConfig("cloud.container.reports") val objectKey = AppConf.getConfig("admin.metrics.cloud.objectKey") val storageConfig = getStorageConfig(container, objectKey) - + print("container for Admin Geo Report: " + container) + print("objectKey for Admin Geo Report: " + objectKey); val organisationDF: DataFrame = loadOrganisationSlugDF() val subOrgDF: DataFrame = generateSubOrgData(organisationDF) val blockData:DataFrame = generateBlockLevelData(subOrgDF) From 3803ee5dd1f24a39107d81cb642e63028790ec96 Mon Sep 17 00:00:00 2001 From: Kenneth Heung Date: Mon, 22 May 2023 10:43:42 +0800 Subject: [PATCH 27/37] Update FunnelReport.scala update config without hardcode reports container --- .../scala/org/sunbird/analytics/sourcing/FunnelReport.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala b/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala index d3aa56a91..78ddc1f4b 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala @@ -122,7 +122,9 @@ object FunnelReport extends IJob with BaseReportsJob { .drop("channel","id","program_id") .persist(StorageLevel.MEMORY_ONLY) - val storageConfig = getStorageConfig("reports", "") + // support to other CSP and report container should be read from configuration + // val storageConfig = getStorageConfig("reports", "") + val storageConfig = getStorageConfig(config, "") saveReportToBlob(funnelReport, configMap, storageConfig, "FunnelReport") funnelReport.unpersist(true) From 6bc541c0cd5a18d3198ec3093b5a93e97982b4e8 Mon Sep 17 00:00:00 2001 From: Kenneth Heung Date: Mon, 22 May 2023 13:52:05 +0800 Subject: [PATCH 28/37] Update FunnelReport.scala convert the param to string --- .../scala/org/sunbird/analytics/sourcing/FunnelReport.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala b/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala index 78ddc1f4b..62dcfaf7b 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala @@ -124,7 +124,9 @@ object FunnelReport extends IJob with BaseReportsJob { // support to other CSP and report container should be read from configuration // val storageConfig = getStorageConfig("reports", "") - val storageConfig = getStorageConfig(config, "") + val modelParams = config.modelParams.getOrElse(Map[String, Option[AnyRef]]()); + val container = modelParams.getOrElse("storageContainer", "reports").asInstanceOf[String] + val storageConfig = getStorageConfig(container, "") saveReportToBlob(funnelReport, configMap, storageConfig, "FunnelReport") funnelReport.unpersist(true) From 281672f0d4107c2bea146fda62317568a7c8607b Mon Sep 17 00:00:00 2001 From: Kenneth Heung Date: Mon, 22 May 2023 23:14:59 +0800 Subject: [PATCH 29/37] Update FunnelReport.scala add logging info --- .../scala/org/sunbird/analytics/sourcing/FunnelReport.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala b/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala index 62dcfaf7b..cf3a5feb2 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala @@ -124,9 +124,11 @@ object FunnelReport extends IJob with BaseReportsJob { // support to other CSP and report container should be read from configuration // val storageConfig = getStorageConfig("reports", "") - val modelParams = config.modelParams.getOrElse(Map[String, Option[AnyRef]]()); + // val modelParams = config.modelParams.getOrElse(Map[String, Option[AnyRef]]()); + val modelParams = config.modelParams.get val container = modelParams.getOrElse("storageContainer", "reports").asInstanceOf[String] val storageConfig = getStorageConfig(container, "") + JobLogger.log(s"saving report in bucket - ${container}", None, Level.INFO) saveReportToBlob(funnelReport, configMap, storageConfig, "FunnelReport") funnelReport.unpersist(true) From a5974a031b34673b8e73ce68673ff19df54b74f6 Mon Sep 17 00:00:00 2001 From: Kenneth Heung Date: Tue, 23 May 2023 05:30:06 +0800 Subject: [PATCH 30/37] Update FunnelReport.scala use globalContainer for report --- .../scala/org/sunbird/analytics/sourcing/FunnelReport.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala b/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala index cf3a5feb2..fa0c32521 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala @@ -43,6 +43,7 @@ object FunnelReport extends IJob with BaseReportsJob { val jobName = "Funnel Report Job" val db = AppConf.getConfig("postgres.db") val url = AppConf.getConfig("postgres.url") + s"$db" + val globalContainer = AppConf.getConfig("cloud.container.reports") val connProperties = CommonUtil.getPostgresConnectionProps val programTable = "program" val nominationTable = "nomination" @@ -127,7 +128,9 @@ object FunnelReport extends IJob with BaseReportsJob { // val modelParams = config.modelParams.getOrElse(Map[String, Option[AnyRef]]()); val modelParams = config.modelParams.get val container = modelParams.getOrElse("storageContainer", "reports").asInstanceOf[String] - val storageConfig = getStorageConfig(container, "") + print("container for Funnel Report: " + container) + // use global container for the report + val storageConfig = getStorageConfig(globalContainer, "") JobLogger.log(s"saving report in bucket - ${container}", None, Level.INFO) saveReportToBlob(funnelReport, configMap, storageConfig, "FunnelReport") From dfad7ae98800a8f0cab96134495d823000bb53e3 Mon Sep 17 00:00:00 2001 From: Kenneth Heung Date: Tue, 23 May 2023 07:04:04 +0800 Subject: [PATCH 31/37] Update FunnelReport.scala - update csp bucket change to bucket specific --- .../org/sunbird/analytics/sourcing/FunnelReport.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala b/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala index fa0c32521..19e5babb1 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala @@ -125,13 +125,13 @@ object FunnelReport extends IJob with BaseReportsJob { // support to other CSP and report container should be read from configuration // val storageConfig = getStorageConfig("reports", "") - // val modelParams = config.modelParams.getOrElse(Map[String, Option[AnyRef]]()); - val modelParams = config.modelParams.get + val modelParams = config.modelParams.getOrElse(Map[String, Option[AnyRef]]()); val container = modelParams.getOrElse("storageContainer", "reports").asInstanceOf[String] print("container for Funnel Report: " + container) - // use global container for the report - val storageConfig = getStorageConfig(globalContainer, "") JobLogger.log(s"saving report in bucket - ${container}", None, Level.INFO) + // use global container for the report - this is only a fallback if required + // val storageConfig = getStorageConfig(globalContainer, "") + val storageConfig = getStorageConfig(container, "") saveReportToBlob(funnelReport, configMap, storageConfig, "FunnelReport") funnelReport.unpersist(true) From b56af0692c1f2344e5efb20a52719251b06db92f Mon Sep 17 00:00:00 2001 From: Kenneth Heung Date: Thu, 25 May 2023 10:19:27 +0800 Subject: [PATCH 32/37] Update FunnelReport.scala - remove debug and cleaning up the code --- .../scala/org/sunbird/analytics/sourcing/FunnelReport.scala | 6 ------ 1 file changed, 6 deletions(-) diff --git a/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala b/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala index 19e5babb1..e1b809fa4 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/sourcing/FunnelReport.scala @@ -43,7 +43,6 @@ object FunnelReport extends IJob with BaseReportsJob { val jobName = "Funnel Report Job" val db = AppConf.getConfig("postgres.db") val url = AppConf.getConfig("postgres.url") + s"$db" - val globalContainer = AppConf.getConfig("cloud.container.reports") val connProperties = CommonUtil.getPostgresConnectionProps val programTable = "program" val nominationTable = "nomination" @@ -124,13 +123,8 @@ object FunnelReport extends IJob with BaseReportsJob { .persist(StorageLevel.MEMORY_ONLY) // support to other CSP and report container should be read from configuration - // val storageConfig = getStorageConfig("reports", "") val modelParams = config.modelParams.getOrElse(Map[String, Option[AnyRef]]()); val container = modelParams.getOrElse("storageContainer", "reports").asInstanceOf[String] - print("container for Funnel Report: " + container) - JobLogger.log(s"saving report in bucket - ${container}", None, Level.INFO) - // use global container for the report - this is only a fallback if required - // val storageConfig = getStorageConfig(globalContainer, "") val storageConfig = getStorageConfig(container, "") saveReportToBlob(funnelReport, configMap, storageConfig, "FunnelReport") From f940bcece993a47727298126403d9aecab475b45 Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Fri, 4 Aug 2023 09:05:20 +1000 Subject: [PATCH 33/37] removed temp repo Signed-off-by: Deepak Devadathan --- data-products/pom.xml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/data-products/pom.xml b/data-products/pom.xml index 7df163a48..442ed0d27 100644 --- a/data-products/pom.xml +++ b/data-products/pom.xml @@ -27,11 +27,7 @@ jcenter-repo Jcenter Repo https://jcenter.bintray.com/ - - - cloud-store - https://oss.sonatype.org/content/repositories/orgsunbird-1144/ - + From b3ac01c609723089743a6b22874ca50424f4a986 Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Fri, 4 Aug 2023 09:06:53 +1000 Subject: [PATCH 34/37] combined two identical switch statements Signed-off-by: Deepak Devadathan --- .../scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala index d0ad5d54f..055202ab9 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/BaseReportsJob.scala @@ -52,10 +52,7 @@ trait BaseReportsJob { val storageSecret = modelParams.getOrElse("storageSecretConfig", "reports_storage_secret").asInstanceOf[String]; val storageEndpoint = modelParams.getOrElse("storageEndpoint", "reports_storage_endpoint").asInstanceOf[String]; store.toLowerCase() match { - case "s3" => - spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", AppConf.getConfig(storageKey)); - spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", AppConf.getConfig(storageSecret)); - case "oci" => + case "s3" | "oci" => spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", AppConf.getConfig(storageKey)); spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", AppConf.getConfig(storageSecret)); case "azure" => From 21f65bba3287a56f3d38a23e3a26b62ef1e2decf Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Fri, 4 Aug 2023 09:07:47 +1000 Subject: [PATCH 35/37] combined switch case Signed-off-by: Deepak Devadathan --- .../org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala index 17227b40b..e98fb8690 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala @@ -180,9 +180,7 @@ trait OnDemandExhaustJob { fc.getHadoopFileUtil().delete(conf, tempDir); val filePrefix = storageConfig.store.toLowerCase() match { // $COVERAGE-OFF$ Disabling scoverage - case "s3" => - CommonUtil.getS3File(storageConfig.container, ""); - case "oci" => + case "s3" | "oci" => CommonUtil.getS3File(storageConfig.container, ""); case "azure" => CommonUtil.getAzureFile(storageConfig.container, "", storageConfig.accountKey.getOrElse("azure_storage_key")) From bbb1f8aa722dc1699db18beb0c9ec614f77da29b Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Fri, 4 Aug 2023 09:09:57 +1000 Subject: [PATCH 36/37] removed unwanted logs Signed-off-by: Deepak Devadathan --- .../org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala | 7 ------- 1 file changed, 7 deletions(-) diff --git a/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala b/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala index e98fb8690..2bf47835a 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/exhaust/OnDemandExhaustJob.scala @@ -172,8 +172,6 @@ trait OnDemandExhaustJob { private def zipAndEncrypt(url: String, storageConfig: StorageConfig, request: JobRequest)(implicit conf: Configuration, fc: FrameworkContext): String = { val path = Paths.get(url); - JobLogger.log("Request is zipAndEncrypt1", Some(Map("requestId" -> request.request_id, "key" -> storageConfig.accountKey.getOrElse(""))), INFO) - JobLogger.log("Request is zipAndEncrypt1", Some(Map("requestId" -> request.request_id, "secret" -> storageConfig.secretKey.getOrElse(""))), INFO) val storageService = fc.getStorageService(storageConfig.store, storageConfig.accountKey.getOrElse(""), storageConfig.secretKey.getOrElse("")); val tempDir = AppConf.getConfig("spark_output_temp_dir") + request.request_id + "/" val localPath = tempDir + path.getFileName; @@ -189,11 +187,6 @@ trait OnDemandExhaustJob { storageConfig.fileName } val objKey = url.replace(filePrefix, ""); - JobLogger.log("Request is zipAndEncrypt1", Some(Map("requestId" -> request.request_id, "url" -> url)), INFO) - JobLogger.log("Request is zipAndEncrypt1", Some(Map("requestId" -> request.request_id, "filePrefix" -> filePrefix)), INFO) - JobLogger.log("Request is zipAndEncrypt1", Some(Map("requestId" -> request.request_id, "localPath" -> localPath)), INFO) - JobLogger.log("Request is zipAndEncrypt1", Some(Map("requestId" -> request.request_id, "objKey" -> objKey)), INFO) - JobLogger.log("Request is zipAndEncrypt1", Some(Map("requestId" -> request.request_id, "objKey" -> objKey)), INFO) if (storageConfig.store.equals("local")) { fc.getHadoopFileUtil().copy(filePrefix, localPath, conf) } From 063abde731f5cddaf6c3ce379948f73e809ea214 Mon Sep 17 00:00:00 2001 From: Deepak Devadathan Date: Fri, 4 Aug 2023 09:11:48 +1000 Subject: [PATCH 37/37] removed debug prints Signed-off-by: Deepak Devadathan --- .../sunbird/analytics/job/report/StateAdminGeoReportJob.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/data-products/src/main/scala/org/sunbird/analytics/job/report/StateAdminGeoReportJob.scala b/data-products/src/main/scala/org/sunbird/analytics/job/report/StateAdminGeoReportJob.scala index fd69842b3..e7d9978c0 100644 --- a/data-products/src/main/scala/org/sunbird/analytics/job/report/StateAdminGeoReportJob.scala +++ b/data-products/src/main/scala/org/sunbird/analytics/job/report/StateAdminGeoReportJob.scala @@ -45,8 +45,6 @@ object StateAdminGeoReportJob extends IJob with StateAdminReportHelper { val container = AppConf.getConfig("cloud.container.reports") val objectKey = AppConf.getConfig("admin.metrics.cloud.objectKey") val storageConfig = getStorageConfig(container, objectKey) - print("container for Admin Geo Report: " + container) - print("objectKey for Admin Geo Report: " + objectKey); val organisationDF: DataFrame = loadOrganisationSlugDF() val subOrgDF: DataFrame = generateSubOrgData(organisationDF) val blockData:DataFrame = generateBlockLevelData(subOrgDF)