Skip to content

Commit 513d437

Browse files
parthosaparthosa
committed
Remove support for cloud based event logs
Signed-off-by: Partho Sarthi <[email protected]>
1 parent 6782797 commit 513d437

3 files changed

+14
-200
lines changed

‎tools/databricks/[RAPIDS Accelerator for Apache Spark] Profiling Tool Notebook Template.ipynb

Lines changed: 4 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -45,14 +45,10 @@
4545
},
4646
"outputs": [],
4747
"source": [
48-
"import warnings\n",
4948
"DEFAULT_TOOLS_VER = \"24.02.4\"\n",
5049
"dbutils.widgets.text(\"Tools Version\", DEFAULT_TOOLS_VER)\n",
5150
"TOOLS_VER=dbutils.widgets.get(\"Tools Version\")\n",
52-
"\n",
5351
"print(f\"Using Tools Version: {TOOLS_VER}\")\n",
54-
"if TOOLS_VER != DEFAULT_TOOLS_VER:\n",
55-
" print(f\"Warning: Requested tools version ({TOOLS_VER}) differs from the default version ({DEFAULT_TOOLS_VER}). This may require different versions of depedencies for compatibility.\")\n",
5652
"\n",
5753
"dbutils.widgets.dropdown(\"CSP\", \"aws\", [\"aws\", \"azure\"])\n",
5854
"CSP=dbutils.widgets.get(\"CSP\")\n",
@@ -110,24 +106,10 @@
110106
"TOOL_JAR_LOCAL_PATH = f'{OUTPUT_PATH}/rapids-4-spark-tools.jar'\n",
111107
"download_file(TOOL_JAR_URL, TOOL_JAR_LOCAL_PATH)\n",
112108
"\n",
113-
"# Download dependency jars\n",
114-
"HADOOP_JAR_VER = '3.3.4'\n",
115-
"HADOOP_JAR_URL = f'https://repo.maven.apache.org/maven2/org/apache/hadoop/hadoop-{CSP}/{HADOOP_JAR_VER}/hadoop-{CSP}-{HADOOP_JAR_VER}.jar'\n",
116-
"HADOOP_JAR_LOCAL_PATH = f'{OUTPUT_PATH}/hadoop-{CSP}-{HADOOP_JAR_VER}.jar'\n",
117-
"download_file(HADOOP_JAR_URL, HADOOP_JAR_LOCAL_PATH)\n",
118-
"\n",
119-
"if CSP=='aws':\n",
120-
" AWS_JAVA_VER = '1.12.262'\n",
121-
" AWS_JAVA_URL = f'https://repo.maven.apache.org/maven2/com/amazonaws/aws-java-sdk/{AWS_JAVA_VER}/aws-java-sdk-{AWS_JAVA_VER}.jar'\n",
122-
" AWS_JAVA_LOCAL_PATH = f'{OUTPUT_PATH}/aws-java-sdk-{AWS_JAVA_VER}.jar'\n",
123-
" download_file(AWS_JAVA_URL, AWS_JAVA_LOCAL_PATH)\n",
124-
" os.environ[\"AWS_JAVA_LOCAL_PATH\"] = AWS_JAVA_LOCAL_PATH\n",
125-
"\n",
126109
"os.environ[\"CSP\"] = CSP\n",
127110
"os.environ[\"EVENTLOG_PATH\"] = EVENTLOG_PATH\n",
128111
"os.environ[\"OUTPUT_PATH\"] = OUTPUT_PATH\n",
129112
"os.environ[\"TOOL_JAR_LOCAL_PATH\"] = TOOL_JAR_LOCAL_PATH\n",
130-
"os.environ[\"HADOOP_JAR_LOCAL_PATH\"] = HADOOP_JAR_LOCAL_PATH\n",
131113
"\n",
132114
"WORKER_INFO_PATH = os.path.join(OUTPUT_PATH, \"worker_info.yaml\")\n",
133115
"os.environ[\"WORKER_INFO_PATH\"] = WORKER_INFO_PATH"
@@ -176,56 +158,6 @@
176158
" f.write(worker_info)"
177159
]
178160
},
179-
{
180-
"cell_type": "code",
181-
"execution_count": 0,
182-
"metadata": {
183-
"application/vnd.databricks.v1+cell": {
184-
"cellMetadata": {
185-
"byteLimit": 2048000,
186-
"rowLimit": 10000
187-
},
188-
"inputWidgets": {},
189-
"nuid": "476b1278-afb7-43c3-95d8-99a4821bc20b",
190-
"showTitle": true,
191-
"title": "CSP Credentials"
192-
}
193-
},
194-
"outputs": [],
195-
"source": [
196-
"# Secrets required if accessing from s3\n",
197-
"os.environ[\"AWS_ACCESS_KEY_ID\"] = 'your_aws_access_key_id'\n",
198-
"os.environ[\"AWS_SECRET_ACCESS_KEY\"] = 'your_aws_secret_access_key'\n",
199-
"# Secrets required if accessing from abfs\n",
200-
"os.environ[\"AZURE_STORAGE_ACCOUNT_NAME\"] = 'your_azure_storage_account_name'\n",
201-
"os.environ[\"AZURE_STORAGE_ACCOUNT_KEY\"] = 'your_azure_storage_account_key'"
202-
]
203-
},
204-
{
205-
"cell_type": "code",
206-
"execution_count": 0,
207-
"metadata": {
208-
"application/vnd.databricks.v1+cell": {
209-
"cellMetadata": {
210-
"byteLimit": 2048000,
211-
"rowLimit": 10000
212-
},
213-
"inputWidgets": {},
214-
"nuid": "73f07b7f-41f4-4751-a97d-074c55cbb2f1",
215-
"showTitle": true,
216-
"title": "Set Java Options"
217-
},
218-
"jupyter": {
219-
"source_hidden": true
220-
}
221-
},
222-
"outputs": [],
223-
"source": [
224-
"if CSP=='azure':\n",
225-
" JAVA_OPTS=f' -Drapids.tools.hadoop.fs.azure.account.key.{os.environ[\"AZURE_STORAGE_ACCOUNT_NAME\"]}.dfs.core.windows.net={os.environ[\"AZURE_STORAGE_ACCOUNT_KEY\"]}'\n",
226-
" os.environ[\"_JAVA_OPTIONS\"] = JAVA_OPTS"
227-
]
228-
},
229161
{
230162
"cell_type": "code",
231163
"execution_count": 0,
@@ -269,7 +201,7 @@
269201
"outputs": [],
270202
"source": [
271203
"%sh\n",
272-
"java -Xmx10g -cp $TOOL_JAR_LOCAL_PATH:$HADOOP_JAR_LOCAL_PATH:$AWS_JAVA_LOCAL_PATH:/databricks/jars/* com.nvidia.spark.rapids.tool.profiling.ProfileMain --platform databricks-$CSP --csv --worker-info $WORKER_INFO_PATH --auto-tuner -o $OUTPUT_PATH $EVENTLOG_PATH > $CONSOLEOUTPUT_PATH"
204+
"java -Xmx10g -cp $TOOL_JAR_LOCAL_PATH:/databricks/jars/* com.nvidia.spark.rapids.tool.profiling.ProfileMain --platform databricks-$CSP --csv --worker-info $WORKER_INFO_PATH --auto-tuner -o $OUTPUT_PATH $EVENTLOG_PATH > $CONSOLEOUTPUT_PATH"
273205
]
274206
},
275207
{
@@ -425,7 +357,7 @@
425357
"stack": true
426358
},
427359
"nuid": "91c1bfb2-695a-4e5c-8a25-848a433108dc",
428-
"origId": 451087074559338,
360+
"origId": 2466908328410530,
429361
"title": "Executive View",
430362
"version": "DashboardViewV1",
431363
"width": 1600
@@ -439,7 +371,7 @@
439371
"stack": true
440372
},
441373
"nuid": "62243296-4562-4f06-90ac-d7a609f19c16",
442-
"origId": 451087074559340,
374+
"origId": 2466908328410531,
443375
"title": "App View",
444376
"version": "DashboardViewV1",
445377
"width": 1920
@@ -448,7 +380,7 @@
448380
"language": "python",
449381
"notebookMetadata": {
450382
"mostRecentlyExecutedCommandWithImplicitDF": {
451-
"commandId": 451087074559324,
383+
"commandId": 2466908328410506,
452384
"dataframes": [
453385
"_sqldf"
454386
]

‎tools/databricks/[RAPIDS Accelerator for Apache Spark] Qualification Tool Notebook Template.ipynb

Lines changed: 6 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -73,10 +73,7 @@
7373
"DEFAULT_TOOLS_VER = \"24.02.4\"\n",
7474
"dbutils.widgets.text(\"Tools Version\", DEFAULT_TOOLS_VER)\n",
7575
"TOOLS_VER=dbutils.widgets.get(\"Tools Version\")\n",
76-
"\n",
7776
"print(f\"Using Tools Version: {TOOLS_VER}\")\n",
78-
"if TOOLS_VER != DEFAULT_TOOLS_VER:\n",
79-
" print(f\"Warning: Requested tools version ({TOOLS_VER}) differs from the default version ({DEFAULT_TOOLS_VER}). This may require different versions of depedencies for compatibility.\")\n",
8077
"\n",
8178
"dbutils.widgets.dropdown(\"CSP\", \"aws\", [\"aws\", \"azure\"])\n",
8279
"CSP=dbutils.widgets.get(\"CSP\")\n",
@@ -134,74 +131,10 @@
134131
"TOOL_JAR_LOCAL_PATH = f'{OUTPUT_PATH}/rapids-4-spark-tools.jar'\n",
135132
"download_file(TOOL_JAR_URL, TOOL_JAR_LOCAL_PATH)\n",
136133
"\n",
137-
"# Download dependency jars\n",
138-
"HADOOP_JAR_VER = '3.3.4'\n",
139-
"HADOOP_JAR_URL = f'https://repo.maven.apache.org/maven2/org/apache/hadoop/hadoop-{CSP}/{HADOOP_JAR_VER}/hadoop-{CSP}-{HADOOP_JAR_VER}.jar'\n",
140-
"HADOOP_JAR_LOCAL_PATH = f'{OUTPUT_PATH}/hadoop-{CSP}-{HADOOP_JAR_VER}.jar'\n",
141-
"download_file(HADOOP_JAR_URL, HADOOP_JAR_LOCAL_PATH)\n",
142-
"\n",
143-
"if CSP=='aws':\n",
144-
" AWS_JAVA_VER = '1.12.262'\n",
145-
" AWS_JAVA_URL = f'https://repo.maven.apache.org/maven2/com/amazonaws/aws-java-sdk/{AWS_JAVA_VER}/aws-java-sdk-{AWS_JAVA_VER}.jar'\n",
146-
" AWS_JAVA_LOCAL_PATH = f'{OUTPUT_PATH}/aws-java-sdk-{AWS_JAVA_VER}.jar'\n",
147-
" download_file(AWS_JAVA_URL, AWS_JAVA_LOCAL_PATH)\n",
148-
" os.environ[\"AWS_JAVA_LOCAL_PATH\"] = AWS_JAVA_LOCAL_PATH\n",
149-
"\n",
150134
"os.environ[\"CSP\"] = CSP\n",
151135
"os.environ[\"EVENTLOG_PATH\"] = EVENTLOG_PATH\n",
152136
"os.environ[\"OUTPUT_PATH\"] = OUTPUT_PATH\n",
153-
"os.environ[\"TOOL_JAR_LOCAL_PATH\"] = TOOL_JAR_LOCAL_PATH\n",
154-
"os.environ[\"HADOOP_JAR_LOCAL_PATH\"] = HADOOP_JAR_LOCAL_PATH"
155-
]
156-
},
157-
{
158-
"cell_type": "code",
159-
"execution_count": 0,
160-
"metadata": {
161-
"application/vnd.databricks.v1+cell": {
162-
"cellMetadata": {
163-
"byteLimit": 2048000,
164-
"rowLimit": 10000
165-
},
166-
"inputWidgets": {},
167-
"nuid": "429e0ce8-961b-4b53-9878-82d2fdc56426",
168-
"showTitle": true,
169-
"title": "CSP Credentials"
170-
}
171-
},
172-
"outputs": [],
173-
"source": [
174-
"# Secrets required if accessing from s3\n",
175-
"os.environ[\"AWS_ACCESS_KEY_ID\"] = 'your_aws_access_key_id'\n",
176-
"os.environ[\"AWS_SECRET_ACCESS_KEY\"] = 'your_aws_secret_access_key'\n",
177-
"# Secrets required if accessing from abfs\n",
178-
"os.environ[\"AZURE_STORAGE_ACCOUNT_NAME\"] = 'your_azure_storage_account_name'\n",
179-
"os.environ[\"AZURE_STORAGE_ACCOUNT_KEY\"] = 'your_azure_storage_account_key'"
180-
]
181-
},
182-
{
183-
"cell_type": "code",
184-
"execution_count": 0,
185-
"metadata": {
186-
"application/vnd.databricks.v1+cell": {
187-
"cellMetadata": {
188-
"byteLimit": 2048000,
189-
"rowLimit": 10000
190-
},
191-
"inputWidgets": {},
192-
"nuid": "c09f8d10-2f60-45cd-843b-ef82eab9a097",
193-
"showTitle": true,
194-
"title": "Set Java Options"
195-
},
196-
"jupyter": {
197-
"source_hidden": true
198-
}
199-
},
200-
"outputs": [],
201-
"source": [
202-
"if CSP=='azure':\n",
203-
" JAVA_OPTS=f' -Drapids.tools.hadoop.fs.azure.account.key.{os.environ[\"AZURE_STORAGE_ACCOUNT_NAME\"]}.dfs.core.windows.net={os.environ[\"AZURE_STORAGE_ACCOUNT_KEY\"]}'\n",
204-
" os.environ[\"_JAVA_OPTIONS\"] = JAVA_OPTS"
137+
"os.environ[\"TOOL_JAR_LOCAL_PATH\"] = TOOL_JAR_LOCAL_PATH"
205138
]
206139
},
207140
{
@@ -247,7 +180,7 @@
247180
"outputs": [],
248181
"source": [
249182
"%sh\n",
250-
"java -Xmx10g -cp $TOOL_JAR_LOCAL_PATH:$HADOOP_JAR_LOCAL_PATH:$AWS_JAVA_LOCAL_PATH:/databricks/jars/* com.nvidia.spark.rapids.tool.qualification.QualificationMain --platform databricks-$CSP -o $OUTPUT_PATH $EVENTLOG_PATH > $CONSOLEOUTPUT_PATH"
183+
"java -Xmx10g -cp $TOOL_JAR_LOCAL_PATH:/databricks/jars/* com.nvidia.spark.rapids.tool.qualification.QualificationMain --platform databricks-$CSP -o $OUTPUT_PATH $EVENTLOG_PATH > $CONSOLEOUTPUT_PATH"
251184
]
252185
},
253186
{
@@ -431,7 +364,7 @@
431364
"stack": true
432365
},
433366
"nuid": "91c1bfb2-695a-4e5c-8a25-848a433108dc",
434-
"origId": 451087074559309,
367+
"origId": 2466908328410528,
435368
"title": "Executive View",
436369
"version": "DashboardViewV1",
437370
"width": 1600
@@ -445,7 +378,7 @@
445378
"stack": true
446379
},
447380
"nuid": "62243296-4562-4f06-90ac-d7a609f19c16",
448-
"origId": 451087074559310,
381+
"origId": 2466908328410529,
449382
"title": "App View",
450383
"version": "DashboardViewV1",
451384
"width": 1920
@@ -454,7 +387,7 @@
454387
"language": "python",
455388
"notebookMetadata": {
456389
"mostRecentlyExecutedCommandWithImplicitDF": {
457-
"commandId": 451087074559302,
390+
"commandId": 2466908328410490,
458391
"dataframes": [
459392
"_sqldf"
460393
]
@@ -483,7 +416,7 @@
483416
}
484417
]
485418
},
486-
"notebookName": "[RAPIDS Accelerator for Apache Spark] Qualification Tool Notebook Template-2",
419+
"notebookName": "[RAPIDS Accelerator for Apache Spark] Qualification Tool Notebook Template",
487420
"widgets": {
488421
"CSP": {
489422
"currentValue": "aws",

‎tools/databricks/[RAPIDS Accelerator for Apache Spark] Qualification User Tool Notebook Template.ipynb

Lines changed: 4 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,6 @@
7676
"DEFAULT_TOOLS_VER = \"24.02.4\"\n",
7777
"dbutils.widgets.text(\"Tools Version\", DEFAULT_TOOLS_VER)\n",
7878
"TOOLS_VER=dbutils.widgets.get(\"Tools Version\")\n",
79-
"\n",
8079
"print(f\"Using Tools Version: {TOOLS_VER}\")"
8180
]
8281
},
@@ -158,56 +157,6 @@
158157
"os.environ[\"OUTPUT_PATH\"] = OUTPUT_PATH"
159158
]
160159
},
161-
{
162-
"cell_type": "code",
163-
"execution_count": 0,
164-
"metadata": {
165-
"application/vnd.databricks.v1+cell": {
166-
"cellMetadata": {
167-
"byteLimit": 2048000,
168-
"rowLimit": 10000
169-
},
170-
"inputWidgets": {},
171-
"nuid": "f24a9f4d-9021-4658-ab8f-451206978257",
172-
"showTitle": true,
173-
"title": "CSP Credentials"
174-
}
175-
},
176-
"outputs": [],
177-
"source": [
178-
"# Secrets required if accessing from s3\n",
179-
"os.environ[\"AWS_ACCESS_KEY_ID\"] = 'your_aws_access_key_id'\n",
180-
"os.environ[\"AWS_SECRET_ACCESS_KEY\"] = 'your_aws_secret_access_key'\n",
181-
"# Secrets required if accessing from abfs\n",
182-
"os.environ[\"AZURE_STORAGE_ACCOUNT_NAME\"] = 'your_azure_storage_account_name'\n",
183-
"os.environ[\"AZURE_STORAGE_ACCOUNT_KEY\"] = 'your_azure_storage_account_key'"
184-
]
185-
},
186-
{
187-
"cell_type": "code",
188-
"execution_count": 0,
189-
"metadata": {
190-
"application/vnd.databricks.v1+cell": {
191-
"cellMetadata": {
192-
"byteLimit": 2048000,
193-
"rowLimit": 10000
194-
},
195-
"inputWidgets": {},
196-
"nuid": "c83817b1-afcc-4321-b3d6-3f6ac6a5affe",
197-
"showTitle": true,
198-
"title": "Set Java Options"
199-
},
200-
"jupyter": {
201-
"source_hidden": true
202-
}
203-
},
204-
"outputs": [],
205-
"source": [
206-
"if CSP=='azure':\n",
207-
" JAVA_OPTS=f' -Drapids.tools.hadoop.fs.azure.account.key.{os.environ[\"AZURE_STORAGE_ACCOUNT_NAME\"]}.dfs.core.windows.net={os.environ[\"AZURE_STORAGE_ACCOUNT_KEY\"]}'\n",
208-
" os.environ[\"_JAVA_OPTIONS\"] = JAVA_OPTS"
209-
]
210-
},
211160
{
212161
"cell_type": "code",
213162
"execution_count": 0,
@@ -465,7 +414,7 @@
465414
"stack": true
466415
},
467416
"nuid": "91c1bfb2-695a-4e5c-8a25-848a433108dc",
468-
"origId": 451087074559339,
417+
"origId": 2466908328410532,
469418
"title": "Executive View",
470419
"version": "DashboardViewV1",
471420
"width": 1600
@@ -479,7 +428,7 @@
479428
"stack": true
480429
},
481430
"nuid": "62243296-4562-4f06-90ac-d7a609f19c16",
482-
"origId": 451087074559341,
431+
"origId": 2466908328410533,
483432
"title": "App View",
484433
"version": "DashboardViewV1",
485434
"width": 1920
@@ -488,7 +437,7 @@
488437
"language": "python",
489438
"notebookMetadata": {
490439
"mostRecentlyExecutedCommandWithImplicitDF": {
491-
"commandId": 451087074559330,
440+
"commandId": 2466908328410520,
492441
"dataframes": [
493442
"_sqldf"
494443
]
@@ -517,7 +466,7 @@
517466
}
518467
]
519468
},
520-
"notebookName": "[RAPIDS Accelerator for Apache Spark] Qualification User Tool Notebook Template-2",
469+
"notebookName": "[RAPIDS Accelerator for Apache Spark] Qualification User Tool Notebook Template",
521470
"widgets": {
522471
"CSP": {
523472
"currentValue": "aws",

0 commit comments

Comments
 (0)