changes kettle-report.xslt

should fix issues 37 and 40 (except for plug-ins) more details in changes are found in change_log.txt
khots · Apr 28, 2013 · 7060ef6 · 7060ef6
1 parent 27aa23a
commit 7060ef6
Show file tree

Hide file tree

Showing 89 changed files with 15,747 additions and 2,896 deletions.
diff --git a/change_log.txt b/change_log.txt
@@ -0,0 +1,57 @@
+This file lists changes done ontop of r79
+
+xslt/Kettle_Report.xslt
+   - added imported io_steps.xslt
+   - unconditionally display connections (<xsl:if test="$usages"> removed)
+   - fixed some bugs
+       - removed some un-used code
+       - some incorrect XPaths
+       - connections now reported as links
+       - steps in highlevel flow diagram now reported as links
+       - info section for transformations was missing
+       - Highlevel flow diagram now contains links to steps
+       - outcommented flat files (not yet supported)
+   - improved reporting of steps:
+       - steps separated with horizontal lines
+       - steps now report hops (to/from) in a separate sub-section
+       - mappings (ETL Metadata Injection), lookups, database tables
+         (this solves most of issue 37), all tabs of 'Select Values' step
+
+xslt/files.xslt
+   - define variable 'modified_date' based on file property 
+
+xslt/shared.xslt
+   - template description changed 
+     - maintain formatting with <pre>
+     - different font if 'no description'
+
+xslt/toc.xslt
+   - outcommented yet unsupported sections
+
+document-folder.kjb
+   - none
+
+process-files.ktr
+   - added (but disabled) flow for full re-build (instead of 'make')
+
+template/index.html
+   - space reserved for frame on left adapted
+
+template/default.css
+   - a:hover constrained to [href]
+
+template/kettle.css
+   - changed font for .description, .extended-description to courier (blue)
+
+template/kettle.css, job.css
+   - created with constructor job
+
+template/images
+   - updated to kettle 4.4.0
+
+constructor/*
+   - used to adapt kettle-cookbook to new versions of kettle
+     (see job descritption of Create_Files.kjb for details)
+   - contains sub-dir ./AllOperators with a bunch of jobs and transformations
+     simply instantiating all operators - helps visualizing what the constructor
+     misses (namely, plug-ins)
diff --git a/constructor/AllOperators/Jobs/BigData.kjb b/constructor/AllOperators/Jobs/BigData.kjb
@@ -0,0 +1,253 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<job>
+  <name>BigData</name>
+    <description/>
+    <extended_description/>
+    <job_version/>
+  <directory>&#47;</directory>
+  <created_user>-</created_user>
+  <created_date>2013&#47;04&#47;10 22:24:27.968</created_date>
+  <modified_user>-</modified_user>
+  <modified_date>2013&#47;04&#47;10 22:24:27.968</modified_date>
+    <parameters>
+    </parameters>
+    <slaveservers>
+    </slaveservers>
+<job-log-table><connection/>
+<schema/>
+<table/>
+<size_limit_lines/>
+<interval/>
+<timeout_days/>
+<field><id>ID_JOB</id><enabled>Y</enabled><name>ID_JOB</name></field><field><id>CHANNEL_ID</id><enabled>Y</enabled><name>CHANNEL_ID</name></field><field><id>JOBNAME</id><enabled>Y</enabled><name>JOBNAME</name></field><field><id>STATUS</id><enabled>Y</enabled><name>STATUS</name></field><field><id>LINES_READ</id><enabled>Y</enabled><name>LINES_READ</name></field><field><id>LINES_WRITTEN</id><enabled>Y</enabled><name>LINES_WRITTEN</name></field><field><id>LINES_UPDATED</id><enabled>Y</enabled><name>LINES_UPDATED</name></field><field><id>LINES_INPUT</id><enabled>Y</enabled><name>LINES_INPUT</name></field><field><id>LINES_OUTPUT</id><enabled>Y</enabled><name>LINES_OUTPUT</name></field><field><id>LINES_REJECTED</id><enabled>Y</enabled><name>LINES_REJECTED</name></field><field><id>ERRORS</id><enabled>Y</enabled><name>ERRORS</name></field><field><id>STARTDATE</id><enabled>Y</enabled><name>STARTDATE</name></field><field><id>ENDDATE</id><enabled>Y</enabled><name>ENDDATE</name></field><field><id>LOGDATE</id><enabled>Y</enabled><name>LOGDATE</name></field><field><id>DEPDATE</id><enabled>Y</enabled><name>DEPDATE</name></field><field><id>REPLAYDATE</id><enabled>Y</enabled><name>REPLAYDATE</name></field><field><id>LOG_FIELD</id><enabled>Y</enabled><name>LOG_FIELD</name></field></job-log-table>
+<jobentry-log-table><connection/>
+<schema/>
+<table/>
+<timeout_days/>
+<field><id>ID_BATCH</id><enabled>Y</enabled><name>ID_BATCH</name></field><field><id>CHANNEL_ID</id><enabled>Y</enabled><name>CHANNEL_ID</name></field><field><id>LOG_DATE</id><enabled>Y</enabled><name>LOG_DATE</name></field><field><id>JOBNAME</id><enabled>Y</enabled><name>TRANSNAME</name></field><field><id>JOBENTRYNAME</id><enabled>Y</enabled><name>STEPNAME</name></field><field><id>LINES_READ</id><enabled>Y</enabled><name>LINES_READ</name></field><field><id>LINES_WRITTEN</id><enabled>Y</enabled><name>LINES_WRITTEN</name></field><field><id>LINES_UPDATED</id><enabled>Y</enabled><name>LINES_UPDATED</name></field><field><id>LINES_INPUT</id><enabled>Y</enabled><name>LINES_INPUT</name></field><field><id>LINES_OUTPUT</id><enabled>Y</enabled><name>LINES_OUTPUT</name></field><field><id>LINES_REJECTED</id><enabled>Y</enabled><name>LINES_REJECTED</name></field><field><id>ERRORS</id><enabled>Y</enabled><name>ERRORS</name></field><field><id>RESULT</id><enabled>Y</enabled><name>RESULT</name></field><field><id>NR_RESULT_ROWS</id><enabled>Y</enabled><name>NR_RESULT_ROWS</name></field><field><id>NR_RESULT_FILES</id><enabled>Y</enabled><name>NR_RESULT_FILES</name></field><field><id>LOG_FIELD</id><enabled>N</enabled><name>LOG_FIELD</name></field><field><id>COPY_NR</id><enabled>N</enabled><name>COPY_NR</name></field></jobentry-log-table>
+<channel-log-table><connection/>
+<schema/>
+<table/>
+<timeout_days/>
+<field><id>ID_BATCH</id><enabled>Y</enabled><name>ID_BATCH</name></field><field><id>CHANNEL_ID</id><enabled>Y</enabled><name>CHANNEL_ID</name></field><field><id>LOG_DATE</id><enabled>Y</enabled><name>LOG_DATE</name></field><field><id>LOGGING_OBJECT_TYPE</id><enabled>Y</enabled><name>LOGGING_OBJECT_TYPE</name></field><field><id>OBJECT_NAME</id><enabled>Y</enabled><name>OBJECT_NAME</name></field><field><id>OBJECT_COPY</id><enabled>Y</enabled><name>OBJECT_COPY</name></field><field><id>REPOSITORY_DIRECTORY</id><enabled>Y</enabled><name>REPOSITORY_DIRECTORY</name></field><field><id>FILENAME</id><enabled>Y</enabled><name>FILENAME</name></field><field><id>OBJECT_ID</id><enabled>Y</enabled><name>OBJECT_ID</name></field><field><id>OBJECT_REVISION</id><enabled>Y</enabled><name>OBJECT_REVISION</name></field><field><id>PARENT_CHANNEL_ID</id><enabled>Y</enabled><name>PARENT_CHANNEL_ID</name></field><field><id>ROOT_CHANNEL_ID</id><enabled>Y</enabled><name>ROOT_CHANNEL_ID</name></field></channel-log-table>
+   <pass_batchid>N</pass_batchid>
+   <shared_objects_file/>
+  <entries>
+    <entry>
+      <name>Amazon EMR Job Executor</name>
+      <description/>
+      <type>EMRJobExecutorPlugin</type>
+      <hadoop_job_name/>
+      <hadoop_job_flow_id/>
+      <jar_url/>
+      <access_key>Encrypted </access_key>
+      <secret_key>Encrypted </secret_key>
+      <staging_dir/>
+      <num_instances>2</num_instances>
+      <master_instance_type>Small [m1.small]</master_instance_type>
+      <slave_instance_type>Small [m1.small]</slave_instance_type>
+      <command_line_args/>
+      <blocking>N</blocking>
+      <logging_interval>60</logging_interval>
+      <hadoop_job_name/>
+      <parallel>N</parallel>
+      <draw>Y</draw>
+      <nr>0</nr>
+      <xloc>90</xloc>
+      <yloc>30</yloc>
+      </entry>
+    <entry>
+      <name>Amazon Hive Job Executor</name>
+      <description/>
+      <type>HiveJobExecutorPlugin</type>
+      <hadoop_job_name/>
+      <hadoop_job_flow_id/>
+      <q_url/>
+      <access_key>Encrypted </access_key>
+      <secret_key>Encrypted </secret_key>
+      <bootstrap_actions/>
+      <staging_dir/>
+      <num_instances>2</num_instances>
+      <master_instance_type>Small [m1.small]</master_instance_type>
+      <slave_instance_type>Small [m1.small]</slave_instance_type>
+      <command_line_args/>
+      <alive>N</alive>
+      <blocking>N</blocking>
+      <logging_interval>60</logging_interval>
+      <hadoop_job_name/>
+      <parallel>N</parallel>
+      <draw>Y</draw>
+      <nr>0</nr>
+      <xloc>220</xloc>
+      <yloc>30</yloc>
+      </entry>
+    <entry>
+      <name>Hadoop Copy Files</name>
+      <description/>
+      <type>HadoopCopyFilesPlugin</type>
+      <copy_empty_folders>Y</copy_empty_folders>
+      <arg_from_previous>N</arg_from_previous>
+      <overwrite_files>N</overwrite_files>
+      <include_subfolders>N</include_subfolders>
+      <remove_source_files>N</remove_source_files>
+      <add_result_filesname>N</add_result_filesname>
+      <destination_is_a_file>N</destination_is_a_file>
+      <create_destination_folder>N</create_destination_folder>
+      <fields>
+      </fields>
+      <parallel>N</parallel>
+      <draw>Y</draw>
+      <nr>0</nr>
+      <xloc>350</xloc>
+      <yloc>30</yloc>
+      </entry>
+    <entry>
+      <name>Hadoop Job Executor</name>
+      <description/>
+      <type>HadoopJobExecutorPlugin</type>
+      <hadoop_job_name/>
+      <simple>Y</simple>
+      <jar_url/>
+      <command_line_args/>
+      <simple_blocking>N</simple_blocking>
+      <blocking>N</blocking>
+      <logging_interval>60</logging_interval>
+      <simple_logging_interval>60</simple_logging_interval>
+      <hadoop_job_name/>
+      <mapper_class/>
+      <combiner_class/>
+      <reducer_class/>
+      <input_path/>
+      <input_format_class/>
+      <output_path/>
+      <output_key_class/>
+      <output_value_class/>
+      <output_format_class/>
+      <hdfs_hostname/>
+      <hdfs_port/>
+      <job_tracker_hostname/>
+      <job_tracker_port/>
+      <num_map_tasks>1</num_map_tasks>
+      <num_reduce_tasks>1</num_reduce_tasks>
+      <user_defined_list>
+      </user_defined_list>
+      <parallel>N</parallel>
+      <draw>Y</draw>
+      <nr>0</nr>
+      <xloc>470</xloc>
+      <yloc>30</yloc>
+      </entry>
+    <entry>
+      <name>Oozie Job Executor</name>
+      <description/>
+      <type>OozieJobExecutor</type>
+    <jobEntryName>Oozie Job Executor</jobEntryName>
+    <blockingPollingInterval>300</blockingPollingInterval>
+    <blockingExecution>true</blockingExecution>
+      <parallel>N</parallel>
+      <draw>Y</draw>
+      <nr>0</nr>
+      <xloc>590</xloc>
+      <yloc>30</yloc>
+      </entry>
+    <entry>
+      <name>Pentaho MapReduce</name>
+      <description/>
+      <type>HadoopTransJobExecutorPlugin</type>
+      <hadoop_job_name/>
+      <map_trans_repo_dir/>
+      <map_trans_repo_file/>
+      <map_trans_repo_reference/>
+      <map_trans/>
+      <combiner_trans_repo_dir/>
+      <combiner_trans_repo_file/>
+      <combiner_trans_repo_reference/>
+      <combiner_trans/>
+      <combiner_single_threaded>Y</combiner_single_threaded>
+      <reduce_trans_repo_dir/>
+      <reduce_trans_repo_file/>
+      <reduce_trans_repo_reference/>
+      <reduce_trans/>
+      <reduce_single_threaded>Y</reduce_single_threaded>
+      <map_input_step_name/>
+      <map_output_step_name/>
+      <combiner_input_step_name/>
+      <combiner_output_step_name/>
+      <reduce_input_step_name/>
+      <reduce_output_step_name/>
+      <blocking>N</blocking>
+      <logging_interval>60</logging_interval>
+      <input_path/>
+      <input_format_class/>
+      <output_path/>
+      <clean_output_path>N</clean_output_path>
+      <suppress_output_map_key>N</suppress_output_map_key>
+      <suppress_output_map_value>N</suppress_output_map_value>
+      <suppress_output_key>N</suppress_output_key>
+      <suppress_output_value>N</suppress_output_value>
+      <output_format_class/>
+      <hdfs_hostname/>
+      <hdfs_port/>
+      <job_tracker_hostname/>
+      <job_tracker_port/>
+      <num_map_tasks>1</num_map_tasks>
+      <num_reduce_tasks>1</num_reduce_tasks>
+      <user_defined_list>
+      </user_defined_list>
+      <parallel>N</parallel>
+      <draw>Y</draw>
+      <nr>0</nr>
+      <xloc>90</xloc>
+      <yloc>110</yloc>
+      </entry>
+    <entry>
+      <name>Pig Script Executor</name>
+      <description/>
+      <type>HadoopPigScriptExecutorPlugin</type>
+    <hdfs_hostname/>
+    <hdfs_port/>
+    <jobtracker_hostname/>
+    <jobtracker_port/>
+    <script_file/>
+    <enable_blocking>N</enable_blocking>
+    <local_execution>N</local_execution>
+    <script_parameters>
+    </script_parameters>
+      <parallel>N</parallel>
+      <draw>Y</draw>
+      <nr>0</nr>
+      <xloc>220</xloc>
+      <yloc>110</yloc>
+      </entry>
+    <entry>
+      <name>Sqoop Export</name>
+      <description/>
+      <type>SqoopExport</type>
+    <jobEntryName>Sqoop Export</jobEntryName>
+    <blockingPollingInterval>300</blockingPollingInterval>
+    <blockingExecution>true</blockingExecution>
+      <parallel>N</parallel>
+      <draw>Y</draw>
+      <nr>0</nr>
+      <xloc>350</xloc>
+      <yloc>110</yloc>
+      </entry>
+    <entry>
+      <name>Sqoop Import</name>
+      <description/>
+      <type>SqoopImport</type>
+    <jobEntryName>Sqoop Import</jobEntryName>
+    <blockingPollingInterval>300</blockingPollingInterval>
+    <blockingExecution>true</blockingExecution>
+      <parallel>N</parallel>
+      <draw>Y</draw>
+      <nr>0</nr>
+      <xloc>470</xloc>
+      <yloc>110</yloc>
+      </entry>
+  </entries>
+  <hops>
+  </hops>
+  <notepads>
+  </notepads>
+</job>