Merge pull request #72 from aamine/reduce_txn

Reduces number of transactions
bricolages · Jun 14, 2016 · 84e4ad1 · 84e4ad1
2 parents 63bf93b + cd02f4e
commit 84e4ad1
Show file tree

Hide file tree

Showing 7 changed files with 65 additions and 49 deletions.
diff --git a/jobclass/create.rb b/jobclass/create.rb
@@ -14,10 +14,12 @@
 
   script {|params, script|
     script.task(params['data-source']) {|task|
-      task.drop_force_if params['drop']
-      task.exec params['table-def']
-      task.analyze_if params['analyze']
-      task.grant_if params['grant'], params['dest-table']
+      task.transaction {
+        task.drop_force_if params['drop']
+        task.exec params['table-def']
+        task.analyze_if params['analyze']
+        task.grant_if params['grant'], params['dest-table']
+      }
     }
   }
 }
diff --git a/jobclass/createview.rb b/jobclass/createview.rb
@@ -14,9 +14,11 @@
 
   script {|params, script|
     script.task(params['data-source']) {|task|
-      task.drop_view_force_if params['drop']
-      task.exec params['sql-file']
-      task.grant_if params['grant'], params['dest-table']
+      task.transaction {
+        task.drop_view_force_if params['drop']
+        task.exec params['sql-file']
+        task.grant_if params['grant'], params['dest-table']
+      }
     }
   }
 }
diff --git a/jobclass/my-import.rb b/jobclass/my-import.rb
@@ -53,23 +53,27 @@
         prev_table = '${dest_table}_old'
         work_table = '${dest_table}_wk'
 
-        # CREATE
-        task.drop_force prev_table
-        task.drop_force work_table
-        task.exec params['table-def'].replace(/\$\{?dest_table\}?\b/, work_table)
+        task.transaction {
+          # CREATE
+          task.drop_force prev_table
+          task.drop_force work_table
+          task.exec params['table-def'].replace(/\$\{?dest_table\}?\b/, work_table)
+
+          # COPY
+          task.load params['s3-ds'], params['s3-prefix'], work_table,
+              'json', nil, params['options'].merge('gzip' => params['gzip'])
 
-        # COPY
-        task.load params['s3-ds'], params['s3-prefix'], work_table,
-            'json', nil, params['options'].merge('gzip' => params['gzip'])
+          # GRANT
+          task.grant_if params['grant'], work_table
+        }
 
-        # VACUUM, ANALYZE, GRANT
+        # VACUUM, ANALYZE
         task.vacuum_if params['vacuum'], params['vacuum-sort'], work_table
         task.analyze_if params['analyze'], work_table
-        task.grant_if params['grant'], work_table
 
         # RENAME
-        task.create_dummy_table '${dest_table}'
         task.transaction {
+          task.create_dummy_table '${dest_table}'
           task.rename_table params['dest-table'].to_s, "#{params['dest-table'].name}_old"
           task.rename_table work_table, params['dest-table'].name
         }

diff --git a/jobclass/my-migrate.rb b/jobclass/my-migrate.rb
@@ -75,23 +75,27 @@
         prev_table = '${dest_table}_old'
         work_table = '${dest_table}_wk'
 
-        # CREATE
-        task.drop_force prev_table
-        task.drop_force work_table
-        task.exec params['table-def'].replace(/\$\{?dest_table\}?\b/, work_table)
+        task.transaction {
+          # CREATE
+          task.drop_force prev_table
+          task.drop_force work_table
+          task.exec params['table-def'].replace(/\$\{?dest_table\}?\b/, work_table)
+
+          # COPY
+          task.load params['s3-ds'], params['s3-file'], work_table,
+              'json', nil, params['options'].merge('gzip' => params['gzip'])
 
-        # COPY
-        task.load params['s3-ds'], params['s3-file'], work_table,
-            'json', nil, params['options'].merge('gzip' => params['gzip'])
+          # GRANT
+          task.grant_if params['grant'], work_table
+        }
 
-        # VACUUM, ANALYZE, GRANT
+        # VACUUM, ANALYZE
         task.vacuum_if params['vacuum'], params['vacuum-sort'], work_table
         task.analyze_if params['analyze'], work_table
-        task.grant_if params['grant'], work_table
 
         # RENAME
-        task.create_dummy_table '${dest_table}'
         task.transaction {
+          task.create_dummy_table '${dest_table}'
           task.rename_table params['dest-table'].to_s, "#{params['dest-table'].name}_old"
           task.rename_table work_table, params['dest-table'].name
         }

diff --git a/jobclass/rebuild-drop.rb b/jobclass/rebuild-drop.rb
@@ -21,17 +21,21 @@
 
   script {|params, script|
     script.task(params['data-source']) {|task|
-      # CREATE
-      task.drop_force params['dest-table']
-      task.exec params['table-def']
+      task.transaction {
+        # CREATE
+        task.drop_force params['dest-table']
+        task.exec params['table-def']
 
-      # INSERT
-      task.exec params['sql-file']
+        # INSERT
+        task.exec params['sql-file']
 
-      # VACUUM, ANALYZE, GRANT
+        # GRANT
+        task.grant_if params['grant'], params['dest-table']
+      }
+
+      # VACUUM, ANALYZE
       task.vacuum_if params['vacuum'], params['vacuum-sort'], params['dest-table']
       task.analyze_if params['analyze'], params['dest-table']
-      task.grant_if params['grant'], params['dest-table']
     }
   }
 }
diff --git a/jobclass/rebuild-rename.rb b/jobclass/rebuild-rename.rb
@@ -24,22 +24,26 @@
       prev_table = '${dest_table}_old'
       work_table = '${dest_table}_wk'
 
-      # CREATE
-      task.drop_force prev_table
-      task.drop_force work_table
-      task.exec params['table-def'].replace(/\$\{?dest_table\}?\b/, work_table)
+      task.transaction {
+        # CREATE
+        task.drop_force prev_table
+        task.drop_force work_table
+        task.exec params['table-def'].replace(/\$\{?dest_table\}?\b/, work_table)
+
+        # INSERT
+        task.exec params['sql-file'].replace(/\$\{?dest_table\}?\b/, work_table)
 
-      # INSERT
-      task.exec params['sql-file'].replace(/\$\{?dest_table\}?\b/, work_table)
+        # GRANT
+        task.grant_if params['grant'], work_table
+      }
 
-      # VACUUM, ANALYZE, GRANT
+      # VACUUM, ANALYZE
       task.vacuum_if params['vacuum'], params['vacuum-sort'], work_table
       task.analyze_if params['analyze'], work_table
-      task.grant_if params['grant'], work_table
 
       # RENAME
-      task.create_dummy_table '$dest_table'
       task.transaction {
+        task.create_dummy_table '$dest_table'
         dest_table = params['dest-table']
         task.rename_table dest_table.to_s, "#{dest_table.name}_old"
         task.rename_table "#{dest_table}_wk", dest_table.name

diff --git a/lib/bricolage/psqldatasource.rb b/lib/bricolage/psqldatasource.rb
@@ -218,9 +218,7 @@ def support_explain?(statement_kind)
 
     def create_dummy_table(target)
       exec SQLStatement.for_string(
-        "\\set ON_ERROR_STOP false\n" +
-        "create table #{target} (x int);\n" +
-        "\\set ON_ERROR_STOP true\n"
+        "create table if not exists #{target} (x int);\n"
       )
     end
 
@@ -234,9 +232,7 @@ def drop_if(enabled)
 
     def drop_obj_force(type, name)
       exec SQLStatement.for_string(
-        "\\set ON_ERROR_STOP false\n" +
-        "drop #{type} #{name} cascade;\n" +
-        "\\set ON_ERROR_STOP true\n"
+        "drop #{type} if exists #{name} cascade;\n"
       )
     end