fix notice, split dump files by days, disable triggers (#34)

ufal · Nov 11, 2021 · 4e86a8e · 4e86a8e
1 parent 506ad7d
commit 4e86a8e
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 10 deletions.
diff --git a/scripts/database-patch01.sh b/scripts/database-patch01.sh
@@ -129,7 +129,7 @@ DECLARE
 BEGIN
   -- add to correct record or create new one
   SELECT INTO units coalesce(max(units),0); -- set 0 if NULL
-  RAISE NOTICE 'ENDPOINT ID = %      period_start = %', endpoint, period_start;
+  -- RAISE NOTICE 'ENDPOINT ID = %      period_start = %', endpoint, period_start;
   UPDATE log_aggr
   SET
     cnt_requests = log_aggr.cnt_requests + requests,
@@ -186,7 +186,7 @@ DECLARE
   row_exists BOOLEAN := false;
 BEGIN
   -- add to correct record or create new one
-  RAISE NOTICE '(IP , SERVICE,  units, body_bytes_sent) = (%, %, %, %)', in_ip, service, units, body_bytes_sent;
+  -- RAISE NOTICE '(IP , SERVICE,  units, body_bytes_sent) = (%, %, %, %)', in_ip, service, units, body_bytes_sent;
   UPDATE log_ip_aggr
   SET
     cnt_requests = log_ip_aggr.cnt_requests + 1,

diff --git a/scripts/database-setup.sh b/scripts/database-setup.sh
@@ -174,7 +174,7 @@ CREATE OR REPLACE FUNCTION trigger_update_log_files()
 RETURNS TRIGGER AS
 	\$\$
 	BEGIN
-	  RAISE NOTICE 'Updating file : % ', NEW.file_id;
+	  -- RAISE NOTICE 'Updating file : % ', NEW.file_id;
       UPDATE log_files
       SET
         last_read_line_checksum = NEW.line_checksum,

diff --git a/scripts/log2sql.pl b/scripts/log2sql.pl
@@ -18,7 +18,7 @@
 my $dbport = 5432;
 my $ignore = qr/\.(gif|jpg|jpeg|tiff|png|js|css|eot|ico|svg)$/;
 my @services;
-
+my @print_sql;
 
 GetOptions (
             'in-files=s{1,}' => \@logfiles,
@@ -75,12 +75,14 @@
     next;
   }
   $file_id++;
+  push @print_sql,"DO \$\$ BEGIN RAISE NOTICE 'LOG FILE ($log_file) ID=$file_id'; END; \$\$; \n";
+
   my ($file_name) = $log_file =~ m/^(.*)\./;
   my $sql_file = "$file_name.sql";
-  my $dump_file = "$file_name.dump";
+###  my $dump_file = "$file_name.dump";
   my ($first_line_checksum, $last_read_line_checksum, $lines_read, $lines_valid) = (undef, undef, 0, 0);
   open LOG, "<$log_file_path" or die "Could not open $log_file_path: $!";
-  open DUMP, ">".File::Spec->catfile($outdir,$dump_file) or die "Could not open $dump_file: $!";
+  my $prev_date='';
   while(my $line = <LOG>){
     $lines_read++;
     $line =~ s/\n$//;
@@ -100,6 +102,28 @@
         }
       }
       next unless defined $service_id;
+      my ($act_date) = join('-',split('/',substr($time_local,0,11)));
+      unless($act_date eq $prev_date){
+
+        unless($lines_valid){
+          close DUMP;
+        }
+        my $dump_file = "$file_name.$act_date.dump";
+        push @print_sql,"
+DO
+\$\$
+BEGIN
+  RAISE NOTICE '\%', NOW();
+  RAISE NOTICE 'starting importing from $dump_file';
+  RAISE NOTICE 'log file line: $lines_read';
+END;
+\$\$;
+";
+        push @print_sql,"\\copy log_file_entries(file_id, service_id, line_number, line_checksum, remote_addr, remote_user, time_local, method, request, protocol, status, body_bytes_sent, http_referer, http_user_agent, unit) from '$dump_file'";
+        open DUMP, ">".File::Spec->catfile($outdir,$dump_file) or die "Could not open $dump_file: $!";
+        $prev_date = $act_date;
+      }
+
       $lines_valid++;
 ##      print STDERR "$service_id: $request\n";
       print DUMP join("\t", ($file_id,$service_id,$lines_valid,$last_read_line_checksum,$remote_addr,$remote_user,$time_local, $method, $request, $protocol, $status, $body_bytes_sent,$http_referer, $http_user_agent, $unit)),"\n";
@@ -108,15 +132,24 @@
   close DUMP;
   close LOG;
 
-  open FILE, ">".File::Spec->catfile($outdir,$sql_file) or die "Could not open $sql_file: $!";
-  print FILE " -- $log_file sql dump
+  open SQL, ">".File::Spec->catfile($outdir,$sql_file) or die "Could not open $sql_file: $!";
+  print SQL " -- $log_file sql dump
+ALTER TABLE log_file_entries DISABLE TRIGGER log_files_lines_read;
+ALTER TABLE log_file_entries DISABLE TRIGGER log_files_lines_read_aggr;
+
+
 INSERT
 INTO log_files(file_id, file_name, first_line_checksum, last_read_line_checksum, lines_read, lines_valid, tail)
 VALUES($file_id,'$log_file','$first_line_checksum', '$last_read_line_checksum', $lines_read, $lines_valid,FALSE);
+";
+  print SQL "$_\n" for @print_sql;
 
-\\copy log_file_entries(file_id, service_id, line_number, line_checksum, remote_addr, remote_user, time_local, method, request, protocol, status, body_bytes_sent, http_referer, http_user_agent, unit) from '$dump_file'
+  print SQL "
+ALTER TABLE log_file_entries ENABLE TRIGGER log_files_lines_read;
+ALTER TABLE log_file_entries ENABLE TRIGGER log_files_lines_read_aggr;
 ";
-  close FILE;
+  print SQL "DO \$\$ BEGIN RAISE NOTICE 'TODO: run aggregations for:::  SELECT * FROM log_files_entries WHERE file_id=$file_id'; END; \$\$; \n";
+  close SQL;
 }
 
 $dbi->disconnect();