Merge to master: sync with AgroPortal 2.4.0

.gitignore

@@ -33,3 +33,8 @@ logs/
 .java-version
 
 processed_files/
+
+queries.txt
+
+graph_comparison.csv
+

Gemfile

@@ -52,3 +52,5 @@ group :development do
 end
 
 gem "binding_of_caller", "~> 1.0"
+gem 'net-smtp'
+gem 'net-ftp'

Gemfile.lock

@@ -1,6 +1,6 @@
 GIT
   remote: https://github.com/ontoportal-lirmm/goo.git
-  revision: 8d108c23a043039e9675b36f8f444d29a87b11fe
+  revision: 5825dc1f9d0ff439b1ba9d8f78fa7bb20b1c65d0
   branch: development
   specs:
     goo (0.0.2)
@@ -29,7 +29,7 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git
-  revision: 0aa6219c44143b94135e01c78eb94ad99a5e8b32
+  revision: 194fcfb9a1c4660dabef738d16f32c210a23c343
   branch: development
   specs:
     ontologies_linked_data (0.0.1)
@@ -49,7 +49,7 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/sparql-client.git
-  revision: 24bccbd0f4a5150fa6ce2af50d7c378c681027ea
+  revision: d4a226e75eb4aeaaf42720eac4f23f55380a0bd3
   branch: development
   specs:
     sparql-client (3.2.2)
@@ -84,7 +84,8 @@ GEM
     base64 (0.2.0)
     bcrypt (3.1.20)
     bcrypt_pbkdf (1.1.1)
-    bigdecimal (3.1.8)
+    bcrypt_pbkdf (1.1.1-arm64-darwin)
+    bigdecimal (3.1.9)
     binding_of_caller (1.0.1)
       debug_inspector (>= 1.2.0)
     builder (3.3.0)
@@ -105,6 +106,7 @@ GEM
     connection_pool (2.5.0)
     cube-ruby (0.0.3)
     dante (0.2.0)
+    date (3.4.1)
     debug_inspector (1.2.0)
     declarative (0.0.20)
     docile (1.4.1)
@@ -152,6 +154,8 @@ GEM
     google-cloud-env (2.1.1)
       faraday (>= 1.0, < 3.a)
     google-cloud-errors (1.4.0)
+    google-protobuf (3.25.3)
+    google-protobuf (3.25.3-arm64-darwin)
     google-protobuf (3.25.3-x86_64-linux)
     googleapis-common-protos (1.6.0)
       google-protobuf (>= 3.18, < 5.a)
@@ -166,6 +170,12 @@ GEM
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (>= 0.16, < 2.a)
+    grpc (1.65.2)
+      google-protobuf (>= 3.25, < 5.0)
+      googleapis-common-protos-types (~> 1.0)
+    grpc (1.65.2-arm64-darwin)
+      google-protobuf (>= 3.25, < 5.0)
+      googleapis-common-protos-types (~> 1.0)
     grpc (1.65.2-x86_64-linux)
       google-protobuf (>= 3.25, < 5.0)
       googleapis-common-protos-types (~> 1.0)
@@ -207,12 +217,19 @@ GEM
       redis
     multi_json (1.15.0)
     mutex_m (0.3.0)
+    net-ftp (0.3.8)
+      net-protocol
+      time
     net-http-persistent (4.0.5)
       connection_pool (~> 2.2)
-    net-scp (4.0.0)
+    net-protocol (0.2.2)
+      timeout
+    net-scp (4.1.0)
       net-ssh (>= 2.6.5, < 8.0.0)
     net-sftp (4.0.0)
       net-ssh (>= 5.0.0, < 8.0.0)
+    net-smtp (0.5.0)
+      net-protocol
     net-ssh (7.3.0)
     netrc (0.11.0)
     oj (3.16.9)
@@ -230,7 +247,7 @@ GEM
       coderay (~> 1.1)
       method_source (~> 1.0)
     public_suffix (5.1.1)
-    rack (3.1.8)
+    rack (3.1.9)
     rack-test (2.2.0)
       rack (>= 1.3)
     rake (13.2.1)
@@ -299,6 +316,9 @@ GEM
     systemu (2.6.5)
     test-unit-minitest (0.9.1)
       minitest (~> 4.7)
+    time (0.4.1)
+      date
+    timeout (0.4.3)
     trailblazer-option (0.1.2)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
@@ -307,6 +327,8 @@ GEM
       macaddr (~> 1.0)
 
 PLATFORMS
+  arm64-darwin-24
+  ruby
   x86_64-linux
 
 DEPENDENCIES
@@ -331,6 +353,8 @@ DEPENDENCIES
   multi_json
   ncbo_annotator!
   ncbo_cron!
+  net-ftp
+  net-smtp
   oj
   ontologies_linked_data!
   parallel
@@ -348,4 +372,4 @@ DEPENDENCIES
   test-unit-minitest
 
 BUNDLED WITH
-   2.3.23
+   2.4.22

bin/migrations/compare_counts.rb

@@ -86,7 +86,7 @@ def build_graphs_file_hash(folder_path = PROCESSED_DIR)
     if filename.end_with?('.graph')
       file_path = File.join(folder_path, filename)
       line = File.open(file_path, "r").readlines.first
-      graphs[line.strip] = filename.to_s.gsub('.graph','')
+      graphs[line.strip] = filename.to_s.gsub('.graph', '')
     end
   end
   graphs
@@ -107,7 +107,6 @@ def compare_graphs_with_files(graph_triples)
 
       # Construct the expected file name based on the graph URI
       file_name = "#{PROCESSED_DIR}/#{graph_filename}"
-
       # puts "count lines of the file #{file_name} for the graph #{graph_uri}"
       if File.exist?(file_name)
         file_lines_count = count_file_lines(file_name)
@@ -128,58 +127,9 @@ def compare_graphs_with_files(graph_triples)
 end
 
 # Main execution
-
 Goo.sparql_query_client.cache.redis_cache.flushdb
 puts "Redis cache flushed"
 
 puts "Comparing graph triple counts with file lines and exporting to CSV..."
 graph_triples = get_all_graphs_counts
 compare_graphs_with_files(graph_triples)
-
-count = 0
-attr_ontology = []
-time = Benchmark.realtime do
-  attr_ontology = LinkedData::Models::Ontology.attributes(:all)
-  count = LinkedData::Models::Ontology.where.include(attr_ontology).all.count
-end
-puts "Ontologies count: #{count} with display=all in #{format("%.4f", time)}s"
-count = 0
-time = Benchmark.realtime do
-  count = LinkedData::Models::OntologySubmission.where.all.count
-end
-puts "Submissions count: #{count} with no display in #{format("%.4f", time)}s"
-
-count = 0
-time = Benchmark.realtime do
-  attr = LinkedData::Models::OntologySubmission.attributes(:all)
-  attr << {ontology: attr_ontology}
-  count = LinkedData::Models::OntologySubmission.where.include(attr).all.count
-end
-puts "Submissions count: #{count} with display=all in #{format("%.4f", time)}s"
-
-count = 0
-time = Benchmark.realtime do
-  attr = LinkedData::Models::Agent.attributes(:all)
-  count = LinkedData::Models::Agent.where.include(attr).all.count
-end
-puts "Agent count: #{count} with display=all in #{format("%.4f", time)}s"
-
-count = 0
-time = Benchmark.realtime do
-  attr = LinkedData::Models::MappingCount.attributes(:all)
-  count = LinkedData::Models::MappingCount.where.include(attr).all.count
-end
-puts "MappingsCount count: #{count} with display=all in #{format("%.4f", time)}s"
-
-count = 0
-time = Benchmark.realtime do
-  count += LinkedData::Models::RestBackupMapping.where.all.count
-end
-puts "RestMappings count: #{count} with no display in #{format("%.4f", time)}s"
-
-count = 0
-time = Benchmark.realtime do
-  attr = LinkedData::Models::RestBackupMapping.attributes(:all) + LinkedData::Models::MappingProcess.attributes(:all)
-  count += LinkedData::Models::RestBackupMapping.where.include(attr).all.count
-end
-puts "RestMappings count: #{count} with display=all in #{format("%.4f", time)}s"

bin/migrations/count_graph_triples.rb

@@ -0,0 +1,60 @@
+# require 'bundler/setup'
+require 'pry'
+require 'benchmark'
+require 'ncbo_annotator'
+require 'ncbo_cron'
+require 'ontologies_linked_data'
+
+graph = ARGV[1]
+profile = ARGV[2]
+
+if graph.nil?
+  puts "Error: Missing arguments. Please provide the graph name."
+  exit(1)
+end
+
+case profile
+when 'ag'
+  # AllegroGraph backend
+  ENV['GOO_BACKEND_NAME'] = 'allegrograph'
+  ENV['GOO_PORT'] = '10035'
+  ENV['GOO_PATH_QUERY'] = '/repositories/ontoportal_test'
+  ENV['GOO_PATH_DATA'] = '/repositories/ontoportal_test/statements'
+  ENV['GOO_PATH_UPDATE'] = '/repositories/ontoportal_test/statements'
+  ENV['COMPOSE_PROFILES'] = 'ag'
+
+when 'fs'
+  # 4store backend
+  ENV['GOO_PORT'] = '9000'
+  ENV['COMPOSE_PROFILES'] = 'fs'
+
+when 'vo'
+  # Virtuoso backend
+  ENV['GOO_BACKEND_NAME'] = 'virtuoso'
+  ENV['GOO_PORT'] = '8890'
+  ENV['GOO_PATH_QUERY'] = '/sparql'
+  ENV['GOO_PATH_DATA'] = '/sparql'
+  ENV['GOO_PATH_UPDATE'] = '/sparql'
+  ENV['COMPOSE_PROFILES'] = 'vo'
+
+when 'gb'
+  # Graphdb backend
+  ENV['GOO_BACKEND_NAME'] = 'graphdb'
+  ENV['GOO_PORT'] = '7200'
+  ENV['GOO_PATH_QUERY'] = '/repositories/ontoportal'
+  ENV['GOO_PATH_DATA'] = '/repositories/ontoportal/statements'
+  ENV['GOO_PATH_UPDATE'] = '/repositories/ontoportal/statements'
+
+else
+  puts "Will import to default config set in config/config.rb"
+end
+
+require_relative '../../config/config'
+count = 0
+time = Benchmark.realtime do
+  rs = Goo.sparql_query_client.query("SELECT (COUNT(?s) as ?count) FROM <#{graph_uri}> WHERE { ?s ?p ?o }")
+  rs = rs.solutions.first
+  count = rs[:count].to_i  if rs
+end
+
+puts 'Imported triples in ' + format("%.4f", time) + 's with total count: ' + count.to_s

bin/migrations/import_metadata_graphs_to_store

@@ -49,14 +49,15 @@ begin
     puts "Start importing #{file} into graph <#{graph_uri}> of line count #{line_count}"
     result = false
     time = Benchmark.realtime do
-      result = system("ruby bin/migrations/import_nt_file.rb #{file} #{graph_uri} #{profile} > /dev/null 2>&1")
+      result = system("ruby bin/migrations/import_nt_file.rb #{file} #{graph_uri} #{profile} >> ./process_log.log 2>&1")
     end
 
     file_count += 1
     total_time += time
 
     if !result
       puts "Error importing #{file} into graph <#{graph_uri}>"
+      exit 1
     else
       import_count += 1
       puts "Imported <#{graph_uri}> successfully in #{time.round(2)} seconds"

bin/migrations/virtuoso/virtuso_grant_write_permission.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# Virtuoso database connection credentials
+DB_PORT=1111
+DB_USER="dba"
+DB_PASS="dba"
+VIRTUOSO_DIR=$1
+
+if [ "$#" -ne 1 ]; then
+  VIRTUOSO_DIR="/opt/virtuoso-opensource/"
+fi
+# Connect to Virtuoso using isql and grant EXECUTE permission
+echo "-- Granting EXECUTE permission on DB.DBA.SPARQL_INSERT_DICT_CONTENT..."
+
+$VIRTUOSO_DIR/bin/isql $DB_PORT $DB_USER $DB_PASS <<EOF
+GRANT EXECUTE ON DB.DBA.SPARQL_INSERT_DICT_CONTENT TO "SPARQL";
+EOF
+
+# Check if the operation was successful
+if [ $? -eq 0 ]; then
+  echo "Permission granted successfully."
+else
+  echo "Failed to grant permission."
+  exit 1
+fi
+
+# Optionally, grant the SPARQL_UPDATE role to the user for more permissions
+echo "-- Granting SPARQL_UPDATE role to 'SPARQL'..."
+$VIRTUOSO_DIR/bin/isql $DB_PORT $DB_USER $DB_PASS <<EOF
+GRANT SPARQL_UPDATE TO "SPARQL";
+EOF
+
+if [ $? -eq 0 ]; then
+  echo "SPARQL_UPDATE role granted successfully."
+else
+  echo "Failed to grant SPARQL_UPDATE role."
+  exit 1
+fi
+
+echo "-- Granting WRITE permission on all graphs to SPARQL user..."
+
+$VIRTUOSO_DIR/bin/isql $DB_PORT $DB_USER $DB_PASS <<EOF
+DB.DBA.RDF_DEFAULT_USER_PERMS_SET ('nobody', 7);
+EOF
+
+# Check if the operation was successful
+if [ $? -eq 0 ]; then
+  echo "WRITE permission granted successfully to SPARQL on all graphs."
+else
+  echo "Failed to grant WRITE permission."
+  exit 1
+fi
+
+# Restart Virtuoso to apply changes
+echo "-- Restarting Virtuoso server to apply changes..."
+$VIRTUOSO_DIR/bin/virtuoso-t +wait
+
+echo "Permission changes applied and Virtuoso restarted."