diff --git a/config/Master-template.php b/config/Master-template.php index 305984129..a6de0b38c 100644 --- a/config/Master-template.php +++ b/config/Master-template.php @@ -262,6 +262,12 @@ class Master * IdP::getExternalDBSyncState(); * IdP::setExternalDBId($identifier); * + * We are now prepared to use a local cache database. For the eduroam case a daily + * update is sufficient and is done by the utils/update_monitor_copy.php. In this case + * the EXTERNAL is actually a local DB, and EXTERNAL_SOURCE is used as the actual + * source for caching. You may use the EXTERNAL as a real extermal database and + * do not worry about setting aything in EXTERNAL_SOURCE + * * The code for the consortium "eduroam" is already written and may serve as template. See the functions in question. */ // Marking this as readonly currently has no effect, as we only ever do SELECTs on that database 'EXTERNAL' => [ @@ -270,6 +276,21 @@ class Master 'user' => 'customerservice', 'pass' => '2lame4u', 'readonly' => TRUE,], + /* + * EXTERNAL_SOURCE is the source only used in the caching script, if you + * do not use local caching then thse settins are irrelevant + */ + 'EXTERNAL_SOURCE' => [ + 'host' => 'customerdb.otherhost.example', + 'db' => 'customer_db', + 'user' => 'customerservice', + 'pass' => '2lame4u', + 'readonly' => TRUE,], + + + + + 'enforce-external-sync' => TRUE, ]; /** @@ -295,6 +316,4 @@ class Master 'http://sommeopenid.example/anotheruser', 'I do not care about security!', ]; - const SUPPORT = [ - ]; } diff --git a/core/DBConnection.php b/core/DBConnection.php index 64017161e..2c5150e80 100644 --- a/core/DBConnection.php +++ b/core/DBConnection.php @@ -297,7 +297,6 @@ private function __construct($database) // is ambiguous. Resolve the ambiguity by telling MySQL to always operate // in UTC time. $this->connection->query("SET SESSION time_zone='+00:00'"); - $this->connection->query("SET NAMES 'latin1'"); } } diff --git a/core/ExternalEduroamDBData.php b/core/ExternalEduroamDBData.php index c5d6c3ead..3a04eef6b 100644 --- a/core/ExternalEduroamDBData.php +++ b/core/ExternalEduroamDBData.php @@ -78,7 +78,6 @@ public function __construct() { throw new Exception("Frontend DB is never an array, always a single DB object."); } $this->db = $connHandle; - $this->db->exec("SET NAMES 'latin1'"); } /** diff --git a/utils/update_monitor_copy.php b/utils/update_monitor_copy.php new file mode 100644 index 000000000..680fd3313 --- /dev/null +++ b/utils/update_monitor_copy.php @@ -0,0 +1,183 @@ +/copyright.php after deploying the software + */ + +/** + * This script will download all data from the views in eduroam database and + * update the local copy + */ + +require_once dirname(dirname(__FILE__)) . "/config/_config.php"; +setlocale(LC_CTYPE, "en_US.UTF-8"); + +class updateFromMonitor { + private $db; + private $db_local; + + private $tablesource = [ + 'admin' => 'eduroam', + 'active_institution' => 'eduroam', + 'active_idp_institution' => 'eduroam', + 'active_SP_location_eduroamdb' => 'eduroam', + 'country_eduroamdb' => 'eduroam', + 'tls_inst' => 'eduroamv2', + 'tls_ro' => 'eduroamv2' + ]; + + public $fields = [ + 'admin' => [ + ['id', 'i'], + ['eptid', 's'], + ['email', 's'], + ['common_name', 's'], + ['id_role', 'i'], + ['role', 's'], + ['realm', 's'] + ], + 'active_institution' => [ + ['id_institution', 'i'], + ['ROid', 's'], + ['inst_realm', 's'], + ['country', 's'], + ['name', 's'], + ['contact', 's'], + ['type', 's'] + ], + 'active_idp_institution' => [ + ['id_institution', 'i'], + ['inst_realm', 's'], + ['country', 's'], + ['name', 's'], + ['contact', 's'] + ], + 'active_SP_location_eduroamdb' => [ + ['country', 's'], + ['country_eng', 's'], + ['institutionid', 'i'], + ['inst_name', 's'], + ['sp_location', 's'], + ['sp_location_contact', 's'] + ], + 'country_eduroamdb' => [ + ['country', 's'], + ['country_eng', 's'], + ['map_group', 's'] + ], + 'tls_ro' => [ + ['ROid', 's'], + ['country', 's'], + ['stage', 'i'], + ['servers', 's'], + ['contacts', 's'], + ['ts', 's'] + ], + 'tls_inst' => [ + ['ROid', 's'], + ['country', 's'], + ['instid', 's'], + ['stage', 'i'], + ['type', 'i'], + ['inst_name', 's'], + ['servers', 's'], + ['contacts', 's'], + ['ts', 's'] + ] + ]; + + public function __construct() { + $DB = \config\Master::DB['EXTERNAL_SOURCE']; + $DB_LOCAL = \config\Master::DB['EXTERNAL']; + $this->db = new mysqli($DB['host'], $DB['user'], $DB['pass'], $DB['db']); + $this->db_local = new mysqli($DB_LOCAL['host'], $DB_LOCAL['user'], $DB_LOCAL['pass']); + } + + public function update_table($table_name) { + $table = 'view_'.$table_name; + $tmp_table = 'tmp_'.$table_name; + if ($this->tablesource[$table_name] == 'eduroam') { + $this->db_local->select_db('monitor_copy'); + } elseif($this->tablesource[$table_name] == 'eduroamv2') { + $this->db_local->select_db('eduroamv2'); + } + $this->db_local->query("CREATE TEMPORARY TABLE $tmp_table SELECT * FROM $table LIMIT 0"); + $sourceDB = $this->tablesource[$table_name]; + $this->db->select_db($sourceDB); + $result = $this->db->query("SET NAMES 'utf8'"); + $result = $this->db_local->query("SET NAMES 'utf8mb4'"); + $result = $this->db->query("SELECT * FROM $table"); + $queryFields = implode(',', array_column($this->fields[$table_name],0)); + while ($row = $result->fetch_assoc()) { + $v = []; + foreach ($this->fields[$table_name] as $field) { + if ($field[1] === 's') { + if (isset($row[$field[0]])) { + $v[] = $this->escape($row[$field[0]]); + } else { + $v[] = "NULL"; + } + } else { + if (isset($row[$field[0]])) { + $v[] = $row[$field[0]]; + } else { + $v[] = "NULL"; + } + } + } + $queryValues = implode(',',$v); + $query = "INSERT INTO $tmp_table (".$queryFields.") VALUES (".$queryValues.")"; + $this->db_local->query($query); + } + + } + + public function fill_table($table_name) { + $table = 'view_'.$table_name; + $tmp_table = 'tmp_'.$table_name; + if ($this->tablesource[$table_name] == 'eduroam') { + $this->db_local->select_db('monitor_copy'); + } elseif($this->tablesource[$table_name] == 'eduroamv2') { + $this->db_local->select_db('eduroamv2'); + } + $result = $this->db_local->query("SET NAMES 'utf8mb4'"); + $this->db_local->query("DELETE FROM $table"); + $this->db_local->query("INSERT INTO $table SELECT * from $tmp_table"); + } + + private function escape($inp) { + $out=str_replace('\\','\\\\',$inp); + $out=str_replace('"','\"',$out); + $out=str_replace('?','\?',$out); + $out = 'convert(cast(convert("'.$out.'" using latin1) as binary) using utf8)'; + return($out); + } +} + +$myDB = new updateFromMonitor(); + +foreach (array_keys($myDB->fields) as $table) { + print("$table\n"); + $myDB->update_table($table); +} + +foreach (array_keys($myDB->fields) as $table) { + print("$table\n"); + $myDB->fill_table($table); +} +