Skip to content

Commit 5e57e09

Browse files
k2sk2s
committed
mysql data export
1 parent e4465dc commit 5e57e09

File tree

6 files changed

+179
-41
lines changed

6 files changed

+179
-41
lines changed

‎.gitignore

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
2+
#ignore thumbnails created by windows
3+
Thumbs.db
4+
#Ignore files build by Visual Studio
5+
*.obj
6+
*.exe
7+
*.pdb
8+
*.user
9+
*.aps
10+
*.pch
11+
*.vspscc
12+
*_i.c
13+
*_p.c
14+
*.ncb
15+
*.suo
16+
*.tlb
17+
*.tlh
18+
*.bak
19+
*.cache
20+
*.ilk
21+
*.log
22+
[Bb]in
23+
[Dd]ebug*/
24+
*.lib
25+
*.sbr
26+
obj/
27+
[Rr]elease*/
28+
_ReSharper*/
29+
[Tt]est[Rr]esult*
30+
31+
#InteliJ
32+
#*.iml
33+
*.ipr
34+
*.iws
35+
.idea/

‎storage/Mysql.php

+43-26
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
<?php
2-
class Storage_Mysql extends Storage_Filesystem
2+
class Storage_Mysql extends Storage_Filesystem implements Storage_Mysql_IStore
33
{
44
protected $_db;
55
protected $_driver;
@@ -18,40 +18,22 @@ public function __construct($identity, $engine, $output, $options)
1818

1919
// mysql options
2020

21-
// debug file
22-
$this->_debugFolder = $this->_baseDir.DIRECTORY_SEPARATOR."debug".DIRECTORY_SEPARATOR;
23-
if ($this->_debugFolder) {
24-
$this->clearFolder($this->_debugFolder);
25-
@mkdir($this->_debugFolder);
26-
$f = fopen($this->_debugFolder."1.sql", "w");
27-
fputs($f, <<<SQL
28-
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
29-
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
30-
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
31-
/*!40101 SET NAMES utf8 */;
32-
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
33-
/*!40103 SET TIME_ZONE='+00:00' */;
34-
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
35-
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
36-
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
37-
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
38-
SQL
39-
);
40-
fclose($f);
41-
}
4221
}
4322

44-
protected function clearFolder($str)
23+
protected function _clearFolder($str, $first=true)
4524
{
4625
if(is_file($str)){
4726
return @unlink($str);
4827
}
4928
elseif(is_dir($str)){
5029
$scan = glob(rtrim($str,'/').'/*');
5130
foreach($scan as $index=>$path){
52-
$this->clearFolder($path);
31+
$this->_clearFolder($path, false);
32+
}
33+
if (!$first) {
34+
@rmdir($str);
5335
}
54-
return @rmdir($str);
36+
return;
5537
}
5638
}
5739

@@ -94,6 +76,13 @@ public function storeDbObject($kind, $name, $def)
9476
file_put_contents($fn, $def);
9577
}
9678

79+
public function storeFilenameFor($kind, $name)
80+
{
81+
$fn = $this->_baseDir.$kind;
82+
@mkdir($fn);
83+
return $fn.DIRECTORY_SEPARATOR.$name;
84+
}
85+
9786
public function init($myrole, $drivers)
9887
{
9988
parent::init($myrole, $drivers);
@@ -108,16 +97,44 @@ public function init($myrole, $drivers)
10897
$this->_db = new PDO("mysql:host=localhost;dbname=mysql", 'root', 'klifo', array(PDO::MYSQL_ATTR_INIT_COMMAND => "SET NAMES utf8"));
10998
// let PDO throw exception on errors
11099
$this->_db->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
100+
// because of data amount we shouldn't use buffered queries
101+
$this->_db->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false);
111102

112103
$this->_driver = $this->_getBackupDriver();
113104
$this->_driver->setDatabaseToBackup("informaglobal");
114105

106+
// check/clear target folder
107+
// TODO check if it is empty and if we are allowed to delete it if not
108+
$this->_clearFolder($this->_baseDir);
109+
110+
// prepare debug file if needed
111+
$this->_debugFolder = $this->_baseDir.DIRECTORY_SEPARATOR."debug".DIRECTORY_SEPARATOR;
112+
if ($this->_debugFolder) {
113+
$this->_clearFolder($this->_debugFolder);
114+
@mkdir($this->_debugFolder);
115+
$f = fopen($this->_debugFolder."1.sql", "w");
116+
fputs($f, <<<SQL
117+
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
118+
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
119+
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
120+
/*!40101 SET NAMES utf8 */;
121+
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
122+
/*!40103 SET TIME_ZONE='+00:00' */;
123+
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
124+
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
125+
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
126+
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
127+
SQL
128+
);
129+
fclose($f);
130+
}
131+
115132
// retrieve all objects known in DB
116133
$objects = $this->_driver->listAvailableObjectsToBackup();
117134
// TODO filter objects which should be backed up
118135

119136
// execute backup of DB objects
120-
$this->_driver->doBackup(array($this, 'storeDbObject'));
137+
$this->_driver->doBackup($this);
121138

122139
$this->_out->stop("ok");
123140

‎storage/Mysql/Backup.php

+71-15
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,8 @@ function setDatabaseToBackup($name)
5050

5151
function listAvailableObjectsToBackup($kind=false)
5252
{
53-
/*if ($kind==self::KIND_TABLES) {
54-
return array('_as_linkmanager_links');
53+
/*if ($kind==self::KIND_TABLES || $kind==self::KIND_DATA) {
54+
return array('video');
5555
}*/
5656

5757
if ($kind===false) {
@@ -90,24 +90,24 @@ protected function _prepareBackup()
9090

9191
}
9292

93-
function doBackup($storeCallback)
93+
function doBackup($store)
9494
{
9595
// make sure we already prepared backup
9696
$this->_prepareBackup();
9797

9898
foreach ($this->_kindsToBackup as $kind) {
9999
$funcName = "_backup".ucfirst($kind);
100100
if (method_exists($this, $funcName)) {
101-
$this->$funcName($storeCallback);
101+
$this->$funcName($store);
102102
} else {
103103
echo "don't know of to backup objects of type '$kind'.\n";
104104
}
105105
}
106106
}
107-
function _backupRefs($storeCallback) {}
108-
function _backupIndexes($storeCallback) {}
107+
function _backupRefs($store) {}
108+
function _backupIndexes($store) {}
109109

110-
function _backupTables($storeCallback)
110+
function _backupTables($store)
111111
{
112112
foreach ($this->listAvailableObjectsToBackup(self::KIND_TABLES) as $def) {
113113
// info SHOW TABLE STATUS LIKE 'TABLES';
@@ -166,37 +166,38 @@ function _backupTables($storeCallback)
166166

167167
// store the table
168168
$tbl = implode(PHP_EOL, $tbl);
169-
call_user_func($storeCallback, self::KIND_TABLES, $def, $tbl.';');
169+
170+
$store->storeDbObject(self::KIND_TABLES, $def, $tbl.';');
170171

171172
// store tables indexes
172173
if (count($idx)) {
173174
array_unshift($idx, "ALTER TABLE `$def`");
174175
$idx[count($idx)-1] = rtrim($idx[count($idx)-1], ',');
175176
$idx = implode(PHP_EOL, $idx);
176-
call_user_func($storeCallback, self::KIND_INDEXES, $def, $idx.';');
177+
$store->storeDbObject(self::KIND_INDEXES, $def, $idx.';');
177178
}
178179

179180
// store table constraints
180181
if (count($refs)) {
181182
array_unshift($refs, "ALTER TABLE `$def`");
182183
$refs = implode(PHP_EOL, $refs);
183-
call_user_func($storeCallback, self::KIND_REFS, $def, $refs.';');
184+
$store->storeDbObject(self::KIND_REFS, $def, $refs.';');
184185
}
185186
}
186187
}
187188
}
188189

189-
function _backupData($storeCallback)
190+
function _backupData($store)
190191
{
191192
// TODO detect if server is localhost
192193
if (false) {
193-
$this->_backupDataFromLocal($storeCallback);
194+
$this->_backupDataFromLocal($store);
194195
} else {
195-
$this->_backupDataFromRemote($storeCallback);
196+
$this->_backupDataFromRemote($store);
196197
}
197198
}
198199

199-
function _backupDataFromLocal($storeCallback)
200+
function _backupDataFromLocal($store)
200201
{
201202
// TODO not implemented
202203
/**
@@ -210,10 +211,65 @@ function _backupDataFromLocal($storeCallback)
210211
}
211212
}
212213

213-
function _backupDataFromRemote($storeCallback)
214+
function _backupDataFromRemote($store)
214215
{
215216
foreach ($this->listAvailableObjectsToBackup(self::KIND_DATA) as $def) {
217+
$fn = $store->storeFilenameFor(self::KIND_DATA, $def);
218+
$f = fopen($fn, "w");
219+
$this->_tableDataToCsv($def, $f);
220+
fclose($f);
221+
}
222+
}
216223

224+
protected function _tableDataToCsv($tableName, $f)
225+
{
226+
// TODO it may be better if Storage_Mysql whould hangle storing of array data
227+
$q = $this->_db->query("SELECT * FROM `{$this->_dbName}`.`$tableName`");
228+
//$q = $this->_db->query("SELECT * FROM `a`.`a_export`");
229+
while (false!==($data=$q->fetch(PDO::FETCH_NUM))) {
230+
// we have to convert null fields to \N
231+
foreach ($data as &$c) {
232+
if (is_null($c)) {
233+
$c = "\N";
234+
} else {
235+
$c = strtr($c, array(
236+
"\\" => "\\\\",
237+
"\t" => "\\\t",
238+
"\n" => "\\\n",
239+
"\r" => "\\\r",
240+
));
241+
}
242+
}
243+
244+
// store data in tab delimited format
245+
//fputcsv($f, $data, "\t", ' ');
246+
fwrite($f, implode("\t", $data).chr(13).chr(10));
217247
}
218248
}
249+
219250
}
251+
252+
SELECT * INTO OUTFILE '/tmp/a_export' FROM a_export
253+
254+
255+
256+
257+
258+
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
259+
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
260+
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
261+
/*!40101 SET NAMES utf8 */;
262+
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
263+
/*!40103 SET TIME_ZONE='+00:00' */;
264+
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
265+
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
266+
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
267+
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
268+
TRUNCATE TABLE cms_page;
269+
LOAD DATA LOCAL INFILE '/home/k2s/Backups/xtbackupMysql/data/cms_page' INTO TABLE cms_page CHARACTER SET UTF8;
270+
select count(*) from cms_page;
271+
272+
273+
274+
DROP DATABASE a;
275+
CREATE DATABASE a;

‎storage/Mysql/IBackup.php

+2
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,6 @@
22
interface Storage_Mysql_IBackup
33
{
44
function setConnection($connection);
5+
function listAvailableObjectsToBackup($kind=false);
6+
function doBackup($storeCallback);
57
}

‎storage/Mysql/IStore.php

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
<?php
2+
interface Storage_Mysql_IStore
3+
{
4+
function storeDbObject($kind, $name, $def);
5+
function storeFilenameFor($kind, $name);
6+
}

‎xtbackup.iml

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<module type="JAVA_MODULE" version="4">
3+
<component name="NewModuleRootManager" inherit-compiler-output="true">
4+
<exclude-output />
5+
<content url="file://$MODULE_DIR$">
6+
<sourceFolder url="file://$MODULE_DIR$" isTestSource="false" />
7+
<excludeFolder url="file://$MODULE_DIR$/.idea" />
8+
</content>
9+
<orderEntry type="inheritedJdk" />
10+
<orderEntry type="sourceFolder" forTests="false" />
11+
<orderEntry type="module-library">
12+
<library name="PHP 5.3.6-13ubuntu3.6">
13+
<CLASSES>
14+
<root url="file:///usr/share/pear" />
15+
</CLASSES>
16+
<JAVADOC />
17+
<SOURCES />
18+
</library>
19+
</orderEntry>
20+
</component>
21+
</module>
22+

0 commit comments

Comments
 (0)