Skip to content

Commit

Permalink
Merge pull request snipe#14278 from uberbrady/allowlist_and_db_prefix…
Browse files Browse the repository at this point in the history 
…_for_restore

Allowlist and db prefix for restore
  • Loading branch information
@snipe
snipe authored Feb 15, 2024
2 parents 6bca1e3 + 6a78706 commit aef45a9
Show file tree
Hide file tree
Showing 2 changed files with 186 additions and 22 deletions.
205 changes: 184 additions & 21 deletions app/Console/Commands/RestoreFromBackup.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,165 @@
use Illuminate\Console\Command;
use ZipArchive;

class SQLStreamer {
private $input;
private $output;
// embed the prefix here?
public ?string $prefix;

private bool $reading_beginning_of_line = true;

public static $buffer_size = 1024 * 1024; // use a 1MB buffer, ought to work fine for most cases?
public array $tablenames = [];
private bool $should_guess = false;
private bool $statement_is_permitted = false;

public function __construct($input, $output, string $prefix = null)
{
$this->input = $input;
$this->output = $output;
$this->prefix = $prefix;
}

public function parse_sql(string $line): string {
// take into account the 'start of line or not' setting as an instance variable?
// 'continuation' lines for a permitted statement are PERMITTED.
if($this->statement_is_permitted && $line[0] === ' ') {
return $line;
}

$table_regex = '`?([a-zA-Z0-9_]+)`?';
$allowed_statements = [
"/^(DROP TABLE (?:IF EXISTS )?)`$table_regex(.*)$/" => false,
"/^(CREATE TABLE )$table_regex(.*)$/" => true, //sets up 'continuation'
"/^(LOCK TABLES )$table_regex(.*)$/" => false,
"/^(INSERT INTO )$table_regex(.*)$/" => false,
"/^UNLOCK TABLES/" => false,
// "/^\\) ENGINE=InnoDB AUTO_INCREMENT=16 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;/" => false, // FIXME not sure what to do here?
"/^\\)[a-zA-Z0-9_= ]*;$/" => false
// ^^^^^^ that bit should *exit* the 'perimitted' black
];

foreach($allowed_statements as $statement => $statechange) {
// $this->info("Checking regex: $statement...\n");
$matches = [];
if (preg_match($statement,$line,$matches)) {
$this->statement_is_permitted = $statechange;
// matches are: 1 => first part of the statement, 2 => tablename, 3 => rest of statement
// (with of course 0 being "the whole match")
if (@$matches[2]) {
// print "Found a tablename! It's: ".$matches[2]."\n";
if ($this->should_guess) {
@$this->tablenames[$matches[2]] += 1;
continue; //oh? FIXME
} else {
$cleaned_tablename = \DB::getTablePrefix().preg_replace('/^'.$this->prefix.'/','',$matches[2]);
$line = preg_replace($statement,'$1`'.$cleaned_tablename.'`$3' , $line);
}
} else {
// no explicit tablename in this one, leave the line alone
}
//how do we *replace* the tablename?
// print "RETURNING LINE: $line";
return $line;
}
}
// all that is not allowed is denied.
return "";
}

//this is used in exactly *TWO* places, and in both cases should return a prefix I think?
// first - if you do the --sanitize-only one (which is mostly for testing/development)
// next - when you run *without* a guessed prefix, this is run first to figure out the prefix
// I think we have to *duplicate* the call to be able to run it again?
public static function guess_prefix($input):string
{
$parser = new self($input, null);
$parser->should_guess = true;
$parser->line_aware_piping(); // <----- THIS is doing the heavy lifting!

$check_tables = ['settings' => null, 'migrations' => null /* 'assets' => null */]; //TODO - move to statics?
//can't use 'users' because the 'accessories_users' table?
// can't use 'assets' because 'ver1_components_assets'
foreach($check_tables as $check_table => $_ignore) {
foreach ($parser->tablenames as $tablename => $_count) {
// print "Comparing $tablename to $check_table\n";
if (str_ends_with($tablename,$check_table)) {
// print "Found one!\n";
$check_tables[$check_table] = substr($tablename,0,-strlen($check_table));
}
}
}
$guessed_prefix = null;
foreach ($check_tables as $clean_table => $prefix_guess) {
if(is_null($prefix_guess)) {
print("Couldn't find table $clean_table\n");
die();
}
if(is_null($guessed_prefix)) {
$guessed_prefix = $prefix_guess;
} else {
if ($guessed_prefix != $prefix_guess) {
print("Prefix mismatch! Had guessed $guessed_prefix but got $prefix_guess\n");
die();
}
}
}

return $guessed_prefix;

}

public function line_aware_piping(): int
{
$bytes_read = 0;
if (! $this->input) {
throw new \Exception("No Input available for line_aware_piping");
}

while (($buffer = fgets($this->input, SQLStreamer::$buffer_size)) !== false) {
$bytes_read += strlen($buffer);
if ($this->reading_beginning_of_line) {
// \Log::debug("Buffer is: '$buffer'");
$cleaned_buffer = $this->parse_sql($buffer);
if ($this->output) {
$bytes_written = fwrite($this->output, $cleaned_buffer);

if ($bytes_written === false) {
throw new \Exception("Unable to write to pipe");
}
}
}
// if we got a newline at the end of this, then the _next_ read is the beginning of a line
if($buffer[strlen($buffer)-1] === "\n") {
$this->reading_beginning_of_line = true;
} else {
$this->reading_beginning_of_line = false;
}

}
return $bytes_read;

}
}



class RestoreFromBackup extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
// FIXME - , stripping prefixes and nonstandard SQL statements. Without --prefix, guess and return the correct prefix to strip
protected $signature = 'snipeit:restore
{--force : Skip the danger prompt; assuming you enter "y"}
{filename : The zip file to be migrated}
{--no-progress : Don\'t show a progress bar}';
{--no-progress : Don\'t show a progress bar}
{--sanitize-guess-prefix : Guess and output the table-prefix needed to "sanitize" the SQL}
{--sanitize-with-prefix= : "Sanitize" the SQL, using the passed-in table prefix (can be learned from --sanitize-guess-prefix). Pass as just \'--sanitize-with-prefix=\' to use no prefix}';

/**
* The console command description.
Expand All @@ -34,8 +182,6 @@ public function __construct()
parent::__construct();
}

public static $buffer_size = 1024 * 1024; // use a 1MB buffer, ought to work fine for most cases?
/**
* Execute the console command.
*
Expand All @@ -55,7 +201,7 @@ public function handle()
return $this->error('Missing required filename');
}

if (! $this->option('force') && ! $this->confirm('Are you sure you wish to restore from the given backup file? This can lead to MASSIVE DATA LOSS!')) {
if (! $this->option('force') && ! $this->option('sanitize-guess-prefix') && ! $this->confirm('Are you sure you wish to restore from the given backup file? This can lead to MASSIVE DATA LOSS!')) {
return $this->error('Data loss not confirmed');
}

Expand Down Expand Up @@ -158,11 +304,11 @@ public function handle()
}

foreach (array_merge($private_dirs, $public_dirs) as $dir) {
$last_pos = strrpos($raw_path, $dir.'/');
$last_pos = strrpos($raw_path, $dir . '/');
if ($last_pos !== false) {
//print("INTERESTING - last_pos is $last_pos when searching $raw_path for $dir - last_pos+strlen(\$dir) is: ".($last_pos+strlen($dir))." and strlen(\$rawpath) is: ".strlen($raw_path)."\n");
//print("We would copy $raw_path to $dir.\n"); //FIXME append to a path?
$interesting_files[$raw_path] = ['dest' =>$dir, 'index' => $i];
$interesting_files[$raw_path] = ['dest' => $dir, 'index' => $i];
continue 2;
if ($last_pos + strlen($dir) + 1 == strlen($raw_path)) {
// we don't care about that; we just want files with the appropriate prefix
Expand All @@ -171,7 +317,7 @@ public function handle()
}
}
$good_extensions = ['png', 'gif', 'jpg', 'svg', 'jpeg', 'doc', 'docx', 'pdf', 'txt',
'zip', 'rar', 'xls', 'xlsx', 'lic', 'xml', 'rtf', 'webp', 'key', 'ico', ];
'zip', 'rar', 'xls', 'xlsx', 'lic', 'xml', 'rtf', 'webp', 'key', 'ico',];
foreach (array_merge($private_files, $public_files) as $file) {
$has_wildcard = (strpos($file, '*') !== false);
if ($has_wildcard) {
Expand All @@ -180,8 +326,8 @@ public function handle()
$last_pos = strrpos($raw_path, $file); // no trailing slash!
if ($last_pos !== false) {
$extension = strtolower(pathinfo($raw_path, PATHINFO_EXTENSION));
if (! in_array($extension, $good_extensions)) {
$this->warn('Potentially unsafe file '.$raw_path.' is being skipped');
if (!in_array($extension, $good_extensions)) {
$this->warn('Potentially unsafe file ' . $raw_path . ' is being skipped');
$boring_files[] = $raw_path;
continue 2;
}
Expand All @@ -196,7 +342,6 @@ public function handle()
}
$boring_files[] = $raw_path; //if we've gotten to here and haven't continue'ed our way into the next iteration, we don't want this file
} // end of pre-processing the ZIP file for-loop

// print_r($interesting_files);exit(-1);

if (count($sqlfiles) != 1) {
Expand All @@ -208,6 +353,17 @@ public function handle()
//older Snipe-IT installs don't have the db-dumps subdirectory component
}

$sql_stat = $za->statIndex($sqlfile_indices[0]);
//$this->info("SQL Stat is: ".print_r($sql_stat,true));
$sql_contents = $za->getStream($sql_stat['name']); // maybe copy *THIS* thing?
// OKAY, now that we *found* the sql file if we're doing just the guess-prefix thing, we can do that *HERE* I think?
if ($this->option('sanitize-guess-prefix')) {
$prefix = SQLStreamer::guess_prefix($sql_contents);
$this->line($prefix);
return $this->info("Re-run this command with '--sanitize-with-prefix=".$prefix."' to see an attempt to sanitze your SQL.");
}

//how to invoke the restore?
$pipes = [];

Expand All @@ -228,6 +384,7 @@ public function handle()
return $this->error('Unable to invoke mysql via CLI');
}

// I'm not sure about these?
stream_set_blocking($pipes[1], false); // use non-blocking reads for stdout
stream_set_blocking($pipes[2], false); // use non-blocking reads for stderr

Expand All @@ -238,9 +395,9 @@ public function handle()

//$sql_contents = fopen($sqlfiles[0], "r"); //NOPE! This isn't a real file yet, silly-billy!

$sql_stat = $za->statIndex($sqlfile_indices[0]);
//$this->info("SQL Stat is: ".print_r($sql_stat,true));
$sql_contents = $za->getStream($sql_stat['name']);
// FIXME - this feels like it wants to go somewhere else?
// and it doesn't seem 'right' - if you can't get a stream to the .sql file,
// why do we care what's happening with pipes and stdout and stderr?!
if ($sql_contents === false) {
$stdout = fgets($pipes[1]);
$this->info($stdout);
Expand All @@ -249,20 +406,27 @@ public function handle()

return false;
}
$bytes_read = 0;

try {
while (($buffer = fgets($sql_contents, self::$buffer_size)) !== false) {
$bytes_read += strlen($buffer);
// \Log::debug("Buffer is: '$buffer'");
if ( $this->option('sanitize-with-prefix') === null) {
// "Legacy" direct-piping
$bytes_read = 0;
while (($buffer = fgets($sql_contents, SQLStreamer::$buffer_size)) !== false) {
$bytes_read += strlen($buffer);
// \Log::debug("Buffer is: '$buffer'");
$bytes_written = fwrite($pipes[0], $buffer);

if ($bytes_written === false) {
throw new Exception("Unable to write to pipe");
if ($bytes_written === false) {
throw new Exception("Unable to write to pipe");
}
}
} else {
$sql_importer = new SQLStreamer($sql_contents, $pipes[0], $this->option('sanitize-with-prefix'));
$bytes_read = $sql_importer->line_aware_piping();
}
} catch (\Exception $e) {
\Log::error("Error during restore!!!! ".$e->getMessage());
// FIXME - put these back and/or put them in the right places?!
$err_out = fgets($pipes[1]);
$err_err = fgets($pipes[2]);
\Log::error("Error OUTPUT: ".$err_out);
Expand All @@ -271,7 +435,6 @@ public function handle()
$this->error($err_err);
throw $e;
}

if (!feof($sql_contents) || $bytes_read == 0) {
return $this->error("Not at end of file for sql file, or zero bytes read. aborting!");
}
Expand Down Expand Up @@ -303,7 +466,7 @@ public function handle()
$fp = $za->getStream($ugly_file_name);
//$this->info("Weird problem, here are file details? ".print_r($file_details,true));
$migrated_file = fopen($file_details['dest'].'/'.basename($pretty_file_name), 'w');
while (($buffer = fgets($fp, self::$buffer_size)) !== false) {
while (($buffer = fgets($fp, SQLStreamer::$buffer_size)) !== false) {
fwrite($migrated_file, $buffer);
}
fclose($migrated_file);
Expand Down
3 changes: 2 additions & 1 deletion composer.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,8 @@
"watson/validating": "^6.1"
},
"suggest": {
"ext-ldap": "*"
"ext-ldap": "*",
"ext-zip": "*"
},
"require-dev": {
"brianium/paratest": "^6.6",
Expand Down

0 comments on commit aef45a9

Please sign in to comment.