Skip to content

Commit

Permalink
1. fix mysql and oracle index create script error
Browse files Browse the repository at this point in the history 
2. fix bulkcopy: only one table data copied
3. enhance pivot parse
4. fix function translate issue: miss declare for plsql to tsql
5. sort table columns before convert db schema
  • Loading branch information
victor committed May 15, 2020
1 parent bc42a45 commit b46fe03
Show file tree
Hide file tree
Showing 26 changed files with 384 additions and 131 deletions.
5 changes: 5 additions & 0 deletions DatabaseConverter/DatabaseConverter.Core/Config/FunctionMapping.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,4 +123,9 @@
<MySql>TRUNCATE</MySql>
<Oracle>TRUNC</Oracle>
</mapping>
<mapping>
<SqlServer direction="IN" args="EXP AS INT">CAST</SqlServer>
<MySql direction="IN" args="EXP AS UNSIGNED">CAST</MySql>
<Oracle>TO_NUMBER</Oracle>
</mapping>
</mappings>
2 changes: 2 additions & 0 deletions DatabaseConverter/DatabaseConverter.Core/Converter/DbConverter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,8 @@ private async Task InternalConvert(SchemaInfo schemaInfo = null)
return;
}

sourceSchemaInfo.TableColumns = DbObjectHelper.ResortTableColumns(sourceSchemaInfo.Tables, sourceSchemaInfo.TableColumns);

if (SettingManager.Setting.NotCreateIfExists)
{
this.Target.DbInterpreter.Option.GetTableAllObjects = false;
Expand Down
10 changes: 9 additions & 1 deletion DatabaseConverter/DatabaseConverter.Core/Manager/FunctionMappingManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,17 @@ public static List<IEnumerable<FunctionMapping>> GetFunctionMappings()
{
DbType = t.Name.ToString(),
Function = t.Value,
Direction = string.IsNullOrEmpty(t.Attribute("direction")?.Value) ? FunctionMappingDirection.INOUT : (FunctionMappingDirection)Enum.Parse(typeof(FunctionMappingDirection), t.Attribute("direction").Value.ToUpper())
Direction = ParseDirection(t),
Args = t.Attribute("args")?.Value
}))
.ToList();
}

private static FunctionMappingDirection ParseDirection(XElement element)
{
return string.IsNullOrEmpty(element.Attribute("direction")?.Value) ?
FunctionMappingDirection.INOUT :
(FunctionMappingDirection)Enum.Parse(typeof(FunctionMappingDirection), element.Attribute("direction").Value.ToUpper());
}
}
}
7 changes: 7 additions & 0 deletions DatabaseConverter/DatabaseConverter.Core/Model/FunctionMapping.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ public class FunctionMapping
public string DbType { get; set; }
public string Function { get; set; }
public FunctionMappingDirection Direction { get; set; }
public string Args { get; set; }
}

public enum FunctionMappingDirection
Expand All @@ -13,4 +14,10 @@ public enum FunctionMappingDirection
IN = 1,
OUT = 2
}

public struct MappingFunctionInfo
{
public string Name { get; set; }
public string Args { get; set; }
}
}
2 changes: 1 addition & 1 deletion DatabaseConverter/DatabaseConverter.Core/ScriptToken/ScriptTokenExtracter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ private void ExtractTokens(dynamic obj)
this.ExtractTokens(v);
}
}
else if (value is Statement || value is TemporaryTable)
else if (value is Statement || value is TemporaryTable || value is StatementItem)
{
this.ExtractTokens(value);
}
Expand Down
7 changes: 6 additions & 1 deletion DatabaseConverter/DatabaseConverter.Core/ScriptToken/ScriptTokenProcessor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ private void ProcessTokens()

if (keywords.Contains(token.Symbol.ToUpper()) || functions.Contains(token.Symbol.ToUpper()))
{
token.Symbol = "_" + token.Symbol;
token.Symbol = token.Symbol + "_";
}

hasChanged = true;
Expand Down Expand Up @@ -226,6 +226,11 @@ private void ProcessTokens()
{
foreach (Match match in Regex.Matches(token.Symbol, pattern))
{
if (kp.Value.StartsWith("@") && token.Symbol.Contains(kp.Value))
{
continue;
}

token.Symbol = Regex.Replace(token.Symbol, pattern, kp.Value);
}
}
Expand Down
17 changes: 14 additions & 3 deletions DatabaseConverter/DatabaseConverter.Core/Translator/ColumnTranslator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -444,10 +444,19 @@ public void ConvertDefaultValue(TableColumn column)

public void ConvertComputeExpression(TableColumn column)
{
if (this.sourceDbType == DatabaseType.Oracle)
{
column.ComputeExp = column.ComputeExp.Replace("U'", "'");
}
else if (this.sourceDbType == DatabaseType.SqlServer)
{
column.ComputeExp = column.ComputeExp.Replace("N'", "'");
}

column.ComputeExp = this.ParseDefinition(column.ComputeExp);
}

private async void CheckComputeExpression()
private void CheckComputeExpression()
{
IEnumerable<Function> customFunctions = this.SourceSchemaInfo?.Functions;

Expand All @@ -468,7 +477,9 @@ private async void CheckComputeExpression()

bool setToNull = false;

bool isReferToSpecialDataType = this.columns.Any(item => item != column
var tableColumns = this.columns.Where(item => item.TableName == column.TableName);

bool isReferToSpecialDataType = tableColumns.Any(item => item.Name != column.Name
&& DataTypeHelper.SpecialDataTypes.Any(t => t.ToLower().Contains(item.DataType.ToLower()))
&& Regex.IsMatch(column.ComputeExp, $@"\b({item.Name})\b", RegexOptions.IgnoreCase));

Expand All @@ -481,7 +492,7 @@ private async void CheckComputeExpression()
{
if (customFunctions == null || customFunctions.Count() == 0)
{
customFunctions = await this.sourceDbInterpreter.GetFunctionsAsync();
customFunctions = this.sourceDbInterpreter.GetFunctionsAsync().Result;
}

if (customFunctions != null)
Expand Down
42 changes: 26 additions & 16 deletions DatabaseConverter/DatabaseConverter.Core/Translator/DbObjectTokenTranslator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ namespace DatabaseConverter.Core
{
public class DbObjectTokenTranslator : DbObjectTranslator
{
private List<string> dataTypes = new List<string>();
private List<string> convertedDataTypes = new List<string>();
private List<string> convertedFunctions = new List<string>();

private List<FunctionSpecification> sourceFuncSpecs;
private List<FunctionSpecification> targetFuncSpecs;
Expand Down Expand Up @@ -67,7 +68,7 @@ protected string HandleDefinition(string definition, List<TSQLToken> tokens, out
switch (text.ToUpper())
{
case "CAST":

case "TO_NUMBER":
functionExpression = this.GetFunctionExpression(token, definition);

break;
Expand All @@ -81,13 +82,14 @@ protected string HandleDefinition(string definition, List<TSQLToken> tokens, out

if (!string.IsNullOrEmpty(functionExpression))
{
string targetFunctionName = this.GetMappedFunctionName(text);
bool useBrackets = false;
MappingFunctionInfo targetFunctionInfo= this.GetMappingFunctionInfo(text, out useBrackets);

FunctionFomular fomular = new FunctionFomular(functionExpression);

Dictionary<string, string> dictDataType = null;

string newExpression = this.ParseFomular(this.sourceFuncSpecs, this.targetFuncSpecs, fomular, targetFunctionName, out dictDataType);
string newExpression = this.ParseFomular(this.sourceFuncSpecs, this.targetFuncSpecs, fomular, targetFunctionInfo, out dictDataType);

if (newExpression != fomular.Expression)
{
Expand All @@ -98,7 +100,15 @@ protected string HandleDefinition(string definition, List<TSQLToken> tokens, out

if (dictDataType != null)
{
this.dataTypes.AddRange(dictDataType.Values);
this.convertedDataTypes.AddRange(dictDataType.Values);
}

if(!string.IsNullOrEmpty(targetFunctionInfo.Args) && changed)
{
if(!this.convertedFunctions.Contains(targetFunctionInfo.Name))
{
this.convertedFunctions.Add(targetFunctionInfo.Name);
}
}
}
}
Expand Down Expand Up @@ -191,7 +201,7 @@ public string BuildDefinition(List<TSQL.Tokens.TSQLToken> tokens)
}
}

if (dataTypes.Contains(text))
if (convertedDataTypes.Contains(text))
{
sb.Append(text);
continue;
Expand All @@ -207,21 +217,21 @@ public string BuildDefinition(List<TSQL.Tokens.TSQLToken> tokens)
}
else if (nextToken != null && nextToken.Text.Trim() == "(") //function handle
{
string textWithBrackets = text.ToLower() + "()";

bool useBrackets = false;

if (this.functionMappings.Any(item => item.Any(t => t.Function.ToLower() == textWithBrackets)))
if (this.convertedFunctions.Contains(text))
{
useBrackets = true;
text = textWithBrackets;
sb.Append(text);
continue;
}

IEnumerable<FunctionMapping> funcMappings = this.functionMappings.FirstOrDefault(item => item.Any(t => t.DbType == sourceDbInterpreter.DatabaseType.ToString() && t.Function.Split(',').Any(m => m.ToLower() == text.ToLower())));
string textWithBrackets = text.ToLower() + "()";

bool useBrackets = false;

MappingFunctionInfo targetFunctionInfo = this.GetMappingFunctionInfo(text, out useBrackets);

if (funcMappings != null)
if (targetFunctionInfo.Name.ToLower() !=text.ToLower())
{
string targetFunction = funcMappings.FirstOrDefault(item => item.DbType == targetDbInterpreter.DatabaseType.ToString())?.Function.Split(',')?.FirstOrDefault();
string targetFunction = targetFunctionInfo.Name;

if (!string.IsNullOrEmpty(targetFunction))
{
Expand Down
49 changes: 31 additions & 18 deletions DatabaseConverter/DatabaseConverter.Core/Translator/DbObjectTranslator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,23 +185,28 @@ public string ReplaceVariables(string script)
}

public string ParseFomular(List<FunctionSpecification> sourceFuncSpecs, List<FunctionSpecification> targetFuncSpecs,
FunctionFomular fomular, string targetFunctionName, out Dictionary<string, string> dictDataType)
FunctionFomular fomular, MappingFunctionInfo targetFunctionInfo, out Dictionary<string, string> dictDataType)
{
dictDataType = new Dictionary<string, string>();

string name = fomular.Name;

FunctionSpecification sourceFuncSpec = sourceFuncSpecs.FirstOrDefault(item => item.Name.ToUpper() == name.ToUpper());
FunctionSpecification targetFuncSpec = targetFuncSpecs.FirstOrDefault(item => item.Name.ToUpper() == targetFunctionName.ToUpper());
FunctionSpecification targetFuncSpec = targetFuncSpecs.FirstOrDefault(item => item.Name.ToUpper() == targetFunctionInfo.Name.ToUpper());

string newExpression = fomular.Expression;

if (sourceFuncSpec != null && targetFuncSpec != null)
{
string delimiter = sourceFuncSpec.Delimiter == "," ? "," : $" {sourceFuncSpec.Delimiter} ";
fomular.Delimiter = delimiter;

List<string> fomularArgs = fomular.Args;

Dictionary<int, string> sourceTokens = this.GetFunctionArgumentTokens(sourceFuncSpec, fomularArgs.Count);
Dictionary<int, string> targetTokens = this.GetFunctionArgumentTokens(targetFuncSpec, fomularArgs.Count);
int fetchCount = string.IsNullOrEmpty(targetFunctionInfo.Args) ? fomularArgs.Count : -1;

Dictionary<int, string> sourceTokens = this.GetFunctionArgumentTokens(sourceFuncSpec, null, fetchCount);
Dictionary<int, string> targetTokens = this.GetFunctionArgumentTokens(targetFuncSpec, targetFunctionInfo.Args, fetchCount);

bool ignore = false;

Expand All @@ -212,10 +217,6 @@ public string ParseFomular(List<FunctionSpecification> sourceFuncSpecs, List<Fun

if (!ignore)
{
string delimiter = sourceFuncSpec.Delimiter == "," ? "," : $" {sourceFuncSpec.Delimiter} ";

fomular.Delimiter = delimiter;

List<string> args = new List<string>();

foreach (var kp in targetTokens)
Expand Down Expand Up @@ -252,26 +253,32 @@ public string ParseFomular(List<FunctionSpecification> sourceFuncSpecs, List<Fun
args.Add(newArg);
}
}
else if (!string.IsNullOrEmpty(targetFunctionInfo.Args))
{
args.Add(token);
}
}

string targetDelimiter = targetFuncSpec.Delimiter == "," ? "," : $" {targetFuncSpec.Delimiter} ";

string strArgs = string.Join(targetDelimiter, args);

newExpression = $"{targetFunctionName}{ (targetFuncSpec.NoParenthesess ? "" : $"({strArgs})") }";
newExpression = $"{targetFunctionInfo.Name}{ (targetFuncSpec.NoParenthesess ? "" : $"({strArgs})") }";
}
}

return newExpression;
}

public Dictionary<int, string> GetFunctionArgumentTokens(FunctionSpecification spec, int fetchCount = -1)
public Dictionary<int, string> GetFunctionArgumentTokens(FunctionSpecification spec, string functionArgs, int fetchCount = -1)
{
Dictionary<int, string> dictTokenIndex = new Dictionary<int, string>();

if (!spec.Args.EndsWith("..."))
string specArgs = string.IsNullOrEmpty(functionArgs) ? spec.Args : functionArgs;

if (!specArgs.EndsWith("..."))
{
string str = Regex.Replace(spec.Args, @"[\[\]]", "");
string str = Regex.Replace(specArgs, @"[\[\]]", "");

string[] args = str.Split(new string[] { spec.Delimiter, " " }, StringSplitOptions.RemoveEmptyEntries);

Expand All @@ -288,35 +295,41 @@ public Dictionary<int, string> GetFunctionArgumentTokens(FunctionSpecification s
return dictTokenIndex;
}

public string GetMappedFunctionName(string name)
public MappingFunctionInfo GetMappingFunctionInfo(string name, out bool useBrackets)
{
useBrackets = false;

string text = name;
string textWithBrackets = name.ToLower() + "()";

if (this.functionMappings.Any(item => item.Any(t => t.Function.ToLower() == textWithBrackets)))
{
text = textWithBrackets;
useBrackets = true;
}

string targetFunctionName = name;

MappingFunctionInfo functionInfo = new MappingFunctionInfo() { Name = name };

IEnumerable<FunctionMapping> funcMappings = this.functionMappings.FirstOrDefault(item => item.Any(t =>
(t.Direction == FunctionMappingDirection.OUT || t.Direction == FunctionMappingDirection.INOUT)
&& t.DbType == sourceDbInterpreter.DatabaseType.ToString() && t.Function.Split(',').Any(m => m.ToLower() == text.ToLower())));

if (funcMappings != null)
{
targetFunctionName = funcMappings.FirstOrDefault(item =>
FunctionMapping mapping = funcMappings.FirstOrDefault(item =>
(item.Direction == FunctionMappingDirection.IN || item.Direction == FunctionMappingDirection.INOUT)
&& item.DbType == targetDbInterpreter.DatabaseType.ToString())?.Function.Split(',')?.FirstOrDefault();
&& item.DbType == targetDbInterpreter.DatabaseType.ToString());

if (string.IsNullOrEmpty(targetFunctionName))
if (mapping != null)
{
targetFunctionName = name;
functionInfo.Name = mapping.Function.Split(',')?.FirstOrDefault();
functionInfo.Args = mapping.Args;
}
}

return targetFunctionName;
return functionInfo;
}

public void Subscribe(IObserver<FeedbackInfo> observer)
Expand Down
Loading

0 comments on commit b46fe03

Please sign in to comment.