araxiaonline/MyDBC
3
0
Fork 0
mirror of https://github.com/araxiaonline/MyDBC.git synced 2026-06-13 03:12:30 -04:00
Code Issues Packages Projects Releases Wiki Activity

Initial Commit

Browse Source
This commit is contained in:
barncastle
2021-04-12 17:19:51 +01:00
commit 8db0ff56ef
20 changed files with 1292 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

227
.gitignore vendored Normal file
View File

@@ -0,0 +1,227 @@
# The following command works for downloading when using Git for Windows:
# curl -LOf http://gist.githubusercontent.com/kmorcinek/2710267/raw/.gitignore
#
# Download this file using PowerShell v3 under Windows with the following comand:
# Invoke-WebRequest https://gist.githubusercontent.com/kmorcinek/2710267/raw/ -OutFile .gitignore
#
# or wget:
# wget --no-check-certificate http://gist.githubusercontent.com/kmorcinek/2710267/raw/.gitignore
# User-specific files
*.suo
*.user
*.sln.docstates
# Build results
[Dd]ebug/
[Rr]elease/
x64/
[Bb]in/
[Oo]bj/
# build folder is nowadays used for build scripts and should not be ignored
#build/
# NuGet Packages
*.nupkg
# The packages folder can be ignored because of Package Restore
**/packages/*
# except build/, which is used as an MSBuild target.
!**/packages/build/
# Uncomment if necessary however generally it will be regenerated when needed
#!**/packages/repositories.config
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
*_i.c
*_p.c
*.ilk
*.meta
*.obj
*.pch
*.pdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*.log
*.vspscc
*.vssscc
.builds
*.pidb
*.log
*.scc
# OS generated files #
.DS_Store*
Icon?
# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opensdf
*.sdf
*.cachefile
# Visual Studio profiler
*.psess
*.vsp
*.vspx
# Guidance Automation Toolkit
*.gpState
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
# TeamCity is a build add-in
_TeamCity*
# DotCover is a Code Coverage Tool
*.dotCover
# NCrunch
*.ncrunch*
.*crunch*.local.xml
# Installshield output folder
[Ee]xpress/
# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html
# Click-Once directory
publish/
# Publish Web Output
*.Publish.xml
# Windows Azure Build Output
csx
*.build.csdef
# Windows Store app package directory
AppPackages/
# Others
*.Cache
ClientBin/
[Ss]tyle[Cc]op.*
~$*
*~
*.dbmdl
*.[Pp]ublish.xml
*.pfx
*.publishsettings
modulesbin/
tempbin/
# EPiServer Site file (VPP)
AppData/
# RIA/Silverlight projects
Generated_Code/
# Backup & report files from converting an old project file to a newer
# Visual Studio version. Backup files are not needed, because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
# vim
*.txt~
*.swp
*.swo
# Temp files when opening LibreOffice on ubuntu
.~lock.*
# svn
.svn
# CVS - Source Control
**/CVS/
# Remainings from resolving conflicts in Source Control
*.orig
# SQL Server files
**/App_Data/*.mdf
**/App_Data/*.ldf
**/App_Data/*.sdf
#LightSwitch generated files
GeneratedArtifacts/
_Pvt_Extensions/
ModelManifest.xml
# =========================
# Windows detritus
# =========================
# Windows image file caches
Thumbs.db
ehthumbs.db
# Folder config file
Desktop.ini
# Recycle Bin used on file shares
$RECYCLE.BIN/
# Mac desktop service store files
.DS_Store
# SASS Compiler cache
.sass-cache
# Visual Studio 2014 CTP
**/*.sln.ide
# Visual Studio temp something
.vs/
# dotnet stuff
project.lock.json
# VS 2015+
*.vc.vc.opendb
*.vc.db
# Rider
.idea/
# Visual Studio Code
.vscode/
# Output folder used by Webpack or other FE stuff
**/node_modules/*
**/wwwroot/*
# SpecFlow specific
*.feature.cs
*.feature.xlsx.*
*.Specs_*.html
#####
# End of core ignore list, below put you custom 'per project' settings (patterns or path)
#####
launchSettings.json

3
.gitmodules vendored Normal file
View File

@@ -0,0 +1,3 @@
[submodule "DBCD"]
path = DBCD
url = https://github.com/wowdev/DBCD.git

1
DBCD Submodule

Submodule DBCD added at d0946ff4da

31
MyDBC.sln Normal file
View File

@@ -0,0 +1,31 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.29306.81
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MyDBC", "MyDBC\MyDBC.csproj", "{5FDF204B-7062-482B-830E-5C6CF7A1A9F3}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DBFileReaderLib", "DBCD\DBFileReaderLib\DBFileReaderLib.csproj", "{23A8914D-15D8-4CA6-A905-7966D0320263}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{5FDF204B-7062-482B-830E-5C6CF7A1A9F3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5FDF204B-7062-482B-830E-5C6CF7A1A9F3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5FDF204B-7062-482B-830E-5C6CF7A1A9F3}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5FDF204B-7062-482B-830E-5C6CF7A1A9F3}.Release|Any CPU.Build.0 = Release|Any CPU
{23A8914D-15D8-4CA6-A905-7966D0320263}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{23A8914D-15D8-4CA6-A905-7966D0320263}.Debug|Any CPU.Build.0 = Debug|Any CPU
{23A8914D-15D8-4CA6-A905-7966D0320263}.Release|Any CPU.ActiveCfg = Release|Any CPU
{23A8914D-15D8-4CA6-A905-7966D0320263}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {5E7E29BB-395A-4238-80FB-6AA3271F7267}
EndGlobalSection
EndGlobal

197
MyDBC/BulkLoader.cs Normal file
View File

@@ -0,0 +1,197 @@
using DBFileReaderLib.Attributes;
using MyDBC.Definition;
using MyDBC.SQL;
using MySqlConnector;
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
namespace MyDBC
{
class BulkLoader
{
public IReadOnlyDictionary<string, SqlColumn> Columns => _columns;
public string PrimaryKey { get; private set; }
public (string, string) ForeignKey { get; private set; }
private readonly string _tableName;
private readonly IEnumerator _source;
private readonly Dictionary<string, SqlColumn> _columns;
private readonly Dictionary<string, string> _comments;
public BulkLoader(string tableName, IEnumerable source)
{
_tableName = tableName;
_source = source.GetEnumerator();
_columns = new Dictionary<string, SqlColumn>();
_comments = new Dictionary<string, string>();
}
public async Task ExportToMySql(MySqlConnection connection, string filename)
{
var loader = new MySqlBulkLoader(connection)
{
TableName = _tableName,
FileName = filename,
NumberOfLinesToSkip = 1,
FieldTerminator = ",",
LineTerminator = "\n",
CharacterSet = "utf8",
EscapeCharacter = '\b', // use an impossible value
};
await loader.LoadAsync();
File.Delete(filename);
}
public async Task<string> ExportToFile(string filename = null)
{
filename ??= Path.GetTempFileName();
using var fs = File.CreateText(filename);
fs.NewLine = "\n";
// validate the collecton isn't empty
if (!_source.MoveNext() || _source.Current == null)
{
await fs.DisposeAsync();
File.Delete(filename);
throw new ArgumentException("Source IEnumerable is empty");
}
// generate the serializer
var serializer = GenerateSerializer(_source.Current);
// write header line
await fs.WriteLineAsync(GetHeaderRow());
// serialize the collection
do await fs.WriteLineAsync(serializer(_source.Current));
while (_source.MoveNext() && _source.Current != null);
await fs.FlushAsync();
await fs.DisposeAsync();
return filename;
}
[Obsolete("Broken in the current version of MySqlConnector", true)]
public Stream GetStream()
{
var ms = new MemoryStream(0x10000);
using var sw = new StreamWriter(ms, Encoding.UTF8, 4096, true)
{
NewLine = "\n"
};
// validate the collecton isn't empty
if (!_source.MoveNext() || _source.Current == null)
throw new ArgumentException("Source IEnumerable is empty");
// generate the serializer
var serializer = GenerateSerializer(_source.Current);
// serialize the collection
do sw.WriteLine(serializer(_source.Current));
while (_source.MoveNext() && _source.Current != null);
ms.Position = 0;
return ms;
}
private Func<object, string> GenerateSerializer(object current)
{
var type = current.GetType();
var fields = type.GetFields();
var expressions = new List<Expression>(fields.Length);
var ownerParameter = Expression.Parameter(typeof(object), "o");
foreach (var field in fields)
{
var isPrimaryKey = PrimaryKey == null && field.GetCustomAttribute<IndexAttribute>() != null;
var foreignKeyAttr = field.GetCustomAttribute<ForeignKeyAttribute>();
var commentAttr = field.GetCustomAttribute<CommentAttribute>();
var fieldExpression = Expression.Field(Expression.Convert(ownerParameter, type), field);
var fieldType = field.FieldType;
if (isPrimaryKey)
PrimaryKey = field.Name;
if (field.FieldType.IsArray)
{
fieldType = fieldType.GetElementType();
// explode arrays into seperate fields
var cardinality = ((Array)field.GetValue(current)).Length;
var fieldName = field.Name + (cardinality > 1 ? "_{0}" : "");
for (var i = 0; i < cardinality; i++)
{
expressions.Add(EscapeOrConvert(Expression.ArrayIndex(fieldExpression, Expression.Constant(i))));
_columns[string.Format(fieldName, i + 1)] = new SqlColumn(fieldType, commentAttr?.Comment);
}
}
else
{
expressions.Add(EscapeOrConvert(fieldExpression));
_columns[field.Name] = new SqlColumn(fieldType, commentAttr?.Comment);
if (foreignKeyAttr != null)
ForeignKey = (field.Name, foreignKeyAttr.ToString());
}
}
// finally concat all the values into a single csv line
var joinMethod = typeof(string).GetMethod("Join", new Type[] { typeof(char), typeof(string[]) });
var arrayExpression = Expression.NewArrayInit(typeof(string), expressions);
var concatExpression = Expression.Call(joinMethod, Expression.Constant(','), arrayExpression);
var lambdaExpression = Expression.Lambda<Func<object, string>>(concatExpression, ownerParameter);
return lambdaExpression.Compile();
}
#region Helpers
private string GetHeaderRow()
{
var sb = new StringBuilder(0x100);
foreach (var column in Columns)
sb.Append($"{EscapeStringImpl(column.Key)},");
return sb.ToString();
}
private static Expression EscapeOrConvert(Expression expression)
{
if (expression.Type == typeof(string))
return Expression.Invoke(EscapeString, expression);
else
return Expression.Call(expression, "ToString", null);
}
private static readonly Expression<Func<string, string>> EscapeString = (s) => EscapeStringImpl(s);
private static readonly char[] EscapeChars = new char[] { '"', ',', '\r', '\n' };
private static string EscapeStringImpl(string value)
{
// starts/ends with space or has quoteable char
var shouldQuote = value != "" &&
(
value[0] == ' ' || value[^1] == ' ' || value.IndexOfAny(EscapeChars) > -1
);
return shouldQuote ? '"' + value.Replace("\"", "\"\"") + '"' : value;
}
#endregion
}
}

25
MyDBC/CSV/CSVObjectWriter.cs Normal file
View File

@@ -0,0 +1,25 @@
using MyDBC.Definition;
using System;
using System.Collections;
using System.IO;
using System.Threading.Tasks;
namespace MyDBC.CSV
{
public sealed class CSVObjectWriter
{
public static async Task WriteToFile(Options options, DefinitionBuilder builder, IEnumerable source)
{
var filename = Path.Combine(options.OutputDirectory, builder.Name + ".csv");
var directory = new DirectoryInfo(options.OutputDirectory);
if (!directory.Exists && directory.Parent != null)
directory.Create();
var dbLoader = new BulkLoader(builder.Name, source);
await dbLoader.ExportToFile(filename);
Console.WriteLine($"Exported {builder.Name} to CSV");
}
}
}

78
MyDBC/DBC/GithubDBDProvider.cs Normal file
View File

@@ -0,0 +1,78 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Net.Http;
using System.Net.Http.Json;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
namespace MyDBC.DBC
{
internal class GithubDBDProvider
{
private const string GitTreeApi = "https://api.github.com/repos/wowdev/WoWDBDefs/git/trees/";
private const string BaseAddress = "https://raw.githubusercontent.com/wowdev/WoWDBDefs/master/definitions/";
private readonly HttpClient Client = new();
private readonly IReadOnlyDictionary<string, string> UriLookup;
public GithubDBDProvider()
{
Client.DefaultRequestHeaders.Add("Accept", "application/vnd.github.v3+json");
Client.DefaultRequestHeaders.Add("User-Agent", "Barncastle/MyDBC");
// git is casesensitive and filenames are unreliable
// so generate a lookup of dbd -> raw git url via git's api
UriLookup = TryLoadUriLookup().Result;
}
public async Task<Stream> StreamForTableName(string tableName)
{
if (!UriLookup.TryGetValue(tableName, out var dbdName))
dbdName = $"{BaseAddress}{tableName}.dbd"; // fallback url
var buffer = await Client.GetByteArrayAsync(dbdName);
return new MemoryStream(buffer);
}
private async Task<IReadOnlyDictionary<string, string>> TryLoadUriLookup()
{
var lookup = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
try
{
// find the definition folder sha and the definition file list
var tree = await Client.GetFromJsonAsync<Tree>(GitTreeApi + "master");
var definition = Array.Find(tree.Elements, e => e.Path == "definitions");
var definitionTree = await Client.GetFromJsonAsync<Tree>(definition.Url);
// load all DBD files and their raw urls
foreach (var ele in definitionTree.Elements)
lookup[Path.GetFileNameWithoutExtension(ele.Path)] = $"{BaseAddress}{ele.Path}";
Console.WriteLine("Loaded DBD lookup");
}
catch
{
Console.WriteLine("Unable to load DBD lookup");
}
return lookup;
}
private class Tree
{
[JsonPropertyName("tree")]
public TreeElement[] Elements { get; set; }
public class TreeElement
{
[JsonPropertyName("path")]
public string Path { get; set; }
[JsonPropertyName("url")]
public string Url { get; set; }
}
}
}
}

26
MyDBC/DBC/LocalDBCProvider.cs Normal file
View File

@@ -0,0 +1,26 @@
using System.IO;
using System.Threading.Tasks;
namespace MyDBC.DBC
{
internal class LocalDBCProvider
{
private readonly string Directory;
public LocalDBCProvider(string directory)
{
Directory = directory;
}
public async Task<Stream> StreamForTableName(string tableName)
{
var filename = Path.Combine(Directory, tableName + ".db2");
if (!File.Exists(filename))
filename = Path.ChangeExtension(filename, ".dbc");
if (!File.Exists(filename))
throw new FileNotFoundException($"Unable to load {tableName}");
return await Task.FromResult(File.OpenRead(filename));
}
}
}

11
MyDBC/Definition/CommentAttribute.cs Normal file
View File

@@ -0,0 +1,11 @@
using System;
namespace MyDBC.Definition
{
public class CommentAttribute : Attribute
{
public readonly string Comment;
public CommentAttribute(string comment) => Comment = comment;
}
}

193
MyDBC/Definition/DefinitionBuilder.cs Normal file
View File

@@ -0,0 +1,193 @@
using DBDefsLib;
using DBFileReaderLib;
using DBFileReaderLib.Attributes;
using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection;
using System.Reflection.Emit;
namespace MyDBC.Definition
{
public class DefinitionBuilder
{
private static ModuleBuilder ModuleBuilder;
public readonly string Name;
public readonly string Build;
public readonly Locale Locale;
private int LocStringSize = 1;
public DefinitionBuilder(string name, string build = null, Locale locale = Locale.None)
{
Name = name;
Build = build;
Locale = locale;
if (ModuleBuilder == null)
{
var assemblyName = new AssemblyName("DBCDefinitons");
var assemblyBuilder = AssemblyBuilder.DefineDynamicAssembly(assemblyName, AssemblyBuilderAccess.Run);
ModuleBuilder = assemblyBuilder.DefineDynamicModule(assemblyName.Name);
}
}
public Type Generate(DBReader dbcReader, Stream dbd, Dictionary<string, string> dbs)
{
var dbdReader = new DBDReader();
var databaseDefinition = dbdReader.Read(dbd);
Structs.VersionDefinitions? versionDefinition = null;
if (!string.IsNullOrWhiteSpace(Build))
{
var dbBuild = new Build(Build);
LocStringSize = GetLocStringSize(dbBuild);
Utils.GetVersionDefinitionByBuild(databaseDefinition, dbBuild, out versionDefinition);
}
if (versionDefinition == null && dbcReader.LayoutHash != 0)
{
var layoutHash = dbcReader.LayoutHash.ToString("X8");
Utils.GetVersionDefinitionByLayoutHash(databaseDefinition, layoutHash, out versionDefinition);
}
if (versionDefinition == null)
throw new FileNotFoundException("No definition found for this file.");
if (LocStringSize > 1 && (int)Locale >= LocStringSize)
throw new FormatException("Invalid locale for this file.");
var typeBuilder = ModuleBuilder.DefineType(Name, TypeAttributes.Public);
var fields = versionDefinition.Value.definitions;
var localiseStrings = Locale != Locale.None;
foreach (var fieldDefinition in fields)
{
var columnInfo = databaseDefinition.columnDefinitions[fieldDefinition.name];
var isLocalisedString = columnInfo.type == "locstring" && LocStringSize > 1;
var fieldType = FieldDefinitionToType(fieldDefinition, columnInfo, localiseStrings);
var field = typeBuilder.DefineField(fieldDefinition.name, fieldType, FieldAttributes.Public);
if (fieldDefinition.isID)
AddAttribute<IndexAttribute>(field, fieldDefinition.isNonInline);
if (fieldDefinition.arrLength > 1)
AddAttribute<CardinalityAttribute>(field, fieldDefinition.arrLength);
if (fieldDefinition.isRelation && fieldDefinition.isNonInline)
{
var metaDataFieldType = FieldDefinitionToType(fieldDefinition, columnInfo, localiseStrings);
AddAttribute<NonInlineRelationAttribute>(field, metaDataFieldType);
}
if (isLocalisedString)
{
if (localiseStrings)
{
AddAttribute<LocaleAttribute>(field, (int)Locale, LocStringSize);
}
else
{
AddAttribute<CardinalityAttribute>(field, LocStringSize);
typeBuilder.DefineField(fieldDefinition.name + "_mask", typeof(uint), FieldAttributes.Public);
}
}
// export comments
if (!string.IsNullOrEmpty(columnInfo.comment))
AddAttribute<CommentAttribute>(field, columnInfo.comment);
// only add foreign keys that can be created
if (fieldDefinition.isRelation &&
columnInfo.foreignTable != null &&
dbs.ContainsKey(columnInfo.foreignTable))
AddAttribute<ForeignKeyAttribute>(field, columnInfo.foreignTable, columnInfo.foreignColumn);
}
return typeBuilder.CreateTypeInfo();
}
private static int GetLocStringSize(Build build)
{
if (build.expansion >= 4 || build.build > 12340) // post wotlk
return 1;
else if (build.build >= 6692) // tbc - wotlk
return 16;
else
return 8; // alpha - vanilla
}
private static void AddAttribute<T>(FieldBuilder field, params object[] parameters) where T : Attribute
{
var constructorParameters = Array.ConvertAll(parameters, x => x.GetType());
var constructorInfo = typeof(T).GetConstructor(constructorParameters);
var attributeBuilder = new CustomAttributeBuilder(constructorInfo, parameters);
field.SetCustomAttribute(attributeBuilder);
}
private Type FieldDefinitionToType(Structs.Definition field, Structs.ColumnDefinition column, bool localiseStrings)
{
var isArray = field.arrLength != 0;
if (field.isRelation)
return isArray ? typeof(int[]) : typeof(int);
switch (column.type)
{
case "int":
{
var type = field.size switch
{
8 => field.isSigned ? typeof(sbyte) : typeof(byte),
16 => field.isSigned ? typeof(short) : typeof(ushort),
32 => field.isSigned ? typeof(int) : typeof(uint),
64 => field.isSigned ? typeof(long) : typeof(ulong),
_ => throw new NotImplementedException("Unhandled field size of " + field.size)
};
return isArray ? type.MakeArrayType() : type;
}
case "string":
{
return isArray ? typeof(string[]) : typeof(string);
}
case "locstring":
{
if (isArray && LocStringSize > 1)
throw new NotSupportedException("Localised string arrays are not supported");
return (!localiseStrings && LocStringSize > 1) || isArray ? typeof(string[]) : typeof(string);
}
case "float":
{
return isArray ? typeof(float[]) : typeof(float);
}
default:
throw new ArgumentException("Unable to construct C# type from " + column.type);
}
}
}
public enum Locale
{
None = -1,
EnUS = 0,
EnGB = EnUS,
KoKR = 1,
FrFR = 2,
DeDE = 3,
EnCN = 4,
ZhCN = EnCN,
EnTW = 5,
ZhTW = EnTW,
EsES = 6,
EsMX = 7,
/* Available from TBC 2.1.0.6692 */
RuRU = 8,
PtPT = 10,
PtBR = PtPT,
ItIT = 11,
}
}

21
MyDBC/Definition/ForeignKeyAttribute.cs Normal file
View File

@@ -0,0 +1,21 @@
using System;
namespace MyDBC.Definition
{
public class ForeignKeyAttribute : Attribute
{
public readonly string Table;
public readonly string Column;
public ForeignKeyAttribute(string table, string column)
{
Table = table;
Column = column;
}
public override string ToString()
{
return $"`{Table}` (`{Column}`)";
}
}
}

21
MyDBC/Helpers/Extensions.cs Normal file
View File

@@ -0,0 +1,21 @@
using DBFileReaderLib;
using System;
using System.Collections;
namespace MyDBC.Helpers
{
internal static class Extensions
{
public static IDictionary GetRecords(this DBReader reader, Type type)
{
var methodInfo = typeof(DBReader).GetMethod("GetRecords");
var generic = methodInfo.MakeGenericMethod(type);
return (IDictionary)generic.Invoke(reader, null);
}
public static string Sql(this string value)
{
return value.Replace("'", "''");
}
}
}

25
MyDBC/Helpers/Utils.cs Normal file
View File

@@ -0,0 +1,25 @@
using System.Collections.Generic;
using System.Globalization;
using System.IO;
namespace MyDBC.Helpers
{
internal static class Utils
{
public static void OverrideNumberFormat()
{
var culture = (CultureInfo)CultureInfo.CurrentCulture.Clone();
culture.NumberFormat = new CultureInfo("en-US").NumberFormat;
CultureInfo.CurrentCulture = culture;
}
public static IEnumerable<string> GetFiles(string path, string searchPattern, SearchOption searchOption = default)
{
var files = new List<string>();
foreach (var sp in searchPattern.Split('|'))
files.AddRange(Directory.GetFiles(path, sp, searchOption));
return files;
}
}
}

18
MyDBC/MyDBC.csproj Normal file
View File

@@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net5.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="CommandLineParser" Version="2.8.0" />
<PackageReference Include="DBDefsLib" Version="1.0.0.20" />
<PackageReference Include="MySqlConnector" Version="1.3.2" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\DBCD\DBFileReaderLib\DBFileReaderLib.csproj" />
</ItemGroup>
</Project>

64
MyDBC/Options.cs Normal file
View File

@@ -0,0 +1,64 @@
using CommandLine;
using System;
namespace MyDBC
{
public class Options
{
public ExportType ExportType { get; private set; }
[Option('d', "directory", HelpText = "Location of the DB files")]
public string Directory { get; set; } = System.IO.Directory.GetCurrentDirectory();
[Option('b', "build", HelpText = "Client build number if targeting pre-Legion e.g. 0.5.3.3368")]
public string Build { get; set; }
#region SQL Options
[Option('c', "connection", HelpText = "SQL connection string")]
public string ConnectionString { get; set; }
[Option("drop", HelpText = "Drops existing tables if conflicts occur")]
public bool DropAndCreate { get; set; }
[Option("fk", HelpText = "Exports Relationship fields as foreign keys")]
public bool ExportForeignKeys { get; set; }
#endregion
#region CSV Options
[Option('o', "output", HelpText = "CSV export directory")]
public string OutputDirectory { get; set; }
#endregion
public void Validate()
{
if (!System.IO.Directory.Exists(Directory))
throw new ArgumentException("Directory not found", nameof(Directory));
if (!string.IsNullOrWhiteSpace(ConnectionString))
{
ExportType |= ExportType.SQL;
// append file import conn string setting
if (!ConnectionString.Contains("AllowLoadLocalInfile", StringComparison.OrdinalIgnoreCase))
ConnectionString += "AllowLoadLocalInfile=true;";
}
if(!string.IsNullOrWhiteSpace(OutputDirectory))
ExportType |= ExportType.CSV;
if (ExportType == 0)
throw new ArgumentException("Either ConnectionString or FileName must be set");
}
}
[Flags]
public enum ExportType
{
SQL = 1,
CSV = 2
}
}

76
MyDBC/Program.cs Normal file
View File

@@ -0,0 +1,76 @@
using CommandLine;
using DBFileReaderLib;
using MyDBC.CSV;
using MyDBC.DBC;
using MyDBC.Definition;
using MyDBC.Helpers;
using MyDBC.SQL;
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
namespace MyDBC
{
class Program
{
private static async Task Main(string[] args)
{
Utils.OverrideNumberFormat();
using var parser = new Parser(s =>
{
s.HelpWriter = Console.Error;
s.CaseInsensitiveEnumValues = true;
s.AutoVersion = false;
});
await parser
.ParseArguments<Options>(args)
.MapResult(Run, Task.FromResult);
Console.WriteLine("Press any key to exit...");
Console.ReadKey();
}
private static async Task Run(Options options)
{
options.Validate();
var dbdProvider = new GithubDBDProvider();
var dbcProvider = new LocalDBCProvider(options.Directory);
var dbs = Utils.GetFiles(options.Directory, "*.db2|*.dbc")
.ToDictionary(Path.GetFileNameWithoutExtension);
foreach (var db in dbs)
{
var dbcStream = await dbcProvider.StreamForTableName(db.Key);
var dbdStream = await dbdProvider.StreamForTableName(db.Key);
var dbReader = new DBReader(dbcStream);
var builder = new DefinitionBuilder(db.Key, options.Build);
var definition = builder.Generate(dbReader, dbdStream, dbs);
var storage = dbReader.GetRecords(definition);
if (storage.Count == 0)
{
Console.WriteLine($"Skipping {db.Key} - empty");
continue;
}
if (options.ExportType.HasFlag(ExportType.SQL))
await DBObjectWriter.WriteToServer(options, builder, storage.Values);
if (options.ExportType.HasFlag(ExportType.CSV))
await CSVObjectWriter.WriteToFile(options, builder, storage.Values);
}
// append foreign keys to the database
if (options.ExportType.HasFlag(ExportType.SQL) && options.ExportForeignKeys)
{
Console.WriteLine("Generating foreign keys");
await DBObjectWriter.WriteForeignKeys(options);
}
}
}
}

93
MyDBC/SQL/DBObjectReader.cs Normal file
View File

@@ -0,0 +1,93 @@
using MySqlConnector;
using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using System.Linq.Expressions;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace MyDBC.SQL
{
public sealed class DBObjectReader
{
public static async Task<List<T>> ReadFromServer<T>(string connectionString, string table)
{
var resultSet = new List<T>(0x10000);
using var conn = new MySqlConnection(connectionString);
if (conn.State != ConnectionState.Open)
await conn.OpenAsync();
using var cmd = new MySqlCommand($"SELECT * FROM `{table}`", conn);
using var rdr = await cmd.ExecuteReaderAsync();
var serializer = GenerateSerializer<T>(rdr);
while (rdr.HasRows && await rdr.ReadAsync())
resultSet.Add(serializer(rdr));
return resultSet;
}
private static Func<MySqlDataReader, T> GenerateSerializer<T>(MySqlDataReader reader)
{
var type = typeof(T);
var fields = type.GetFields();
var expressions = new List<MemberAssignment>(fields.Length);
var readerParameter = Expression.Parameter(typeof(MySqlDataReader), "r");
var ownerParameter = Expression.Parameter(typeof(T), "o");
// calls the MySqlDataReader.GetFieldValue method
Expression ReadValue(Type t, int index) => Expression.Call(readerParameter, "GetFieldValue", new[] { t }, Expression.Constant(index));
foreach (var column in GetSchema(reader))
{
var field = fields.Single(x => x.Name.Equals(column.Key, StringComparison.OrdinalIgnoreCase));
var fieldType = field.FieldType;
if (field.FieldType.IsArray)
{
fieldType = fieldType.GetElementType();
// store a GetFieldValue call for each array item
var readExpressions = new Expression[column.Value.Length];
for (int i = 0; i < readExpressions.Length; i++)
readExpressions[i] = ReadValue(fieldType, column.Value[i]);
// create and bind a new array from the above
expressions.Add(Expression.Bind(field, Expression.NewArrayInit(fieldType, readExpressions)));
}
else
{
// bind a direct GetFieldValue call
expressions.Add(Expression.Bind(field, ReadValue(fieldType, column.Value[0])));
}
}
// create a new T: MemberInit can always be reduced
var initExpression = Expression.MemberInit(Expression.New(typeof(T)), expressions).Reduce();
var lambdaExpression = Expression.Lambda<Func<MySqlDataReader, T>>(initExpression, readerParameter);
return lambdaExpression.Compile();
}
/// <summary>
/// Creates a schema map that maps indicies to fields
/// </summary>
/// <param name="reader"></param>
/// <returns></returns>
private static Dictionary<string, int[]> GetSchema(MySqlDataReader reader)
{
var ordinalRemover = new Regex("(_\\d+$)", RegexOptions.Compiled);
return Enumerable.Range(0, reader.FieldCount)
.Select(i => (Name: reader.GetName(i), Ordinal: i))
.OrderBy(n => n.Name)
.GroupBy(n => ordinalRemover.Replace(n.Name, ""))
.ToDictionary(n => n.Key, n => n.Select(x => x.Ordinal).ToArray());
}
}
}

128
MyDBC/SQL/DBObjectWriter.cs Normal file
View File

@@ -0,0 +1,128 @@
using MyDBC.Definition;
using MyDBC.Helpers;
using MySqlConnector;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Data;
using System.Text;
using System.Threading.Tasks;
namespace MyDBC.SQL
{
public sealed class DBObjectWriter
{
private static readonly Dictionary<string, (string Col, string Ref)> ForeignKeys = new();
public static async Task WriteToServer(Options options, DefinitionBuilder builder, IEnumerable source)
{
using var conn = new MySqlConnection(options.ConnectionString);
var dbLoader = new BulkLoader(builder.Name, source);
var fileName = await dbLoader.ExportToFile();
if (conn.State != ConnectionState.Open)
await conn.OpenAsync();
// create the table
var createTableQuery = GetCreateTableStatement(dbLoader, builder, options.DropAndCreate);
using var cmd = new MySqlCommand(createTableQuery, conn);
await cmd.ExecuteNonQueryAsync();
// bulk insert
await dbLoader.ExportToMySql(conn, fileName);
await conn.CloseAsync();
if (dbLoader.ForeignKey != default)
ForeignKeys.Add(builder.Name, dbLoader.ForeignKey);
Console.WriteLine($"Exported {builder.Name} to SQL");
}
public static async Task WriteForeignKeys(Options options)
{
if (ForeignKeys.Count == 0)
return;
using var conn = new MySqlConnection(options.ConnectionString);
if (conn.State != ConnectionState.Open)
await conn.OpenAsync();
// alter the tables
var alterTableQuery = GetAlterTableStatement();
using var cmd = new MySqlCommand(alterTableQuery, conn);
await cmd.ExecuteNonQueryAsync();
await conn.CloseAsync();
}
private static string GetCreateTableStatement(BulkLoader loader, DefinitionBuilder builder, bool dropAndCreate)
{
var sb = new StringBuilder(0x200);
// append drop if applicable
if (dropAndCreate)
sb.AppendLine($"DROP TABLE IF EXISTS `{builder.Name}`;");
// build table structure
sb.Append($"CREATE TABLE `{builder.Name}` (");
foreach (var column in loader.Columns)
{
if (!DataTypeMap.TryGetValue(column.Value.Type, out string dataType))
throw new ArgumentException($"Unable to map {column.Value} to a MySQL type");
// append this field
sb.Append($" `{column.Key}` {dataType} NOT NULL,");
// insert default field value
if (column.Key == loader.PrimaryKey)
sb.Insert(sb.Length - 1, " AUTO_INCREMENT"); // PK
else if (dataType != "TEXT")
sb.Insert(sb.Length - 1, " DEFAULT '0'"); // numeric
if (!string.IsNullOrWhiteSpace(column.Value.Comment))
sb.Insert(sb.Length - 1, $" COMMENT '{column.Value.Comment.Sql()}'");
}
// remove the trailing comma
sb[^1] = ' ';
// append primary key
if (!string.IsNullOrEmpty(loader.PrimaryKey))
sb.AppendLine($", PRIMARY KEY ({loader.PrimaryKey})");
// add engine and charset
sb.AppendLine(") ENGINE=InnoDB DEFAULT CHARSET=utf8;");
return sb.ToString();
}
private static string GetAlterTableStatement()
{
var sb = new StringBuilder(0x200);
foreach (var table in ForeignKeys)
{
sb.Append($"ALTER TABLE `{table.Key}`");
sb.AppendLine($"ADD FOREIGN KEY (`{table.Value.Col}`) REFERENCES {table.Value.Ref};");
}
return sb.ToString();
}
private static readonly Dictionary<Type, string> DataTypeMap = new()
{
[typeof(ulong)] = "BIGINT UNSIGNED",
[typeof(long)] = "BIGINT",
[typeof(float)] = "DOUBLE",
[typeof(int)] = "INT",
[typeof(uint)] = "INT UNSIGNED",
[typeof(short)] = "SMALLINT",
[typeof(ushort)] = "SMALLINT UNSIGNED",
[typeof(sbyte)] = "TINYINT",
[typeof(byte)] = "TINYINT UNSIGNED",
[typeof(string)] = "TEXT",
};
}
}

16
MyDBC/SQL/SqlColumn.cs Normal file
View File

@@ -0,0 +1,16 @@
using System;
namespace MyDBC.SQL
{
internal struct SqlColumn
{
public Type Type;
public string Comment;
public SqlColumn(Type type, string comment)
{
Type = type;
Comment = comment;
}
}
}

38
README.md Normal file
View File

@@ -0,0 +1,38 @@
# MyDBC
A command line tool for exporting World of Warcraft's various DB files to SQL and/or CSV. This tool is powered by [DBCD](https://github.com/wowdev/DBCD) and [WoWDBDefs](https://github.com/wowdev/WoWDBDefs) so supports all DB formats which are almost all named.
#### Project Prerequisites
- .Net Core 5
#### Arguments
| Long Name | Short Name | Description |
| ------- | :---- | ----- |
| --directory | --d | Directory containing DB files, defaults to the current one |
| --build | --b | Client build string e.g. "0.5.3.3368" (see notes) |
| --connection | --c | SQL connection string for SQL exports |
| --output | --o | Output directory for CSV exports |
| --drop | | Drops and recreates tables (SQL) |
| --fk | | Exports [Relationship](https://github.com/wowdev/WoWDBDefs#column-annotations) fields as foreign keys (SQL) |
| --help | | Shows this table |
#### Usage
Exporting the current directory to SQL with foreign keys and table drop:
`MyDBC.exe --c "Server=localhost;Database=test;Uid=root;Pwd=;" --drop true --fk=true`
Exporting the current directory to CSV:
`MyDBC.exe --o "D:\Test"`
#### Notes
- All tables and CSV files will be named as per their source filename.
- `--connection` and `--output` can be used simultaneously.
- `--build` is required for all DBs before Legion so that DBCD can load the correct structure.
- The tool uses MySQL's `LOAD DATA` command which by default, appends to an existing table. You will need to use the `--drop` argument if this is not desired.
- Unfortunately WoWDBDef "foreign keys" are not supported, only "relations", due to them not lending themselves well to MySQL's optional foreign key constraints. In short; WoW uses `0` whereas MySQL uses `NULL` to dictate a missing reference.