Xpl0itR
/
protodec
mirror of https://github.com/Xpl0itR/protodec.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

mission complete

This commit is contained in:
Xpl0itR 2024-10-08 23:54:00 +01:00
parent b3ff1fe608
commit 8f22ee243e
13 changed files with 121 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -22,7 +22,7 @@ Options:
Commands:
il2cpp Use LibCpp2IL backend to directly load Il2Cpp compiled game assembly. EXPERIMENTAL.
lua Use Lua AST backend to load Lua source files.
lua Use Loretta backend to load Lua source files.
```
See per-command help message for more info.

3
src/LibProtodec/LibProtodec.csproj
View File

@ -18,12 +18,13 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Google.Protobuf" Version="3.28.2" />
<PackageReference Include="Loretta.CodeAnalysis.Lua" Version="0.2.12" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.0-preview.6.24327.7" />
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="8.0.0" PrivateAssets="All" />
<PackageReference Include="Samboy063.LibCpp2IL" Version="2022.1.0-development.1027" />
<PackageReference Include="System.Reflection.MetadataLoadContext" Version="9.0.0-preview.6.24327.7" />
<PackageReference Include="Xpl0itR.SystemEx" Version="1.2.0" />
<PackageReference Include="Xpl0itR.SystemEx" Version="1.2.1" />
</ItemGroup>
</Project>

16
src/LibProtodec/Loaders/LuaSourceLoader.cs
View File

@ -16,19 +16,27 @@ namespace LibProtodec.Loaders;
public sealed class LuaSourceLoader
{
public IReadOnlyList<SyntaxTree> LoadedSyntaxTrees { get; }
public IReadOnlyDictionary<string, SyntaxTree> LoadedSyntaxTrees { get; }
public LuaSourceLoader(string sourcePath, ILogger<LuaSourceLoader>? logger = null)
{
LoadedSyntaxTrees = File.Exists(sourcePath)
? [LoadSyntaxTreeFromSourceFile(sourcePath)]
? new Dictionary<string, SyntaxTree> { { Path.GetFileNameWithoutExtension(sourcePath) , LoadSyntaxTreeFromSourceFile(sourcePath) } }
: Directory.EnumerateFiles(sourcePath, searchPattern: "*.lua")
.Select(LoadSyntaxTreeFromSourceFile)
.ToList();
.Select(static sourcePath =>
(Path.GetFileNameWithoutExtension(sourcePath), LoadSyntaxTreeFromSourceFile(sourcePath)))
.ToDictionary();
logger?.LogLoadedLuaSyntaxTrees(LoadedSyntaxTrees.Count);
}
public SyntaxTree ResolveImport(string import)
{
import = Path.GetFileNameWithoutExtension(import);
return LoadedSyntaxTrees[import];
}
private static SyntaxTree LoadSyntaxTreeFromSourceFile(string filePath)
{
using FileStream fileStream = File.OpenRead(filePath);

11
src/LibProtodec/Models/Protobuf/Fields/MessageField.cs
View File

@ -13,7 +13,9 @@ namespace LibProtodec.Models.Protobuf.Fields;
public sealed class MessageField
{
public required IProtobufType Type { get; init; }
private bool? _isRepeated;
public IProtobufType? Type { get; set; }
public Message? DeclaringMessage { get; set; }
public string? Name { get; set; }
@ -23,6 +25,11 @@ public sealed class MessageField
public bool IsRequired { get; set; }
public bool IsObsolete { get; init; }
public bool HasHasProp { get; init; }
public bool IsRepeated
{
get => _isRepeated ??= Type is Repeated;
set => _isRepeated = value;
}
public void WriteTo(TextWriter writer, bool isOneOf)
{
@ -30,7 +37,7 @@ public sealed class MessageField
Guard.IsNotNull(Name);
Guard.IsNotNull(DeclaringMessage);
if (IsOptional || (HasHasProp && !isOneOf && Type is not Repeated))
if (IsOptional || (HasHasProp && !isOneOf && !IsRepeated))
{
writer.Write("optional ");
}

1
src/LibProtodec/Models/Protobuf/Fields/ServiceMethod.cs
View File

@ -50,6 +50,7 @@ public sealed class ServiceMethod(Service declaringService)
writer.Indent++;
Protobuf.WriteOptionTo(writer, "deprecated", "true");
writer.WriteLine();
writer.Indent--;
writer.WriteLine('}');

9
src/LibProtodec/Models/Protobuf/Protobuf.cs
View File

@ -69,24 +69,25 @@ public sealed class Protobuf
if (_imports is not null)
{
writer.WriteLine();
foreach (string import in _imports)
{
writer.WriteLine();
writer.Write("import \"");
writer.Write(import);
writer.WriteLine("\";");
writer.Write("\";");
}
}
if (CilNamespace is not null)
{
writer.WriteLine();
writer.WriteLine();
WriteOptionTo(writer, "csharp_namespace", CilNamespace, true);
}
foreach (TopLevel topLevel in TopLevels)
{
writer.WriteLine();
writer.WriteLine();
topLevel.WriteTo(writer);
}
@ -110,6 +111,6 @@ public sealed class Protobuf
writer.Write('\"');
}
writer.WriteLine(';');
writer.Write(';');
}
}

3
src/LibProtodec/Models/Protobuf/TopLevels/Enum.cs
View File

@ -26,16 +26,19 @@ public sealed class Enum : TopLevel, INestableType
if (ContainsDuplicateFieldId)
{
Protobuf.WriteOptionTo(writer, "allow_alias", "true");
writer.WriteLine();
}
if (this.IsObsolete)
{
Protobuf.WriteOptionTo(writer, "deprecated", "true");
writer.WriteLine();
}
if (IsClosed)
{
Protobuf.WriteOptionTo(writer, "features.enum_type", "CLOSED");
writer.WriteLine();
}
foreach (EnumField field in Fields)

1
src/LibProtodec/Models/Protobuf/TopLevels/Message.cs
View File

@ -28,6 +28,7 @@ public sealed class Message : TopLevel, INestableType
if (this.IsObsolete)
{
Protobuf.WriteOptionTo(writer, "deprecated", "true");
writer.WriteLine();
}
int[] oneOfs = OneOfs.SelectMany(static oneOf => oneOf.Value).ToArray();

1
src/LibProtodec/Models/Protobuf/TopLevels/Service.cs
View File

@ -24,6 +24,7 @@ public sealed class Service : TopLevel
if (this.IsObsolete)
{
Protobuf.WriteOptionTo(writer, "deprecated", "true");
writer.WriteLine();
}
foreach (ServiceMethod method in Methods)

54
src/LibProtodec/Models/Protobuf/Types/Descriptor.cs
View File

@ -1,54 +0,0 @@
// Copyright © 2024 Xpl0itR
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
using CommunityToolkit.Diagnostics;
using LibProtodec.Models.Protobuf.TopLevels;
namespace LibProtodec.Models.Protobuf.Types;
public sealed class Descriptor : IProtobufType
{
public int TypeIndex { get; set; }
public bool IsRepeated { get; set; }
public /*TopLevel*/IProtobufType? TopLevelType { get; set; }
public string Name =>
Type.Name;
public IProtobufType Type
{
get
{
IProtobufType type = TopLevelType as IProtobufType ?? TypeIndex switch
{
1 => Scalar.Double,
2 => Scalar.Float,
3 => Scalar.Int64,
4 => Scalar.UInt64,
5 => Scalar.Int32,
6 => Scalar.Fixed64,
7 => Scalar.Fixed32,
8 => Scalar.Bool,
9 => Scalar.String,
10 => ThrowHelper.ThrowNotSupportedException<IProtobufType>("Parsing proto2 groups are not supported. Open an issue if you need this."),
12 => Scalar.Bytes,
13 => Scalar.UInt32,
15 => Scalar.SFixed32,
16 => Scalar.SFixed64,
17 => Scalar.SInt32,
18 => Scalar.SInt64,
_ => ThrowHelper.ThrowArgumentOutOfRangeException<IProtobufType>()
};
if (IsRepeated)
{
type = new Repeated(type);
}
return type;
}
}
}

100
src/LibProtodec/ProtodecContext.Lua.cs
View File

@ -5,28 +5,39 @@
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
using System.Collections.Generic;
using System.IO;
using System.Linq;
using SystemEx;
using CommunityToolkit.Diagnostics;
using LibProtodec.Loaders;
using LibProtodec.Models.Protobuf;
using LibProtodec.Models.Protobuf.Fields;
using LibProtodec.Models.Protobuf.TopLevels;
using LibProtodec.Models.Protobuf.Types;
using Loretta.CodeAnalysis;
using Loretta.CodeAnalysis.Lua.Syntax;
using SystemEx;
using FdpTypes = Google.Protobuf.Reflection.FieldDescriptorProto.Types;
namespace LibProtodec;
// TODO: add debug logging
partial class ProtodecContext
{
public Protobuf ParseLuaSyntaxTree(SyntaxTree ast)
private readonly Dictionary<string, Dictionary<string, object>> _parsedPbTables = [];
private readonly Dictionary<string, Protobuf> _parsedProtobufs = [];
public Protobuf ParseLuaSyntaxTree(LuaSourceLoader loader, SyntaxTree ast, ParserOptions options = ParserOptions.None)
{
if (_parsedProtobufs.TryGetValue(ast.FilePath, out Protobuf? parsedProto))
{
return parsedProto;
}
CompilationUnitSyntax root = (CompilationUnitSyntax)ast.GetRoot();
SyntaxList<StatementSyntax> statements = root.Statements.Statements;
bool importedProtobufLib = false;
Dictionary<string, object> pbTable = [];
Dictionary<string, object> pbTable = _parsedPbTables[ast.FilePath] = [];
Dictionary<string, Dictionary<string, object>> imports = [];
Protobuf protobuf = new()
{
Version = 2,
@ -39,6 +50,7 @@ partial class ProtodecContext
{
case LocalVariableDeclarationStatementSyntax
{
Names: [{ IdentifierName.Name: { } importKey }],
EqualsValues.Values: [FunctionCallExpressionSyntax { Expression: IdentifierNameSyntax { Name: "require" } } call]
}:
switch (call.Argument)
@ -46,9 +58,11 @@ partial class ProtodecContext
case StringFunctionArgumentSyntax { Expression.Token.ValueText: "protobuf/protobuf" }:
importedProtobufLib = true;
break;
case ExpressionListFunctionArgumentSyntax { Expressions: [LiteralExpressionSyntax { Token.ValueText: {} import }] }:
import = Path.GetFileNameWithoutExtension(import).TrimEnd("_pb"); //todo: handle imports properly
protobuf.Imports.Add($"{import}.proto");
case ExpressionListFunctionArgumentSyntax { Expressions: [LiteralExpressionSyntax { Token.ValueText: { } import }] }:
SyntaxTree importedAst = loader.ResolveImport(import);
Protobuf importedProto = ParseLuaSyntaxTree(loader, importedAst);
imports.Add(importKey, _parsedPbTables[importedAst.FilePath]);
protobuf.Imports.Add(importedProto.FileName);
break;
}
break;
@ -57,9 +71,9 @@ partial class ProtodecContext
Variables: [MemberAccessExpressionSyntax
{
Expression: IdentifierNameSyntax,
MemberName.ValueText: {} tableKey
MemberName.ValueText: { } tableKey
}],
EqualsValues.Values: [FunctionCallExpressionSyntax { Expression: MemberAccessExpressionSyntax { MemberName.ValueText: {} factory } }]
EqualsValues.Values: [FunctionCallExpressionSyntax { Expression: MemberAccessExpressionSyntax { MemberName.ValueText: { } factory } }]
}:
switch (factory)
{
@ -82,7 +96,7 @@ partial class ProtodecContext
pbTable.Add(tableKey, @enum);
break;
case "FieldDescriptor":
pbTable.Add(tableKey, new MessageField { Type = new Descriptor() });
pbTable.Add(tableKey, new MessageField());
break;
case "EnumValueDescriptor":
pbTable.Add(tableKey, new EnumField());
@ -93,7 +107,7 @@ partial class ProtodecContext
{
Variables: [MemberAccessExpressionSyntax
{
Expression: MemberAccessExpressionSyntax { MemberName.ValueText: {} tableKey },
Expression: MemberAccessExpressionSyntax { MemberName.ValueText: { } tableKey },
MemberName.ValueText: { } memberName
}],
EqualsValues.Values: [{ } valueExpr]
@ -103,7 +117,6 @@ partial class ProtodecContext
.Cast<UnkeyedTableFieldSyntax>()
.Select(static element => element.Value);
object? valueLiteral = (valueExpr as LiteralExpressionSyntax)?.Token.Value;
switch (pbTable[tableKey])
{
case Message message:
@ -152,7 +165,6 @@ partial class ProtodecContext
}
break;
case MessageField messageField:
Descriptor descriptor = (Descriptor)messageField.Type;
switch (memberName)
{
case "name":
@ -162,29 +174,36 @@ partial class ProtodecContext
messageField.Id = (int)(double)valueLiteral!;
break;
case "label":
switch ((int)(double)valueLiteral!)
switch ((FdpTypes.Label)(double)valueLiteral!)
{
case 1:
case FdpTypes.Label.Optional:
messageField.IsOptional = true;
break;
case 2:
case FdpTypes.Label.Required:
messageField.IsRequired = true;
break;
case 3:
descriptor.IsRepeated = true;
case FdpTypes.Label.Repeated:
messageField.IsRepeated = true;
break;
}
break;
case "type":
descriptor.TypeIndex = (int)(double)valueLiteral!;
case "enum_type" when (options & ParserOptions.SkipEnums) > 0:
messageField.Type = Scalar.Int32;
break;
case "message_type":
case "enum_type":
string typeTableKey = ((MemberAccessExpressionSyntax)valueExpr).MemberName.ValueText;
if (pbTable.TryGetValue(typeTableKey, out object? topLevel)) //TODO: if this is false, then the top level is from an import, we will need to handle this
descriptor.TopLevelType = (IProtobufType)topLevel;
else
descriptor.TopLevelType = new Scalar(typeTableKey); // temporary hack
case "message_type":
MemberAccessExpressionSyntax memberAccessExpr = (MemberAccessExpressionSyntax)valueExpr;
string importKey = ((IdentifierNameSyntax)memberAccessExpr.Expression).Name;
Dictionary<string, object> table = imports.GetValueOrDefault(importKey, pbTable);
string typeTableKey = memberAccessExpr.MemberName.ValueText;
IProtobufType scalar = (IProtobufType)table[typeTableKey];
messageField.Type = messageField.IsRepeated
? new Repeated(scalar)
: scalar;
break;
case "type":
messageField.Type ??= ParseFieldType(
(FdpTypes.Type)(double)valueLiteral!, messageField.IsRepeated);
break;
case "has_default_value":
if ((bool)valueLiteral!)
@ -220,6 +239,35 @@ partial class ProtodecContext
}
this.Protobufs.Add(protobuf);
_parsedProtobufs.Add(ast.FilePath, protobuf);
return protobuf;
}
protected static IProtobufType ParseFieldType(FdpTypes.Type type, bool isRepeated)
{
IProtobufType scalar = type switch
{
FdpTypes.Type.Double => Scalar.Double,
FdpTypes.Type.Float => Scalar.Float,
FdpTypes.Type.Int64 => Scalar.Int64,
FdpTypes.Type.Uint64 => Scalar.UInt64,
FdpTypes.Type.Int32 => Scalar.Int32,
FdpTypes.Type.Fixed64 => Scalar.Fixed64,
FdpTypes.Type.Fixed32 => Scalar.Fixed32,
FdpTypes.Type.Bool => Scalar.Bool,
FdpTypes.Type.String => Scalar.String,
FdpTypes.Type.Bytes => Scalar.Bytes,
FdpTypes.Type.Uint32 => Scalar.UInt32,
FdpTypes.Type.Sfixed32 => Scalar.SFixed32,
FdpTypes.Type.Sfixed64 => Scalar.SFixed64,
FdpTypes.Type.Sint32 => Scalar.SInt32,
FdpTypes.Type.Sint64 => Scalar.SInt64,
FdpTypes.Type.Group => ThrowHelper.ThrowNotSupportedException<IProtobufType>(
"Parsing proto2 groups are not supported. Open an issue if you need this."),
};
return isRepeated
? new Repeated(scalar)
: scalar;
}
}

7
src/LibProtodec/ProtodecContext.cs
View File

@ -36,14 +36,13 @@ public partial class ProtodecContext
public void WriteAllTo(IndentedTextWriter writer)
{
writer.WriteLine("// Decompiled with protodec");
writer.WriteLine();
writer.Write("// Decompiled with protodec");
foreach (TopLevel topLevel in Protobufs.SelectMany(static proto => proto.TopLevels))
{
writer.WriteLine();
writer.WriteLine();
topLevel.WriteTo(writer);
writer.WriteLine();
writer.WriteLine();
}
}

12
src/protodec/Program.cs
View File

@ -117,17 +117,23 @@ internal sealed class Commands
}
/// <summary>
/// Use Lua AST backend to load Lua source files.
/// Use Loretta backend to load Lua source files.
/// </summary>
/// <param name="targetPath">Either the path to the target lua file or a directory of lua files, all of which be parsed.</param>
/// <param name="outPath">An existing directory to output into individual files, otherwise output to a single file.</param>
/// <param name="skipEnums">Skip parsing enums and replace references to them with int32.</param>
/// <param name="logLevel">Logging severity level.</param>
[Command("lua")]
public void Lua(
[Argument] string targetPath,
[Argument] string outPath,
bool skipEnums,
LogLevel logLevel = LogLevel.Information)
{
ParserOptions options = ParserOptions.None;
if (skipEnums)
options |= ParserOptions.SkipEnums;
using ILoggerFactory loggerFactory = CreateLoggerFactory(logLevel);
ILogger logger = CreateProtodecLogger(loggerFactory);
@ -140,9 +146,9 @@ internal sealed class Commands
};
logger.LogInformation("Parsing Lua syntax trees...");
foreach (SyntaxTree ast in loader.LoadedSyntaxTrees)
foreach (SyntaxTree ast in loader.LoadedSyntaxTrees.Values)
{
ctx.ParseLuaSyntaxTree(ast);
ctx.ParseLuaSyntaxTree(loader, ast, options);
}
// NOTE: I'm duplicating this code rather than refactoring because I have a lot of uncommited changes on another computer,