From 8f22ee243e3d21d44b9b0dd4ba79a94933053667 Mon Sep 17 00:00:00 2001 From: Xpl0itR Date: Tue, 8 Oct 2024 23:54:00 +0100 Subject: [PATCH] mission complete --- README.md | 2 +- src/LibProtodec/LibProtodec.csproj | 3 +- src/LibProtodec/Loaders/LuaSourceLoader.cs | 16 ++- .../Models/Protobuf/Fields/MessageField.cs | 11 +- .../Models/Protobuf/Fields/ServiceMethod.cs | 1 + src/LibProtodec/Models/Protobuf/Protobuf.cs | 9 +- .../Models/Protobuf/TopLevels/Enum.cs | 3 + .../Models/Protobuf/TopLevels/Message.cs | 1 + .../Models/Protobuf/TopLevels/Service.cs | 1 + .../Models/Protobuf/Types/Descriptor.cs | 54 ---------- src/LibProtodec/ProtodecContext.Lua.cs | 100 +++++++++++++----- src/LibProtodec/ProtodecContext.cs | 7 +- src/protodec/Program.cs | 12 ++- 13 files changed, 121 insertions(+), 99 deletions(-) delete mode 100644 src/LibProtodec/Models/Protobuf/Types/Descriptor.cs diff --git a/README.md b/README.md index b7f032a..7e3e44f 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Options: Commands: il2cpp Use LibCpp2IL backend to directly load Il2Cpp compiled game assembly. EXPERIMENTAL. - lua Use Lua AST backend to load Lua source files. + lua Use Loretta backend to load Lua source files. ``` See per-command help message for more info. diff --git a/src/LibProtodec/LibProtodec.csproj b/src/LibProtodec/LibProtodec.csproj index cabbf8f..e637b0c 100644 --- a/src/LibProtodec/LibProtodec.csproj +++ b/src/LibProtodec/LibProtodec.csproj @@ -18,12 +18,13 @@ + - + \ No newline at end of file diff --git a/src/LibProtodec/Loaders/LuaSourceLoader.cs b/src/LibProtodec/Loaders/LuaSourceLoader.cs index 2d15ff8..1076c93 100644 --- a/src/LibProtodec/Loaders/LuaSourceLoader.cs +++ b/src/LibProtodec/Loaders/LuaSourceLoader.cs @@ -16,19 +16,27 @@ namespace LibProtodec.Loaders; public sealed class LuaSourceLoader { - public IReadOnlyList LoadedSyntaxTrees { get; } + public IReadOnlyDictionary LoadedSyntaxTrees { get; } public LuaSourceLoader(string sourcePath, ILogger? logger = null) { LoadedSyntaxTrees = File.Exists(sourcePath) - ? [LoadSyntaxTreeFromSourceFile(sourcePath)] + ? new Dictionary { { Path.GetFileNameWithoutExtension(sourcePath) , LoadSyntaxTreeFromSourceFile(sourcePath) } } : Directory.EnumerateFiles(sourcePath, searchPattern: "*.lua") - .Select(LoadSyntaxTreeFromSourceFile) - .ToList(); + .Select(static sourcePath => + (Path.GetFileNameWithoutExtension(sourcePath), LoadSyntaxTreeFromSourceFile(sourcePath))) + .ToDictionary(); logger?.LogLoadedLuaSyntaxTrees(LoadedSyntaxTrees.Count); } + public SyntaxTree ResolveImport(string import) + { + import = Path.GetFileNameWithoutExtension(import); + + return LoadedSyntaxTrees[import]; + } + private static SyntaxTree LoadSyntaxTreeFromSourceFile(string filePath) { using FileStream fileStream = File.OpenRead(filePath); diff --git a/src/LibProtodec/Models/Protobuf/Fields/MessageField.cs b/src/LibProtodec/Models/Protobuf/Fields/MessageField.cs index 2940931..0a7cc29 100644 --- a/src/LibProtodec/Models/Protobuf/Fields/MessageField.cs +++ b/src/LibProtodec/Models/Protobuf/Fields/MessageField.cs @@ -13,7 +13,9 @@ namespace LibProtodec.Models.Protobuf.Fields; public sealed class MessageField { - public required IProtobufType Type { get; init; } + private bool? _isRepeated; + + public IProtobufType? Type { get; set; } public Message? DeclaringMessage { get; set; } public string? Name { get; set; } @@ -23,6 +25,11 @@ public sealed class MessageField public bool IsRequired { get; set; } public bool IsObsolete { get; init; } public bool HasHasProp { get; init; } + public bool IsRepeated + { + get => _isRepeated ??= Type is Repeated; + set => _isRepeated = value; + } public void WriteTo(TextWriter writer, bool isOneOf) { @@ -30,7 +37,7 @@ public sealed class MessageField Guard.IsNotNull(Name); Guard.IsNotNull(DeclaringMessage); - if (IsOptional || (HasHasProp && !isOneOf && Type is not Repeated)) + if (IsOptional || (HasHasProp && !isOneOf && !IsRepeated)) { writer.Write("optional "); } diff --git a/src/LibProtodec/Models/Protobuf/Fields/ServiceMethod.cs b/src/LibProtodec/Models/Protobuf/Fields/ServiceMethod.cs index 1f436ad..473bbbc 100644 --- a/src/LibProtodec/Models/Protobuf/Fields/ServiceMethod.cs +++ b/src/LibProtodec/Models/Protobuf/Fields/ServiceMethod.cs @@ -50,6 +50,7 @@ public sealed class ServiceMethod(Service declaringService) writer.Indent++; Protobuf.WriteOptionTo(writer, "deprecated", "true"); + writer.WriteLine(); writer.Indent--; writer.WriteLine('}'); diff --git a/src/LibProtodec/Models/Protobuf/Protobuf.cs b/src/LibProtodec/Models/Protobuf/Protobuf.cs index 1e3543a..b4526df 100644 --- a/src/LibProtodec/Models/Protobuf/Protobuf.cs +++ b/src/LibProtodec/Models/Protobuf/Protobuf.cs @@ -69,24 +69,25 @@ public sealed class Protobuf if (_imports is not null) { - writer.WriteLine(); - foreach (string import in _imports) { + writer.WriteLine(); writer.Write("import \""); writer.Write(import); - writer.WriteLine("\";"); + writer.Write("\";"); } } if (CilNamespace is not null) { + writer.WriteLine(); writer.WriteLine(); WriteOptionTo(writer, "csharp_namespace", CilNamespace, true); } foreach (TopLevel topLevel in TopLevels) { + writer.WriteLine(); writer.WriteLine(); topLevel.WriteTo(writer); } @@ -110,6 +111,6 @@ public sealed class Protobuf writer.Write('\"'); } - writer.WriteLine(';'); + writer.Write(';'); } } \ No newline at end of file diff --git a/src/LibProtodec/Models/Protobuf/TopLevels/Enum.cs b/src/LibProtodec/Models/Protobuf/TopLevels/Enum.cs index d538bc7..b25240c 100644 --- a/src/LibProtodec/Models/Protobuf/TopLevels/Enum.cs +++ b/src/LibProtodec/Models/Protobuf/TopLevels/Enum.cs @@ -26,16 +26,19 @@ public sealed class Enum : TopLevel, INestableType if (ContainsDuplicateFieldId) { Protobuf.WriteOptionTo(writer, "allow_alias", "true"); + writer.WriteLine(); } if (this.IsObsolete) { Protobuf.WriteOptionTo(writer, "deprecated", "true"); + writer.WriteLine(); } if (IsClosed) { Protobuf.WriteOptionTo(writer, "features.enum_type", "CLOSED"); + writer.WriteLine(); } foreach (EnumField field in Fields) diff --git a/src/LibProtodec/Models/Protobuf/TopLevels/Message.cs b/src/LibProtodec/Models/Protobuf/TopLevels/Message.cs index 458e780..9c55142 100644 --- a/src/LibProtodec/Models/Protobuf/TopLevels/Message.cs +++ b/src/LibProtodec/Models/Protobuf/TopLevels/Message.cs @@ -28,6 +28,7 @@ public sealed class Message : TopLevel, INestableType if (this.IsObsolete) { Protobuf.WriteOptionTo(writer, "deprecated", "true"); + writer.WriteLine(); } int[] oneOfs = OneOfs.SelectMany(static oneOf => oneOf.Value).ToArray(); diff --git a/src/LibProtodec/Models/Protobuf/TopLevels/Service.cs b/src/LibProtodec/Models/Protobuf/TopLevels/Service.cs index 42fa5bc..941a275 100644 --- a/src/LibProtodec/Models/Protobuf/TopLevels/Service.cs +++ b/src/LibProtodec/Models/Protobuf/TopLevels/Service.cs @@ -24,6 +24,7 @@ public sealed class Service : TopLevel if (this.IsObsolete) { Protobuf.WriteOptionTo(writer, "deprecated", "true"); + writer.WriteLine(); } foreach (ServiceMethod method in Methods) diff --git a/src/LibProtodec/Models/Protobuf/Types/Descriptor.cs b/src/LibProtodec/Models/Protobuf/Types/Descriptor.cs deleted file mode 100644 index 974bf53..0000000 --- a/src/LibProtodec/Models/Protobuf/Types/Descriptor.cs +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright © 2024 Xpl0itR -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -using CommunityToolkit.Diagnostics; -using LibProtodec.Models.Protobuf.TopLevels; - -namespace LibProtodec.Models.Protobuf.Types; - -public sealed class Descriptor : IProtobufType -{ - public int TypeIndex { get; set; } - public bool IsRepeated { get; set; } - public /*TopLevel*/IProtobufType? TopLevelType { get; set; } - - public string Name => - Type.Name; - - public IProtobufType Type - { - get - { - IProtobufType type = TopLevelType as IProtobufType ?? TypeIndex switch - { - 1 => Scalar.Double, - 2 => Scalar.Float, - 3 => Scalar.Int64, - 4 => Scalar.UInt64, - 5 => Scalar.Int32, - 6 => Scalar.Fixed64, - 7 => Scalar.Fixed32, - 8 => Scalar.Bool, - 9 => Scalar.String, - 10 => ThrowHelper.ThrowNotSupportedException("Parsing proto2 groups are not supported. Open an issue if you need this."), - 12 => Scalar.Bytes, - 13 => Scalar.UInt32, - 15 => Scalar.SFixed32, - 16 => Scalar.SFixed64, - 17 => Scalar.SInt32, - 18 => Scalar.SInt64, - _ => ThrowHelper.ThrowArgumentOutOfRangeException() - }; - - if (IsRepeated) - { - type = new Repeated(type); - } - - return type; - } - } -} \ No newline at end of file diff --git a/src/LibProtodec/ProtodecContext.Lua.cs b/src/LibProtodec/ProtodecContext.Lua.cs index 9f2d330..f5ec624 100644 --- a/src/LibProtodec/ProtodecContext.Lua.cs +++ b/src/LibProtodec/ProtodecContext.Lua.cs @@ -5,28 +5,39 @@ // file, You can obtain one at http://mozilla.org/MPL/2.0/. using System.Collections.Generic; -using System.IO; using System.Linq; +using SystemEx; using CommunityToolkit.Diagnostics; +using LibProtodec.Loaders; using LibProtodec.Models.Protobuf; using LibProtodec.Models.Protobuf.Fields; using LibProtodec.Models.Protobuf.TopLevels; using LibProtodec.Models.Protobuf.Types; using Loretta.CodeAnalysis; using Loretta.CodeAnalysis.Lua.Syntax; -using SystemEx; +using FdpTypes = Google.Protobuf.Reflection.FieldDescriptorProto.Types; namespace LibProtodec; +// TODO: add debug logging partial class ProtodecContext { - public Protobuf ParseLuaSyntaxTree(SyntaxTree ast) + private readonly Dictionary> _parsedPbTables = []; + private readonly Dictionary _parsedProtobufs = []; + + public Protobuf ParseLuaSyntaxTree(LuaSourceLoader loader, SyntaxTree ast, ParserOptions options = ParserOptions.None) { + if (_parsedProtobufs.TryGetValue(ast.FilePath, out Protobuf? parsedProto)) + { + return parsedProto; + } + CompilationUnitSyntax root = (CompilationUnitSyntax)ast.GetRoot(); SyntaxList statements = root.Statements.Statements; bool importedProtobufLib = false; - Dictionary pbTable = []; + Dictionary pbTable = _parsedPbTables[ast.FilePath] = []; + Dictionary> imports = []; Protobuf protobuf = new() { Version = 2, @@ -39,6 +50,7 @@ partial class ProtodecContext { case LocalVariableDeclarationStatementSyntax { + Names: [{ IdentifierName.Name: { } importKey }], EqualsValues.Values: [FunctionCallExpressionSyntax { Expression: IdentifierNameSyntax { Name: "require" } } call] }: switch (call.Argument) @@ -46,9 +58,11 @@ partial class ProtodecContext case StringFunctionArgumentSyntax { Expression.Token.ValueText: "protobuf/protobuf" }: importedProtobufLib = true; break; - case ExpressionListFunctionArgumentSyntax { Expressions: [LiteralExpressionSyntax { Token.ValueText: {} import }] }: - import = Path.GetFileNameWithoutExtension(import).TrimEnd("_pb"); //todo: handle imports properly - protobuf.Imports.Add($"{import}.proto"); + case ExpressionListFunctionArgumentSyntax { Expressions: [LiteralExpressionSyntax { Token.ValueText: { } import }] }: + SyntaxTree importedAst = loader.ResolveImport(import); + Protobuf importedProto = ParseLuaSyntaxTree(loader, importedAst); + imports.Add(importKey, _parsedPbTables[importedAst.FilePath]); + protobuf.Imports.Add(importedProto.FileName); break; } break; @@ -57,9 +71,9 @@ partial class ProtodecContext Variables: [MemberAccessExpressionSyntax { Expression: IdentifierNameSyntax, - MemberName.ValueText: {} tableKey + MemberName.ValueText: { } tableKey }], - EqualsValues.Values: [FunctionCallExpressionSyntax { Expression: MemberAccessExpressionSyntax { MemberName.ValueText: {} factory } }] + EqualsValues.Values: [FunctionCallExpressionSyntax { Expression: MemberAccessExpressionSyntax { MemberName.ValueText: { } factory } }] }: switch (factory) { @@ -82,7 +96,7 @@ partial class ProtodecContext pbTable.Add(tableKey, @enum); break; case "FieldDescriptor": - pbTable.Add(tableKey, new MessageField { Type = new Descriptor() }); + pbTable.Add(tableKey, new MessageField()); break; case "EnumValueDescriptor": pbTable.Add(tableKey, new EnumField()); @@ -93,7 +107,7 @@ partial class ProtodecContext { Variables: [MemberAccessExpressionSyntax { - Expression: MemberAccessExpressionSyntax { MemberName.ValueText: {} tableKey }, + Expression: MemberAccessExpressionSyntax { MemberName.ValueText: { } tableKey }, MemberName.ValueText: { } memberName }], EqualsValues.Values: [{ } valueExpr] @@ -103,7 +117,6 @@ partial class ProtodecContext .Cast() .Select(static element => element.Value); object? valueLiteral = (valueExpr as LiteralExpressionSyntax)?.Token.Value; - switch (pbTable[tableKey]) { case Message message: @@ -152,7 +165,6 @@ partial class ProtodecContext } break; case MessageField messageField: - Descriptor descriptor = (Descriptor)messageField.Type; switch (memberName) { case "name": @@ -162,29 +174,36 @@ partial class ProtodecContext messageField.Id = (int)(double)valueLiteral!; break; case "label": - switch ((int)(double)valueLiteral!) + switch ((FdpTypes.Label)(double)valueLiteral!) { - case 1: + case FdpTypes.Label.Optional: messageField.IsOptional = true; break; - case 2: + case FdpTypes.Label.Required: messageField.IsRequired = true; break; - case 3: - descriptor.IsRepeated = true; + case FdpTypes.Label.Repeated: + messageField.IsRepeated = true; break; } break; - case "type": - descriptor.TypeIndex = (int)(double)valueLiteral!; + case "enum_type" when (options & ParserOptions.SkipEnums) > 0: + messageField.Type = Scalar.Int32; break; - case "message_type": case "enum_type": - string typeTableKey = ((MemberAccessExpressionSyntax)valueExpr).MemberName.ValueText; - if (pbTable.TryGetValue(typeTableKey, out object? topLevel)) //TODO: if this is false, then the top level is from an import, we will need to handle this - descriptor.TopLevelType = (IProtobufType)topLevel; - else - descriptor.TopLevelType = new Scalar(typeTableKey); // temporary hack + case "message_type": + MemberAccessExpressionSyntax memberAccessExpr = (MemberAccessExpressionSyntax)valueExpr; + string importKey = ((IdentifierNameSyntax)memberAccessExpr.Expression).Name; + Dictionary table = imports.GetValueOrDefault(importKey, pbTable); + string typeTableKey = memberAccessExpr.MemberName.ValueText; + IProtobufType scalar = (IProtobufType)table[typeTableKey]; + messageField.Type = messageField.IsRepeated + ? new Repeated(scalar) + : scalar; + break; + case "type": + messageField.Type ??= ParseFieldType( + (FdpTypes.Type)(double)valueLiteral!, messageField.IsRepeated); break; case "has_default_value": if ((bool)valueLiteral!) @@ -220,6 +239,35 @@ partial class ProtodecContext } this.Protobufs.Add(protobuf); + _parsedProtobufs.Add(ast.FilePath, protobuf); return protobuf; } + + protected static IProtobufType ParseFieldType(FdpTypes.Type type, bool isRepeated) + { + IProtobufType scalar = type switch + { + FdpTypes.Type.Double => Scalar.Double, + FdpTypes.Type.Float => Scalar.Float, + FdpTypes.Type.Int64 => Scalar.Int64, + FdpTypes.Type.Uint64 => Scalar.UInt64, + FdpTypes.Type.Int32 => Scalar.Int32, + FdpTypes.Type.Fixed64 => Scalar.Fixed64, + FdpTypes.Type.Fixed32 => Scalar.Fixed32, + FdpTypes.Type.Bool => Scalar.Bool, + FdpTypes.Type.String => Scalar.String, + FdpTypes.Type.Bytes => Scalar.Bytes, + FdpTypes.Type.Uint32 => Scalar.UInt32, + FdpTypes.Type.Sfixed32 => Scalar.SFixed32, + FdpTypes.Type.Sfixed64 => Scalar.SFixed64, + FdpTypes.Type.Sint32 => Scalar.SInt32, + FdpTypes.Type.Sint64 => Scalar.SInt64, + FdpTypes.Type.Group => ThrowHelper.ThrowNotSupportedException( + "Parsing proto2 groups are not supported. Open an issue if you need this."), + }; + + return isRepeated + ? new Repeated(scalar) + : scalar; + } } \ No newline at end of file diff --git a/src/LibProtodec/ProtodecContext.cs b/src/LibProtodec/ProtodecContext.cs index ff4ab85..a2e7843 100644 --- a/src/LibProtodec/ProtodecContext.cs +++ b/src/LibProtodec/ProtodecContext.cs @@ -36,14 +36,13 @@ public partial class ProtodecContext public void WriteAllTo(IndentedTextWriter writer) { - writer.WriteLine("// Decompiled with protodec"); - writer.WriteLine(); + writer.Write("// Decompiled with protodec"); foreach (TopLevel topLevel in Protobufs.SelectMany(static proto => proto.TopLevels)) { + writer.WriteLine(); + writer.WriteLine(); topLevel.WriteTo(writer); - writer.WriteLine(); - writer.WriteLine(); } } diff --git a/src/protodec/Program.cs b/src/protodec/Program.cs index 7e4a40c..8e961a6 100644 --- a/src/protodec/Program.cs +++ b/src/protodec/Program.cs @@ -117,17 +117,23 @@ internal sealed class Commands } /// - /// Use Lua AST backend to load Lua source files. + /// Use Loretta backend to load Lua source files. /// /// Either the path to the target lua file or a directory of lua files, all of which be parsed. /// An existing directory to output into individual files, otherwise output to a single file. + /// Skip parsing enums and replace references to them with int32. /// Logging severity level. [Command("lua")] public void Lua( [Argument] string targetPath, [Argument] string outPath, + bool skipEnums, LogLevel logLevel = LogLevel.Information) { + ParserOptions options = ParserOptions.None; + if (skipEnums) + options |= ParserOptions.SkipEnums; + using ILoggerFactory loggerFactory = CreateLoggerFactory(logLevel); ILogger logger = CreateProtodecLogger(loggerFactory); @@ -140,9 +146,9 @@ internal sealed class Commands }; logger.LogInformation("Parsing Lua syntax trees..."); - foreach (SyntaxTree ast in loader.LoadedSyntaxTrees) + foreach (SyntaxTree ast in loader.LoadedSyntaxTrees.Values) { - ctx.ParseLuaSyntaxTree(ast); + ctx.ParseLuaSyntaxTree(loader, ast, options); } // NOTE: I'm duplicating this code rather than refactoring because I have a lot of uncommited changes on another computer,