From b3ff1fe60893ec73c3c8c8b7c9e470fd274e626a Mon Sep 17 00:00:00 2001 From: Xpl0itR Date: Tue, 8 Oct 2024 04:46:49 +0100 Subject: [PATCH] nearly done --- src/LibProtodec/LoggerMessages.cs | 3 + .../Models/Protobuf/Fields/EnumField.cs | 8 +- .../Models/Protobuf/Fields/MessageField.cs | 24 ++- src/LibProtodec/Models/Protobuf/Protobuf.cs | 29 ++- .../Models/Protobuf/TopLevels/TopLevel.cs | 2 +- .../Models/Protobuf/Types/Descriptor.cs | 54 +++++ src/LibProtodec/ProtodecContext.Lua.cs | 195 +++++++++++++++--- src/LibProtodec/ProtodecContext.cs | 9 +- 8 files changed, 281 insertions(+), 43 deletions(-) create mode 100644 src/LibProtodec/Models/Protobuf/Types/Descriptor.cs diff --git a/src/LibProtodec/LoggerMessages.cs b/src/LibProtodec/LoggerMessages.cs index 632b930..8d786f5 100644 --- a/src/LibProtodec/LoggerMessages.cs +++ b/src/LibProtodec/LoggerMessages.cs @@ -15,6 +15,9 @@ internal static partial class LoggerMessages [LoggerMessage(Level = LogLevel.Warning, Message = "Failed to locate corresponding id field; likely stripped or otherwise obfuscated.")] internal static partial void LogFailedToLocateIdField(this ILogger logger); + [LoggerMessage(Level = LogLevel.Warning, Message = "{msg} is not yet implemented. Open an issue if this is something you need.")] + internal static partial void LogNotImplemented(this ILogger logger, string msg); + [LoggerMessage(Level = LogLevel.Information, Message = "Loaded {typeCount} types from {assemblyCount} assemblies for parsing.")] internal static partial void LogLoadedTypeAndAssemblyCount(this ILogger logger, int typeCount, int assemblyCount); diff --git a/src/LibProtodec/Models/Protobuf/Fields/EnumField.cs b/src/LibProtodec/Models/Protobuf/Fields/EnumField.cs index bf7b615..7dd8e53 100644 --- a/src/LibProtodec/Models/Protobuf/Fields/EnumField.cs +++ b/src/LibProtodec/Models/Protobuf/Fields/EnumField.cs @@ -4,17 +4,21 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. +using CommunityToolkit.Diagnostics; + namespace LibProtodec.Models.Protobuf.Fields; public sealed class EnumField { - public required string Name { get; init; } - public required int Id { get; init; } + public string? Name { get; set; } + public int Id { get; set; } public bool IsObsolete { get; init; } public void WriteTo(System.IO.TextWriter writer) { + Guard.IsNotNull(Name); + writer.Write(Name); writer.Write(" = "); writer.Write(Id); diff --git a/src/LibProtodec/Models/Protobuf/Fields/MessageField.cs b/src/LibProtodec/Models/Protobuf/Fields/MessageField.cs index 7c96f4f..2940931 100644 --- a/src/LibProtodec/Models/Protobuf/Fields/MessageField.cs +++ b/src/LibProtodec/Models/Protobuf/Fields/MessageField.cs @@ -5,29 +5,43 @@ // file, You can obtain one at http://mozilla.org/MPL/2.0/. using System.IO; +using CommunityToolkit.Diagnostics; using LibProtodec.Models.Protobuf.TopLevels; using LibProtodec.Models.Protobuf.Types; namespace LibProtodec.Models.Protobuf.Fields; -public sealed class MessageField(Message declaringMessage) +public sealed class MessageField { public required IProtobufType Type { get; init; } - public required string Name { get; init; } - public required int Id { get; init; } + public Message? DeclaringMessage { get; set; } + public string? Name { get; set; } + public int Id { get; set; } + + public bool IsOptional { get; set; } + public bool IsRequired { get; set; } public bool IsObsolete { get; init; } public bool HasHasProp { get; init; } public void WriteTo(TextWriter writer, bool isOneOf) { - if (HasHasProp && !isOneOf && Type is not Repeated) + Guard.IsNotNull(Type); + Guard.IsNotNull(Name); + Guard.IsNotNull(DeclaringMessage); + + if (IsOptional || (HasHasProp && !isOneOf && Type is not Repeated)) { writer.Write("optional "); } + if (IsRequired) + { + writer.Write("required "); + } + writer.Write( - declaringMessage.QualifyTypeName(Type)); + DeclaringMessage.QualifyTypeName(Type)); writer.Write(' '); writer.Write(Name); writer.Write(" = "); diff --git a/src/LibProtodec/Models/Protobuf/Protobuf.cs b/src/LibProtodec/Models/Protobuf/Protobuf.cs index eaf12af..1e3543a 100644 --- a/src/LibProtodec/Models/Protobuf/Protobuf.cs +++ b/src/LibProtodec/Models/Protobuf/Protobuf.cs @@ -8,20 +8,23 @@ using System.CodeDom.Compiler; using System.Collections.Generic; using System.IO; using System.Linq; +using CommunityToolkit.Diagnostics; using LibProtodec.Models.Protobuf.TopLevels; namespace LibProtodec.Models.Protobuf; public sealed class Protobuf { + private int _version = 3; private string? _fileName; private HashSet? _imports; public readonly List TopLevels = []; - public string? Edition { get; set; } public string? AssemblyName { get; init; } - public string? Namespace { get; init; } + public string? SourceName { get; init; } + public string? Edition { get; set; } + public string? CilNamespace { get; init; } public string FileName { @@ -29,6 +32,16 @@ public sealed class Protobuf set => _fileName = value; } + public int Version + { + get => _version; + set + { + Guard.IsBetweenOrEqualTo(value, 2, 3); + _version = value; + } + } + public HashSet Imports => _imports ??= []; @@ -42,10 +55,16 @@ public sealed class Protobuf writer.WriteLine(AssemblyName); } + if (SourceName is not null) + { + writer.Write("// Source: "); + writer.WriteLine(SourceName); + } + writer.WriteLine(); writer.WriteLine( Edition is null - ? """syntax = "proto3";""" + ? $"""syntax = "proto{Version}";""" : $"""edition = "{Edition}";"""); if (_imports is not null) @@ -60,10 +79,10 @@ public sealed class Protobuf } } - if (Namespace is not null) + if (CilNamespace is not null) { writer.WriteLine(); - WriteOptionTo(writer, "csharp_namespace", Namespace, true); + WriteOptionTo(writer, "csharp_namespace", CilNamespace, true); } foreach (TopLevel topLevel in TopLevels) diff --git a/src/LibProtodec/Models/Protobuf/TopLevels/TopLevel.cs b/src/LibProtodec/Models/Protobuf/TopLevels/TopLevel.cs index 755560d..113e8bd 100644 --- a/src/LibProtodec/Models/Protobuf/TopLevels/TopLevel.cs +++ b/src/LibProtodec/Models/Protobuf/TopLevels/TopLevel.cs @@ -12,7 +12,7 @@ namespace LibProtodec.Models.Protobuf.TopLevels; public abstract class TopLevel { - public required string Name { get; init; } + public required string Name { get; set; } public bool IsObsolete { get; init; } public Protobuf? Protobuf { get; set; } diff --git a/src/LibProtodec/Models/Protobuf/Types/Descriptor.cs b/src/LibProtodec/Models/Protobuf/Types/Descriptor.cs new file mode 100644 index 0000000..974bf53 --- /dev/null +++ b/src/LibProtodec/Models/Protobuf/Types/Descriptor.cs @@ -0,0 +1,54 @@ +// Copyright © 2024 Xpl0itR +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +using CommunityToolkit.Diagnostics; +using LibProtodec.Models.Protobuf.TopLevels; + +namespace LibProtodec.Models.Protobuf.Types; + +public sealed class Descriptor : IProtobufType +{ + public int TypeIndex { get; set; } + public bool IsRepeated { get; set; } + public /*TopLevel*/IProtobufType? TopLevelType { get; set; } + + public string Name => + Type.Name; + + public IProtobufType Type + { + get + { + IProtobufType type = TopLevelType as IProtobufType ?? TypeIndex switch + { + 1 => Scalar.Double, + 2 => Scalar.Float, + 3 => Scalar.Int64, + 4 => Scalar.UInt64, + 5 => Scalar.Int32, + 6 => Scalar.Fixed64, + 7 => Scalar.Fixed32, + 8 => Scalar.Bool, + 9 => Scalar.String, + 10 => ThrowHelper.ThrowNotSupportedException("Parsing proto2 groups are not supported. Open an issue if you need this."), + 12 => Scalar.Bytes, + 13 => Scalar.UInt32, + 15 => Scalar.SFixed32, + 16 => Scalar.SFixed64, + 17 => Scalar.SInt32, + 18 => Scalar.SInt64, + _ => ThrowHelper.ThrowArgumentOutOfRangeException() + }; + + if (IsRepeated) + { + type = new Repeated(type); + } + + return type; + } + } +} \ No newline at end of file diff --git a/src/LibProtodec/ProtodecContext.Lua.cs b/src/LibProtodec/ProtodecContext.Lua.cs index 54689d7..9f2d330 100644 --- a/src/LibProtodec/ProtodecContext.Lua.cs +++ b/src/LibProtodec/ProtodecContext.Lua.cs @@ -4,28 +4,34 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. +using System.Collections.Generic; +using System.IO; +using System.Linq; using CommunityToolkit.Diagnostics; +using LibProtodec.Models.Protobuf; +using LibProtodec.Models.Protobuf.Fields; +using LibProtodec.Models.Protobuf.TopLevels; +using LibProtodec.Models.Protobuf.Types; using Loretta.CodeAnalysis; using Loretta.CodeAnalysis.Lua.Syntax; +using SystemEx; namespace LibProtodec; partial class ProtodecContext { - public void ParseLuaSyntaxTree(SyntaxTree ast) + public Protobuf ParseLuaSyntaxTree(SyntaxTree ast) { CompilationUnitSyntax root = (CompilationUnitSyntax)ast.GetRoot(); SyntaxList statements = root.Statements.Statements; - LocalVariableDeclarationStatementSyntax pbTableDeclaration = - (LocalVariableDeclarationStatementSyntax)statements[0]; - - string pbTableName = pbTableDeclaration.Names[0].Name; - Guard.IsTrue( - pbTableName.EndsWith("_pbTable")); - bool importedProtobufLib = false; - string? returns = null; + Dictionary pbTable = []; + Protobuf protobuf = new() + { + Version = 2, + SourceName = ast.FilePath + }; foreach (StatementSyntax statement in statements) { @@ -33,7 +39,6 @@ partial class ProtodecContext { case LocalVariableDeclarationStatementSyntax { - Names: [{ Name: { } varName }], EqualsValues.Values: [FunctionCallExpressionSyntax { Expression: IdentifierNameSyntax { Name: "require" } } call] }: switch (call.Argument) @@ -42,39 +47,179 @@ partial class ProtodecContext importedProtobufLib = true; break; case ExpressionListFunctionArgumentSyntax { Expressions: [LiteralExpressionSyntax { Token.ValueText: {} import }] }: - // TODO: handle imported protos + import = Path.GetFileNameWithoutExtension(import).TrimEnd("_pb"); //todo: handle imports properly + protobuf.Imports.Add($"{import}.proto"); break; } break; - case ExpressionStatementSyntax + case AssignmentStatementSyntax { - Expression: FunctionCallExpressionSyntax + Variables: [MemberAccessExpressionSyntax { - Expression: IdentifierNameSyntax { Name: "module" }, - Argument: ExpressionListFunctionArgumentSyntax - { - Expressions: [LiteralExpressionSyntax literal] - } - } + Expression: IdentifierNameSyntax, + MemberName.ValueText: {} tableKey + }], + EqualsValues.Values: [FunctionCallExpressionSyntax { Expression: MemberAccessExpressionSyntax { MemberName.ValueText: {} factory } }] }: - string moduleName = literal.Token.ValueText; + switch (factory) + { + case "Descriptor": + Message message = new() + { + Name = tableKey, + Protobuf = protobuf + }; + protobuf.TopLevels.Add(message); + pbTable.Add(tableKey, message); + break; + case "EnumDescriptor": + Enum @enum = new() + { + Name = tableKey, + Protobuf = protobuf + }; + protobuf.TopLevels.Add(@enum); + pbTable.Add(tableKey, @enum); + break; + case "FieldDescriptor": + pbTable.Add(tableKey, new MessageField { Type = new Descriptor() }); + break; + case "EnumValueDescriptor": + pbTable.Add(tableKey, new EnumField()); + break; + } break; case AssignmentStatementSyntax { - Variables: [MemberAccessExpressionSyntax varExpr], - EqualsValues.Values: [{} valueExpr] + Variables: [MemberAccessExpressionSyntax + { + Expression: MemberAccessExpressionSyntax { MemberName.ValueText: {} tableKey }, + MemberName.ValueText: { } memberName + }], + EqualsValues.Values: [{ } valueExpr] }: - // TODO: build protos + var valueTableElements = + (valueExpr as TableConstructorExpressionSyntax)?.Fields + .Cast() + .Select(static element => element.Value); + object? valueLiteral = (valueExpr as LiteralExpressionSyntax)?.Token.Value; + + switch (pbTable[tableKey]) + { + case Message message: + switch (memberName) + { + case "name": + message.Name = (string)valueLiteral!; + break; + case "fields": + foreach (MessageField messageField in valueTableElements! + .Cast() + .Select(x => pbTable[x.MemberName.ValueText]) + .Cast()) + { + messageField.DeclaringMessage = message; + message.Fields.Add(messageField.Id, messageField); + } + break; + case "nested_types": + if (valueTableElements!.Any()) + Logger?.LogNotImplemented("Parsing nested messages from lua"); + break; + case "enum_types": + if (valueTableElements!.Any()) + Logger?.LogNotImplemented("Parsing nested enums from lua"); + break; + case "is_extendable": + if ((bool)valueLiteral!) + Logger?.LogNotImplemented("Parsing message extensions from lua"); + break; + } + break; + case Enum @enum: + switch (memberName) + { + case "name": + @enum.Name = (string)valueLiteral!; + break; + case "values": + @enum.Fields.AddRange( + valueTableElements! + .Cast() + .Select(x => pbTable[x.MemberName.ValueText]) + .Cast()); + break; + } + break; + case MessageField messageField: + Descriptor descriptor = (Descriptor)messageField.Type; + switch (memberName) + { + case "name": + messageField.Name = (string)valueLiteral!; + break; + case "number": + messageField.Id = (int)(double)valueLiteral!; + break; + case "label": + switch ((int)(double)valueLiteral!) + { + case 1: + messageField.IsOptional = true; + break; + case 2: + messageField.IsRequired = true; + break; + case 3: + descriptor.IsRepeated = true; + break; + } + break; + case "type": + descriptor.TypeIndex = (int)(double)valueLiteral!; + break; + case "message_type": + case "enum_type": + string typeTableKey = ((MemberAccessExpressionSyntax)valueExpr).MemberName.ValueText; + if (pbTable.TryGetValue(typeTableKey, out object? topLevel)) //TODO: if this is false, then the top level is from an import, we will need to handle this + descriptor.TopLevelType = (IProtobufType)topLevel; + else + descriptor.TopLevelType = new Scalar(typeTableKey); // temporary hack + break; + case "has_default_value": + if ((bool)valueLiteral!) + Logger?.LogNotImplemented("Parsing default field values from lua"); + break; + } + break; + case EnumField enumField: + switch (memberName) + { + case "name": + enumField.Name = (string)valueLiteral!; + break; + case "number" when valueExpr is UnaryExpressionSyntax { Operand: LiteralExpressionSyntax { Token.Value: { } negativeNumber } }: + enumField.Id = -(int)(double)negativeNumber; + break; + case "number": + enumField.Id = (int)(double)valueLiteral!; + break; + } + break; + } break; case ReturnStatementSyntax { Expressions: [IdentifierNameSyntax identifier] }: - returns = identifier.Name; + protobuf.FileName = $"{identifier.Name.TrimEnd("_pbTable")}.proto"; break; } } - if (!importedProtobufLib || returns != pbTableName) + if (!importedProtobufLib) { ThrowHelper.ThrowInvalidDataException(); } + + this.Protobufs.Add(protobuf); + return protobuf; } } \ No newline at end of file diff --git a/src/LibProtodec/ProtodecContext.cs b/src/LibProtodec/ProtodecContext.cs index 48b28a7..ff4ab85 100644 --- a/src/LibProtodec/ProtodecContext.cs +++ b/src/LibProtodec/ProtodecContext.cs @@ -38,8 +38,6 @@ public partial class ProtodecContext { writer.WriteLine("// Decompiled with protodec"); writer.WriteLine(); - writer.WriteLine("""syntax = "proto3";"""); - writer.WriteLine(); foreach (TopLevel topLevel in Protobufs.SelectMany(static proto => proto.TopLevels)) { @@ -119,13 +117,14 @@ public partial class ProtodecContext continue; } - MessageField field = new(message) + MessageField field = new() { Type = ParseFieldType(propertyType, options, protobuf), Name = TranslateMessageFieldName(property.Name), Id = (int)idFields[fi].ConstantValue!, IsObsolete = HasObsoleteAttribute(property.CustomAttributes), - HasHasProp = msgFieldHasHasProp + HasHasProp = msgFieldHasHasProp, + DeclaringMessage = message }; Logger?.LogParsedField(field.Name, field.Id, field.Type.Name); @@ -499,7 +498,7 @@ public partial class ProtodecContext Protobuf protobuf = new() { AssemblyName = topLevelType.DeclaringAssemblyName, - Namespace = topLevelType.Namespace + CilNamespace = topLevelType.Namespace }; topLevel.Protobuf = protobuf;