mission complete

README.md

@@ -22,7 +22,7 @@ Options:
 
 Commands:
   il2cpp    Use LibCpp2IL backend to directly load Il2Cpp compiled game assembly. EXPERIMENTAL.
-  lua       Use Lua AST backend to load Lua source files.
+  lua       Use Loretta backend to load Lua source files.
 ```
 See per-command help message for more info.
 

src/LibProtodec/LibProtodec.csproj

@@ -18,12 +18,13 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Google.Protobuf" Version="3.28.2" />
     <PackageReference Include="Loretta.CodeAnalysis.Lua" Version="0.2.12" />
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.0-preview.6.24327.7" />
     <PackageReference Include="Microsoft.SourceLink.GitHub" Version="8.0.0" PrivateAssets="All" />
     <PackageReference Include="Samboy063.LibCpp2IL" Version="2022.1.0-development.1027" />
     <PackageReference Include="System.Reflection.MetadataLoadContext" Version="9.0.0-preview.6.24327.7" />
-    <PackageReference Include="Xpl0itR.SystemEx" Version="1.2.0" />
+    <PackageReference Include="Xpl0itR.SystemEx" Version="1.2.1" />
   </ItemGroup>
 
 </Project>

src/LibProtodec/Loaders/LuaSourceLoader.cs

@@ -16,19 +16,27 @@ namespace LibProtodec.Loaders;
 
 public sealed class LuaSourceLoader
 {
-    public IReadOnlyList<SyntaxTree> LoadedSyntaxTrees { get; }
+    public IReadOnlyDictionary<string, SyntaxTree> LoadedSyntaxTrees { get; }
 
     public LuaSourceLoader(string sourcePath, ILogger<LuaSourceLoader>? logger = null)
     {
         LoadedSyntaxTrees = File.Exists(sourcePath)
-            ? [LoadSyntaxTreeFromSourceFile(sourcePath)]
+            ? new Dictionary<string, SyntaxTree> { { Path.GetFileNameWithoutExtension(sourcePath) , LoadSyntaxTreeFromSourceFile(sourcePath) } }
             : Directory.EnumerateFiles(sourcePath, searchPattern: "*.lua")
-                       .Select(LoadSyntaxTreeFromSourceFile)
+                       .Select(static sourcePath =>
-                       .ToList();
+                           (Path.GetFileNameWithoutExtension(sourcePath), LoadSyntaxTreeFromSourceFile(sourcePath)))
+                       .ToDictionary();
 
         logger?.LogLoadedLuaSyntaxTrees(LoadedSyntaxTrees.Count);
     }
 
+    public SyntaxTree ResolveImport(string import)
+    {
+        import = Path.GetFileNameWithoutExtension(import);
+
+        return LoadedSyntaxTrees[import];
+    }
+
     private static SyntaxTree LoadSyntaxTreeFromSourceFile(string filePath)
     {
         using FileStream fileStream = File.OpenRead(filePath);

src/LibProtodec/Models/Protobuf/Fields/MessageField.cs

@@ -13,7 +13,9 @@ namespace LibProtodec.Models.Protobuf.Fields;
 
 public sealed class MessageField
 {
-    public required IProtobufType Type { get; init; }
+    private bool? _isRepeated;
+
+    public IProtobufType? Type { get; set; }
     public Message? DeclaringMessage { get; set; }
 
     public string? Name { get; set; }
@@ -23,6 +25,11 @@ public sealed class MessageField
     public bool IsRequired { get; set;  }
     public bool IsObsolete { get; init; }
     public bool HasHasProp { get; init; }
+    public bool IsRepeated
+    {
+        get => _isRepeated ??= Type is Repeated;
+        set => _isRepeated = value;
+    }
 
     public void WriteTo(TextWriter writer, bool isOneOf)
     {
@@ -30,7 +37,7 @@ public sealed class MessageField
         Guard.IsNotNull(Name);
         Guard.IsNotNull(DeclaringMessage);
 
-        if (IsOptional || (HasHasProp && !isOneOf && Type is not Repeated))
+        if (IsOptional || (HasHasProp && !isOneOf && !IsRepeated))
         {
             writer.Write("optional ");
         }

src/LibProtodec/Models/Protobuf/Fields/ServiceMethod.cs

@@ -50,6 +50,7 @@ public sealed class ServiceMethod(Service declaringService)
             writer.Indent++;
 
             Protobuf.WriteOptionTo(writer, "deprecated", "true");
+            writer.WriteLine();
 
             writer.Indent--;
             writer.WriteLine('}');

src/LibProtodec/Models/Protobuf/Protobuf.cs

@@ -69,24 +69,25 @@ public sealed class Protobuf
 
         if (_imports is not null)
         {
-            writer.WriteLine();
-            
             foreach (string import in _imports)
             {
+                writer.WriteLine();
                 writer.Write("import \"");
                 writer.Write(import);
-                writer.WriteLine("\";");
+                writer.Write("\";");
             }
         }
 
         if (CilNamespace is not null)
         {
+            writer.WriteLine();
             writer.WriteLine();
             WriteOptionTo(writer, "csharp_namespace", CilNamespace, true);
         }
 
         foreach (TopLevel topLevel in TopLevels)
         {
+            writer.WriteLine();
             writer.WriteLine();
             topLevel.WriteTo(writer);
         }
@@ -110,6 +111,6 @@ public sealed class Protobuf
             writer.Write('\"');
         }
 
-        writer.WriteLine(';');
+        writer.Write(';');
     }
 }

src/LibProtodec/Models/Protobuf/TopLevels/Enum.cs

@@ -26,16 +26,19 @@ public sealed class Enum : TopLevel, INestableType
         if (ContainsDuplicateFieldId)
         {
             Protobuf.WriteOptionTo(writer, "allow_alias", "true");
+            writer.WriteLine();
         }
 
         if (this.IsObsolete)
         {
             Protobuf.WriteOptionTo(writer, "deprecated", "true");
+            writer.WriteLine();
         }
 
         if (IsClosed)
         {
             Protobuf.WriteOptionTo(writer, "features.enum_type", "CLOSED");
+            writer.WriteLine();
         }
 
         foreach (EnumField field in Fields)

src/LibProtodec/Models/Protobuf/TopLevels/Message.cs

@@ -28,6 +28,7 @@ public sealed class Message : TopLevel, INestableType
         if (this.IsObsolete)
         {
             Protobuf.WriteOptionTo(writer, "deprecated", "true");
+            writer.WriteLine();
         }
 
         int[] oneOfs = OneOfs.SelectMany(static oneOf => oneOf.Value).ToArray();

src/LibProtodec/Models/Protobuf/TopLevels/Service.cs

@@ -24,6 +24,7 @@ public sealed class Service : TopLevel
         if (this.IsObsolete)
         {
             Protobuf.WriteOptionTo(writer, "deprecated", "true");
+            writer.WriteLine();
         }
 
         foreach (ServiceMethod method in Methods)

src/LibProtodec/Models/Protobuf/Types/Descriptor.cs

@@ -1,54 +0,0 @@
-// Copyright © 2024 Xpl0itR
-// 
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-using CommunityToolkit.Diagnostics;
-using LibProtodec.Models.Protobuf.TopLevels;
-
-namespace LibProtodec.Models.Protobuf.Types;
-
-public sealed class Descriptor : IProtobufType
-{
-    public int TypeIndex { get; set; }
-    public bool IsRepeated { get; set; }
-    public /*TopLevel*/IProtobufType? TopLevelType { get; set; }
-
-    public string Name =>
-        Type.Name;
-
-    public IProtobufType Type
-    {
-        get
-        {
-            IProtobufType type = TopLevelType as IProtobufType ?? TypeIndex switch
-            {
-                1  => Scalar.Double,
-                2  => Scalar.Float,
-                3  => Scalar.Int64,
-                4  => Scalar.UInt64,
-                5  => Scalar.Int32,
-                6  => Scalar.Fixed64,
-                7  => Scalar.Fixed32,
-                8  => Scalar.Bool,
-                9  => Scalar.String,
-                10 => ThrowHelper.ThrowNotSupportedException<IProtobufType>("Parsing proto2 groups are not supported. Open an issue if you need this."),
-                12 => Scalar.Bytes,
-                13 => Scalar.UInt32,
-                15 => Scalar.SFixed32,
-                16 => Scalar.SFixed64,
-                17 => Scalar.SInt32,
-                18 => Scalar.SInt64,
-                _ => ThrowHelper.ThrowArgumentOutOfRangeException<IProtobufType>()
-            };
-
-            if (IsRepeated)
-            {
-                type = new Repeated(type);
-            }
-
-            return type;
-        }
-    }
-}

src/LibProtodec/ProtodecContext.Lua.cs

@@ -5,28 +5,39 @@
 // file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 using System.Collections.Generic;
-using System.IO;
 using System.Linq;
+using SystemEx;
 using CommunityToolkit.Diagnostics;
+using LibProtodec.Loaders;
 using LibProtodec.Models.Protobuf;
 using LibProtodec.Models.Protobuf.Fields;
 using LibProtodec.Models.Protobuf.TopLevels;
 using LibProtodec.Models.Protobuf.Types;
 using Loretta.CodeAnalysis;
 using Loretta.CodeAnalysis.Lua.Syntax;
-using SystemEx;
+using FdpTypes = Google.Protobuf.Reflection.FieldDescriptorProto.Types;
 
 namespace LibProtodec;
 
+// TODO: add debug logging
 partial class ProtodecContext
 {
-    public Protobuf ParseLuaSyntaxTree(SyntaxTree ast)
+    private readonly Dictionary<string, Dictionary<string, object>> _parsedPbTables = [];
+    private readonly Dictionary<string, Protobuf> _parsedProtobufs = [];
+
+    public Protobuf ParseLuaSyntaxTree(LuaSourceLoader loader, SyntaxTree ast, ParserOptions options = ParserOptions.None)
     {
+        if (_parsedProtobufs.TryGetValue(ast.FilePath, out Protobuf? parsedProto))
+        {
+            return parsedProto;
+        }
+
         CompilationUnitSyntax root = (CompilationUnitSyntax)ast.GetRoot();
         SyntaxList<StatementSyntax> statements = root.Statements.Statements;
 
         bool importedProtobufLib = false;
-        Dictionary<string, object> pbTable = [];
+        Dictionary<string, object> pbTable = _parsedPbTables[ast.FilePath] = [];
+        Dictionary<string, Dictionary<string, object>> imports = [];
         Protobuf protobuf = new()
         {
             Version    = 2,
@@ -39,6 +50,7 @@ partial class ProtodecContext
             {
                 case LocalVariableDeclarationStatementSyntax
                 {
+                    Names: [{ IdentifierName.Name: { } importKey }],
                     EqualsValues.Values: [FunctionCallExpressionSyntax { Expression: IdentifierNameSyntax { Name: "require" } } call]
                 }:
                     switch (call.Argument)
@@ -46,9 +58,11 @@ partial class ProtodecContext
                         case StringFunctionArgumentSyntax { Expression.Token.ValueText: "protobuf/protobuf" }:
                             importedProtobufLib = true;
                             break;
-                        case ExpressionListFunctionArgumentSyntax { Expressions: [LiteralExpressionSyntax { Token.ValueText: {} import }] }:
+                        case ExpressionListFunctionArgumentSyntax { Expressions: [LiteralExpressionSyntax { Token.ValueText: { } import }] }:
-                            import = Path.GetFileNameWithoutExtension(import).TrimEnd("_pb"); //todo: handle imports properly
+                            SyntaxTree importedAst = loader.ResolveImport(import);
-                            protobuf.Imports.Add($"{import}.proto");
+                            Protobuf importedProto = ParseLuaSyntaxTree(loader, importedAst);
+                            imports.Add(importKey, _parsedPbTables[importedAst.FilePath]);
+                            protobuf.Imports.Add(importedProto.FileName);
                             break;
                     }
                     break;
@@ -57,9 +71,9 @@ partial class ProtodecContext
                     Variables: [MemberAccessExpressionSyntax
                     {
                         Expression: IdentifierNameSyntax,
-                        MemberName.ValueText: {} tableKey
+                        MemberName.ValueText: { } tableKey
                     }],
-                    EqualsValues.Values: [FunctionCallExpressionSyntax { Expression: MemberAccessExpressionSyntax { MemberName.ValueText: {} factory } }]
+                    EqualsValues.Values: [FunctionCallExpressionSyntax { Expression: MemberAccessExpressionSyntax { MemberName.ValueText: { } factory } }]
                 }:
                     switch (factory)
                     {
@@ -82,7 +96,7 @@ partial class ProtodecContext
                             pbTable.Add(tableKey, @enum);
                             break;
                         case "FieldDescriptor":
-                            pbTable.Add(tableKey, new MessageField { Type = new Descriptor() });
+                            pbTable.Add(tableKey, new MessageField());
                             break;
                         case "EnumValueDescriptor":
                             pbTable.Add(tableKey, new EnumField());
@@ -93,7 +107,7 @@ partial class ProtodecContext
                 {
                     Variables: [MemberAccessExpressionSyntax
                     {
-                        Expression: MemberAccessExpressionSyntax { MemberName.ValueText: {} tableKey },
+                        Expression: MemberAccessExpressionSyntax { MemberName.ValueText: { } tableKey },
                         MemberName.ValueText: { } memberName
                     }],
                     EqualsValues.Values: [{ } valueExpr]
@@ -103,7 +117,6 @@ partial class ProtodecContext
                         .Cast<UnkeyedTableFieldSyntax>()
                         .Select(static element => element.Value);
                     object? valueLiteral = (valueExpr as LiteralExpressionSyntax)?.Token.Value;
-
                     switch (pbTable[tableKey])
                     {
                         case Message message:
@@ -152,7 +165,6 @@ partial class ProtodecContext
                             }
                             break;
                         case MessageField messageField:
-                            Descriptor descriptor = (Descriptor)messageField.Type;
                             switch (memberName)
                             {
                                 case "name":
@@ -162,29 +174,36 @@ partial class ProtodecContext
                                     messageField.Id = (int)(double)valueLiteral!;
                                     break;
                                 case "label":
-                                    switch ((int)(double)valueLiteral!)
+                                    switch ((FdpTypes.Label)(double)valueLiteral!)
                                     {
-                                        case 1:
+                                        case FdpTypes.Label.Optional:
                                             messageField.IsOptional = true;
                                             break;
-                                        case 2:
+                                        case FdpTypes.Label.Required:
                                             messageField.IsRequired = true;
                                             break;
-                                        case 3:
+                                        case FdpTypes.Label.Repeated:
-                                            descriptor.IsRepeated = true;
+                                            messageField.IsRepeated = true;
                                             break;
                                     }
                                     break;
-                                case "type":
+                                case "enum_type" when (options & ParserOptions.SkipEnums) > 0:
-                                    descriptor.TypeIndex = (int)(double)valueLiteral!;
+                                    messageField.Type = Scalar.Int32;
                                     break;
-                                case "message_type":
                                 case "enum_type":
-                                    string typeTableKey = ((MemberAccessExpressionSyntax)valueExpr).MemberName.ValueText;
+                                case "message_type":
-                                    if (pbTable.TryGetValue(typeTableKey, out object? topLevel)) //TODO: if this is false, then the top level is from an import, we will need to handle this
+                                    MemberAccessExpressionSyntax memberAccessExpr = (MemberAccessExpressionSyntax)valueExpr;
-                                        descriptor.TopLevelType = (IProtobufType)topLevel;
+                                    string importKey = ((IdentifierNameSyntax)memberAccessExpr.Expression).Name;
-                                    else
+                                    Dictionary<string, object> table = imports.GetValueOrDefault(importKey, pbTable);
-                                        descriptor.TopLevelType = new Scalar(typeTableKey); // temporary hack
+                                    string typeTableKey = memberAccessExpr.MemberName.ValueText;
+                                    IProtobufType scalar = (IProtobufType)table[typeTableKey];
+                                    messageField.Type = messageField.IsRepeated
+                                        ? new Repeated(scalar)
+                                        : scalar;
+                                    break;
+                                case "type":
+                                    messageField.Type ??= ParseFieldType(
+                                        (FdpTypes.Type)(double)valueLiteral!, messageField.IsRepeated);
                                     break;
                                 case "has_default_value":
                                     if ((bool)valueLiteral!)
@@ -220,6 +239,35 @@ partial class ProtodecContext
         }
 
         this.Protobufs.Add(protobuf);
+        _parsedProtobufs.Add(ast.FilePath, protobuf);
         return protobuf;
     }
+
+    protected static IProtobufType ParseFieldType(FdpTypes.Type type, bool isRepeated)
+    {
+        IProtobufType scalar = type switch
+        {
+            FdpTypes.Type.Double   => Scalar.Double,
+            FdpTypes.Type.Float    => Scalar.Float,
+            FdpTypes.Type.Int64    => Scalar.Int64,
+            FdpTypes.Type.Uint64   => Scalar.UInt64,
+            FdpTypes.Type.Int32    => Scalar.Int32,
+            FdpTypes.Type.Fixed64  => Scalar.Fixed64,
+            FdpTypes.Type.Fixed32  => Scalar.Fixed32,
+            FdpTypes.Type.Bool     => Scalar.Bool,
+            FdpTypes.Type.String   => Scalar.String,
+            FdpTypes.Type.Bytes    => Scalar.Bytes,
+            FdpTypes.Type.Uint32   => Scalar.UInt32,
+            FdpTypes.Type.Sfixed32 => Scalar.SFixed32,
+            FdpTypes.Type.Sfixed64 => Scalar.SFixed64,
+            FdpTypes.Type.Sint32   => Scalar.SInt32,
+            FdpTypes.Type.Sint64   => Scalar.SInt64,
+            FdpTypes.Type.Group    => ThrowHelper.ThrowNotSupportedException<IProtobufType>(
+                "Parsing proto2 groups are not supported. Open an issue if you need this."),
+        };
+
+        return isRepeated
+            ? new Repeated(scalar)
+            : scalar;
+    }
 }

src/LibProtodec/ProtodecContext.cs

@@ -36,14 +36,13 @@ public partial class ProtodecContext
 
     public void WriteAllTo(IndentedTextWriter writer)
     {
-        writer.WriteLine("// Decompiled with protodec");
+        writer.Write("// Decompiled with protodec");
-        writer.WriteLine();
 
         foreach (TopLevel topLevel in Protobufs.SelectMany(static proto => proto.TopLevels))
         {
+            writer.WriteLine();
+            writer.WriteLine();
             topLevel.WriteTo(writer);
-            writer.WriteLine();
-            writer.WriteLine();
         }
     }
 

src/protodec/Program.cs

@@ -117,17 +117,23 @@ internal sealed class Commands
     }
 
     /// <summary>
-    ///     Use Lua AST backend to load Lua source files.
+    ///     Use Loretta backend to load Lua source files.
     /// </summary>
     /// <param name="targetPath">Either the path to the target lua file or a directory of lua files, all of which be parsed.</param>
     /// <param name="outPath">An existing directory to output into individual files, otherwise output to a single file.</param>
+    /// <param name="skipEnums">Skip parsing enums and replace references to them with int32.</param>
     /// <param name="logLevel">Logging severity level.</param>
     [Command("lua")]
     public void Lua(
         [Argument] string targetPath,
         [Argument] string outPath,
+        bool     skipEnums,
         LogLevel logLevel = LogLevel.Information)
     {
+        ParserOptions options = ParserOptions.None;
+        if (skipEnums)
+            options |= ParserOptions.SkipEnums;
+
         using ILoggerFactory loggerFactory = CreateLoggerFactory(logLevel);
         ILogger logger = CreateProtodecLogger(loggerFactory);
 
@@ -140,9 +146,9 @@ internal sealed class Commands
         };
 
         logger.LogInformation("Parsing Lua syntax trees...");
-        foreach (SyntaxTree ast in loader.LoadedSyntaxTrees)
+        foreach (SyntaxTree ast in loader.LoadedSyntaxTrees.Values)
         {
-            ctx.ParseLuaSyntaxTree(ast);
+            ctx.ParseLuaSyntaxTree(loader, ast, options);
         }
 
         // NOTE: I'm duplicating this code rather than refactoring because I have a lot of uncommited changes on another computer,