Improve attribute parsing (#82)

balazssomogyi · sombalazs · xoofx · web-flow · commit 7b1e0da9d13a · 2023-10-14T17:55:44.000+02:00
Adding support for parsing attributes defined as a macro,
and parsing function attributes that are on a separate line
before the function declaration.

Co-authored-by: Balazs Somogyi &lt;balazs.somogyi@cognex.com&gt;
Co-authored-by: Alexandre Mutel &lt;alexandre_mutel@live.com&gt;
diff --git a/src/CppAst.Tests/AttributesTest/TestTokenAttributes.cs b/src/CppAst.Tests/AttributesTest/TestTokenAttributes.cs
@@ -151,6 +151,34 @@ struct [[deprecated(""old"")]] TestMessage{
           );
         }
 
+        [Test]
+        public void TestCpp11StructAttributesWithMacro()
+        {
+            ParseAssert(@"
+#define CLASS_ATTRIBUTE [[complex_attribute::attribute_name(""attribute_argument"")]]
+struct
+CLASS_ATTRIBUTE
+Test{
+    int a;
+    int b;
+};", compilation =>
+                {
+                    Assert.False(compilation.HasErrors);
+
+                    Assert.AreEqual(1, compilation.Classes.Count);
+                    Assert.AreEqual(1, compilation.Classes[0].TokenAttributes.Count);
+                    {
+                        var attr = compilation.Classes[0].TokenAttributes[0];
+                        Assert.AreEqual("complex_attribute", attr.Scope);
+                        Assert.AreEqual("attribute_name", attr.Name);
+                        Assert.AreEqual("\"attribute_argument\"", attr.Arguments);
+                    }
+                },
+                // we are using a C++14 attribute because it can be used everywhere
+                new CppParserOptions() { AdditionalArguments = { "-std=c++14" }, ParseTokenAttributes = true }
+            );
+        }
+
         [Test]
         public void TestCpp11VariablesAttributes()
         {
@@ -204,6 +232,27 @@ public void TestCpp11FunctionsAttributes()
           );
         }
 
+        [Test]
+        public void TestCpp11FunctionsAttributesOnNewLine()
+        {
+            ParseAssert(@"
+[[noreturn]]
+void x() {};", compilation =>
+                {
+                    Assert.False(compilation.HasErrors);
+
+                    Assert.AreEqual(1, compilation.Functions.Count);
+                    Assert.AreEqual(1, compilation.Functions[0].TokenAttributes.Count);
+                    {
+                        var attr = compilation.Functions[0].TokenAttributes[0];
+                        Assert.AreEqual("noreturn", attr.Name);
+                    }
+                },
+                // we are using a C++14 attribute because it can be used everywhere
+                new CppParserOptions() { AdditionalArguments = { "-std=c++14" }, ParseTokenAttributes = true }
+            );
+        }
+
         [Test]
         public void TestCpp11NamespaceAttributes()
         {
diff --git a/src/CppAst/CppModelBuilder.cs b/src/CppAst/CppModelBuilder.cs
@@ -1541,19 +1541,19 @@ private void ParseAttributes(CXCursor cursor, ICppAttributeContainer attrContain
                 bool hasOnlineAttribute = CppTokenUtil.TryToSeekOnlineAttributes(cursor, out var onLineRange);
                 if (hasOnlineAttribute)
                 {
-                    CppTokenUtil.ParseAttributesInRange(cursor.TranslationUnit, onLineRange, ref tokenAttributes);
+                    CppTokenUtil.ParseAttributesInRange(_rootContainerContext.Container as CppGlobalDeclarationContainer, cursor.TranslationUnit, onLineRange, ref tokenAttributes);
                 }
             }
 
             //Parse attributes contains in cursor
             if (attrContainer is CppFunction)
             {
                 var func = attrContainer as CppFunction;
-                CppTokenUtil.ParseFunctionAttributes(cursor, func.Name, ref tokenAttributes);
+                CppTokenUtil.ParseFunctionAttributes(_rootContainerContext.Container as CppGlobalDeclarationContainer, cursor, func.Name, ref tokenAttributes);
             }
             else
             {
-                CppTokenUtil.ParseCursorAttributs(cursor, ref tokenAttributes);
+                CppTokenUtil.ParseCursorAttributs(_rootContainerContext.Container as CppGlobalDeclarationContainer, cursor, ref tokenAttributes);
             }
 
             attrContainer.TokenAttributes.AddRange(tokenAttributes);
diff --git a/src/CppAst/CppTokenUtil.cs b/src/CppAst/CppTokenUtil.cs
@@ -14,7 +14,7 @@ namespace CppAst
 {
     static internal unsafe class CppTokenUtil
     {
-        public static void ParseCursorAttributs(CXCursor cursor, ref List<CppAttribute> attributes)
+        public static void ParseCursorAttributs(CppGlobalDeclarationContainer globalContainer, CXCursor cursor, ref List<CppAttribute> attributes)
         {
             var tokenizer = new AttributeTokenizer(cursor);
             var tokenIt = new TokenIterator(tokenizer);
@@ -25,7 +25,7 @@ public static void ParseCursorAttributs(CXCursor cursor, ref List<CppAttribute>
 
             while (tokenIt.CanPeek)
             {
-                if (ParseAttributes(tokenIt, ref attributes))
+                if (ParseAttributes(globalContainer, tokenIt, ref attributes))
                 {
                     continue;
                 }
@@ -42,7 +42,7 @@ public static void ParseCursorAttributs(CXCursor cursor, ref List<CppAttribute>
         }
 
 
-        public static void ParseFunctionAttributes(CXCursor cursor, string functionName, ref List<CppAttribute> attributes)
+        public static void ParseFunctionAttributes(CppGlobalDeclarationContainer globalContainer, CXCursor cursor, string functionName, ref List<CppAttribute> attributes)
         {
             // TODO: This function is not 100% correct when parsing tokens up to the function name
             // we assume to find the function name immediately followed by a `(`
@@ -58,7 +58,7 @@ public static void ParseFunctionAttributes(CXCursor cursor, string functionName,
             // Parse leading attributes
             while (tokenIt.CanPeek)
             {
-                if (ParseAttributes(tokenIt, ref attributes))
+                if (ParseAttributes(globalContainer, tokenIt, ref attributes))
                 {
                     continue;
                 }
@@ -103,7 +103,7 @@ public static void ParseFunctionAttributes(CXCursor cursor, string functionName,
 
             while (tokenIt.CanPeek)
             {
-                if (ParseAttributes(tokenIt, ref attributes))
+                if (ParseAttributes(globalContainer, tokenIt, ref attributes))
                 {
                     continue;
                 }
@@ -115,7 +115,7 @@ public static void ParseFunctionAttributes(CXCursor cursor, string functionName,
         }
 
 
-        public static void ParseAttributesInRange(CXTranslationUnit tu, CXSourceRange range, ref List<CppAttribute> collectAttributes)
+        public static void ParseAttributesInRange(CppGlobalDeclarationContainer globalContainer, CXTranslationUnit tu, CXSourceRange range, ref List<CppAttribute> collectAttributes)
         {
             var tokenizer = new AttributeTokenizer(tu, range);
             var tokenIt = new TokenIterator(tokenizer);
@@ -134,7 +134,7 @@ public static void ParseAttributesInRange(CXTranslationUnit tu, CXSourceRange ra
 
             while (tokenIt.CanPeek)
             {
-                if (ParseAttributes(tokenIt, ref collectAttributes))
+                if (ParseAttributes(globalContainer, tokenIt, ref collectAttributes))
                 {
                     continue;
                 }
@@ -615,17 +615,17 @@ bool HasInlineTypeDefinition(CXCursor varDecl)
                 CXSourceLocation GetNextLocation(CXSourceLocation loc, int inc = 1)
                 {
                     CXSourceLocation value;
-                    loc.GetSpellingLocation(out var f, out var u, out var z, out var originalOffset);
-                    var offset = IncOffset(inc, z);
-                    var shouldUseLine = (z != 0 && (offset != 0 || offset != uint.MaxValue));
+                    loc.GetSpellingLocation(out var file, out var line, out var column, out var originalOffset);
+                    var signedOffset = (int)column + inc;
+                    var shouldUseLine = (column != 0 && signedOffset > 0);
                     if (shouldUseLine)
                     {
-                        value = tu.GetLocation(f, u, offset);
+                        value = tu.GetLocation(file, line, (uint)signedOffset);
                     }
                     else
                     {
-                        offset = IncOffset(inc, originalOffset);
-                        value = tu.GetLocationForOffset(f, offset);
+                        var offset = IncOffset(inc, originalOffset);
+                        value = tu.GetLocationForOffset(file, offset);
                     }
 
                     return value;
@@ -911,8 +911,42 @@ private enum AttributeLexerParseStatus
             Error,
         }
 
+        private static (string, string) GetNameSpaceAndAttribute(string fullAttribute)
+        {
+            string[] colons = { "::" };
+            string[] tokens = fullAttribute.Split(colons, System.StringSplitOptions.None);
+            if (tokens.Length == 2)
+            {
+                return (tokens[0], tokens[1]);
+            }
+            else
+            {
+                return (null, tokens[0]);
+            }
+        }
+
+
+        private static (string, string) GetNameAndArguments(string name)
+        {
+            if (name.Contains("("))
+            {
+                Char[] seperator = { '(' };
+                var argumentTokens = name.Split(seperator, 2);
+                var length = argumentTokens[1].LastIndexOf(')');
+                string argument = null;
+                if (length > 0)
+                {
+                    argument = argumentTokens[1].Substring(0, length);
+                }
+                return (argumentTokens[0], argument);
+            }
+            else
+            {
+                return (name, null);
+            }
+        }
 
-        private static bool ParseAttributes(TokenIterator tokenIt, ref List<CppAttribute> attributes)
+        private static bool ParseAttributes(CppGlobalDeclarationContainer globalContainer, TokenIterator tokenIt, ref List<CppAttribute> attributes)
         {
             // Parse C++ attributes
             // [[<attribute>]]
@@ -985,6 +1019,32 @@ private static bool ParseAttributes(TokenIterator tokenIt, ref List<CppAttribute
                 return tokenIt.Skip(")"); ;
             }
 
+            // See if we have a macro
+            var value = tokenIt.PeekText();
+            var macro = globalContainer.Macros.Find(v => v.Name == value);
+            if (macro != null)
+            {
+                if (macro.Value.StartsWith("[[") && macro.Value.EndsWith("]]"))
+                {
+                    CppAttribute attribute = null;
+                    var fullAttribute = macro.Value.Substring(2, macro.Value.Length - 4);
+                    var (scope, name) = GetNameSpaceAndAttribute(fullAttribute);
+                    var (attributeName, arguments) = GetNameAndArguments(name);
+
+                    attribute = new CppAttribute(attributeName, AttributeKind.TokenAttribute);
+                    attribute.Scope = scope;
+                    attribute.Arguments = arguments;
+
+                    if (attributes == null)
+                    {
+                        attributes = new List<CppAttribute>();
+                    }
+                    attributes.Add(attribute);
+                    tokenIt.Next();
+                    return true;
+                }
+            }
+
             return false;
         }
 

Original file line number	Diff line number	Diff line change
`@@ -1541,19 +1541,19 @@ private void ParseAttributes(CXCursor cursor, ICppAttributeContainer attrContain`
`1541`	`1541`	`bool hasOnlineAttribute = CppTokenUtil.TryToSeekOnlineAttributes(cursor, out var onLineRange);`
`1542`	`1542`	`if (hasOnlineAttribute)`
`1543`	`1543`	`{`
`1544`		`- CppTokenUtil.ParseAttributesInRange(cursor.TranslationUnit, onLineRange, ref tokenAttributes);`
	`1544`	`+ CppTokenUtil.ParseAttributesInRange(_rootContainerContext.Container as CppGlobalDeclarationContainer, cursor.TranslationUnit, onLineRange, ref tokenAttributes);`
`1545`	`1545`	`}`
`1546`	`1546`	`}`
`1547`	`1547`
`1548`	`1548`	`//Parse attributes contains in cursor`
`1549`	`1549`	`if (attrContainer is CppFunction)`
`1550`	`1550`	`{`
`1551`	`1551`	`var func = attrContainer as CppFunction;`
`1552`		`- CppTokenUtil.ParseFunctionAttributes(cursor, func.Name, ref tokenAttributes);`
	`1552`	`+ CppTokenUtil.ParseFunctionAttributes(_rootContainerContext.Container as CppGlobalDeclarationContainer, cursor, func.Name, ref tokenAttributes);`
`1553`	`1553`	`}`
`1554`	`1554`	`else`
`1555`	`1555`	`{`
`1556`		`- CppTokenUtil.ParseCursorAttributs(cursor, ref tokenAttributes);`
	`1556`	`+ CppTokenUtil.ParseCursorAttributs(_rootContainerContext.Container as CppGlobalDeclarationContainer, cursor, ref tokenAttributes);`
`1557`	`1557`	`}`
`1558`	`1558`
`1559`	`1559`	`attrContainer.TokenAttributes.AddRange(tokenAttributes);`
Original file line number	Diff line number	Diff line change
`@@ -14,7 +14,7 @@ namespace CppAst`
`14`	`14`	`{`
`15`	`15`	`static internal unsafe class CppTokenUtil`
`16`	`16`	`{`
`17`		`- public static void ParseCursorAttributs(CXCursor cursor, ref List<CppAttribute> attributes)`
	`17`	`+ public static void ParseCursorAttributs(CppGlobalDeclarationContainer globalContainer, CXCursor cursor, ref List<CppAttribute> attributes)`
`18`	`18`	`{`
`19`	`19`	`var tokenizer = new AttributeTokenizer(cursor);`
`20`	`20`	`var tokenIt = new TokenIterator(tokenizer);`
`@@ -25,7 +25,7 @@ public static void ParseCursorAttributs(CXCursor cursor, ref List<CppAttribute>`
`25`	`25`
`26`	`26`	`while (tokenIt.CanPeek)`
`27`	`27`	`{`
`28`		`- if (ParseAttributes(tokenIt, ref attributes))`
	`28`	`+ if (ParseAttributes(globalContainer, tokenIt, ref attributes))`
`29`	`29`	`{`
`30`	`30`	`continue;`
`31`	`31`	`}`
`@@ -42,7 +42,7 @@ public static void ParseCursorAttributs(CXCursor cursor, ref List<CppAttribute>`
`42`	`42`	`}`
`43`	`43`
`44`	`44`
`45`		`- public static void ParseFunctionAttributes(CXCursor cursor, string functionName, ref List<CppAttribute> attributes)`
	`45`	`+ public static void ParseFunctionAttributes(CppGlobalDeclarationContainer globalContainer, CXCursor cursor, string functionName, ref List<CppAttribute> attributes)`
`46`	`46`	`{`
`47`	`47`	`// TODO: This function is not 100% correct when parsing tokens up to the function name`
`48`	`48`	// we assume to find the function name immediately followed by a `(`
`@@ -58,7 +58,7 @@ public static void ParseFunctionAttributes(CXCursor cursor, string functionName,`
`58`	`58`	`// Parse leading attributes`
`59`	`59`	`while (tokenIt.CanPeek)`
`60`	`60`	`{`
`61`		`- if (ParseAttributes(tokenIt, ref attributes))`
	`61`	`+ if (ParseAttributes(globalContainer, tokenIt, ref attributes))`
`62`	`62`	`{`
`63`	`63`	`continue;`
`64`	`64`	`}`
`@@ -103,7 +103,7 @@ public static void ParseFunctionAttributes(CXCursor cursor, string functionName,`
`103`	`103`
`104`	`104`	`while (tokenIt.CanPeek)`
`105`	`105`	`{`
`106`		`- if (ParseAttributes(tokenIt, ref attributes))`
	`106`	`+ if (ParseAttributes(globalContainer, tokenIt, ref attributes))`
`107`	`107`	`{`
`108`	`108`	`continue;`
`109`	`109`	`}`
`@@ -115,7 +115,7 @@ public static void ParseFunctionAttributes(CXCursor cursor, string functionName,`
`115`	`115`	`}`
`116`	`116`
`117`	`117`
`118`		`- public static void ParseAttributesInRange(CXTranslationUnit tu, CXSourceRange range, ref List<CppAttribute> collectAttributes)`
	`118`	`+ public static void ParseAttributesInRange(CppGlobalDeclarationContainer globalContainer, CXTranslationUnit tu, CXSourceRange range, ref List<CppAttribute> collectAttributes)`
`119`	`119`	`{`
`120`	`120`	`var tokenizer = new AttributeTokenizer(tu, range);`
`121`	`121`	`var tokenIt = new TokenIterator(tokenizer);`
`@@ -134,7 +134,7 @@ public static void ParseAttributesInRange(CXTranslationUnit tu, CXSourceRange ra`
`134`	`134`
`135`	`135`	`while (tokenIt.CanPeek)`
`136`	`136`	`{`
`137`		`- if (ParseAttributes(tokenIt, ref collectAttributes))`
	`137`	`+ if (ParseAttributes(globalContainer, tokenIt, ref collectAttributes))`
`138`	`138`	`{`
`139`	`139`	`continue;`
`140`	`140`	`}`
`@@ -615,17 +615,17 @@ bool HasInlineTypeDefinition(CXCursor varDecl)`
`615`	`615`	`CXSourceLocation GetNextLocation(CXSourceLocation loc, int inc = 1)`
`616`	`616`	`{`
`617`	`617`	`CXSourceLocation value;`
`618`		`- loc.GetSpellingLocation(out var f, out var u, out var z, out var originalOffset);`
`619`		`- var offset = IncOffset(inc, z);`
`620`		`- var shouldUseLine = (z != 0 && (offset != 0 \|\| offset != uint.MaxValue));`
	`618`	`+ loc.GetSpellingLocation(out var file, out var line, out var column, out var originalOffset);`
	`619`	`+ var signedOffset = (int)column + inc;`
	`620`	`+ var shouldUseLine = (column != 0 && signedOffset > 0);`
`621`	`621`	`if (shouldUseLine)`
`622`	`622`	`{`
`623`		`- value = tu.GetLocation(f, u, offset);`
	`623`	`+ value = tu.GetLocation(file, line, (uint)signedOffset);`
`624`	`624`	`}`
`625`	`625`	`else`
`626`	`626`	`{`
`627`		`- offset = IncOffset(inc, originalOffset);`
`628`		`- value = tu.GetLocationForOffset(f, offset);`
	`627`	`+ var offset = IncOffset(inc, originalOffset);`
	`628`	`+ value = tu.GetLocationForOffset(file, offset);`
`629`	`629`	`}`
`630`	`630`
`631`	`631`	`return value;`
`@@ -911,8 +911,42 @@ private enum AttributeLexerParseStatus`
`911`	`911`	`Error,`
`912`	`912`	`}`
`913`	`913`
	`914`	`+ private static (string, string) GetNameSpaceAndAttribute(string fullAttribute)`
	`915`	`+ {`
	`916`	`+ string[] colons = { "::" };`
	`917`	`+ string[] tokens = fullAttribute.Split(colons, System.StringSplitOptions.None);`
	`918`	`+ if (tokens.Length == 2)`
	`919`	`+ {`
	`920`	`+ return (tokens[0], tokens[1]);`
	`921`	`+ }`
	`922`	`+ else`
	`923`	`+ {`
	`924`	`+ return (null, tokens[0]);`
	`925`	`+ }`
	`926`	`+ }`
	`927`	`+`
	`928`	`+`
	`929`	`+ private static (string, string) GetNameAndArguments(string name)`
	`930`	`+ {`
	`931`	`+ if (name.Contains("("))`
	`932`	`+ {`
	`933`	`+ Char[] seperator = { '(' };`
	`934`	`+ var argumentTokens = name.Split(seperator, 2);`
	`935`	`+ var length = argumentTokens[1].LastIndexOf(')');`
	`936`	`+ string argument = null;`
	`937`	`+ if (length > 0)`
	`938`	`+ {`
	`939`	`+ argument = argumentTokens[1].Substring(0, length);`
	`940`	`+ }`
	`941`	`+ return (argumentTokens[0], argument);`
	`942`	`+ }`
	`943`	`+ else`
	`944`	`+ {`
	`945`	`+ return (name, null);`
	`946`	`+ }`
	`947`	`+ }`
`914`	`948`
`915`		`- private static bool ParseAttributes(TokenIterator tokenIt, ref List<CppAttribute> attributes)`
	`949`	`+ private static bool ParseAttributes(CppGlobalDeclarationContainer globalContainer, TokenIterator tokenIt, ref List<CppAttribute> attributes)`
`916`	`950`	`{`
`917`	`951`	`// Parse C++ attributes`
`918`	`952`	`// [[<attribute>]]`
`@@ -985,6 +1019,32 @@ private static bool ParseAttributes(TokenIterator tokenIt, ref List<CppAttribute`
`985`	`1019`	`return tokenIt.Skip(")"); ;`
`986`	`1020`	`}`
`987`	`1021`
	`1022`	`+ // See if we have a macro`
	`1023`	`+ var value = tokenIt.PeekText();`
	`1024`	`+ var macro = globalContainer.Macros.Find(v => v.Name == value);`
	`1025`	`+ if (macro != null)`
	`1026`	`+ {`
	`1027`	`+ if (macro.Value.StartsWith("[[") && macro.Value.EndsWith("]]"))`
	`1028`	`+ {`
	`1029`	`+ CppAttribute attribute = null;`
	`1030`	`+ var fullAttribute = macro.Value.Substring(2, macro.Value.Length - 4);`
	`1031`	`+ var (scope, name) = GetNameSpaceAndAttribute(fullAttribute);`
	`1032`	`+ var (attributeName, arguments) = GetNameAndArguments(name);`
	`1033`	`+`
	`1034`	`+ attribute = new CppAttribute(attributeName, AttributeKind.TokenAttribute);`
	`1035`	`+ attribute.Scope = scope;`
	`1036`	`+ attribute.Arguments = arguments;`
	`1037`	`+`
	`1038`	`+ if (attributes == null)`
	`1039`	`+ {`
	`1040`	`+ attributes = new List<CppAttribute>();`
	`1041`	`+ }`
	`1042`	`+ attributes.Add(attribute);`
	`1043`	`+ tokenIt.Next();`
	`1044`	`+ return true;`
	`1045`	`+ }`
	`1046`	`+ }`
	`1047`	`+`
`988`	`1048`	`return false;`
`989`	`1049`	`}`
`990`	`1050`