[PATCH v4 1/2] widl: Prevent infinite loop when structure contains array of itself

[Fabian Maurer]

From: Fabian Maurer

Wine-Bug: https://bugs.winehq.org/show_bug.cgi?id=55115
---
 tools/widl/typegen.c | 62 ++++++++++++++++++++++++++++++++++++++------
 1 file changed, 54 insertions(+), 8 deletions(-)

diff --git a/tools/widl/typegen.c b/tools/widl/typegen.c
index b3373ded11d..332bff87fd4 100644
--- a/tools/widl/typegen.c
+++ b/tools/widl/typegen.c
@@ -746,8 +746,15 @@ static int type_has_pointers(const type_t *type)
     return FALSE;
 }

+struct visited_struct_array
+{
+    const type_t **structs;
+    int count;
+    int capacity;
+};
+
 static int type_has_full_pointer(const type_t *type, const attr_list_t *attrs,
-                                 int toplevel_param)
+                                 int toplevel_param, struct visited_struct_array *visited_structs)
 {
     switch (typegen_detect_type(type, NULL, TDT_IGNORE_STRINGS))
     {
@@ -762,17 +769,55 @@ static int type_has_full_pointer(const type_t *type, const attr_list_t *attrs,
         if (get_pointer_fc(type, attrs, toplevel_param) == FC_FP)
             return TRUE;
         else
-            return type_has_full_pointer(type_array_get_element_type(type), NULL, FALSE);
+            return type_has_full_pointer(type_array_get_element_type(type), NULL, FALSE, visited_structs);
     case TGT_STRUCT:
     {
+        int i;
+        int ret = FALSE;
         var_list_t *fields = type_struct_get_fields(type);
         const var_t *field;
+
+        if (!visited_structs->structs)
+        {
+            visited_structs->count = 0;
+            visited_structs->capacity = 100;
+            visited_structs->structs = xmalloc(sizeof (type_t*) * visited_structs->capacity);
+        }
+
+        if (visited_structs->count >= visited_structs->capacity)
+        {
+            visited_structs->capacity *= 2;
+            visited_structs->structs = xrealloc(visited_structs->structs, sizeof (type_t*) * visited_structs->capacity);
+        }
+
+        visited_structs->structs[visited_structs->count] = type;
+        for (i = 0; i < visited_structs->count; i++)
+        {
+            if (visited_structs->structs[i] == type)
+            {
+                /* Found struct we visited already, abort to prevent infinite loop.
+                 * Can't be at the first struct we visit, so we can skip cleanup and just return */
+               return FALSE;
+            }
+        }
+
+        visited_structs->count++;
         if (fields) LIST_FOR_EACH_ENTRY( field, fields, const var_t, entry )
         {
-            if (type_has_full_pointer(field->declspec.type, field->attrs, FALSE))
-                return TRUE;
+            if (type_has_full_pointer(field->declspec.type, field->attrs, FALSE, visited_structs))
+            {
+                ret = TRUE;
+                break;
+            }
         }
-        break;
+        visited_structs->count--;
+        if (!visited_structs->count)
+        {
+            /* End of recursion, clean up */
+            free(visited_structs->structs);
+            visited_structs->structs = 0;
+        }
+        return ret;
     }
     case TGT_UNION:
     {
@@ -781,7 +826,7 @@ static int type_has_full_pointer(const type_t *type, const attr_list_t *attrs,
         fields = type_union_get_cases(type);
         if (fields) LIST_FOR_EACH_ENTRY( field, fields, const var_t, entry )
         {
-            if (field->declspec.type && type_has_full_pointer(field->declspec.type, field->attrs, FALSE))
+            if (field->declspec.type && type_has_full_pointer(field->declspec.type, field->attrs, FALSE, visited_structs))
                 return TRUE;
         }
         break;
@@ -2082,12 +2127,13 @@ static unsigned int type_buffer_alignment(const type_t *t)
 int is_full_pointer_function(const var_t *func)
 {
     const var_t *var;
-    if (type_has_full_pointer(type_function_get_rettype(func->declspec.type), func->attrs, TRUE))
+    struct visited_struct_array visited_structs = {0};
+    if (type_has_full_pointer(type_function_get_rettype(func->declspec.type), func->attrs, TRUE, &visited_structs))
         return TRUE;
     if (!type_function_get_args(func->declspec.type))
         return FALSE;
     LIST_FOR_EACH_ENTRY( var, type_function_get_args(func->declspec.type), const var_t, entry )
-        if (type_has_full_pointer( var->declspec.type, var->attrs, TRUE ))
+        if (type_has_full_pointer( var->declspec.type, var->attrs, TRUE, &visited_structs ))
             return TRUE;
     return FALSE;
 }