Add check for overflow in queue size calculation in RTOS compatibility layer. (#339)

* Add check for overflow in queue size calculation.

* Update release data and version.

1 files changed, 18 insertions, 0 deletions

diff --git a/utility/rtos_compatibility_layers/FreeRTOS/tx_freertos.c b/utility/rtos_compatibility_layers/FreeRTOS/tx_freertos.c
index 9882b654..b963f31f 100644
--- a/utility/rtos_compatibility_layers/FreeRTOS/tx_freertos.c
+++ b/utility/rtos_compatibility_layers/FreeRTOS/tx_freertos.c
@@ -33,6 +33,10 @@
 /*                                            start flag, corrected stack */
 /*                                            allocation size,            */
 /*                                            resulting in version 6.1.12 */
+/*  12-31-2023     Xiuwen Cai               Modified comment(s), and      */
+/*                                            added check for overflow in */
+/*                                            queue size calculation,     */
+/*                                            resulting in version 6.4.0  */
 /*                                                                        */
 /**************************************************************************/
 
@@ -1526,6 +1530,13 @@ QueueHandle_t xQueueCreate(UBaseType_t uxQueueLength, UBaseType_t uxItemSize)
     }
 #endif
 
+    if ((uxQueueLength > (SIZE_MAX / uxItemSize)) ||
+        (uxQueueLength > (ULONG_MAX / uxItemSize))) {
+
+        /* Integer overflow in queue size */
+        return NULL;
+    }
+
     p_queue = txfr_malloc(sizeof(txfr_queue_t));
     if(p_queue == NULL) {
         return NULL;
@@ -2692,6 +2703,13 @@ QueueSetHandle_t xQueueCreateSet(const UBaseType_t uxEventQueueLength)
     }
 #endif
 
+    if ((uxEventQueueLength > (SIZE_MAX / sizeof(void *))) ||
+        (uxEventQueueLength > (ULONG_MAX / sizeof(void *)))) {
+
+        /* Integer overflow in queue size */
+        return NULL;
+    }
+
     p_set = txfr_malloc(sizeof(txfr_queueset_t));
     if(p_set == NULL) {
         return NULL;