/* * The MIT License (MIT) * * Copyright (c) 2019 Ha Thach (tinyusb.org) * Copyright (c) 2020 Reinhard Panhuber - rework to unmasked pointers * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * This file is part of the TinyUSB stack. */ #include "osal/osal.h" #include "tusb_fifo.h" #define TU_FIFO_DBG 0 // Suppress IAR warning // Warning[Pa082]: undefined behavior: the order of volatile accesses is undefined in this statement #if defined(__ICCARM__) #pragma diag_suppress = Pa082 #endif #if OSAL_MUTEX_REQUIRED TU_ATTR_ALWAYS_INLINE static inline void ff_lock(osal_mutex_t mutex) { if (mutex != NULL) { osal_mutex_lock(mutex, OSAL_TIMEOUT_WAIT_FOREVER); } } TU_ATTR_ALWAYS_INLINE static inline void ff_unlock(osal_mutex_t mutex) { if (mutex != NULL) { osal_mutex_unlock(mutex); } } #else #define ff_lock(_mutex) #define ff_unlock(_mutex) #endif //--------------------------------------------------------------------+ // Setup API //--------------------------------------------------------------------+ bool tu_fifo_config(tu_fifo_t *f, void *buffer, uint16_t depth, uint16_t item_size, bool overwritable) { // Limit index space to 2*depth - this allows for a fast "modulo" calculation // but limits the maximum depth to 2^16/2 = 2^15 and buffer overflows are detectable // only if overflow happens once (important for unsupervised DMA applications) if (depth > 0x8000) { return false; } ff_lock(f->mutex_wr); ff_lock(f->mutex_rd); f->buffer = (uint8_t *)buffer; f->depth = depth; f->item_size = (uint16_t)(item_size & 0x7FFFu); f->overwritable = overwritable; f->rd_idx = 0u; f->wr_idx = 0u; ff_unlock(f->mutex_wr); ff_unlock(f->mutex_rd); return true; } // clear fifo by resetting read and write indices void tu_fifo_clear(tu_fifo_t *f) { ff_lock(f->mutex_wr); ff_lock(f->mutex_rd); f->rd_idx = 0; f->wr_idx = 0; ff_unlock(f->mutex_wr); ff_unlock(f->mutex_rd); } // Change the fifo overwritable mode void tu_fifo_set_overwritable(tu_fifo_t *f, bool overwritable) { if (f->overwritable == overwritable) { return; } ff_lock(f->mutex_wr); ff_lock(f->mutex_rd); f->overwritable = overwritable; ff_unlock(f->mutex_wr); ff_unlock(f->mutex_rd); } //--------------------------------------------------------------------+ // Pull & Push // copy data to/from fifo without updating read/write pointers //--------------------------------------------------------------------+ #ifdef CFG_TUSB_FIFO_ACCESS_FIXED_ADDR_RW32 // Copy to fifo from fixed address buffer (usually a rx register) with TU_FIFO_FIXED_ADDR_RW32 mode static void ff_push_fixed_addr_rw32(uint8_t *ff_buf, const volatile uint32_t *reg_rx, uint16_t len) { // Reading full available 32 bit words from const app address uint16_t full_words = len >> 2; while (full_words--) { const uint32_t tmp32 = *reg_rx; tu_unaligned_write32(ff_buf, tmp32); ff_buf += 4; } // Read the remaining 1-3 bytes from const app address const uint8_t bytes_rem = len & 0x03; if (bytes_rem) { const uint32_t tmp32 = *reg_rx; memcpy(ff_buf, &tmp32, bytes_rem); } } // Copy from fifo to fixed address buffer (usually a tx register) with TU_FIFO_FIXED_ADDR_RW32 mode static void ff_pull_fixed_addr_rw32(volatile uint32_t *reg_tx, const uint8_t *ff_buf, uint16_t len) { // Write full available 32 bit words to const address uint16_t full_words = len >> 2u; while (full_words--) { *reg_tx = tu_unaligned_read32(ff_buf); ff_buf += 4u; } // Write the remaining 1-3 bytes const uint8_t bytes_rem = len & 0x03; if (bytes_rem) { uint32_t tmp32 = 0u; memcpy(&tmp32, ff_buf, bytes_rem); *reg_tx = tmp32; } } #endif // send n items to fifo WITHOUT updating write pointer static void ff_push_n(const tu_fifo_t *f, const void *app_buf, uint16_t n, uint16_t wr_ptr, tu_fifo_access_mode_t copy_mode) { const uint16_t lin_count = f->depth - wr_ptr; const uint16_t wrap_count = n - lin_count; uint16_t lin_bytes = lin_count * f->item_size; uint16_t wrap_bytes = wrap_count * f->item_size; // current buffer of fifo uint8_t *ff_buf = f->buffer + (wr_ptr * f->item_size); switch (copy_mode) { case TU_FIFO_INC_ADDR_RW8: if (n <= lin_count) { // Linear only memcpy(ff_buf, app_buf, n * f->item_size); } else { // Wrap around memcpy(ff_buf, app_buf, lin_bytes); // linear part memcpy(f->buffer, ((const uint8_t *)app_buf) + lin_bytes, wrap_bytes); // wrapped part } break; #ifdef CFG_TUSB_FIFO_ACCESS_FIXED_ADDR_RW32 case TU_FIFO_FIXED_ADDR_RW32: { const volatile uint32_t *reg_rx = (volatile const uint32_t *)app_buf; if (n <= lin_count) { // Linear only ff_push_fixed_addr_rw32(ff_buf, reg_rx, n * f->item_size); } else { // Wrap around // Write full words to linear part of buffer uint16_t lin_4n_bytes = lin_bytes & 0xFFFC; ff_push_fixed_addr_rw32(ff_buf, reg_rx, lin_4n_bytes); ff_buf += lin_4n_bytes; // There could be odd 1-3 bytes before the wrap-around boundary const uint8_t rem = lin_bytes & 0x03; if (rem > 0) { const uint8_t remrem = (uint8_t)tu_min16(wrap_bytes, 4 - rem); const uint32_t tmp32 = *reg_rx; tu_scatter_write32(tmp32, ff_buf, rem, f->buffer, remrem); wrap_bytes -= remrem; ff_buf = f->buffer + remrem; // wrap around } else { ff_buf = f->buffer; // wrap around to beginning } // Write data wrapped part if (wrap_bytes > 0) { ff_push_fixed_addr_rw32(ff_buf, reg_rx, wrap_bytes); } } break; } #endif default: break; // unknown mode } } // get n items from fifo WITHOUT updating read pointer static void ff_pull_n(const tu_fifo_t *f, void *app_buf, uint16_t n, uint16_t rd_ptr, tu_fifo_access_mode_t copy_mode) { const uint16_t lin_count = f->depth - rd_ptr; const uint16_t wrap_count = n - lin_count; // only used if wrapped uint16_t lin_bytes = lin_count * f->item_size; uint16_t wrap_bytes = wrap_count * f->item_size; // current buffer of fifo const uint8_t *ff_buf = f->buffer + (rd_ptr * f->item_size); switch (copy_mode) { case TU_FIFO_INC_ADDR_RW8: if (n <= lin_count) { // Linear only memcpy(app_buf, ff_buf, n * f->item_size); } else { // Wrap around memcpy(app_buf, ff_buf, lin_bytes); // linear part memcpy((uint8_t *)app_buf + lin_bytes, f->buffer, wrap_bytes); // wrapped part } break; #ifdef CFG_TUSB_FIFO_ACCESS_FIXED_ADDR_RW32 case TU_FIFO_FIXED_ADDR_RW32: { volatile uint32_t *reg_tx = (volatile uint32_t *)app_buf; if (n <= lin_count) { // Linear only ff_pull_fixed_addr_rw32(reg_tx, ff_buf, n * f->item_size); } else { // Wrap around case // Read full words from linear part uint16_t lin_4n_bytes = lin_bytes & 0xFFFC; ff_pull_fixed_addr_rw32(reg_tx, ff_buf, lin_4n_bytes); ff_buf += lin_4n_bytes; // There could be odd 1-3 bytes before the wrap-around boundary const uint8_t rem = lin_bytes & 0x03; if (rem > 0) { const uint8_t remrem = (uint8_t)tu_min16(wrap_bytes, 4 - rem); const uint32_t scatter32 = tu_scatter_read32(ff_buf, rem, f->buffer, remrem); *reg_tx = scatter32; wrap_bytes -= remrem; ff_buf = f->buffer + remrem; // wrap around } else { ff_buf = f->buffer; // wrap around to beginning } // Read data wrapped part if (wrap_bytes > 0) { ff_pull_fixed_addr_rw32(reg_tx, ff_buf, wrap_bytes); } } break; } #endif default: break; // unknown mode } } //--------------------------------------------------------------------+ // Index Helper //--------------------------------------------------------------------+ // Advance an absolute index // "absolute" index is only in the range of [0..2*depth) static uint16_t advance_index(uint16_t depth, uint16_t idx, uint16_t offset) { // We limit the index space of p such that a correct wrap around happens // Check for a wrap around or if we are in unused index space - This has to be checked first!! // We are exploiting the wrap around to the correct index uint16_t new_idx = (uint16_t)(idx + offset); if ((idx > new_idx) || (new_idx >= 2 * depth)) { const uint16_t non_used_index_space = (uint16_t)(UINT16_MAX - (2 * depth - 1)); new_idx = (uint16_t)(new_idx + non_used_index_space); } return new_idx; } // index to pointer (0..depth-1), simply a modulo with minus. TU_ATTR_ALWAYS_INLINE static inline uint16_t idx2ptr(uint16_t depth, uint16_t idx) { // Only run at most 3 times since index is limit in the range of [0..2*depth) while (idx >= depth) { idx -= depth; } return idx; } // Works on local copies of w // When an overwritable fifo is overflowed, rd_idx will be re-index so that it forms a full fifo static uint16_t correct_read_index(tu_fifo_t *f, uint16_t wr_idx) { uint16_t rd_idx; if (wr_idx >= f->depth) { rd_idx = wr_idx - f->depth; } else { rd_idx = wr_idx + f->depth; } f->rd_idx = rd_idx; return rd_idx; } //--------------------------------------------------------------------+ // n-API //--------------------------------------------------------------------+ // Works on local copies of w and r // Must be protected by read mutex since in case of an overflow read pointer gets modified uint16_t tu_fifo_peek_n_access_mode(tu_fifo_t *f, void *p_buffer, uint16_t n, uint16_t wr_idx, uint16_t rd_idx, tu_fifo_access_mode_t access_mode) { uint16_t count = tu_ff_overflow_count(f->depth, wr_idx, rd_idx); if (count == 0) { return 0; // nothing to peek } // Check overflow and correct if required if (count > f->depth) { rd_idx = correct_read_index(f, wr_idx); count = f->depth; } if (count < n) { n = count; // limit to available count } const uint16_t rd_ptr = idx2ptr(f->depth, rd_idx); ff_pull_n(f, p_buffer, n, rd_ptr, access_mode); return n; } // Read n items without removing it from the FIFO, correct read pointer if overflowed uint16_t tu_fifo_peek_n(tu_fifo_t *f, void *p_buffer, uint16_t n) { ff_lock(f->mutex_rd); const uint16_t ret = tu_fifo_peek_n_access_mode(f, p_buffer, n, f->wr_idx, f->rd_idx, TU_FIFO_INC_ADDR_RW8); ff_unlock(f->mutex_rd); return ret; } // Read n items from fifo with access mode uint16_t tu_fifo_read_n_access_mode(tu_fifo_t *f, void *buffer, uint16_t n, tu_fifo_access_mode_t access_mode) { ff_lock(f->mutex_rd); // Peek the data: f->rd_idx might get modified in case of an overflow so we can not use a local variable n = tu_fifo_peek_n_access_mode(f, buffer, n, f->wr_idx, f->rd_idx, access_mode); f->rd_idx = advance_index(f->depth, f->rd_idx, n); ff_unlock(f->mutex_rd); return n; } // Write n items to fifo with access mode uint16_t tu_fifo_write_n_access_mode(tu_fifo_t *f, const void *data, uint16_t n, tu_fifo_access_mode_t access_mode) { if (n == 0) { return 0; } ff_lock(f->mutex_wr); uint16_t wr_idx = f->wr_idx; uint16_t rd_idx = f->rd_idx; const uint8_t *buf8 = (const uint8_t *)data; TU_LOG(TU_FIFO_DBG, "rd = %3u, wr = %3u, count = %3u, remain = %3u, n = %3u: ", rd_idx, wr_idx, tu_ff_overflow_count(f->depth, wr_idx, rd_idx), tu_ff_remaining_local(f->depth, wr_idx, rd_idx), n); if (!f->overwritable) { // limit up to full const uint16_t remain = tu_ff_remaining_local(f->depth, wr_idx, rd_idx); n = tu_min16(n, remain); } else { // In over-writable mode, fifo_write() is allowed even when fifo is full. In such case, // oldest data in fifo i.e. at read pointer data will be overwritten // Note: we can modify read buffer contents however we must not modify the read index itself within a write // function! Since it would end up in a race condition with read functions! if (n >= f->depth) { // Only copy last part if (access_mode == TU_FIFO_INC_ADDR_RW8) { buf8 += (n - f->depth) * f->item_size; } else { // TODO should read from hw fifo to discard data, however reading an odd number could // accidentally discard data. } n = f->depth; // We start writing at the read pointer's position since we fill the whole buffer wr_idx = rd_idx; } else { const uint16_t overflowable_count = tu_ff_overflow_count(f->depth, wr_idx, rd_idx); if (overflowable_count + n >= 2 * f->depth) { // Double overflowed // Index is bigger than the allowed range [0,2*depth) // re-position write index to have a full fifo after pushed wr_idx = advance_index(f->depth, rd_idx, f->depth - n); // TODO we should also shift out n bytes from read index since we avoid changing rd index !! // However memmove() is expensive due to actual copying + wrapping consideration. // Also race condition could happen anyway if read() is invoke while moving result in corrupted memory // currently deliberately not implemented --> result in incorrect data read back } else { // normal + single overflowed: // Index is in the range of [0,2*depth) and thus detect and recoverable. Recovering is handled in read() // Therefore we just increase write index // we will correct (re-position) read index later on in fifo_read() function } } } if (n) { const uint16_t wr_ptr = idx2ptr(f->depth, wr_idx); TU_LOG(TU_FIFO_DBG, "actual_n = %u, wr_ptr = %u", n, wr_ptr); ff_push_n(f, buf8, n, wr_ptr, access_mode); f->wr_idx = advance_index(f->depth, wr_idx, n); TU_LOG(TU_FIFO_DBG, "\tnew_wr = %u\r\n", f->wr_idx); } ff_unlock(f->mutex_wr); return n; } uint16_t tu_fifo_discard_n(tu_fifo_t *f, uint16_t n) { const uint16_t count = tu_min16(n, tu_fifo_count(f)); // limit to available count ff_lock(f->mutex_rd); f->rd_idx = advance_index(f->depth, f->rd_idx, count); ff_unlock(f->mutex_rd); return count; } //--------------------------------------------------------------------+ // One API //--------------------------------------------------------------------+ // peek() using local write/read index, correct read index if overflowed // Be careful, caller must not lock mutex, since this Will also try to lock mutex static bool ff_peek_local(tu_fifo_t *f, void *buf, uint16_t wr_idx, uint16_t rd_idx) { const uint16_t ovf_count = tu_ff_overflow_count(f->depth, wr_idx, rd_idx); if (ovf_count == 0) { return false; // nothing to peek } // Correct read index if overflow if (ovf_count > f->depth) { ff_lock(f->mutex_rd); rd_idx = correct_read_index(f, wr_idx); ff_unlock(f->mutex_rd); } const uint16_t rd_ptr = idx2ptr(f->depth, rd_idx); memcpy(buf, f->buffer + (rd_ptr * f->item_size), f->item_size); return true; } // Read one element out of the buffer, correct read index if overflowed bool tu_fifo_read(tu_fifo_t *f, void *buffer) { // Peek the data // f->rd_idx might get modified in case of an overflow so we can not use a local variable const bool ret = ff_peek_local(f, buffer, f->wr_idx, f->rd_idx); if (ret) { ff_lock(f->mutex_rd); f->rd_idx = advance_index(f->depth, f->rd_idx, 1); ff_unlock(f->mutex_rd); } return ret; } // Read one item without removing it from the FIFO, correct read index if overflowed bool tu_fifo_peek(tu_fifo_t *f, void *p_buffer) { return ff_peek_local(f, p_buffer, f->wr_idx, f->rd_idx); } // Write one element into the buffer bool tu_fifo_write(tu_fifo_t *f, const void *data) { bool ret; ff_lock(f->mutex_wr); const uint16_t wr_idx = f->wr_idx; if (tu_fifo_full(f) && !f->overwritable) { ret = false; } else { const uint16_t wr_ptr = idx2ptr(f->depth, wr_idx); memcpy(f->buffer + (wr_ptr * f->item_size), data, f->item_size); f->wr_idx = advance_index(f->depth, wr_idx, 1); ret = true; } ff_unlock(f->mutex_wr); return ret; } //--------------------------------------------------------------------+ // Index API //--------------------------------------------------------------------+ /******************************************************************************/ /*! @brief Advance write pointer - intended to be used in combination with DMA. It is possible to fill the FIFO by use of a DMA in circular mode. Within DMA ISRs you may update the write pointer to be able to read from the FIFO. As long as the DMA is the only process writing into the FIFO this is safe to use. USE WITH CARE - WE DO NOT CONDUCT SAFETY CHECKS HERE! @param[in] f Pointer to the FIFO buffer to manipulate @param[in] n Number of items the write pointer moves forward */ /******************************************************************************/ void tu_fifo_advance_write_pointer(tu_fifo_t *f, uint16_t n) { f->wr_idx = advance_index(f->depth, f->wr_idx, n); } // Correct the read index in case tu_fifo_overflow() returned true! void tu_fifo_correct_read_pointer(tu_fifo_t *f) { ff_lock(f->mutex_rd); correct_read_index(f, f->wr_idx); ff_unlock(f->mutex_rd); } /******************************************************************************/ /*! @brief Advance read pointer - intended to be used in combination with DMA. It is possible to read from the FIFO by use of a DMA in linear mode. Within DMA ISRs you may update the read pointer to be able to again write into the FIFO. As long as the DMA is the only process reading from the FIFO this is safe to use. USE WITH CARE - WE DO NOT CONDUCT SAFETY CHECKS HERE! @param[in] f Pointer to the FIFO buffer to manipulate @param[in] n Number of items the read pointer moves forward */ /******************************************************************************/ void tu_fifo_advance_read_pointer(tu_fifo_t *f, uint16_t n) { f->rd_idx = advance_index(f->depth, f->rd_idx, n); } /******************************************************************************/ /*! @brief Get read info Returns the length and pointer from which bytes can be read in a linear manner. This is of major interest for DMA transmissions. If returned length is zero the corresponding pointer is invalid. The read pointer does NOT get advanced, use tu_fifo_advance_read_pointer() to do so! @param[in] f Pointer to FIFO @param[out] *info Pointer to struct which holds the desired infos */ /******************************************************************************/ void tu_fifo_get_read_info(tu_fifo_t *f, tu_fifo_buffer_info_t *info) { // Operate on temporary values in case they change in between uint16_t wr_idx = f->wr_idx; uint16_t rd_idx = f->rd_idx; uint16_t cnt = tu_ff_overflow_count(f->depth, wr_idx, rd_idx); // Check overflow and correct if required - may happen in case a DMA wrote too fast if (cnt > f->depth) { ff_lock(f->mutex_rd); rd_idx = correct_read_index(f, wr_idx); ff_unlock(f->mutex_rd); cnt = f->depth; } // Check if fifo is empty if (cnt == 0) { info->linear.len = 0; info->wrapped.len = 0; info->linear.ptr = NULL; info->wrapped.ptr = NULL; return; } // Get relative pointers uint16_t wr_ptr = idx2ptr(f->depth, wr_idx); uint16_t rd_ptr = idx2ptr(f->depth, rd_idx); // Copy pointer to buffer to start reading from info->linear.ptr = &f->buffer[rd_ptr]; // Check if there is a wrap around necessary if (wr_ptr > rd_ptr) { // Non wrapping case info->linear.len = cnt; info->wrapped.len = 0; info->wrapped.ptr = NULL; } else { info->linear.len = f->depth - rd_ptr; // Also the case if FIFO was full info->wrapped.len = cnt - info->linear.len; info->wrapped.ptr = f->buffer; } } /******************************************************************************/ /*! @brief Get linear write info Returns the length and pointer to which bytes can be written into FIFO in a linear manner. This is of major interest for DMA transmissions not using circular mode. If a returned length is zero the corresponding pointer is invalid. The returned lengths summed up are the currently free space in the FIFO. The write pointer does NOT get advanced, use tu_fifo_advance_write_pointer() to do so! TAKE CARE TO NOT OVERFLOW THE BUFFER MORE THAN TWO TIMES THE FIFO DEPTH - IT CAN NOT RECOVERE OTHERWISE! @param[in] f Pointer to FIFO @param[out] *info Pointer to struct which holds the desired infos */ /******************************************************************************/ void tu_fifo_get_write_info(tu_fifo_t *f, tu_fifo_buffer_info_t *info) { uint16_t wr_idx = f->wr_idx; uint16_t rd_idx = f->rd_idx; uint16_t remain = tu_ff_remaining_local(f->depth, wr_idx, rd_idx); if (remain == 0) { info->linear.len = 0; info->wrapped.len = 0; info->linear.ptr = NULL; info->wrapped.ptr = NULL; return; } // Get relative pointers uint16_t wr_ptr = idx2ptr(f->depth, wr_idx); uint16_t rd_ptr = idx2ptr(f->depth, rd_idx); // Copy pointer to buffer to start writing to info->linear.ptr = &f->buffer[wr_ptr]; if (wr_ptr < rd_ptr) { // Non wrapping case info->linear.len = rd_ptr - wr_ptr; info->wrapped.len = 0; info->wrapped.ptr = NULL; } else { info->linear.len = f->depth - wr_ptr; info->wrapped.len = remain - info->linear.len; // Remaining length - n already was limited to remain or FIFO depth info->wrapped.ptr = f->buffer; // Always start of buffer } }