diff options
Diffstat (limited to 'components')
56 files changed, 6801 insertions, 612 deletions
diff --git a/components/dfs/Kconfig b/components/dfs/Kconfig index 33bfd3379b..9c0b1c88d5 100644 --- a/components/dfs/Kconfig +++ b/components/dfs/Kconfig @@ -193,6 +193,40 @@ if RT_USING_DFS_V1 endif endif +if RT_USING_DFS_V2 + config RT_USING_PAGECACHE + bool "Enable page cache" + default n + + if RT_USING_PAGECACHE + menu "page cache config" + config RT_PAGECACHE_COUNT + int "page cache max total pages." + default 4096 + + config RT_PAGECACHE_ASPACE_COUNT + int "aspace max active pages." + default 1024 + + config RT_PAGECACHE_PRELOAD + int "max pre load pages." + default 4 + + config RT_PAGECACHE_HASH_NR + int "page cache hash size." + default 1024 + + config RT_PAGECACHE_GC_WORK_LEVEL + int "page cache gc work trigger min percentage, default 90%." + default 90 + + config RT_PAGECACHE_GC_STOP_LEVEL + int "page cache gc to min percentage, default 70%." + default 70 + endmenu + endif +endif + endif endmenu diff --git a/components/dfs/dfs_v1/filesystems/devfs/devfs.c b/components/dfs/dfs_v1/filesystems/devfs/devfs.c index 993f967131..cceb14150e 100644 --- a/components/dfs/dfs_v1/filesystems/devfs/devfs.c +++ b/components/dfs/dfs_v1/filesystems/devfs/devfs.c @@ -286,11 +286,10 @@ int dfs_device_fs_unlink(struct dfs_filesystem *fs, const char *path) int dfs_device_fs_stat(struct dfs_filesystem *fs, const char *path, struct stat *st) { + st->st_dev = (dev_t)((size_t)dfs_filesystem_lookup(fs->path)); /* stat root directory */ if ((path[0] == '/') && (path[1] == '\0')) { - st->st_dev = 0; - st->st_mode = S_IFREG | S_IRUSR | S_IRGRP | S_IROTH | S_IWUSR | S_IWGRP | S_IWOTH; st->st_mode &= ~S_IFREG; @@ -308,8 +307,6 @@ int dfs_device_fs_stat(struct dfs_filesystem *fs, const char *path, struct stat dev_id = rt_device_find(&path[1]); if (dev_id != RT_NULL) { - st->st_dev = 0; - st->st_mode = S_IRUSR | S_IRGRP | S_IROTH | S_IWUSR | S_IWGRP | S_IWOTH; diff --git a/components/dfs/dfs_v1/filesystems/tmpfs/dfs_tmpfs.c b/components/dfs/dfs_v1/filesystems/tmpfs/dfs_tmpfs.c index e59721a714..c87be8239b 100644 --- a/components/dfs/dfs_v1/filesystems/tmpfs/dfs_tmpfs.c +++ b/components/dfs/dfs_v1/filesystems/tmpfs/dfs_tmpfs.c @@ -198,9 +198,11 @@ int dfs_tmpfs_ioctl(struct dfs_file *file, int cmd, void *args) { return -RT_ENOMEM; } + else if (mmap2->lwp == RT_NULL) + return -RT_EINVAL; LOG_D("tmpfile mmap ptr:%x , size:%d\n", d_file->data, mmap2->length); - mmap2->ret = lwp_map_user_phy(lwp_self(), RT_NULL, d_file->data, mmap2->length, 0); + mmap2->ret = lwp_map_user_phy(mmap2->lwp, mmap2->addr, d_file->data, mmap2->length, 0); } return RT_EOK; break; diff --git a/components/dfs/dfs_v1/include/dfs_file.h b/components/dfs/dfs_v1/include/dfs_file.h index d36b2ce89b..e125fb25ac 100644 --- a/components/dfs/dfs_v1/include/dfs_file.h +++ b/components/dfs/dfs_v1/include/dfs_file.h @@ -64,6 +64,7 @@ struct dfs_file void *data; /* Specific fd data */ }; +#ifdef RT_USING_SMART struct dfs_mmap2_args { void *addr; @@ -72,8 +73,10 @@ struct dfs_mmap2_args int flags; off_t pgoffset; + struct rt_lwp *lwp; void *ret; }; +#endif void dfs_vnode_mgr_init(void); int dfs_vnode_init(struct dfs_vnode *vnode, int type, const struct dfs_file_ops *fops); @@ -92,8 +95,9 @@ off_t dfs_file_lseek(struct dfs_file *fd, off_t offset); int dfs_file_stat(const char *path, struct stat *buf); int dfs_file_rename(const char *oldpath, const char *newpath); int dfs_file_ftruncate(struct dfs_file *fd, off_t length); +#ifdef RT_USING_SMART int dfs_file_mmap2(struct dfs_file *fd, struct dfs_mmap2_args *mmap2); - +#endif /* 0x5254 is just a magic number to make these relatively unique ("RT") */ #define RT_FIOFTRUNCATE 0x52540000U #define RT_FIOGETADDR 0x52540001U diff --git a/components/dfs/dfs_v1/src/dfs_file.c b/components/dfs/dfs_v1/src/dfs_file.c index a22f7b3f31..acdc124f9a 100644 --- a/components/dfs/dfs_v1/src/dfs_file.c +++ b/components/dfs/dfs_v1/src/dfs_file.c @@ -713,6 +713,7 @@ int dfs_file_ftruncate(struct dfs_file *fd, off_t length) return result; } +#ifdef RT_USING_SMART int dfs_file_mmap2(struct dfs_file *fd, struct dfs_mmap2_args *mmap2) { int ret = 0; @@ -736,6 +737,7 @@ int dfs_file_mmap2(struct dfs_file *fd, struct dfs_mmap2_args *mmap2) return ret; } +#endif #ifdef RT_USING_FINSH #include <finsh.h> diff --git a/components/dfs/dfs_v2/SConscript b/components/dfs/dfs_v2/SConscript index a8f2811a18..ae7432f71a 100644 --- a/components/dfs/dfs_v2/SConscript +++ b/components/dfs/dfs_v2/SConscript @@ -2,20 +2,12 @@ from building import * import os # The set of source files associated with this SConscript file. -src = Split(''' -src/dfs.c -src/dfs_file.c -src/dfs_fs.c -src/dfs_dentry.c -src/dfs_vnode.c -src/dfs_mnt.c -src/dfs_posix.c -''') +src = Glob('src/*.c') + Glob('src/*.cpp') cwd = GetCurrentDir() CPPPATH = [cwd + "/include"] -if GetDepend('RT_USING_POSIX'): - src += ['src/poll.c', 'src/select.c'] +if not GetDepend('RT_USING_SMART'): + SrcRemove(src, ['src/dfs_file_mmap.c']) group = DefineGroup('Filesystem', src, depend = ['RT_USING_DFS', 'RT_USING_DFS_V2'], CPPPATH = CPPPATH) diff --git a/components/dfs/dfs_v2/filesystems/elmfat/dfs_elm.c b/components/dfs/dfs_v2/filesystems/elmfat/dfs_elm.c index c6281c55b7..c453058d72 100644 --- a/components/dfs/dfs_v2/filesystems/elmfat/dfs_elm.c +++ b/components/dfs/dfs_v2/filesystems/elmfat/dfs_elm.c @@ -32,6 +32,24 @@ #include <dfs_file.h> #include <dfs_mnt.h> +#ifdef RT_USING_PAGECACHE +#include "dfs_pcache.h" +#endif + + +static int dfs_elm_free_vnode(struct dfs_vnode *vnode); + +#ifdef RT_USING_PAGECACHE +static ssize_t dfs_elm_page_read(struct dfs_file *file, struct dfs_page *page); +static ssize_t dfs_elm_page_write(struct dfs_page *page); + +static struct dfs_aspace_ops dfs_elm_aspace_ops = +{ + .read = dfs_elm_page_read, + .write = dfs_elm_page_write, +}; +#endif + #undef SS #if FF_MAX_SS == FF_MIN_SS #define SS(fs) ((UINT)FF_MAX_SS) /* Fixed sector size */ @@ -361,7 +379,7 @@ int dfs_elm_open(struct dfs_file *file) extern int elm_get_vol(FATFS * fat); RT_ASSERT(file->vnode->ref_count > 0); - if (file->vnode->ref_count > 1) + if (file->vnode->data) { if (file->vnode->type == FT_DIRECTORY && !(file->flags & O_DIRECTORY)) @@ -425,6 +443,7 @@ int dfs_elm_open(struct dfs_file *file) } file->vnode->data = dir; + rt_mutex_init(&file->vnode->lock, file->dentry->pathname, RT_IPC_FLAG_PRIO); return RT_EOK; } else @@ -465,6 +484,7 @@ int dfs_elm_open(struct dfs_file *file) file->vnode->size = f_size(fd); file->vnode->type = FT_REGULAR; file->vnode->data = fd; + rt_mutex_init(&file->vnode->lock, file->dentry->pathname, RT_IPC_FLAG_PRIO); if (file->flags & O_APPEND) { @@ -516,6 +536,9 @@ int dfs_elm_close(struct dfs_file *file) rt_free(fd); } + file->vnode->data = RT_NULL; + rt_mutex_detach(&file->vnode->lock); + return elm_result_to_dfs(result); } @@ -558,7 +581,7 @@ int dfs_elm_ioctl(struct dfs_file *file, int cmd, void *args) ssize_t dfs_elm_read(struct dfs_file *file, void *buf, size_t len, off_t *pos) { FIL *fd; - FRESULT result; + FRESULT result = FR_OK; UINT byte_read; if (file->vnode->type == FT_DIRECTORY) @@ -566,14 +589,19 @@ ssize_t dfs_elm_read(struct dfs_file *file, void *buf, size_t len, off_t *pos) return -EISDIR; } - fd = (FIL *)(file->vnode->data); - RT_ASSERT(fd != RT_NULL); - - result = f_read(fd, buf, len, &byte_read); - /* update position */ - *pos = fd->fptr; - if (result == FR_OK) - return byte_read; + if (file->vnode->size > *pos) + { + fd = (FIL *)(file->vnode->data); + RT_ASSERT(fd != RT_NULL); + rt_mutex_take(&file->vnode->lock, RT_WAITING_FOREVER); + f_lseek(fd, *pos); + result = f_read(fd, buf, len, &byte_read); + /* update position */ + *pos = fd->fptr; + rt_mutex_release(&file->vnode->lock); + if (result == FR_OK) + return byte_read; + } return elm_result_to_dfs(result); } @@ -591,11 +619,13 @@ ssize_t dfs_elm_write(struct dfs_file *file, const void *buf, size_t len, off_t fd = (FIL *)(file->vnode->data); RT_ASSERT(fd != RT_NULL); - + rt_mutex_take(&file->vnode->lock, RT_WAITING_FOREVER); + f_lseek(fd, *pos); result = f_write(fd, buf, len, &byte_write); /* update position and file size */ *pos = fd->fptr; file->vnode->size = f_size(fd); + rt_mutex_release(&file->vnode->lock); if (result == FR_OK) return byte_write; @@ -642,8 +672,9 @@ off_t dfs_elm_lseek(struct dfs_file *file, off_t offset, int wherece) /* regular file type */ fd = (FIL *)(file->vnode->data); RT_ASSERT(fd != RT_NULL); - + rt_mutex_take(&file->vnode->lock, RT_WAITING_FOREVER); result = f_lseek(fd, offset); + rt_mutex_release(&file->vnode->lock); if (result == FR_OK) { /* return current position */ @@ -657,8 +688,9 @@ off_t dfs_elm_lseek(struct dfs_file *file, off_t offset, int wherece) dir = (DIR *)(file->vnode->data); RT_ASSERT(dir != RT_NULL); - + rt_mutex_take(&file->vnode->lock, RT_WAITING_FOREVER); result = f_seekdir(dir, offset / sizeof(struct dirent)); + rt_mutex_release(&file->vnode->lock); if (result == FR_OK) { /* update file position */ @@ -827,21 +859,35 @@ int dfs_elm_stat(struct dfs_dentry *dentry, struct stat *st) st->st_dev = (dev_t)(size_t)(dentry->mnt->dev_id); st->st_ino = (ino_t)dfs_dentry_full_path_crc32(dentry); - st->st_mode = S_IFREG | S_IRUSR | S_IRGRP | S_IROTH | - S_IWUSR | S_IWGRP | S_IWOTH; if (file_info.fattrib & AM_DIR) { - st->st_mode &= ~S_IFREG; - st->st_mode |= S_IFDIR | S_IXUSR | S_IXGRP | S_IXOTH; + st->st_mode = S_IFDIR | (S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); + } + else + { + st->st_mode = S_IFREG | (S_IRWXU | S_IRWXG | S_IRWXO); } + if (file_info.fattrib & AM_RDO) st->st_mode &= ~(S_IWUSR | S_IWGRP | S_IWOTH); - st->st_size = file_info.fsize; + if (S_IFDIR & st->st_mode) + { + st->st_size = file_info.fsize; + } + else + { +#ifdef RT_USING_PAGECACHE + st->st_size = (dentry->vnode && dentry->vnode->aspace) ? dentry->vnode->size : file_info.fsize; +#else + st->st_size = file_info.fsize; +#endif + } + st->st_blksize = fat->csize * SS(fat); if (file_info.fattrib & AM_ARC) { - st->st_blocks = file_info.fsize ? ((file_info.fsize - 1) / SS(fat) / fat->csize + 1) : 0; + st->st_blocks = st->st_size ? ((st->st_size - 1) / SS(fat) / fat->csize + 1) : 0; st->st_blocks *= (st->st_blksize / 512); // man say st_blocks is number of 512B blocks allocated } else @@ -901,20 +947,23 @@ static struct dfs_vnode *dfs_elm_lookup(struct dfs_dentry *dentry) vnode = dfs_vnode_create(); if (vnode) { + vnode->mnt = dentry->mnt; + vnode->size = st.st_size; + vnode->data = NULL; + if (S_ISDIR(st.st_mode)) { - vnode->mode = S_IFDIR | (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); + vnode->mode = S_IFDIR | (S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); vnode->type = FT_DIRECTORY; } else { - vnode->mode = S_IFREG | S_IXUSR | (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); + vnode->mode = S_IFREG | (S_IRWXU | S_IRWXG | S_IRWXO); vnode->type = FT_REGULAR; +#ifdef RT_USING_PAGECACHE + vnode->aspace = dfs_aspace_create(dentry, vnode, &dfs_elm_aspace_ops); +#endif } - - vnode->mnt = dentry->mnt; - vnode->data = NULL; - vnode->size = 0; } return vnode; @@ -934,13 +983,18 @@ static struct dfs_vnode *dfs_elm_create_vnode(struct dfs_dentry *dentry, int typ { if (type == FT_DIRECTORY) { - vnode->mode = S_IFDIR | mode; + /* fat directory force mode 0555 */ + vnode->mode = S_IFDIR | (S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); vnode->type = FT_DIRECTORY; } else { - vnode->mode = S_IFREG | mode; + /* fat REGULAR file mode force mode 0777 */ + vnode->mode = S_IFREG | (S_IRWXU | S_IRWXG | S_IRWXO); vnode->type = FT_REGULAR; +#ifdef RT_USING_PAGECACHE + vnode->aspace = dfs_aspace_create(dentry, vnode, &dfs_elm_aspace_ops); +#endif } vnode->mnt = dentry->mnt; @@ -962,6 +1016,46 @@ static int dfs_elm_free_vnode(struct dfs_vnode *vnode) return 0; } +#ifdef RT_USING_PAGECACHE +static ssize_t dfs_elm_page_read(struct dfs_file *file, struct dfs_page *page) +{ + int ret = -EINVAL; + + if (page->page) + { + off_t fpos = page->fpos; + ret = dfs_elm_read(file, page->page, page->size, &fpos); + } + + return ret; +} + +ssize_t dfs_elm_page_write(struct dfs_page *page) +{ + FIL *fd; + FRESULT result; + UINT byte_write; + + if (page->aspace->vnode->type == FT_DIRECTORY) + { + return -EISDIR; + } + + fd = (FIL *)(page->aspace->vnode->data); + RT_ASSERT(fd != RT_NULL); + rt_mutex_take(&page->aspace->vnode->lock, RT_WAITING_FOREVER); + f_lseek(fd, page->fpos); + result = f_write(fd, page->page, page->len, &byte_write); + rt_mutex_release(&page->aspace->vnode->lock); + if (result == FR_OK) + { + return byte_write; + } + + return elm_result_to_dfs(result); +} +#endif + static const struct dfs_file_ops dfs_elm_fops = { .open = dfs_elm_open, diff --git a/components/dfs/dfs_v2/filesystems/mqueue/dfs_mqueue.c b/components/dfs/dfs_v2/filesystems/mqueue/dfs_mqueue.c index 13719f386a..0bc61c635c 100644 --- a/components/dfs/dfs_v2/filesystems/mqueue/dfs_mqueue.c +++ b/components/dfs/dfs_v2/filesystems/mqueue/dfs_mqueue.c @@ -149,7 +149,7 @@ static struct dfs_vnode *dfs_mqueue_create_vnode(struct dfs_dentry *dentry, int dfs_mqueue_insert_after(&(mq_file->list)); } - vnode->mode = S_IFREG | mode; + vnode->mode = S_IFREG | (S_IRWXU | S_IRWXG | S_IRWXO); vnode->type = FT_REGULAR; rt_mq_t mq = rt_mq_create(dentry->pathname + 1, mq_file->msg_size, mq_file->max_msgs, RT_IPC_FLAG_FIFO); @@ -191,7 +191,7 @@ struct dfs_vnode *_dfs_mqueue_lookup(struct dfs_dentry *dentry) { vnode = dfs_vnode_create(); if (mq_file && mq_file->data) { - vnode->mode = S_IFREG | S_IRUSR | S_IWUSR | S_IXUSR; + vnode->mode = S_IFREG | (S_IRWXU | S_IRWXG | S_IRWXO); vnode->type = FT_REGULAR; vnode->mnt = dentry->mnt; vnode->data = mq_file; @@ -202,7 +202,7 @@ struct dfs_vnode *_dfs_mqueue_lookup(struct dfs_dentry *dentry) { vnode->fops = &_mqueue_fops; vnode->mnt = dentry->mnt; vnode->type = FT_DIRECTORY; - vnode->mode = S_IFDIR | S_IRUSR | S_IWUSR | S_IXUSR; + vnode->mode = S_IFDIR | (S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); } return vnode; } diff --git a/components/dfs/dfs_v2/filesystems/romfs/dfs_romfs.c b/components/dfs/dfs_v2/filesystems/romfs/dfs_romfs.c index 81382ae69b..43db9b6ec3 100644 --- a/components/dfs/dfs_v2/filesystems/romfs/dfs_romfs.c +++ b/components/dfs/dfs_v2/filesystems/romfs/dfs_romfs.c @@ -191,12 +191,12 @@ static struct dfs_vnode *dfs_romfs_lookup (struct dfs_dentry *dentry) vnode->size = dirent->size; if (dirent->type == ROMFS_DIRENT_DIR) { - vnode->mode = romfs_modemap[ROMFS_DIRENT_DIR] | S_IRUSR; + vnode->mode = romfs_modemap[ROMFS_DIRENT_DIR] | (S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); vnode->type = FT_DIRECTORY; } else if (dirent->type == ROMFS_DIRENT_FILE) { - vnode->mode = romfs_modemap[ROMFS_DIRENT_FILE] | S_IRUSR | S_IXUSR; + vnode->mode = romfs_modemap[ROMFS_DIRENT_FILE] | (S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); vnode->type = FT_REGULAR; } diff --git a/components/dfs/dfs_v2/filesystems/tmpfs/dfs_tmpfs.c b/components/dfs/dfs_v2/filesystems/tmpfs/dfs_tmpfs.c index 3db3823926..36d1ea692b 100644 --- a/components/dfs/dfs_v2/filesystems/tmpfs/dfs_tmpfs.c +++ b/components/dfs/dfs_v2/filesystems/tmpfs/dfs_tmpfs.c @@ -28,6 +28,20 @@ #define DBG_TAG "tmpfs" #define DBG_LVL DBG_INFO #include <rtdbg.h> +#ifdef RT_USING_PAGECACHE +#include "dfs_pcache.h" +#endif + +#ifdef RT_USING_PAGECACHE +static ssize_t dfs_tmp_page_read(struct dfs_file *file, struct dfs_page *page); +static ssize_t dfs_tmp_page_write(struct dfs_page *page); + +static struct dfs_aspace_ops dfs_tmp_aspace_ops = +{ + .read = dfs_tmp_page_read, + .write = dfs_tmp_page_write, +}; +#endif static int _path_separate(const char *path, char *parent_path, char *file_name) { @@ -285,6 +299,8 @@ static ssize_t dfs_tmpfs_read(struct dfs_file *file, void *buf, size_t count, of d_file = (struct tmpfs_file *)file->vnode->data; RT_ASSERT(d_file != NULL); + rt_mutex_take(&file->vnode->lock, RT_WAITING_FOREVER); + if (count < file->vnode->size - *pos) length = count; else @@ -296,6 +312,8 @@ static ssize_t dfs_tmpfs_read(struct dfs_file *file, void *buf, size_t count, of /* update file current position */ *pos += length; + rt_mutex_release(&file->vnode->lock); + return length; } @@ -310,12 +328,15 @@ static ssize_t dfs_tmpfs_write(struct dfs_file *file, const void *buf, size_t co superblock = d_file->sb; RT_ASSERT(superblock != NULL); + rt_mutex_take(&file->vnode->lock, RT_WAITING_FOREVER); + if (count + *pos > file->vnode->size) { rt_uint8_t *ptr; ptr = rt_realloc(d_file->data, *pos + count); if (ptr == NULL) { + rt_mutex_release(&file->vnode->lock); rt_set_errno(-ENOMEM); return 0; } @@ -335,6 +356,7 @@ static ssize_t dfs_tmpfs_write(struct dfs_file *file, const void *buf, size_t co /* update file current position */ *pos += count; + rt_mutex_release(&file->vnode->lock); return count; } @@ -371,6 +393,7 @@ static int dfs_tmpfs_close(struct dfs_file *file) struct tmpfs_file *d_file; RT_ASSERT(file->vnode->ref_count > 0); + if (file->vnode->ref_count != 1) return 0; @@ -390,6 +413,8 @@ static int dfs_tmpfs_close(struct dfs_file *file) rt_free(d_file); } + rt_mutex_detach(&file->vnode->lock); + return RT_EOK; } @@ -424,6 +449,12 @@ static int dfs_tmpfs_open(struct dfs_file *file) file->fpos = 0; } + RT_ASSERT(file->vnode->ref_count > 0); + if(file->vnode->ref_count == 1) + { + rt_mutex_init(&file->vnode->lock, file->dentry->pathname, RT_IPC_FLAG_PRIO); + } + return 0; } @@ -442,12 +473,13 @@ static int dfs_tmpfs_stat(struct dfs_dentry *dentry, struct stat *st) st->st_dev = (dev_t)(size_t)(dentry->mnt->dev_id); st->st_ino = (ino_t)dfs_dentry_full_path_crc32(dentry); - st->st_mode = S_IFREG | S_IRUSR | S_IRGRP | S_IROTH | - S_IWUSR | S_IWGRP | S_IWOTH; if (d_file->type == TMPFS_TYPE_DIR) { - st->st_mode &= ~S_IFREG; - st->st_mode |= S_IFDIR | S_IXUSR | S_IXGRP | S_IXOTH; + st->st_mode = S_IFDIR | (S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); + } + else + { + st->st_mode = S_IFREG | (S_IRWXU | S_IRWXG | S_IRWXO); } st->st_size = d_file->size; @@ -468,13 +500,18 @@ static int dfs_tmpfs_getdents(struct dfs_file *file, d_file = (struct tmpfs_file *)file->vnode->data; + rt_mutex_take(&file->vnode->lock, RT_WAITING_FOREVER); + superblock = d_file->sb; RT_ASSERT(superblock != RT_NULL); /* make integer count */ count = (count / sizeof(struct dirent)); if (count == 0) + { + rt_mutex_release(&file->vnode->lock); return -EINVAL; + } end = file->fpos + count; index = 0; @@ -507,6 +544,7 @@ static int dfs_tmpfs_getdents(struct dfs_file *file, break; } } + rt_mutex_release(&file->vnode->lock); return count * sizeof(struct dirent); } @@ -607,13 +645,16 @@ static struct dfs_vnode *_dfs_tmpfs_lookup(struct dfs_dentry *dentry) { if (d_file->type == TMPFS_TYPE_DIR) { - vnode->mode = S_IFDIR | S_IRUSR | S_IWUSR | S_IXUSR; + vnode->mode = S_IFDIR | (S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); vnode->type = FT_DIRECTORY; } else { - vnode->mode = S_IFREG | S_IRUSR | S_IWUSR | S_IXUSR; + vnode->mode = S_IFREG | (S_IRWXU | S_IRWXG | S_IRWXO); vnode->type = FT_REGULAR; +#ifdef RT_USING_PAGECACHE + vnode->aspace = dfs_aspace_create(dentry, vnode, &dfs_tmp_aspace_ops); +#endif } vnode->mnt = dentry->mnt; @@ -681,14 +722,17 @@ static struct dfs_vnode *dfs_tmpfs_create_vnode(struct dfs_dentry *dentry, int t if (type == FT_DIRECTORY) { d_file->type = TMPFS_TYPE_DIR; - vnode->mode = S_IFDIR | mode; + vnode->mode = S_IFDIR | (S_IRUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); vnode->type = FT_DIRECTORY; } else { d_file->type = TMPFS_TYPE_FILE; - vnode->mode = S_IFREG | mode; + vnode->mode = S_IFREG | (S_IRWXU | S_IRWXG | S_IRWXO); vnode->type = FT_REGULAR; +#ifdef RT_USING_PAGECACHE + vnode->aspace = dfs_aspace_create(dentry, vnode, &dfs_tmp_aspace_ops); +#endif } rt_spin_lock(&superblock->lock); rt_list_insert_after(&(p_file->subdirs), &(d_file->sibling)); @@ -713,6 +757,41 @@ static int dfs_tmpfs_free_vnode(struct dfs_vnode *vnode) return 0; } +#ifdef RT_USING_PAGECACHE +static ssize_t dfs_tmp_page_read(struct dfs_file *file, struct dfs_page *page) +{ + int ret = -EINVAL; + + if (page->page) + { + off_t fpos = page->fpos; + ret = dfs_tmpfs_read(file, page->page, page->size, &fpos); + } + + return ret; +} + +ssize_t dfs_tmp_page_write(struct dfs_page *page) +{ + struct tmpfs_file *d_file; + + if (page->aspace->vnode->type == FT_DIRECTORY) + { + return -EISDIR; + } + + d_file = (struct tmpfs_file *)(page->aspace->vnode->data); + RT_ASSERT(d_file != RT_NULL); + rt_mutex_take(&page->aspace->vnode->lock, RT_WAITING_FOREVER); + if (page->len > 0) + memcpy(d_file->data + page->fpos, page->page, page->len); + + rt_mutex_release(&page->aspace->vnode->lock); + + return F_OK; +} +#endif + static int dfs_tmpfs_truncate(struct dfs_file *file, off_t offset) { struct tmpfs_file *d_file = RT_NULL; diff --git a/components/dfs/dfs_v2/include/dfs_file.h b/components/dfs/dfs_v2/include/dfs_file.h index bb6551c8eb..4486401333 100644 --- a/components/dfs/dfs_v2/include/dfs_file.h +++ b/components/dfs/dfs_v2/include/dfs_file.h @@ -33,6 +33,7 @@ struct rt_pollreq; struct dirent; struct lwp_avl_struct; struct file_lock; +struct dfs_aspace; struct dfs_file_ops { @@ -73,6 +74,9 @@ struct dfs_vnode struct timespec mtime; struct timespec ctime; + struct dfs_aspace *aspace; + struct rt_mutex lock; + void *data; /* private data of this file system */ }; @@ -93,6 +97,8 @@ struct dfs_file struct dfs_dentry *dentry; /* dentry of this file */ struct dfs_vnode *vnode; /* vnode of this file */ + void *mmap_context; /* used by mmap routine */ + void *data; }; @@ -122,7 +128,7 @@ struct dfs_vnode *dfs_vnode_ref(struct dfs_vnode *vnode); void dfs_vnode_unref(struct dfs_vnode *vnode); /*dfs_file.c*/ - +#ifdef RT_USING_SMART struct dfs_mmap2_args { void *addr; @@ -131,8 +137,10 @@ struct dfs_mmap2_args int flags; off_t pgoffset; + struct rt_lwp *lwp; void *ret; }; +#endif void dfs_file_init(struct dfs_file *file); void dfs_file_deinit(struct dfs_file *file); @@ -166,8 +174,13 @@ int dfs_file_isdir(const char *path); int dfs_file_access(const char *path, mode_t mode); int dfs_file_chdir(const char *path); char *dfs_file_getcwd(char *buf, size_t size); + +#ifdef RT_USING_SMART int dfs_file_mmap2(struct dfs_file *file, struct dfs_mmap2_args *mmap2); +int dfs_file_mmap(struct dfs_file *file, struct dfs_mmap2_args *mmap2); +#endif + /* 0x5254 is just a magic number to make these relatively unique ("RT") */ #define RT_FIOFTRUNCATE 0x52540000U #define RT_FIOGETADDR 0x52540001U diff --git a/components/dfs/dfs_v2/include/dfs_mnt.h b/components/dfs/dfs_v2/include/dfs_mnt.h index 0d73051868..207c9b6bfd 100644 --- a/components/dfs/dfs_v2/include/dfs_mnt.h +++ b/components/dfs/dfs_v2/include/dfs_mnt.h @@ -53,6 +53,7 @@ int dfs_mnt_destroy(struct dfs_mnt* mnt); int dfs_mnt_list(struct dfs_mnt* mnt); int dfs_mnt_insert(struct dfs_mnt* mnt, struct dfs_mnt* child); +struct dfs_mnt *dfs_mnt_dev_lookup(rt_device_t dev_id); struct dfs_mnt *dfs_mnt_lookup(const char *path); const char *dfs_mnt_get_mounted_path(struct rt_device *device); diff --git a/components/dfs/dfs_v2/include/dfs_pcache.h b/components/dfs/dfs_v2/include/dfs_pcache.h new file mode 100644 index 0000000000..356df6c4f9 --- /dev/null +++ b/components/dfs/dfs_v2/include/dfs_pcache.h @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-05-05 RTT Implement dentry in dfs v2.0 + */ + +#ifndef DFS_PAGE_CACHE_H__ +#define DFS_PAGE_CACHE_H__ + +#include <rtthread.h> + +#ifdef RT_USING_PAGECACHE + +#include <dfs_file.h> +#include <avl.h> + +#ifdef __cplusplus +extern "C" +{ +#endif + +struct dfs_vnode; +struct dfs_dentry; +struct dfs_aspace; + +struct dfs_mmap +{ + rt_list_t mmap_node; + struct rt_varea *varea; +}; + +struct dfs_page +{ + rt_list_t space_node; + rt_list_t dirty_node; + struct util_avl_struct avl_node; + rt_list_t mmap_head; + + rt_atomic_t ref_count; + + void *page; + off_t fpos; + size_t size; + size_t len; + int is_dirty; + rt_tick_t tick_ms; + + struct dfs_aspace *aspace; +}; + +struct dfs_aspace_ops +{ + ssize_t (*read)(struct dfs_file *file, struct dfs_page *page); + ssize_t (*write)(struct dfs_page *page); +}; + +struct dfs_aspace +{ + rt_list_t hash_node, cache_node; + char *fullpath, *pathname; + struct dfs_mnt *mnt; + + rt_list_t list_active, list_inactive; + rt_list_t list_dirty; + size_t pages_count; + + struct util_avl_root avl_root; + struct dfs_page *avl_page; + + rt_bool_t is_active; + + struct rt_mutex lock; + rt_atomic_t ref_count; + + struct dfs_vnode *vnode; + const struct dfs_aspace_ops *ops; +}; + +#ifndef RT_PAGECACHE_HASH_NR +#define RT_PAGECACHE_HASH_NR 1024 +#endif + +struct dfs_pcache +{ + rt_list_t head[RT_PAGECACHE_HASH_NR]; + rt_list_t list_active, list_inactive; + rt_atomic_t pages_count; + struct rt_mutex lock; + struct rt_messagequeue *mqueue; + rt_tick_t last_time_wb; +}; + +struct dfs_aspace *dfs_aspace_create(struct dfs_dentry *dentry, struct dfs_vnode *vnode, const struct dfs_aspace_ops *ops); +int dfs_aspace_destroy(struct dfs_aspace *aspace); + +int dfs_aspace_read(struct dfs_file *file, void *buf, size_t count, off_t *pos); +int dfs_aspace_write(struct dfs_file *file, const void *buf, size_t count, off_t *pos); +int dfs_aspace_flush(struct dfs_aspace *aspace); +int dfs_aspace_clean(struct dfs_aspace *aspace); + +void *dfs_aspace_mmap(struct dfs_file *file, struct rt_varea *varea, void *vaddr); +int dfs_aspace_unmap(struct dfs_file *file, struct rt_varea *varea); +int dfs_aspace_page_unmap(struct dfs_file *file, struct rt_varea *varea, void *vaddr); +int dfs_aspace_page_dirty(struct dfs_file *file, struct rt_varea *varea, void *vaddr); + +off_t dfs_aspace_fpos(struct rt_varea *varea, void *vaddr); +void *dfs_aspace_vaddr(struct rt_varea *varea, off_t fpos); + +int dfs_aspace_mmap_read(struct dfs_file *file, struct rt_varea *varea, void *data); +int dfs_aspace_mmap_write(struct dfs_file *file, struct rt_varea *varea, void *data); + +void dfs_pcache_release(size_t count); +void dfs_pcache_unmount(struct dfs_mnt *mnt); + +#ifdef __cplusplus +} +#endif + +#endif + +#endif diff --git a/components/dfs/dfs_v2/include/dfs_seq_file.h b/components/dfs/dfs_v2/include/dfs_seq_file.h new file mode 100644 index 0000000000..6dbc30f24d --- /dev/null +++ b/components/dfs/dfs_v2/include/dfs_seq_file.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + */ + +#ifndef __DFS_SEQ_FILE_H__ +#define __DFS_SEQ_FILE_H__ + +#include <dfs.h> +#include <dfs_fs.h> + +struct dfs_seq_ops; + +struct dfs_seq_file +{ + char *buf; + size_t size; + size_t from; + size_t count; + size_t pad_until; + off_t index; + off_t read_pos; + struct rt_mutex lock; + const struct dfs_seq_ops *ops; + const struct dfs_file *file; + void *data; +}; + +struct dfs_seq_ops +{ + void *(*start)(struct dfs_seq_file *seq, off_t *index); + void (*stop)(struct dfs_seq_file *seq, void *data); + void *(*next)(struct dfs_seq_file *seq, void *data, off_t *index); + int (*show)(struct dfs_seq_file *seq, void *data); +}; + +/** + * check if the buffer is full + */ +static inline rt_bool_t dfs_seq_is_full(struct dfs_seq_file *seq) +{ + return seq->count == seq->size; +} + +/** + * set padding width size + */ +static inline void dfs_seq_setwidth(struct dfs_seq_file *seq, size_t size) +{ + seq->pad_until = seq->count + size; +} + +int dfs_seq_open(struct dfs_file *file, const struct dfs_seq_ops *ops); +ssize_t dfs_seq_read(struct dfs_file *file, void *buf, size_t size, off_t *pos); +ssize_t dfs_seq_lseek(struct dfs_file *file, off_t offset, int whence); +int dfs_seq_release(struct dfs_file *file); +int dfs_seq_write(struct dfs_seq_file *seq, const void *data, size_t len); + +void dfs_seq_vprintf(struct dfs_seq_file *seq, const char *fmt, va_list args); +void dfs_seq_printf(struct dfs_seq_file *seq, const char *fmt, ...); +void dfs_seq_putc(struct dfs_seq_file *seq, char c); +void dfs_seq_puts(struct dfs_seq_file *seq, const char *s); +void dfs_seq_pad(struct dfs_seq_file *seq, char c); + +#endif diff --git a/components/dfs/dfs_v2/src/dfs_dentry.c b/components/dfs/dfs_v2/src/dfs_dentry.c index 6605f094f2..27054b1af5 100644 --- a/components/dfs/dfs_v2/src/dfs_dentry.c +++ b/components/dfs/dfs_v2/src/dfs_dentry.c @@ -197,7 +197,7 @@ struct dfs_dentry *dfs_dentry_lookup(struct dfs_mnt *mnt, const char *path, uint path = "/"; } } - + dfs_file_lock(); dentry = _dentry_hash_lookup(mnt, path); if (!dentry) { @@ -252,7 +252,7 @@ struct dfs_dentry *dfs_dentry_lookup(struct dfs_mnt *mnt, const char *path, uint { DLOG(note, "dentry", "found dentry"); } - + dfs_file_unlock(); return dentry; } diff --git a/components/dfs/dfs_v2/src/dfs_file.c b/components/dfs/dfs_v2/src/dfs_file.c index 544e38bc61..a7baa9323f 100644 --- a/components/dfs/dfs_v2/src/dfs_file.c +++ b/components/dfs/dfs_v2/src/dfs_file.c @@ -19,6 +19,10 @@ #include "dfs_mnt.h" #include "dfs_private.h" +#ifdef RT_USING_PAGECACHE +#include "dfs_pcache.h" +#endif + #define DBG_TAG "DFS.file" #define DBG_LVL DBG_WARNING #include <rtdbg.h> @@ -114,7 +118,7 @@ static void dfs_file_unref(struct dfs_file *file) { if (file->vnode->ref_count > 1) { - file->vnode->ref_count--; + rt_atomic_sub(&(file->vnode->ref_count), 1); } else if (file->vnode->ref_count == 1) { @@ -448,6 +452,7 @@ int dfs_file_open(struct dfs_file *file, const char *path, int oflags, mode_t mo struct dfs_vnode *vnode = RT_NULL; DLOG(msg, "dfs_file", "dentry", DLOG_MSG, "dfs_dentry_create(%s)", fullpath); + dfs_file_lock(); dentry = dfs_dentry_create(mnt, fullpath); if (dentry) { @@ -472,6 +477,7 @@ int dfs_file_open(struct dfs_file *file, const char *path, int oflags, mode_t mo dentry = RT_NULL; } } + dfs_file_unlock(); } } } @@ -518,7 +524,9 @@ int dfs_file_open(struct dfs_file *file, const char *path, int oflags, mode_t mo if (dfs_is_mounted(file->vnode->mnt) == 0) { + dfs_file_lock(); ret = file->fops->open(file); + dfs_file_unlock(); } else { @@ -572,6 +580,12 @@ int dfs_file_open(struct dfs_file *file, const char *path, int oflags, mode_t mo if (dfs_is_mounted(file->vnode->mnt) == 0) { +#ifdef RT_USING_PAGECACHE + if (file->vnode->aspace) + { + dfs_aspace_clean(file->vnode->aspace); + } +#endif ret = file->fops->truncate(file, 0); } else @@ -612,7 +626,12 @@ int dfs_file_close(struct dfs_file *file) if (ref_count == 1 && file->fops && file->fops->close) { DLOG(msg, "dfs_file", file->dentry->mnt->fs_ops->name, DLOG_MSG, "fops->close(file)"); - +#ifdef RT_USING_PAGECACHE + if (file->vnode->aspace) + { + dfs_aspace_flush(file->vnode->aspace); + } +#endif ret = file->fops->close(file); if (ret == 0) /* close file sucessfully */ @@ -665,7 +684,16 @@ ssize_t dfs_file_read(struct dfs_file *file, void *buf, size_t len) if (dfs_is_mounted(file->vnode->mnt) == 0) { - ret = file->fops->read(file, buf, len, &pos); +#ifdef RT_USING_PAGECACHE + if (file->vnode->aspace) + { + ret = dfs_aspace_read(file, buf, len, &pos); + } + else +#endif + { + ret = file->fops->read(file, buf, len, &pos); + } } else { @@ -710,7 +738,16 @@ ssize_t dfs_file_write(struct dfs_file *file, const void *buf, size_t len) if (dfs_is_mounted(file->vnode->mnt) == 0) { - ret = file->fops->write(file, buf, len, &pos); +#ifdef RT_USING_PAGECACHE + if (file->vnode->aspace) + { + ret = dfs_aspace_write(file, buf, len, &pos); + } + else +#endif + { + ret = file->fops->write(file, buf, len, &pos); + } } else { @@ -1047,6 +1084,12 @@ int dfs_file_fsync(struct dfs_file *file) { if (dfs_is_mounted(file->vnode->mnt) == 0) { +#ifdef RT_USING_PAGECACHE + if (file->vnode->aspace) + { + dfs_aspace_flush(file->vnode->aspace); + } +#endif ret = file->fops->flush(file); } else @@ -1089,6 +1132,14 @@ int dfs_file_unlink(const char *path) rt_bool_t has_child = RT_FALSE; has_child = dfs_mnt_has_child_mnt(mnt, fullpath); +#ifdef RT_USING_PAGECACHE + if (dentry->vnode->aspace) + { + dfs_aspace_clean(dentry->vnode->aspace); + } +#endif + dfs_file_lock(); + if (has_child == RT_FALSE) { /* no child mnt point, unlink it */ @@ -1106,6 +1157,7 @@ int dfs_file_unlink(const char *path) { ret = -EBUSY; } + dfs_file_unlock(); /* release this dentry */ dfs_dentry_unref(dentry); @@ -1467,6 +1519,12 @@ int dfs_file_rename(const char *old_file, const char *new_file) { if (dfs_is_mounted(mnt) == 0) { +#ifdef RT_USING_PAGECACHE + if (old_dentry->vnode->aspace) + { + dfs_aspace_clean(old_dentry->vnode->aspace); + } +#endif ret = mnt->fs_ops->rename(old_dentry, new_dentry); } } @@ -1499,6 +1557,12 @@ int dfs_file_ftruncate(struct dfs_file *file, off_t length) { if (dfs_is_mounted(file->vnode->mnt) == 0) { +#ifdef RT_USING_PAGECACHE + if (file->vnode->aspace) + { + dfs_aspace_clean(file->vnode->aspace); + } +#endif ret = file->fops->truncate(file, length); } else @@ -1529,6 +1593,12 @@ int dfs_file_flush(struct dfs_file *file) { if (dfs_is_mounted(file->vnode->mnt) == 0) { +#ifdef RT_USING_PAGECACHE + if (file->vnode->aspace) + { + dfs_aspace_flush(file->vnode->aspace); + } +#endif ret = file->fops->flush(file); } else @@ -1669,13 +1739,23 @@ int dfs_file_access(const char *path, mode_t mode) return ret; } +#ifdef RT_USING_SMART int dfs_file_mmap2(struct dfs_file *file, struct dfs_mmap2_args *mmap2) { - int ret = 0; + int ret = RT_EOK; if (file && mmap2) { - if (file->vnode->type != FT_DEVICE || !file->vnode->fops->ioctl) + if (file->vnode->type == FT_REGULAR) + { + ret = dfs_file_mmap(file, mmap2); + if (ret != 0) + { + ret = ret > 0 ? ret : -ret; + rt_set_errno(ret); + } + } + else if (file->vnode->type != FT_DEVICE || !file->vnode->fops->ioctl) { rt_set_errno(EINVAL); } @@ -1700,6 +1780,7 @@ int dfs_file_mmap2(struct dfs_file *file, struct dfs_mmap2_args *mmap2) return ret; } +#endif #ifdef RT_USING_FINSH diff --git a/components/dfs/dfs_v2/src/dfs_file_mmap.c b/components/dfs/dfs_v2/src/dfs_file_mmap.c new file mode 100644 index 0000000000..6e356ef405 --- /dev/null +++ b/components/dfs/dfs_v2/src/dfs_file_mmap.c @@ -0,0 +1,443 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + */ + +#include "dfs_file.h" +#include "dfs_dentry.h" +#include "dfs_mnt.h" + +#define DBG_TAG "dfs.mmap" +#define DBG_LVL DBG_WARNING +#include <rtdbg.h> + +#if defined(RT_USING_SMART) && defined(ARCH_MM_MMU) && defined(RT_USING_PAGECACHE) + +#include "dfs_pcache.h" + +#include <lwp.h> + +#include <sys/mman.h> + +#include <lwp_user_mm.h> +#include <mm_aspace.h> +#include <mm_fault.h> +#include <mm_flag.h> +#include <mm_page.h> +#include <mmu.h> +#include <page.h> +#include <tlb.h> + +static rt_mem_obj_t dfs_get_mem_obj(struct dfs_file *file); +static void *dfs_mem_obj_get_file(rt_mem_obj_t mem_obj); + +static rt_varea_t _dfs_map_user_varea_data(struct rt_lwp *lwp, void *map_vaddr, size_t map_size, size_t attr, mm_flag_t flags, off_t pgoffset, void *data) +{ + int ret = 0; + rt_varea_t varea; + void *vaddr = map_vaddr; + rt_mem_obj_t mem_obj = dfs_get_mem_obj(data); + + ret = rt_aspace_map(lwp->aspace, &vaddr, map_size, + attr, flags, mem_obj, pgoffset); + if (ret != RT_EOK) + { + varea = RT_NULL; + } + else + { + varea = rt_aspace_query(lwp->aspace, vaddr); + } + + if (ret != RT_EOK) + { + LOG_E("failed to map %lx with size %lx with errno %d", map_vaddr, + map_size, ret); + } + + return varea; +} + +static rt_varea_t dfs_map_user_varea_data(struct dfs_mmap2_args *mmap2, void *data) +{ + rt_varea_t varea = RT_NULL; + size_t offset = 0; + void *map_vaddr = mmap2->addr; + size_t map_size = mmap2->length; + struct rt_lwp *lwp = mmap2->lwp; + rt_size_t k_attr; + rt_size_t k_flags; + + if (map_size) + { + offset = (size_t)map_vaddr & ARCH_PAGE_MASK; + map_size += (offset + ARCH_PAGE_SIZE - 1); + map_size &= ~ARCH_PAGE_MASK; + map_vaddr = (void *)((size_t)map_vaddr & ~ARCH_PAGE_MASK); + + k_flags = lwp_user_mm_flag_to_kernel(mmap2->flags); + k_attr = lwp_user_mm_attr_to_kernel(mmap2->prot); + + varea = _dfs_map_user_varea_data(lwp, map_vaddr, map_size, k_attr, k_flags, mmap2->pgoffset, data); + } + + return varea; +} + +static void hint_free(rt_mm_va_hint_t hint) +{ +} + +static void on_page_fault(struct rt_varea *varea, struct rt_aspace_fault_msg *msg) +{ + void *page; + struct dfs_file *file = dfs_mem_obj_get_file(varea->mem_obj); + + if (file) + { + LOG_I("%s varea: %p", __func__, varea); + LOG_I("varea start: %p size: 0x%x offset: 0x%x attr: 0x%x flag: 0x%x", + varea->start, varea->size, varea->offset, varea->attr, varea->flag); + LOG_I("fault vaddr: %p", msg->fault_vaddr); + + if (file->dentry) + { + LOG_I("file: %s%s", file->dentry->mnt->fullpath, file->dentry->pathname); + } + + page = dfs_aspace_mmap(file, varea, msg->fault_vaddr); + if (page) + { + msg->response.status = MM_FAULT_STATUS_OK_MAPPED; + msg->response.size = ARCH_PAGE_SIZE; + msg->response.vaddr = page; + } + else + { + LOG_E("%s varea %p mmap failed at vaddr %p", __func__, varea, msg->fault_vaddr); + } + } + else + { + LOG_E("%s varea %p not a file, vaddr %p", __func__, varea, varea->start); + } +} + +/* do pre open bushiness like inc a ref */ +static void on_varea_open(struct rt_varea *varea) +{ + struct dfs_file *file = dfs_mem_obj_get_file(varea->mem_obj); + varea->data = RT_NULL; + rt_atomic_add(&(file->ref_count), 1); +} + +/* do post close bushiness like def a ref */ +static void on_varea_close(struct rt_varea *varea) +{ + struct dfs_file *file = dfs_mem_obj_get_file(varea->mem_obj); + + if (file) + { + LOG_I("%s varea: %p", __func__, varea); + LOG_I("varea start: %p size: 0x%x offset: 0x%x attr: 0x%x flag: 0x%x", + varea->start, varea->size, varea->offset, varea->attr, varea->flag); + + if (file->dentry) + { + LOG_I("file: %s%s", file->dentry->mnt->fullpath, file->dentry->pathname); + } + + dfs_aspace_unmap(file, varea); + dfs_file_lock(); + if (rt_atomic_load(&(file->ref_count)) == 1) + { + dfs_file_close(file); + } + else + { + rt_atomic_sub(&(file->ref_count), 1); + } + dfs_file_unlock(); + } + else + { + LOG_E("%s varea %p not a file, vaddr %p", __func__, varea, varea->start); + } +} + +static const char *get_name(rt_varea_t varea) +{ + struct dfs_file *file = dfs_mem_obj_get_file(varea->mem_obj); + + return (file && file->dentry) ? file->dentry->pathname : "file-mapper"; +} + +void page_read(struct rt_varea *varea, struct rt_aspace_io_msg *msg) +{ + rt_ubase_t ret; + struct dfs_file *file = dfs_mem_obj_get_file(varea->mem_obj); + + if (file) + { + LOG_I("%s varea: %p", __func__, varea); + LOG_I("varea start: %p size: 0x%x offset: 0x%x attr: 0x%x flag: 0x%x", + varea->start, varea->size, varea->offset, varea->attr, varea->flag); + + ret = dfs_aspace_mmap_read(file, varea, msg); + if (ret > 0) + { + msg->response.status = MM_FAULT_STATUS_OK; + if (ret < ARCH_PAGE_SIZE) + { + memset((char *)msg->buffer_vaddr + ret, 0, ARCH_PAGE_SIZE - ret); + } + } + } + else + { + LOG_E("%s varea %p not a file, vaddr %p", __func__, varea, varea->start); + } +} + +void page_write(struct rt_varea *varea, struct rt_aspace_io_msg *msg) +{ + rt_ubase_t ret; + struct dfs_file *file = dfs_mem_obj_get_file(varea->mem_obj); + + if (file) + { + LOG_I("%s varea: %p", __func__, varea); + LOG_I("varea start: %p size: 0x%x offset: 0x%x attr: 0x%x flag: 0x%x", + varea->start, varea->size, varea->offset, varea->attr, varea->flag); + + ret = dfs_aspace_mmap_write(file, varea, msg); + if (ret > 0) + { + msg->response.status = MM_FAULT_STATUS_OK; + if (ret < ARCH_PAGE_SIZE) + { + memset((char *)msg->buffer_vaddr + ret, 0, ARCH_PAGE_SIZE - ret); + } + } + } + else + { + LOG_E("%s varea %p not a file, vaddr %p", __func__, varea, varea->start); + } +} + +static rt_err_t unmap_pages(rt_varea_t varea, void *rm_start, void *rm_end) +{ + struct dfs_file *file = dfs_mem_obj_get_file(varea->mem_obj); + + if (file) + { + LOG_I("%s varea: %p start: %p end: %p", __func__, varea, rm_start, rm_end); + + RT_ASSERT(!((rt_ubase_t)rm_start & ARCH_PAGE_MASK)); + RT_ASSERT(!((rt_ubase_t)rm_end & ARCH_PAGE_MASK)); + while (rm_start != rm_end) + { + dfs_aspace_page_unmap(file, varea, rm_start); + rm_start += ARCH_PAGE_SIZE; + } + + return RT_EOK; + } + else + { + LOG_E("%s varea %p not a file, vaddr %p", __func__, varea, varea->start); + } + + return -RT_ERROR; +} + +rt_err_t on_varea_shrink(struct rt_varea *varea, void *new_vaddr, rt_size_t size) +{ + char *varea_start = varea->start; + void *rm_start; + void *rm_end; + + LOG_I("%s varea: %p", __func__, varea); + LOG_I("varea start: %p size: 0x%x offset: 0x%x attr: 0x%x flag: 0x%x", + varea->start, varea->size, varea->offset, varea->attr, varea->flag); + LOG_I("new_vaddr: %p size: %p", new_vaddr, size); + + if (varea_start == (char *)new_vaddr) + { + rm_start = varea_start + size; + rm_end = varea_start + varea->size; + } + else if (varea_start < (char *)new_vaddr) + { + rm_start = varea_start; + rm_end = new_vaddr; + } + + return unmap_pages(varea, rm_start, rm_end); +} + +rt_err_t on_varea_expand(struct rt_varea *varea, void *new_vaddr, rt_size_t size) +{ + LOG_I("%s varea: %p", __func__, varea); + LOG_I("varea start: %p size: 0x%x offset: 0x%x attr: 0x%x flag: 0x%x", + varea->start, varea->size, varea->offset, varea->attr, varea->flag); + LOG_I("new_vaddr: %p size: %p", new_vaddr, size); + + return RT_EOK; +} + +rt_err_t on_varea_split(struct rt_varea *existed, void *unmap_start, rt_size_t unmap_len, struct rt_varea *subset) +{ + struct dfs_file *file = dfs_mem_obj_get_file(existed->mem_obj); + + if (file) + { + LOG_I("%s varea: %p", __func__, existed); + LOG_I("varea start: %p size: 0x%x offset: 0x%x attr: 0x%x flag: 0x%x", + existed->start, existed->size, existed->offset, existed->attr, existed->flag); + LOG_I("unmap_start: %p unmap_len: %p", unmap_start, unmap_len); + + if (file->dentry) + { + LOG_I("file: %s%s", file->dentry->mnt->fullpath, file->dentry->pathname); + } + + unmap_pages(existed, unmap_start, (char *)unmap_start + unmap_len); + + subset->data = existed->data; + rt_atomic_add(&(file->ref_count), 1); + + return RT_EOK; + } + else + { + LOG_E("%s varea %p not a file, vaddr %p", __func__, existed, existed->start); + } + + return -RT_ERROR; +} + +rt_err_t on_varea_merge(struct rt_varea *merge_to, struct rt_varea *merge_from) +{ + struct dfs_file *file = dfs_mem_obj_get_file(merge_from->mem_obj); + + if (file) + { + LOG_I("%s varea: %p", __func__, merge_from); + LOG_I("varea start: %p size: 0x%x offset: 0x%x attr: 0x%x flag: 0x%x", + merge_from->start, merge_from->size, merge_from->offset, merge_from->attr, merge_from->flag); + + if (file->dentry) + { + LOG_I("file: %s%s", file->dentry->mnt->fullpath, file->dentry->pathname); + } + + dfs_aspace_unmap(file, merge_from); + merge_from->data = RT_NULL; + rt_atomic_sub(&(file->ref_count), 1); + + return RT_EOK; + } + else + { + LOG_E("%s varea %p not a file, vaddr %p", __func__, merge_from, merge_from->start); + } + + return -RT_ERROR; +} + +static struct rt_mem_obj _mem_obj = +{ + .hint_free = hint_free, + .on_page_fault = on_page_fault, + .on_varea_open = on_varea_open, + .on_varea_close = on_varea_close, + .get_name = get_name, + + .page_read = page_read, + .page_write = page_write, + + .on_varea_shrink = on_varea_shrink, + .on_varea_expand = on_varea_expand, + .on_varea_split = on_varea_split, + .on_varea_merge = on_varea_merge, +}; + +struct dfs_mem_obj { + struct rt_mem_obj mem_obj; + void *file; +}; + +static rt_mem_obj_t dfs_get_mem_obj(struct dfs_file *file) +{ + rt_mem_obj_t mobj = file->mmap_context; + if (!mobj) + { + struct dfs_mem_obj *dfs_mobj; + dfs_file_lock(); + dfs_mobj = rt_malloc(sizeof(*dfs_mobj)); + if (dfs_mobj) + { + dfs_mobj->file = file; + mobj = &dfs_mobj->mem_obj; + memcpy(mobj, &_mem_obj, sizeof(*mobj)); + file->mmap_context = mobj; + } + dfs_file_unlock(); + } + return mobj; +} + +static void *dfs_mem_obj_get_file(rt_mem_obj_t mem_obj) +{ + struct dfs_mem_obj *dfs_mobj; + dfs_mobj = rt_container_of(mem_obj, struct dfs_mem_obj, mem_obj); + return dfs_mobj->file; +} + +int dfs_file_mmap(struct dfs_file *file, struct dfs_mmap2_args *mmap2) +{ + int ret = -EINVAL; + + LOG_I("mmap2 args addr: %p length: 0x%x prot: %d flags: 0x%x pgoffset: 0x%x", + mmap2->addr, mmap2->length, mmap2->prot, mmap2->flags, mmap2->pgoffset); + if (file && file->vnode && file->vnode->aspace) + { + /* create a va area in user space (lwp) */ + rt_varea_t varea = dfs_map_user_varea_data(mmap2, file); + if (varea) + { + mmap2->ret = varea->start; + LOG_I("%s varea: %p", __func__, varea); + LOG_I("varea start: %p size: 0x%x offset: 0x%x attr: 0x%x flag: 0x%x", + varea->start, varea->size, varea->offset, varea->attr, varea->flag); + LOG_I("file: %s%s", file->dentry->mnt->fullpath, file->dentry->pathname); + ret = RT_EOK; + } + else + { + ret = -ENOMEM; + } + } + else if (file->vnode->aspace == RT_NULL) + { + LOG_E("File mapping is not supported, file: %s%s", file->dentry->mnt->fullpath, file->dentry->pathname); + } + + return ret; +} +#else +int dfs_file_mmap(struct dfs_file *file, struct dfs_mmap2_args *mmap2) +{ + LOG_E("File mapping support is not enabled, file: %s%s", file->dentry->mnt->fullpath, file->dentry->pathname); + LOG_E("mmap2 args addr: %p length: 0x%x prot: %d flags: 0x%x pgoffset: 0x%x", + mmap2->addr, mmap2->length, mmap2->prot, mmap2->flags, mmap2->pgoffset); + + return -EPERM; +} +#endif diff --git a/components/dfs/dfs_v2/src/dfs_fs.c b/components/dfs/dfs_v2/src/dfs_fs.c index 8c28dbe7ef..e65ccee478 100644 --- a/components/dfs/dfs_v2/src/dfs_fs.c +++ b/components/dfs/dfs_v2/src/dfs_fs.c @@ -19,6 +19,10 @@ #include <dfs_mnt.h> #include "dfs_private.h" +#ifdef RT_USING_PAGECACHE +#include "dfs_pcache.h" +#endif + #define DBG_TAG "DFS.fs" #define DBG_LVL DBG_INFO #include <rtdbg.h> @@ -314,6 +318,9 @@ int dfs_umount(const char *specialfile, int flags) if (!(mnt->flags & MNT_IS_LOCKED) && rt_list_isempty(&mnt->child) && (ref_count == 1 || (flags & MNT_FORCE))) { +#ifdef RT_USING_PAGECACHE + dfs_pcache_unmount(mnt); +#endif /* destroy this mount point */ DLOG(msg, "dfs", "mnt", DLOG_MSG, "dfs_mnt_destroy(mnt)"); ret = dfs_mnt_destroy(mnt); @@ -396,6 +403,18 @@ int dfs_mkfs(const char *fs_name, const char *device_name) if (type->fs_ops->mkfs) { ret = type->fs_ops->mkfs(dev_id, type->fs_ops->name); +#ifdef RT_USING_PAGECACHE + if (ret == RT_EOK) + { + struct dfs_mnt *mnt = RT_NULL; + + mnt = dfs_mnt_dev_lookup(dev_id); + if (mnt) + { + dfs_pcache_unmount(mnt); + } + } +#endif } return ret; diff --git a/components/dfs/dfs_v2/src/dfs_mnt.c b/components/dfs/dfs_v2/src/dfs_mnt.c index 379606d9ce..4ad73c6dee 100644 --- a/components/dfs/dfs_v2/src/dfs_mnt.c +++ b/components/dfs/dfs_v2/src/dfs_mnt.c @@ -131,6 +131,53 @@ int dfs_mnt_remove(struct dfs_mnt* mnt) return ret; } +static struct dfs_mnt *_dfs_mnt_dev_lookup(struct dfs_mnt *mnt, rt_device_t dev_id) +{ + struct dfs_mnt *ret = RT_NULL, *iter = RT_NULL; + + rt_list_for_each_entry(iter, &mnt->child, sibling) + { + if (iter->dev_id == dev_id) + { + ret = iter; + break; + } + else + { + ret = _dfs_mnt_dev_lookup(iter, dev_id); + if (ret) + { + break; + } + } + } + + return ret; +} + +struct dfs_mnt *dfs_mnt_dev_lookup(rt_device_t dev_id) +{ + struct dfs_mnt *mnt = _root_mnt; + struct dfs_mnt *ret = RT_NULL; + + if (mnt) + { + dfs_lock(); + + if (mnt->dev_id == dev_id) + { + dfs_unlock(); + return mnt; + } + + ret = _dfs_mnt_dev_lookup(mnt, dev_id); + + dfs_unlock(); + } + + return ret; +} + /** * this function will return the file system mounted on specified path. * @@ -139,24 +186,26 @@ int dfs_mnt_remove(struct dfs_mnt* mnt) * @return the found file system or NULL if no file system mounted on * specified path */ -struct dfs_mnt* dfs_mnt_lookup(const char* fullpath) +struct dfs_mnt *dfs_mnt_lookup(const char *fullpath) { struct dfs_mnt *mnt = _root_mnt; struct dfs_mnt *iter = RT_NULL; if (mnt) { + int mnt_len = rt_strlen(mnt->fullpath); + dfs_lock(); - if (strncmp(mnt->fullpath, fullpath, strlen(fullpath))!= 0) + if ((strncmp(mnt->fullpath, fullpath, mnt_len) == 0) && + (mnt_len == 1 || (fullpath[mnt_len] == '\0') || (fullpath[mnt_len] == '/'))) { while (!rt_list_isempty(&mnt->child)) { rt_list_for_each_entry(iter, &mnt->child, sibling) { - int mnt_len = rt_strlen(iter->fullpath); + mnt_len = rt_strlen(iter->fullpath); if ((strncmp(iter->fullpath, fullpath, mnt_len) == 0) && - ((fullpath[mnt_len] == '\0') || - (fullpath[mnt_len] == '/'))) + ((fullpath[mnt_len] == '\0') || (fullpath[mnt_len] == '/'))) { mnt = iter; break; @@ -166,6 +215,10 @@ struct dfs_mnt* dfs_mnt_lookup(const char* fullpath) if (mnt != iter) break; } } + else + { + mnt = RT_NULL; + } dfs_unlock(); if (mnt) diff --git a/components/dfs/dfs_v2/src/dfs_pcache.c b/components/dfs/dfs_v2/src/dfs_pcache.c new file mode 100644 index 0000000000..0b2492dc7f --- /dev/null +++ b/components/dfs/dfs_v2/src/dfs_pcache.c @@ -0,0 +1,1439 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-05-05 RTT Implement mnt in dfs v2.0 + */ + +#include "dfs_pcache.h" +#include "dfs_dentry.h" +#include "dfs_mnt.h" +#include "mm_page.h" +#include <mmu.h> +#include <tlb.h> + +#ifdef RT_USING_PAGECACHE + +#define DBG_TAG "dfs.pcache" +#define DBG_LVL DBG_WARNING +#include <rtdbg.h> + +#ifndef RT_PAGECACHE_COUNT +#define RT_PAGECACHE_COUNT 4096 +#endif + +#ifndef RT_PAGECACHE_ASPACE_COUNT +#define RT_PAGECACHE_ASPACE_COUNT 1024 +#endif + +#ifndef RT_PAGECACHE_PRELOAD +#define RT_PAGECACHE_PRELOAD 4 +#endif + +#ifndef RT_PAGECACHE_GC_WORK_LEVEL +#define RT_PAGECACHE_GC_WORK_LEVEL 90 +#endif + +#ifndef RT_PAGECACHE_GC_STOP_LEVEL +#define RT_PAGECACHE_GC_STOP_LEVEL 70 +#endif + +#define PCACHE_MQ_GC 1 +#define PCACHE_MQ_WB 2 + +struct dfs_aspace_mmap_obj +{ + rt_uint32_t cmd; + struct rt_mailbox *ack; + struct dfs_file *file; + struct rt_varea *varea; + void *data; +}; + +struct dfs_pcache_mq_obj +{ + struct rt_mailbox *ack; + rt_uint32_t cmd; +}; + +static struct dfs_page *dfs_page_lookup(struct dfs_file *file, off_t pos); +static void dfs_page_ref(struct dfs_page *page); +static int dfs_page_inactive(struct dfs_page *page); +static int dfs_page_remove(struct dfs_page *page); +static void dfs_page_release(struct dfs_page *page); +static int dfs_page_dirty(struct dfs_page *page); + +static int dfs_aspace_release(struct dfs_aspace *aspace); + +static int dfs_aspace_lock(struct dfs_aspace *aspace); +static int dfs_aspace_unlock(struct dfs_aspace *aspace); + +static int dfs_pcache_lock(void); +static int dfs_pcache_unlock(void); + + +static struct dfs_pcache __pcache; + + +static int dfs_aspace_gc(struct dfs_aspace *aspace, int count) +{ + int cnt = count; + + if (aspace) + { + dfs_aspace_lock(aspace); + + if (aspace->pages_count > 0) + { + struct dfs_page *page = RT_NULL; + rt_list_t *node = aspace->list_inactive.next; + + while (cnt && node != &aspace->list_active) + { + page = rt_list_entry(node, struct dfs_page, space_node); + node = node->next; + if (dfs_page_remove(page) == 0) + { + cnt --; + } + } + + node = aspace->list_active.next; + while (cnt && node != &aspace->list_inactive) + { + page = rt_list_entry(node, struct dfs_page, space_node); + node = node->next; + if (dfs_page_remove(page) == 0) + { + cnt --; + } + } + } + + dfs_aspace_unlock(aspace); + } + + return count - cnt; +} + +void dfs_pcache_release(size_t count) +{ + rt_list_t *node = RT_NULL; + struct dfs_aspace *aspace = RT_NULL; + + dfs_pcache_lock(); + + if (count == 0) + { + count = rt_atomic_load(&(__pcache.pages_count)) - RT_PAGECACHE_COUNT * RT_PAGECACHE_GC_STOP_LEVEL / 100; + } + + node = __pcache.list_inactive.next; + while (count && node != &__pcache.list_active) + { + aspace = rt_list_entry(node, struct dfs_aspace, cache_node); + node = node->next; + if (aspace) + { + count -= dfs_aspace_gc(aspace, count); + dfs_aspace_release(aspace); + } + } + + node = __pcache.list_active.next; + while (count && node != &__pcache.list_inactive) + { + aspace = rt_list_entry(node, struct dfs_aspace, cache_node); + node = node->next; + if (aspace) + { + count -= dfs_aspace_gc(aspace, count); + } + } + + dfs_pcache_unlock(); +} + +void dfs_pcache_unmount(struct dfs_mnt *mnt) +{ + rt_list_t *node = RT_NULL; + struct dfs_aspace *aspace = RT_NULL; + + dfs_pcache_lock(); + + node = __pcache.list_inactive.next; + while (node != &__pcache.list_active) + { + aspace = rt_list_entry(node, struct dfs_aspace, cache_node); + node = node->next; + if (aspace && aspace->mnt == mnt) + { + dfs_aspace_clean(aspace); + dfs_aspace_release(aspace); + } + } + + node = __pcache.list_active.next; + while (node != &__pcache.list_inactive) + { + aspace = rt_list_entry(node, struct dfs_aspace, cache_node); + node = node->next; + if (aspace && aspace->mnt == mnt) + { + dfs_aspace_clean(aspace); + dfs_aspace_release(aspace); + } + } + + dfs_pcache_unlock(); +} + +static int dfs_pcache_limit_check(void) +{ + int index = 4; + + while (index && rt_atomic_load(&(__pcache.pages_count)) > RT_PAGECACHE_COUNT * RT_PAGECACHE_GC_WORK_LEVEL / 100) + { + dfs_pcache_release(0); + index --; + } + + return 0; +} + +static void dfs_pcache_thread(void *parameter) +{ + struct dfs_pcache_mq_obj work; + + while (1) + { + if (rt_mq_recv(__pcache.mqueue, &work, sizeof(work), RT_WAITING_FOREVER) == sizeof(work)) + { + if (work.cmd == PCACHE_MQ_GC) + { + dfs_pcache_limit_check(); + } + else if (work.cmd == PCACHE_MQ_WB) + { + int count = 0; + rt_list_t *node; + struct dfs_page *page = 0; + + while (1) + { + /* try to get dirty page */ + dfs_pcache_lock(); + page = 0; + rt_list_for_each(node, &__pcache.list_active) + { + if (node != &__pcache.list_inactive) + { + struct dfs_aspace *aspace = rt_list_entry(node, struct dfs_aspace, cache_node); + dfs_aspace_lock(aspace); + if (aspace->list_dirty.next != &aspace->list_dirty) + { + page = rt_list_entry(aspace->list_dirty.next, struct dfs_page, dirty_node); + dfs_page_ref(page); + dfs_aspace_unlock(aspace); + break; + } + else + { + page = RT_NULL; + } + dfs_aspace_unlock(aspace); + } + } + dfs_pcache_unlock(); + + if (page) + { + struct dfs_aspace *aspace = page->aspace; + + dfs_aspace_lock(aspace); + if (page->is_dirty == 1 && aspace->vnode) + { + if (rt_tick_get_millisecond() - page->tick_ms >= 500) + { + if (aspace->vnode->size < page->fpos + page->size) + { + page->len = aspace->vnode->size - page->fpos; + } + else + { + page->len = page->size; + } + //rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, page->page, page->size); + aspace->ops->write(page); + + page->is_dirty = 0; + + if (page->dirty_node.next != RT_NULL) + { + rt_list_remove(&page->dirty_node); + page->dirty_node.next = RT_NULL; + } + } + } + dfs_page_release(page); + dfs_aspace_unlock(aspace); + } + else + { + break; + } + + rt_thread_mdelay(5); + + count ++; + if (count >= 4) + { + break; + } + } + } + } + } +} + +static int dfs_pcache_init(void) +{ + rt_thread_t tid; + + for (int i = 0; i < RT_PAGECACHE_HASH_NR; i++) + { + rt_list_init(&__pcache.head[i]); + } + + rt_list_init(&__pcache.list_active); + rt_list_init(&__pcache.list_inactive); + rt_list_insert_after(&__pcache.list_active, &__pcache.list_inactive); + + rt_atomic_store(&(__pcache.pages_count), 0); + + rt_mutex_init(&__pcache.lock, "pcache", RT_IPC_FLAG_PRIO); + + __pcache.mqueue = rt_mq_create("pcache", sizeof(struct dfs_pcache_mq_obj), 1024, RT_IPC_FLAG_FIFO); + tid = rt_thread_create("pcache", dfs_pcache_thread, 0, 8192, 25, 5); + if (tid) + { + rt_thread_startup(tid); + } + + __pcache.last_time_wb = rt_tick_get_millisecond(); + + return 0; +} +INIT_PREV_EXPORT(dfs_pcache_init); + +static rt_ubase_t dfs_pcache_mq_work(rt_uint32_t cmd) +{ + rt_err_t err; + struct dfs_pcache_mq_obj work = { 0 }; + + work.cmd = cmd; + + err = rt_mq_send_wait(__pcache.mqueue, (const void *)&work, sizeof(struct dfs_pcache_mq_obj), 0); + + return err; +} + +static int dfs_pcache_lock(void) +{ + rt_mutex_take(&__pcache.lock, RT_WAITING_FOREVER); + return 0; +} + +static int dfs_pcache_unlock(void) +{ + rt_mutex_release(&__pcache.lock); + return 0; +} + +static uint32_t dfs_aspace_hash(struct dfs_mnt *mnt, const char *path) +{ + uint32_t val = 0; + + if (path) + { + while (*path) + { + val = ((val << 5) + val) + *path++; + } + } + + return (val ^ (unsigned long)mnt) & (RT_PAGECACHE_HASH_NR - 1); +} + +static struct dfs_aspace *dfs_aspace_hash_lookup(struct dfs_dentry *dentry, const struct dfs_aspace_ops *ops) +{ + struct dfs_aspace *aspace = RT_NULL; + + dfs_pcache_lock(); + rt_list_for_each_entry(aspace, &__pcache.head[dfs_aspace_hash(dentry->mnt, dentry->pathname)], hash_node) + { + + if (aspace->mnt == dentry->mnt + && aspace->ops == ops + && !strcmp(aspace->pathname, dentry->pathname)) + { + rt_atomic_add(&aspace->ref_count, 1); + dfs_pcache_unlock(); + return aspace; + } + } + dfs_pcache_unlock(); + + return RT_NULL; +} + +static void dfs_aspace_insert(struct dfs_aspace *aspace) +{ + uint32_t val = 0; + + val = dfs_aspace_hash(aspace->mnt, aspace->pathname); + + dfs_pcache_lock(); + rt_atomic_add(&aspace->ref_count, 1); + rt_list_insert_after(&__pcache.head[val], &aspace->hash_node); + rt_list_insert_before(&__pcache.list_inactive, &aspace->cache_node); + dfs_pcache_unlock(); +} + +static void dfs_aspace_remove(struct dfs_aspace *aspace) +{ + dfs_pcache_lock(); + if (aspace->hash_node.next != RT_NULL) + { + rt_list_remove(&aspace->hash_node); + } + if (aspace->cache_node.next != RT_NULL) + { + rt_list_remove(&aspace->cache_node); + } + dfs_pcache_unlock(); +} + +static void dfs_aspace_active(struct dfs_aspace *aspace) +{ + dfs_pcache_lock(); + if (aspace->cache_node.next != RT_NULL) + { + rt_list_remove(&aspace->cache_node); + rt_list_insert_before(&__pcache.list_inactive, &aspace->cache_node); + } + dfs_pcache_unlock(); +} + +static void dfs_aspace_inactive(struct dfs_aspace *aspace) +{ + dfs_pcache_lock(); + if (aspace->cache_node.next != RT_NULL) + { + rt_list_remove(&aspace->cache_node); + rt_list_insert_before(&__pcache.list_active, &aspace->cache_node); + } + dfs_pcache_unlock(); +} + +static struct dfs_aspace *_dfs_aspace_create(struct dfs_dentry *dentry, + struct dfs_vnode *vnode, + const struct dfs_aspace_ops *ops) +{ + struct dfs_aspace *aspace; + + aspace = rt_calloc(1, sizeof(struct dfs_aspace)); + if (aspace) + { + rt_list_init(&aspace->list_active); + rt_list_init(&aspace->list_inactive); + rt_list_init(&aspace->list_dirty); + rt_list_insert_after(&aspace->list_active, &aspace->list_inactive); + + aspace->avl_root.root_node = 0; + aspace->avl_page = 0; + + rt_mutex_init(&aspace->lock, rt_thread_self()->parent.name, RT_IPC_FLAG_PRIO); + rt_atomic_store(&aspace->ref_count, 1); + + aspace->pages_count = 0; + aspace->vnode = vnode; + aspace->ops = ops; + + if (dentry && dentry->mnt) + { + aspace->mnt = dentry->mnt; + aspace->fullpath = rt_strdup(dentry->mnt->fullpath); + aspace->pathname = rt_strdup(dentry->pathname); + } + + dfs_aspace_insert(aspace); + } + + return aspace; +} + +struct dfs_aspace *dfs_aspace_create(struct dfs_dentry *dentry, + struct dfs_vnode *vnode, + const struct dfs_aspace_ops *ops) +{ + struct dfs_aspace *aspace = RT_NULL; + + RT_ASSERT(vnode && ops); + dfs_pcache_lock(); + if (dentry) + { + aspace = dfs_aspace_hash_lookup(dentry, ops); + } + + if (!aspace) + { + aspace = _dfs_aspace_create(dentry, vnode, ops); + } + else + { + aspace->vnode = vnode; + dfs_aspace_active(aspace); + } + dfs_pcache_unlock(); + return aspace; +} + +int dfs_aspace_destroy(struct dfs_aspace *aspace) +{ + int ret = -EINVAL; + + if (aspace) + { + dfs_pcache_lock(); + dfs_aspace_lock(aspace); + rt_atomic_sub(&aspace->ref_count, 1); + RT_ASSERT(rt_atomic_load(&aspace->ref_count) > 0); + dfs_aspace_inactive(aspace); + aspace->vnode = RT_NULL; + if (dfs_aspace_release(aspace) != 0) + { + dfs_aspace_unlock(aspace); + } + dfs_pcache_unlock(); + } + + return ret; +} + +static int dfs_aspace_release(struct dfs_aspace *aspace) +{ + int ret = -1; + + if (aspace) + { + dfs_pcache_lock(); + dfs_aspace_lock(aspace); + + if (rt_atomic_load(&aspace->ref_count) == 1 && aspace->pages_count == 0) + { + dfs_aspace_remove(aspace); + if (aspace->fullpath) + { + rt_free(aspace->fullpath); + } + if (aspace->pathname) + { + rt_free(aspace->pathname); + } + rt_mutex_detach(&aspace->lock); + rt_free(aspace); + ret = 0; + } + else + { + dfs_aspace_unlock(aspace); + } + dfs_pcache_unlock(); + } + + return ret; +} + +static int _dfs_aspace_dump(struct dfs_aspace *aspace, int is_dirty) +{ + if (aspace) + { + rt_list_t *next; + struct dfs_page *page; + + dfs_aspace_lock(aspace); + if (aspace->pages_count > 0) + { + rt_list_for_each(next, &aspace->list_inactive) + { + if (next != &aspace->list_active) + { + page = rt_list_entry(next, struct dfs_page, space_node); + if (is_dirty && page->is_dirty) + { + rt_kprintf(" pages >> fpos: %d index :%d is_dirty: %d\n", page->fpos, page->fpos / ARCH_PAGE_SIZE, page->is_dirty); + } + else if (is_dirty == 0) + { + rt_kprintf(" pages >> fpos: %d index :%d is_dirty: %d\n", page->fpos, page->fpos / ARCH_PAGE_SIZE, page->is_dirty); + } + } + } + } + else + { + rt_kprintf(" pages >> empty\n"); + } + dfs_aspace_unlock(aspace); + } + return 0; +} + +static int dfs_pcache_dump(int argc, char **argv) +{ + int dump = 0; + rt_list_t *node; + struct dfs_aspace *aspace; + + if (argc == 2) + { + if (strcmp(argv[1], "--dump") == 0) + { + dump = 1; + } + else if (strcmp(argv[1], "--dirty") == 0) + { + dump = 2; + } + else + { + rt_kprintf("dfs page cache dump\n"); + rt_kprintf("usage: dfs_cache\n"); + rt_kprintf(" dfs_cache --dump\n"); + rt_kprintf(" dfs_cache --dirty\n"); + return 0; + } + } + + dfs_pcache_lock(); + + rt_kprintf("total pages count: %d / %d\n", rt_atomic_load(&(__pcache.pages_count)), RT_PAGECACHE_COUNT); + + rt_list_for_each(node, &__pcache.list_active) + { + if (node != &__pcache.list_inactive) + { + aspace = rt_list_entry(node, struct dfs_aspace, cache_node); + + if (aspace->mnt) + { + rt_kprintf("file: %s%s pages: %d\n", aspace->fullpath, aspace->pathname, aspace->pages_count); + } + else + { + rt_kprintf("unknown type, pages: %d\n", aspace->pages_count); + } + + if (dump > 0) + { + _dfs_aspace_dump(aspace, dump == 2 ? 1 : 0); + } + } + } + + dfs_pcache_unlock(); + + return 0; +} +MSH_CMD_EXPORT_ALIAS(dfs_pcache_dump, dfs_cache, dump dfs page cache); + +static int dfs_page_unmap(struct dfs_page *page) +{ + rt_list_t *next; + struct dfs_mmap *map; + + next = page->mmap_head.next; + + if (next != &page->mmap_head && page->fpos < page->aspace->vnode->size) + { + dfs_page_dirty(page); + } + + while (next != &page->mmap_head) + { + map = rt_list_entry(next, struct dfs_mmap, mmap_node); + next = next->next; + + if (map) + { + void *vaddr = dfs_aspace_vaddr(map->varea, page->fpos); + //rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, vaddr, ARCH_PAGE_SIZE); + rt_varea_unmap_page(map->varea, vaddr); + + rt_free(map); + } + } + + rt_list_init(&page->mmap_head); + + return 0; +} + +static struct dfs_page *dfs_page_create(void) +{ + struct dfs_page *page = RT_NULL; + + page = rt_calloc(1, sizeof(struct dfs_page)); + if (page) + { + page->page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE); + if (page->page) + { + //memset(page->page, 0x00, ARCH_PAGE_SIZE); + rt_list_init(&page->mmap_head); + rt_atomic_store(&(page->ref_count), 1); + } + else + { + LOG_E("page alloc failed!\n"); + rt_free(page); + page = RT_NULL; + } + } + + return page; +} + +static void dfs_page_ref(struct dfs_page *page) +{ + rt_atomic_add(&(page->ref_count), 1); +} + +static void dfs_page_release(struct dfs_page *page) +{ + struct dfs_aspace *aspace = page->aspace; + + dfs_aspace_lock(aspace); + + rt_atomic_sub(&(page->ref_count), 1); + + if (rt_atomic_load(&(page->ref_count)) == 0) + { + dfs_page_unmap(page); + + if (page->is_dirty == 1 && aspace->vnode) + { + if (aspace->vnode->size < page->fpos + page->size) + { + page->len = aspace->vnode->size - page->fpos; + } + else + { + page->len = page->size; + } + //rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, page->page, page->size); + aspace->ops->write(page); + page->is_dirty = 0; + } + RT_ASSERT(page->is_dirty == 0); + + rt_pages_free(page->page, 0); + page->page = RT_NULL; + rt_free(page); + } + + dfs_aspace_unlock(aspace); +} + +static int dfs_page_compare(off_t fpos, off_t value) +{ + return fpos / ARCH_PAGE_SIZE * ARCH_PAGE_SIZE - value; +} + +static int _dfs_page_insert(struct dfs_aspace *aspace, struct dfs_page *page) +{ + struct dfs_page *tmp; + struct util_avl_struct *current = NULL; + struct util_avl_struct **next = &(aspace->avl_root.root_node); + + /* Figure out where to put new node */ + while (*next) + { + current = *next; + tmp = rt_container_of(current, struct dfs_page, avl_node); + + if (page->fpos < tmp->fpos) + next = &(current->avl_left); + else if (page->fpos > tmp->fpos) + next = &(current->avl_right); + else + return -1; + } + + /* Add new node and rebalance tree. */ + util_avl_link(&page->avl_node, current, next); + util_avl_rebalance(current, &aspace->avl_root); + aspace->avl_page = page; + + return 0; +} + +static void _dfs_page_remove(struct dfs_aspace *aspace, struct dfs_page *page) +{ + if (aspace->avl_page && aspace->avl_page == page) + { + aspace->avl_page = 0; + } + + util_avl_remove(&page->avl_node, &aspace->avl_root); +} + +static int dfs_aspace_lock(struct dfs_aspace *aspace) +{ + rt_mutex_take(&aspace->lock, RT_WAITING_FOREVER); + return 0; +} + +static int dfs_aspace_unlock(struct dfs_aspace *aspace) +{ + rt_mutex_release(&aspace->lock); + return 0; +} + +static int dfs_page_insert(struct dfs_page *page) +{ + struct dfs_aspace *aspace = page->aspace; + + dfs_aspace_lock(aspace); + + rt_list_insert_before(&aspace->list_inactive, &page->space_node); + aspace->pages_count ++; + + RT_ASSERT(_dfs_page_insert(aspace, page) == 0); + + if (aspace->pages_count > RT_PAGECACHE_ASPACE_COUNT) + { + rt_list_t *next = aspace->list_active.next; + + if (next != &aspace->list_inactive) + { + struct dfs_page *tmp = rt_list_entry(next, struct dfs_page, space_node); + dfs_page_inactive(tmp); + } + } + + rt_atomic_add(&(__pcache.pages_count), 1); + + dfs_aspace_unlock(aspace); + + return 0; +} + +static int dfs_page_remove(struct dfs_page *page) +{ + int ret = -1; + struct dfs_aspace *aspace = page->aspace; + + dfs_aspace_lock(aspace); + + if (rt_atomic_load(&(page->ref_count)) == 1) + { + if (page->space_node.next != RT_NULL) + { + rt_list_remove(&page->space_node); + page->space_node.next = RT_NULL; + aspace->pages_count--; + _dfs_page_remove(aspace, page); + } + if (page->dirty_node.next != RT_NULL) + { + rt_list_remove(&page->dirty_node); + page->dirty_node.next = RT_NULL; + } + + rt_atomic_sub(&(__pcache.pages_count), 1); + + dfs_page_release(page); + ret = 0; + } + + dfs_aspace_unlock(aspace); + + return ret; +} + +static int dfs_page_active(struct dfs_page *page) +{ + struct dfs_aspace *aspace = page->aspace; + + dfs_aspace_lock(aspace); + if (page->space_node.next != RT_NULL) + { + rt_list_remove(&page->space_node); + rt_list_insert_before(&aspace->list_inactive, &page->space_node); + } + dfs_aspace_unlock(aspace); + + return 0; +} + +static int dfs_page_inactive(struct dfs_page *page) +{ + struct dfs_aspace *aspace = page->aspace; + + dfs_aspace_lock(aspace); + if (page->space_node.next != RT_NULL) + { + rt_list_remove(&page->space_node); + rt_list_insert_before(&aspace->list_active, &page->space_node); + } + dfs_aspace_unlock(aspace); + + return 0; +} + +static int dfs_page_dirty(struct dfs_page *page) +{ + struct dfs_aspace *aspace = page->aspace; + + dfs_aspace_lock(aspace); + + if (page->dirty_node.next == RT_NULL && page->space_node.next != RT_NULL) + { + rt_list_insert_before(&aspace->list_dirty, &page->dirty_node); + } + + page->is_dirty = 1; + page->tick_ms = rt_tick_get_millisecond(); + + if (rt_tick_get_millisecond() - __pcache.last_time_wb >= 1000) + { + dfs_pcache_mq_work(PCACHE_MQ_WB); + __pcache.last_time_wb = rt_tick_get_millisecond(); + } + + dfs_aspace_unlock(aspace); + + return 0; +} + +static struct dfs_page *dfs_page_search(struct dfs_aspace *aspace, off_t fpos) +{ + int cmp; + struct dfs_page *page; + struct util_avl_struct *avl_node; + + dfs_aspace_lock(aspace); + + if (aspace->avl_page && dfs_page_compare(fpos, aspace->avl_page->fpos) == 0) + { + page = aspace->avl_page; + dfs_page_active(page); + dfs_page_ref(page); + dfs_aspace_unlock(aspace); + return page; + } + + avl_node = aspace->avl_root.root_node; + while (avl_node) + { + page = rt_container_of(avl_node, struct dfs_page, avl_node); + cmp = dfs_page_compare(fpos, page->fpos); + + if (cmp < 0) + { + avl_node = avl_node->avl_left; + } + else if (cmp > 0) + { + avl_node = avl_node->avl_right; + } + else + { + aspace->avl_page = page; + dfs_page_active(page); + dfs_page_ref(page); + dfs_aspace_unlock(aspace); + return page; + } + } + + dfs_aspace_unlock(aspace); + + return RT_NULL; +} + +static struct dfs_page *dfs_aspace_load_page(struct dfs_file *file, off_t pos) +{ + struct dfs_page *page = RT_NULL; + + if (file && file->vnode && file->vnode->aspace) + { + struct dfs_vnode *vnode = file->vnode; + struct dfs_aspace *aspace = vnode->aspace; + + page = dfs_page_create(); + if (page) + { + page->aspace = aspace; + page->size = ARCH_PAGE_SIZE; + page->fpos = pos / ARCH_PAGE_SIZE * ARCH_PAGE_SIZE; + aspace->ops->read(file, page); + //rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, page->page, page->size); + page->ref_count ++; + + dfs_page_insert(page); + } + } + + return page; +} + +static struct dfs_page *dfs_page_lookup(struct dfs_file *file, off_t pos) +{ + struct dfs_page *page = RT_NULL; + struct dfs_aspace *aspace = file->vnode->aspace; + + dfs_aspace_lock(aspace); + page = dfs_page_search(aspace, pos); + if (!page) + { + int count = RT_PAGECACHE_PRELOAD; + struct dfs_page *tmp = RT_NULL; + off_t fpos = pos / ARCH_PAGE_SIZE * ARCH_PAGE_SIZE; + + do + { + page = dfs_aspace_load_page(file, fpos); + if (page) + { + if (tmp == RT_NULL) + { + tmp = page; + } + else + { + dfs_page_release(page); + } + } + else + { + break; + } + + fpos += ARCH_PAGE_SIZE; + page = dfs_page_search(aspace, fpos); + if (page) + { + dfs_page_release(page); + } + count --; + + } while (count && page == RT_NULL); + + page = tmp; + if (page) + { + dfs_aspace_unlock(aspace); + + if (rt_atomic_load(&(__pcache.pages_count)) >= RT_PAGECACHE_COUNT) + { + dfs_pcache_limit_check(); + } + else if (rt_atomic_load(&(__pcache.pages_count)) >= RT_PAGECACHE_COUNT * RT_PAGECACHE_GC_WORK_LEVEL / 100) + { + dfs_pcache_mq_work(PCACHE_MQ_GC); + } + + return page; + } + } + dfs_aspace_unlock(aspace); + + return page; +} + +int dfs_aspace_read(struct dfs_file *file, void *buf, size_t count, off_t *pos) +{ + int ret = -EINVAL; + + if (file && file->vnode && file->vnode->aspace) + { + struct dfs_vnode *vnode = file->vnode; + struct dfs_aspace *aspace = vnode->aspace; + + struct dfs_page *page; + char *ptr = (char *)buf; + + ret = 0; + + while (count) + { + page = dfs_page_lookup(file, *pos); + if (page) + { + off_t len; + + dfs_aspace_lock(aspace); + if (aspace->vnode->size < page->fpos + ARCH_PAGE_SIZE) + { + len = aspace->vnode->size - *pos; + } + else + { + len = page->fpos + ARCH_PAGE_SIZE - *pos; + } + + len = count > len ? len : count; + if (len) + { + //rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, page->page, page->size); + rt_memcpy(ptr, page->page + *pos - page->fpos, len); + ptr += len; + *pos += len; + count -= len; + ret += len; + } + else + { + dfs_page_release(page); + dfs_aspace_unlock(aspace); + break; + } + dfs_page_release(page); + dfs_aspace_unlock(aspace); + } + else + { + break; + } + } + } + + return ret; +} + +int dfs_aspace_write(struct dfs_file *file, const void *buf, size_t count, off_t *pos) +{ + int ret = -EINVAL; + + if (file && file->vnode && file->vnode->aspace) + { + struct dfs_vnode *vnode = file->vnode; + struct dfs_aspace *aspace = vnode->aspace; + + struct dfs_page *page; + char *ptr = (char *)buf; + + ret = 0; + + while (count) + { + page = dfs_page_lookup(file, *pos); + if (page) + { + off_t len; + + dfs_aspace_lock(aspace); + len = page->fpos + ARCH_PAGE_SIZE - *pos; + len = count > len ? len : count; + rt_memcpy(page->page + *pos - page->fpos, ptr, len); + //rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, page->page, page->size); + ptr += len; + *pos += len; + count -= len; + ret += len; + + if (*pos > aspace->vnode->size) + { + aspace->vnode->size = *pos; + } + + dfs_page_dirty(page); + dfs_page_release(page); + dfs_aspace_unlock(aspace); + } + else + { + break; + } + } + } + + return ret; +} + +int dfs_aspace_flush(struct dfs_aspace *aspace) +{ + if (aspace) + { + rt_list_t *next; + struct dfs_page *page; + + dfs_aspace_lock(aspace); + + if (aspace->pages_count > 0 && aspace->vnode) + { + rt_list_for_each(next, &aspace->list_dirty) + { + page = rt_list_entry(next, struct dfs_page, dirty_node); + if (page->is_dirty == 1 && aspace->vnode) + { + if (aspace->vnode->size < page->fpos + page->size) + { + page->len = aspace->vnode->size - page->fpos; + } + else + { + page->len = page->size; + } + //rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, page->page, page->size); + + aspace->ops->write(page); + + page->is_dirty = 0; + } + RT_ASSERT(page->is_dirty == 0); + } + } + + dfs_aspace_unlock(aspace); + } + return 0; +} + +int dfs_aspace_clean(struct dfs_aspace *aspace) +{ + if (aspace) + { + dfs_aspace_lock(aspace); + + if (aspace->pages_count > 0) + { + rt_list_t *next = aspace->list_active.next; + struct dfs_page *page; + + while (next && next != &aspace->list_active) + { + if (next == &aspace->list_inactive) + { + next = next->next; + continue; + } + page = rt_list_entry(next, struct dfs_page, space_node); + next = next->next; + dfs_page_remove(page); + } + } + + dfs_aspace_unlock(aspace); + } + + return 0; +} + +void *dfs_aspace_mmap(struct dfs_file *file, struct rt_varea *varea, void *vaddr) +{ + void *ret = RT_NULL; + struct dfs_page *page; + struct dfs_aspace *aspace = file->vnode->aspace; + + page = dfs_page_lookup(file, dfs_aspace_fpos(varea, vaddr)); + if (page) + { + struct dfs_mmap *map = (struct dfs_mmap *)rt_calloc(1, sizeof(struct dfs_mmap)); + if (map) + { + void *pg_paddr = rt_kmem_v2p(page->page); + int err = rt_varea_map_range(varea, vaddr, pg_paddr, page->size); + if (err == RT_EOK) + { + ret = page->page; + map->varea = varea; + dfs_aspace_lock(aspace); + rt_list_insert_after(&page->mmap_head, &map->mmap_node); + dfs_page_release(page); + dfs_aspace_unlock(aspace); + } + else + { + dfs_page_release(page); + rt_free(map); + } + } + else + { + dfs_page_release(page); + } + } + + return ret; +} + +int dfs_aspace_unmap(struct dfs_file *file, struct rt_varea *varea) +{ + struct dfs_vnode *vnode = file->vnode; + struct dfs_aspace *aspace = vnode->aspace; + + if (aspace) + { + rt_list_t *next; + struct dfs_page *page; + + dfs_aspace_lock(aspace); + if (aspace->pages_count > 0) + { + rt_list_for_each(next, &aspace->list_active) + { + if (next != &aspace->list_inactive) + { + page = rt_list_entry(next, struct dfs_page, space_node); + if (page) + { + rt_list_t *node, *tmp; + struct dfs_mmap *map; + + node = page->mmap_head.next; + + while (node != &page->mmap_head) + { + map = rt_list_entry(node, struct dfs_mmap, mmap_node); + tmp = node; + node = node->next; + + if (map && varea == map->varea) + { + if (varea->attr == MMU_MAP_U_RWCB && page->fpos < page->aspace->vnode->size) + { + dfs_page_dirty(page); + } + rt_list_remove(tmp); + rt_free(map); + break; + } + } + } + } + } + } + dfs_aspace_unlock(aspace); + } + + return 0; +} + +int dfs_aspace_page_unmap(struct dfs_file *file, struct rt_varea *varea, void *vaddr) +{ + struct dfs_page *page; + struct dfs_aspace *aspace = file->vnode->aspace; + + if (aspace) + { + dfs_aspace_lock(aspace); + + page = dfs_page_search(aspace, dfs_aspace_fpos(varea, vaddr)); + if (page) + { + rt_list_t *node, *tmp; + struct dfs_mmap *map; + + node = page->mmap_head.next; + + while (node != &page->mmap_head) + { + map = rt_list_entry(node, struct dfs_mmap, mmap_node); + tmp = node; + node = node->next; + + if (map && varea == map->varea) + { + if (varea->attr == MMU_MAP_U_RWCB) + { + dfs_page_dirty(page); + } + rt_list_remove(tmp); + rt_free(map); + break; + } + } + + dfs_page_release(page); + } + + dfs_aspace_unlock(aspace); + } + + return 0; +} + +int dfs_aspace_page_dirty(struct dfs_file *file, struct rt_varea *varea, void *vaddr) +{ + struct dfs_page *page; + struct dfs_aspace *aspace = file->vnode->aspace; + + if (aspace) + { + dfs_aspace_lock(aspace); + + page = dfs_page_search(aspace, dfs_aspace_fpos(varea, vaddr)); + if (page) + { + dfs_page_dirty(page); + dfs_page_release(page); + } + + dfs_aspace_unlock(aspace); + } + + return 0; +} + +off_t dfs_aspace_fpos(struct rt_varea *varea, void *vaddr) +{ + return (off_t)(intptr_t)vaddr - (off_t)(intptr_t)varea->start + varea->offset * ARCH_PAGE_SIZE; +} + +void *dfs_aspace_vaddr(struct rt_varea *varea, off_t fpos) +{ + return varea->start + fpos - varea->offset * ARCH_PAGE_SIZE; +} + +int dfs_aspace_mmap_read(struct dfs_file *file, struct rt_varea *varea, void *data) +{ + int ret = 0; + + if (file && varea) + { + struct rt_aspace_io_msg *msg = (struct rt_aspace_io_msg *)data; + if (msg) + { + off_t fpos = dfs_aspace_fpos(varea, msg->fault_vaddr); + return dfs_aspace_read(file, msg->buffer_vaddr, ARCH_PAGE_SIZE, &fpos); + } + } + + return ret; +} + +int dfs_aspace_mmap_write(struct dfs_file *file, struct rt_varea *varea, void *data) +{ + int ret = 0; + + if (file && varea) + { + struct rt_aspace_io_msg *msg = (struct rt_aspace_io_msg *)data; + if (msg) + { + off_t fpos = dfs_aspace_fpos(varea, msg->fault_vaddr); + return dfs_aspace_write(file, msg->buffer_vaddr, ARCH_PAGE_SIZE, &fpos); + } + } + + return ret; +} + +#endif diff --git a/components/dfs/dfs_v2/src/dfs_posix.c b/components/dfs/dfs_v2/src/dfs_posix.c index 3319efe2bf..77ffe20516 100644 --- a/components/dfs/dfs_v2/src/dfs_posix.c +++ b/components/dfs/dfs_v2/src/dfs_posix.c @@ -1216,53 +1216,46 @@ FINSH_FUNCTION_EXPORT_ALIAS(chdir, cd, change current working directory); */ int access(const char *path, int amode) { - int fd, ret = -1, flags = 0; - struct stat sb; + struct stat st; if (path == NULL) { - rt_set_errno(-EBADF); + rt_set_errno(-EINVAL); return -1; } - if (amode == F_OK) + if (stat(path, &st) < 0) { - if (stat(path, &sb) < 0) - return -1; /* already sets errno */ - else - return 0; - } - - /* ignore R_OK,W_OK,X_OK condition */ - if (dfs_file_isdir(path) == 0) - { - flags |= O_DIRECTORY; + rt_set_errno(-ENOENT); + return -1; } - if (amode & R_OK) + if (amode == F_OK) { - flags |= O_RDONLY; + return 0; } - if (amode & W_OK) + if ((amode & R_OK) && !(st.st_mode & S_IRUSR)) { - flags |= O_WRONLY; + rt_set_errno(-EACCES); + return -1; } - if (amode & X_OK) + if ((amode & W_OK) && !(st.st_mode & S_IWUSR)) { - flags |= O_EXEC; + rt_set_errno(-EACCES); + return -1; } - fd = open(path, flags, 0); - if (fd >= 0) + if ((amode & X_OK) && !(st.st_mode & S_IXUSR)) { - ret = 0; - close(fd); + rt_set_errno(-EACCES); + return -1; } - return ret; + return 0; } + /** * this function is a POSIX compliant version, which will set current * working directory. diff --git a/components/dfs/dfs_v2/src/dfs_seq_file.c b/components/dfs/dfs_v2/src/dfs_seq_file.c new file mode 100644 index 0000000000..58e80f8c4e --- /dev/null +++ b/components/dfs/dfs_v2/src/dfs_seq_file.c @@ -0,0 +1,494 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + */ + +#include <dfs_seq_file.h> +#include <dfs_dentry.h> + +#define DBG_TAG "DFS.seq" +#define DBG_LVL DBG_WARNING +#include <rtdbg.h> + +#ifndef PAGE_SIZE +#define PAGE_SIZE 4096 +#endif + +static void dfs_seq_overflow(struct dfs_seq_file *seq) +{ + seq->count = seq->size; +} + +static void *dfs_seq_alloc(unsigned long size) +{ + return rt_calloc(1, size); +} + +int dfs_seq_open(struct dfs_file *file, const struct dfs_seq_ops *ops) +{ + struct dfs_seq_file *seq; + + if (!ops) + { + LOG_E("dfs_seq_open: ops = null, pathname: %s\n", file->dentry->pathname); + return -EINVAL; + } + + if (file->data) + { + LOG_W("dfs_seq_open: file->data != null\n"); + } + + seq = rt_calloc(1, sizeof(struct dfs_seq_file)); + if (!seq) + return -ENOMEM; + + file->data = seq; + + rt_mutex_init(&seq->lock, "dfs_seq", RT_IPC_FLAG_PRIO); + + seq->ops = ops; + seq->file = file; + + return 0; +} + +static int dfs_seq_traverse(struct dfs_seq_file *seq, off_t offset) +{ + off_t pos = 0; + int error = 0; + void *p; + + seq->index = 0; + seq->count = seq->from = 0; + if (!offset) + return 0; + + if (!seq->buf) + { + seq->buf = dfs_seq_alloc(seq->size = PAGE_SIZE); + if (!seq->buf) + return -ENOMEM; + } + p = seq->ops->start(seq, &seq->index); + while (p) + { + error = seq->ops->show(seq, p); + if (error < 0) + break; + if (error) + { + error = 0; + seq->count = 0; + } + if (dfs_seq_is_full(seq)) + goto Eoverflow; + + p = seq->ops->next(seq, p, &seq->index); + if (pos + seq->count > offset) + { + seq->from = offset - pos; + seq->count -= seq->from; + break; + } + pos += seq->count; + seq->count = 0; + if (pos == offset) + break; + } + seq->ops->stop(seq, p); + return error; + +Eoverflow: + seq->ops->stop(seq, p); + rt_free(seq->buf); + seq->count = 0; + seq->buf = dfs_seq_alloc(seq->size <<= 1); + return !seq->buf ? -ENOMEM : -EAGAIN; +} + +ssize_t dfs_seq_read(struct dfs_file *file, void *buf, size_t size, off_t *pos) +{ + struct dfs_seq_file *seq = file->data; + size_t copied = 0; + size_t n; + void *p; + int err = 0; + + if (!size) + return 0; + + rt_mutex_take(&seq->lock, RT_WAITING_FOREVER); + + /* + * if request is to read from zero offset, reset iterator to first + * record as it might have been already advanced by previous requests + */ + if (*pos == 0) + { + seq->index = 0; + seq->count = 0; + } + + /* Don't assume ki_pos is where we left it */ + if (*pos != seq->read_pos) + { + while ((err = dfs_seq_traverse(seq, *pos)) == -EAGAIN) + ; + if (err) + { + /* With prejudice... */ + seq->read_pos = 0; + seq->index = 0; + seq->count = 0; + goto Done; + } + else + { + seq->read_pos = *pos; + } + } + + /* grab buffer if we didn't have one */ + if (!seq->buf) + { + seq->buf = dfs_seq_alloc(seq->size = PAGE_SIZE); + if (!seq->buf) + goto Enomem; + } + // something left in the buffer - copy it out first + if (seq->count) + { + n = seq->count > size ? size : seq->count; + rt_memcpy((char *)buf + copied, seq->buf + seq->from, n); + size -= n; + seq->count -= n; + seq->from += n; + copied += n; + if (seq->count) // hadn't managed to copy everything + goto Done; + } + // get a non-empty record in the buffer + seq->from = 0; + p = seq->ops->start(seq, &seq->index); + while (p) + { + err = seq->ops->show(seq, p); + if (err < 0) // hard error + break; + if (err) // ->show() says "skip it" + seq->count = 0; + if (!seq->count) + { // empty record + p = seq->ops->next(seq, p, &seq->index); + continue; + } + if (!dfs_seq_is_full(seq)) // got it + goto Fill; + // need a bigger buffer + seq->ops->stop(seq, p); + rt_free(seq->buf); + seq->count = 0; + seq->buf = dfs_seq_alloc(seq->size <<= 1); + if (!seq->buf) + goto Enomem; + p = seq->ops->start(seq, &seq->index); + } + // EOF or an error + seq->ops->stop(seq, p); + seq->count = 0; + goto Done; +Fill: + // one non-empty record is in the buffer; if they want more, + // try to fit more in, but in any case we need to advance + // the iterator once for every record shown. + while (1) + { + size_t offs = seq->count; + off_t pos = seq->index; + + p = seq->ops->next(seq, p, &seq->index); + if (pos == seq->index) + { + LOG_W(".next function %p did not update position index\n", seq->ops->next); + seq->index++; + } + if (!p) // no next record for us + break; + if (seq->count >= size) + break; + err = seq->ops->show(seq, p); + if (err > 0) + { // ->show() says "skip it" + seq->count = offs; + } + else if (err || dfs_seq_is_full(seq)) + { + seq->count = offs; + break; + } + } + seq->ops->stop(seq, p); + n = seq->count > size ? size : seq->count; + rt_memcpy((char *)buf + copied, seq->buf, n); + size -= n; + copied += n; + seq->count -= n; + seq->from = n; +Done: + if (!copied) + { + copied = seq->count ? -EFAULT : err; + } + else + { + *pos += copied; + seq->read_pos += copied; + } + rt_mutex_release(&seq->lock); + return copied; +Enomem: + err = -ENOMEM; + goto Done; +} + +ssize_t dfs_seq_lseek(struct dfs_file *file, off_t offset, int whence) +{ + struct dfs_seq_file *seq = file->data; + off_t retval = -EINVAL; + + rt_mutex_take(&seq->lock, RT_WAITING_FOREVER); + + switch (whence) + { + case SEEK_CUR: + offset += file->fpos; + case SEEK_SET: + if (offset < 0) + break; + retval = offset; + if (offset != seq->read_pos) + { + while ((retval = dfs_seq_traverse(seq, offset)) == -EAGAIN); + if (retval) + { + /* with extreme prejudice... */ + retval = 0; + seq->read_pos = 0; + seq->index = 0; + seq->count = 0; + } + else + { + seq->read_pos = offset; + retval = offset; + } + } + } + + rt_mutex_release(&seq->lock); + + return retval; +} + +int dfs_seq_release(struct dfs_file *file) +{ + struct dfs_seq_file *seq = file->data; + + if (seq) + { + rt_mutex_detach(&seq->lock); + if (seq->buf) + { + rt_free(seq->buf); + } + rt_free(seq); + } + + return 0; +} + +void dfs_seq_vprintf(struct dfs_seq_file *seq, const char *f, va_list args) +{ + int len; + + if (seq->count < seq->size) + { + len = vsnprintf(seq->buf + seq->count, seq->size - seq->count, f, args); + if (seq->count + len < seq->size) + { + seq->count += len; + return; + } + } + dfs_seq_overflow(seq); +} + +void dfs_seq_printf(struct dfs_seq_file *seq, const char *f, ...) +{ + va_list args; + + va_start(args, f); + dfs_seq_vprintf(seq, f, args); + va_end(args); +} + +/** + * write char to buffer + */ +void dfs_seq_putc(struct dfs_seq_file *seq, char c) +{ + if (seq->count < seq->size) + { + seq->buf[seq->count++] = c; + } +} + +/** + * write string to buffer + */ +void dfs_seq_puts(struct dfs_seq_file *seq, const char *s) +{ + int len = strlen(s); + + if (seq->count + len >= seq->size) + { + dfs_seq_overflow(seq); + return; + } + rt_memcpy(seq->buf + seq->count, s, len); + seq->count += len; +} + +/** + * write arbitrary data to buffer + */ +int dfs_seq_write(struct dfs_seq_file *seq, const void *data, size_t len) +{ + if (seq->count + len < seq->size) + { + rt_memcpy(seq->buf + seq->count, data, len); + seq->count += len; + return 0; + } + dfs_seq_overflow(seq); + return -1; +} + +/** + * write padding spaces to buffer + */ +void dfs_seq_pad(struct dfs_seq_file *seq, char c) +{ + int size = seq->pad_until - seq->count; + + if (size > 0) + { + if (size + seq->count > seq->size) + { + dfs_seq_overflow(seq); + return; + } + rt_memset(seq->buf + seq->count, ' ', size); + seq->count += size; + } + + if (c) + { + dfs_seq_putc(seq, c); + } +} + +#if 1 +/* test demo */ + +static char *txt[4] = { + "text1", + "text2", + "text3", + "text4", +}; + +static void *seq_test_start(struct dfs_seq_file *seq, off_t *index) +{ + off_t i = *index; // seq->index + + if (i >= 0 && i < 4) + { + return txt[i]; + } + + return RT_NULL; +} + +static void seq_test_stop(struct dfs_seq_file *seq, void *data) +{ +} + +static void *seq_test_next(struct dfs_seq_file *seq, void *data, off_t *index) +{ + off_t i = *index + 1; // seq->index + + *index = i; + + if (i >= 0 && i < 4) + { + return txt[i]; + } + + return RT_NULL; +} + +static int seq_test_show(struct dfs_seq_file *seq, void *data) +{ + const char *text = (const char *)data; + + dfs_seq_setwidth(seq, 20); + dfs_seq_puts(seq, "puts "); + dfs_seq_putc(seq, 'c'); + dfs_seq_write(seq, " write", 6); + dfs_seq_printf(seq, " %s", text); + dfs_seq_pad(seq, 0); + + return 0; +} + +static const struct dfs_seq_ops _test_ops = { + .start = seq_test_start, + .stop = seq_test_stop, + .next = seq_test_next, + .show = seq_test_show, +}; + +static int dfs_seq_test(int argc, char **argv) +{ + struct dfs_file file = {0}; + int ret = dfs_seq_open(&file, &_test_ops); + + if (ret == 0) + { + char buf[256] = {0}; + off_t pos = (argc > 1) ? atoi(argv[1]) : 0; + ssize_t len = (argc > 2) ? atoi(argv[2]) : 255; + + if (len > 255) + { + len = 255; + rt_kprintf("buf len is %d, max read is 255\n", 256, len); + } + + len = dfs_seq_read(&file, buf, len, &pos); + buf[len] = '\0'; + + rt_kprintf("show: \"%s\" len: %d\n", buf, len); + dfs_seq_release(&file); + } + + return 0; +} +MSH_CMD_EXPORT_ALIAS(dfs_seq_test, seq_test, seq_test[pos][read_len]); +#endif diff --git a/components/dfs/dfs_v2/src/dfs_vnode.c b/components/dfs/dfs_v2/src/dfs_vnode.c index d3af9b3970..21778e683c 100644 --- a/components/dfs/dfs_v2/src/dfs_vnode.c +++ b/components/dfs/dfs_v2/src/dfs_vnode.c @@ -10,6 +10,9 @@ #include <dfs_file.h> #include <dfs_mnt.h> +#ifdef RT_USING_PAGECACHE +#include "dfs_pcache.h" +#endif #define DBG_TAG "DFS.vnode" #define DBG_LVL DBG_WARNING @@ -58,7 +61,12 @@ int dfs_vnode_destroy(struct dfs_vnode* vnode) if (rt_atomic_load(&(vnode->ref_count)) == 1) { LOG_I("free a vnode: %p", vnode); - +#ifdef RT_USING_PAGECACHE + if (vnode->aspace) + { + dfs_aspace_destroy(vnode->aspace); + } +#endif if (vnode->mnt) { DLOG(msg, "vnode", vnode->mnt->fs_ops->name, DLOG_MSG, "fs_ops->free_vnode"); @@ -106,7 +114,12 @@ void dfs_vnode_unref(struct dfs_vnode *vnode) { rt_atomic_sub(&(vnode->ref_count), 1); DLOG(note, "vnode", "vnode ref_count=%d", rt_atomic_load(&(vnode->ref_count))); - +#ifdef RT_USING_PAGECACHE + if (vnode->aspace) + { + dfs_aspace_destroy(vnode->aspace); + } +#endif if (rt_atomic_load(&(vnode->ref_count)) == 0) { LOG_I("free a vnode: %p", vnode); diff --git a/components/drivers/tty/n_tty.c b/components/drivers/tty/n_tty.c index 1191f57696..067d587746 100644 --- a/components/drivers/tty/n_tty.c +++ b/components/drivers/tty/n_tty.c @@ -2008,12 +2008,22 @@ static int job_control(struct tty_struct *tty) return __tty_check_change(tty, SIGTTIN); } +static struct rt_wqueue *_wait_queue_current_get(struct tty_struct *tty) +{ + struct rt_lwp *lwp; + + lwp = lwp_self(); + if (!lwp || !lwp->tty) + lwp = RT_NULL; + + return wait_queue_get(lwp, tty); +} + static int n_tty_read(struct dfs_file *fd, void *buf, size_t count) { int level = 0; char *b = (char *)buf; struct tty_struct *tty = RT_NULL; - struct rt_lwp *lwp = RT_NULL; struct rt_wqueue *wq = RT_NULL; int wait_ret = 0; int retval = 0; @@ -2031,8 +2041,7 @@ static int n_tty_read(struct dfs_file *fd, void *buf, size_t count) struct n_tty_data *ldata = tty->disc_data; - lwp = (struct rt_lwp *)(rt_thread_self()->lwp); - wq = wait_queue_get(lwp, tty); + wq = _wait_queue_current_get(tty); while(count) { @@ -2171,15 +2180,13 @@ static int n_tty_poll(struct dfs_file *fd, struct rt_pollreq *req) int mask = POLLOUT; struct tty_struct *tty = RT_NULL; struct rt_wqueue *wq = RT_NULL; - struct rt_lwp *lwp = RT_NULL; tty = (struct tty_struct *)fd->vnode->data; RT_ASSERT(tty != RT_NULL); RT_ASSERT(tty->init_flag == TTY_INIT_FLAG_INITED); - lwp = (struct rt_lwp *)(rt_thread_self()->lwp); - wq = wait_queue_get(lwp, tty); + wq = _wait_queue_current_get(tty); rt_poll_add(wq, req); level = rt_hw_interrupt_disable(); diff --git a/components/drivers/tty/tty.c b/components/drivers/tty/tty.c index 30ab00d890..6e05f95033 100644 --- a/components/drivers/tty/tty.c +++ b/components/drivers/tty/tty.c @@ -15,6 +15,7 @@ #include <rtthread.h> #include <tty.h> #include <tty_ldisc.h> +#include <shell.h> #if defined(RT_USING_POSIX_DEVIO) #include <termios.h> @@ -272,9 +273,97 @@ static int tiocsctty(struct tty_struct *tty, int arg) return 0; } +static int tiocswinsz(struct tty_struct *tty, struct winsize *p_winsize) +{ + rt_kprintf("\x1b[8;%d;%dt", p_winsize->ws_col, p_winsize->ws_row); + return 0; +} + +static int tiocgwinsz(struct tty_struct *tty, struct winsize *p_winsize) +{ + if(rt_thread_self() != rt_thread_find(FINSH_THREAD_NAME)) + { + /* only can be used in tshell thread; otherwise, return default size */ + p_winsize->ws_col = 80; + p_winsize->ws_row = 24; + } + else + { + #define _TIO_BUFLEN 20 + char _tio_buf[_TIO_BUFLEN]; + unsigned char cnt1, cnt2, cnt3, i; + char row_s[4], col_s[4]; + char *p; + + rt_memset(_tio_buf, 0, _TIO_BUFLEN); + + /* send the command to terminal for getting the window size of the terminal */ + rt_kprintf("\033[18t"); + + /* waiting for the response from the terminal */ + i = 0; + while(i < _TIO_BUFLEN) + { + _tio_buf[i] = finsh_getchar(); + if(_tio_buf[i] != 't') + { + i ++; + } + else + { + break; + } + } + if(i == _TIO_BUFLEN) + { + /* buffer overloaded, and return default size */ + p_winsize->ws_col = 80; + p_winsize->ws_row = 24; + return 0; + } + + /* interpreting data eg: "\033[8;1;15t" which means row is 1 and col is 15 (unit: size of ONE character) */ + rt_memset(row_s,0,4); + rt_memset(col_s,0,4); + cnt1 = 0; + while(cnt1 < _TIO_BUFLEN && _tio_buf[cnt1] != ';') + { + cnt1++; + } + cnt2 = ++cnt1; + while(cnt2 < _TIO_BUFLEN && _tio_buf[cnt2] != ';') + { + cnt2++; + } + p = row_s; + while(cnt1 < cnt2) + { + *p++ = _tio_buf[cnt1++]; + } + p = col_s; + cnt2++; + cnt3 = rt_strlen(_tio_buf) - 1; + while(cnt2 < cnt3) + { + *p++ = _tio_buf[cnt2++]; + } + + /* load the window size date */ + p_winsize->ws_col = atoi(col_s); + p_winsize->ws_row = atoi(row_s); + #undef _TIO_BUFLEN + } + + p_winsize->ws_xpixel = 0;/* unused */ + p_winsize->ws_ypixel = 0;/* unused */ + + return 0; +} + static int tty_ioctl(struct dfs_file *fd, int cmd, void *args) { int ret = 0; + void *p = (void *)args; struct tty_struct *tty = RT_NULL; struct tty_struct *real_tty = RT_NULL; struct tty_ldisc *ld = RT_NULL; @@ -295,6 +384,10 @@ static int tty_ioctl(struct dfs_file *fd, int cmd, void *args) { case TIOCSCTTY: return tiocsctty(real_tty, 1); + case TIOCGWINSZ: + return tiocgwinsz(real_tty, p); + case TIOCSWINSZ: + return tiocswinsz(real_tty, p); } ld = tty->ldisc; diff --git a/components/libc/posix/io/eventfd/eventfd.c b/components/libc/posix/io/eventfd/eventfd.c index a2f0975128..ec3a88ba47 100644 --- a/components/libc/posix/io/eventfd/eventfd.c +++ b/components/libc/posix/io/eventfd/eventfd.c @@ -14,6 +14,7 @@ #include <stdint.h> #include <unistd.h> #include <dfs_file.h> +#include <dfs.h> #include "poll.h" #include "eventfd.h" diff --git a/components/lwp/Kconfig b/components/lwp/Kconfig index a63c85beef..18505932db 100644 --- a/components/lwp/Kconfig +++ b/components/lwp/Kconfig @@ -68,4 +68,20 @@ if RT_USING_LWP bool "The unix98 PTY debug output" default n endif + + menuconfig RT_USING_LDSO + bool "LDSO: dynamic load shared objects" + default n + + if RT_USING_LDSO + config ELF_DEBUG_ENABLE + bool "Enable ldso debug" + default n + + config ELF_LOAD_RANDOMIZE + bool "Enable random load address" + default n + endif + endif + diff --git a/components/lwp/arch/aarch64/cortex-a/lwp_arch.c b/components/lwp/arch/aarch64/cortex-a/lwp_arch.c index fbb50bfdca..e46b8ac59a 100644 --- a/components/lwp/arch/aarch64/cortex-a/lwp_arch.c +++ b/components/lwp/arch/aarch64/cortex-a/lwp_arch.c @@ -31,22 +31,20 @@ int arch_user_space_init(struct rt_lwp *lwp) { size_t *mmu_table; - mmu_table = (size_t *)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE); - if (!mmu_table) + mmu_table = rt_hw_mmu_pgtbl_create(); + if (mmu_table) { - return -RT_ENOMEM; + lwp->end_heap = USER_HEAP_VADDR; + lwp->aspace = rt_aspace_create( + (void *)USER_VADDR_START, USER_VADDR_TOP - USER_VADDR_START, mmu_table); + if (!lwp->aspace) + { + return -RT_ERROR; + } } - - lwp->end_heap = USER_HEAP_VADDR; - - memset(mmu_table, 0, ARCH_PAGE_SIZE); - rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_table, ARCH_PAGE_SIZE); - - lwp->aspace = rt_aspace_create( - (void *)USER_VADDR_START, USER_VADDR_TOP - USER_VADDR_START, mmu_table); - if (!lwp->aspace) + else { - return -RT_ERROR; + return -RT_ENOMEM; } return 0; diff --git a/components/lwp/arch/aarch64/cortex-a/lwp_arch.h b/components/lwp/arch/aarch64/cortex-a/lwp_arch.h index f9faebfc91..c7d9c8d74e 100644 --- a/components/lwp/arch/aarch64/cortex-a/lwp_arch.h +++ b/components/lwp/arch/aarch64/cortex-a/lwp_arch.h @@ -17,10 +17,11 @@ #ifdef ARCH_MM_MMU #define USER_VADDR_TOP 0x0001000000000000UL -#define USER_HEAP_VEND 0x0000ffffB0000000UL -#define USER_HEAP_VADDR 0x0000ffff80000000UL +#define USER_HEAP_VADDR (0x0000ffff40000000UL) +#define USER_HEAP_VEND USER_STACK_VSTART #define USER_STACK_VSTART 0x0000ffff70000000UL -#define USER_STACK_VEND USER_HEAP_VADDR +#define USER_STACK_VEND (USER_STACK_VSTART + 0x10000000) +#define USER_ARG_VADDR USER_STACK_VEND #define LDSO_LOAD_VADDR 0x60000000UL #define USER_VADDR_START 0x00200000UL #define USER_LOAD_VADDR USER_VADDR_START diff --git a/components/lwp/arch/aarch64/cortex-a/lwp_gcc.S b/components/lwp/arch/aarch64/cortex-a/lwp_gcc.S index 541c360a55..1806dcf097 100644 --- a/components/lwp/arch/aarch64/cortex-a/lwp_gcc.S +++ b/components/lwp/arch/aarch64/cortex-a/lwp_gcc.S @@ -50,10 +50,12 @@ arch_start_umode: mov sp, x3 mov x4, #(SPSR_Mode(0) | SPSR_A64) - mov x3, x2 ;/* user stack top */ msr daifset, #3 dsb sy mrs x30, sp_el0 + /* user stack top */ + msr sp_el0, x2 + mov x3, x2 msr spsr_el1, x4 msr elr_el1, x1 eret diff --git a/components/lwp/arch/arm/cortex-a/lwp_arch.c b/components/lwp/arch/arm/cortex-a/lwp_arch.c index d1fcf63d93..0d8f965429 100644 --- a/components/lwp/arch/arm/cortex-a/lwp_arch.c +++ b/components/lwp/arch/arm/cortex-a/lwp_arch.c @@ -23,13 +23,11 @@ #include <lwp_arch.h> #include <lwp_user_mm.h> -#define KPTE_START (KERNEL_VADDR_START >> ARCH_SECTION_SHIFT) - int arch_user_space_init(struct rt_lwp *lwp) { size_t *mmu_table; - mmu_table = (size_t *)rt_pages_alloc(2); + mmu_table = rt_hw_mmu_pgtbl_create(); if (!mmu_table) { return -RT_ENOMEM; @@ -37,9 +35,6 @@ int arch_user_space_init(struct rt_lwp *lwp) lwp->end_heap = USER_HEAP_VADDR; - rt_memcpy(mmu_table + KPTE_START, (size_t *)rt_kernel_space.page_table + KPTE_START, ARCH_PAGE_SIZE); - rt_memset(mmu_table, 0, 3 * ARCH_PAGE_SIZE); - rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_table, 4 * ARCH_PAGE_SIZE); lwp->aspace = rt_aspace_create((void *)USER_VADDR_START, USER_VADDR_TOP - USER_VADDR_START, mmu_table); if (!lwp->aspace) @@ -87,7 +82,7 @@ void arch_user_space_free(struct rt_lwp *lwp) rt_aspace_delete(lwp->aspace); /* must be freed after aspace delete, pgtbl is required for unmap */ - rt_pages_free(pgtbl, 2); + rt_hw_mmu_pgtbl_delete(pgtbl); lwp->aspace = RT_NULL; } else diff --git a/components/lwp/arch/arm/cortex-a/lwp_arch.h b/components/lwp/arch/arm/cortex-a/lwp_arch.h index def88bf3b2..7262f446c8 100644 --- a/components/lwp/arch/arm/cortex-a/lwp_arch.h +++ b/components/lwp/arch/arm/cortex-a/lwp_arch.h @@ -21,7 +21,7 @@ #define USER_STACK_VSTART 0x70000000UL #define USER_STACK_VEND USER_HEAP_VADDR #define LDSO_LOAD_VADDR 0x60000000UL -#define USER_VADDR_START 0x00100000UL +#define USER_VADDR_START 0x00010000UL #define USER_LOAD_VADDR USER_VADDR_START #ifdef __cplusplus diff --git a/components/lwp/arch/arm/cortex-a/lwp_gcc.S b/components/lwp/arch/arm/cortex-a/lwp_gcc.S index 7e27592c6d..65da7c5313 100644 --- a/components/lwp/arch/arm/cortex-a/lwp_gcc.S +++ b/components/lwp/arch/arm/cortex-a/lwp_gcc.S @@ -43,7 +43,12 @@ arch_start_umode: msr spsr, r9 mov sp, r3 - mov r3, r2 ;/* user stack top */ + /* set user stack top */ + cps #Mode_SYS + mov sp, r2 + cps #Mode_SVC + mov r3, r2 + /* set data address. */ movs pc, r1 diff --git a/components/lwp/arch/risc-v/rv64/lwp_arch.c b/components/lwp/arch/risc-v/rv64/lwp_arch.c index 9a88709668..387a54f073 100644 --- a/components/lwp/arch/risc-v/rv64/lwp_arch.c +++ b/components/lwp/arch/risc-v/rv64/lwp_arch.c @@ -93,17 +93,13 @@ int arch_user_space_init(struct rt_lwp *lwp) { rt_ubase_t *mmu_table; - mmu_table = (rt_ubase_t *)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE); + mmu_table = rt_hw_mmu_pgtbl_create(); if (!mmu_table) { return -RT_ENOMEM; } lwp->end_heap = USER_HEAP_VADDR; - - rt_memcpy(mmu_table, rt_kernel_space.page_table, ARCH_PAGE_SIZE); - rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_table, ARCH_PAGE_SIZE); - lwp->aspace = rt_aspace_create( (void *)USER_VADDR_START, USER_VADDR_TOP - USER_VADDR_START, mmu_table); if (!lwp->aspace) @@ -129,7 +125,7 @@ void arch_user_space_free(struct rt_lwp *lwp) rt_aspace_delete(lwp->aspace); /* must be freed after aspace delete, pgtbl is required for unmap */ - rt_pages_free(pgtbl, 0); + rt_hw_mmu_pgtbl_delete(pgtbl); lwp->aspace = RT_NULL; } else diff --git a/components/lwp/lwp.c b/components/lwp/lwp.c index 9e67c2123e..73847fd856 100644 --- a/components/lwp/lwp.c +++ b/components/lwp/lwp.c @@ -153,6 +153,9 @@ struct process_aux *lwp_argscopy(struct rt_lwp *lwp, int argc, char **argv, char int len; size_t *args_k; struct process_aux *aux; + size_t prot = PROT_READ | PROT_WRITE; + size_t flags = MAP_FIXED | MAP_PRIVATE; + size_t zero = 0; for (i = 0; i < argc; i++) { @@ -179,9 +182,8 @@ struct process_aux *lwp_argscopy(struct rt_lwp *lwp, int argc, char **argv, char return RT_NULL; } - /* args = (int *)lwp_map_user(lwp, 0, size); */ - args = (int *)lwp_map_user(lwp, (void *)(USER_VADDR_TOP - ARCH_PAGE_SIZE), size, 0); - if (args == RT_NULL) + args = lwp_mmap2(lwp, (void *)(USER_STACK_VEND), size, prot, flags, -1, 0); + if (args == RT_NULL || lwp_data_put(lwp, args, &zero, sizeof(zero)) != sizeof(zero)) { return RT_NULL; } @@ -1417,3 +1419,37 @@ void lwp_uthread_ctx_restore(void) thread = rt_thread_self(); thread->user_ctx.ctx = RT_NULL; } + +void rt_update_process_times(void) +{ + struct rt_thread *thread; +#ifdef RT_USING_SMP + struct rt_cpu* pcpu; + + pcpu = rt_cpu_self(); +#endif + + thread = rt_thread_self(); + + if (!IS_USER_MODE(thread)) + { + thread->user_time += 1; +#ifdef RT_USING_SMP + pcpu->cpu_stat.user += 1; +#endif + } + else + { + thread->system_time += 1; +#ifdef RT_USING_SMP + if (thread == pcpu->idle_thread) + { + pcpu->cpu_stat.idle += 1; + } + else + { + pcpu->cpu_stat.system += 1; + } +#endif + } +} diff --git a/components/lwp/lwp.h b/components/lwp/lwp.h index 6b137cff8e..2fcd880bc4 100644 --- a/components/lwp/lwp.h +++ b/components/lwp/lwp.h @@ -240,7 +240,9 @@ struct __pthread { } #endif -#define AUX_ARRAY_ITEMS_NR 6 +#ifndef AUX_ARRAY_ITEMS_NR +#define AUX_ARRAY_ITEMS_NR 32 +#endif /* aux key */ #define AT_NULL 0 diff --git a/components/lwp/lwp_elf.c b/components/lwp/lwp_elf.c new file mode 100644 index 0000000000..0610769b26 --- /dev/null +++ b/components/lwp/lwp_elf.c @@ -0,0 +1,811 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-08-23 zhangsz first version + */ + +#include <rtthread.h> + +#ifdef RT_USING_LDSO + +#include <dfs_file.h> +#include <unistd.h> +#include <stdio.h> +#include <fcntl.h> +#include <lwp_elf.h> + +#include "lwp.h" +#include "lwp_arch.h" + +#ifdef ARCH_MM_MMU +#include <lwp_user_mm.h> +#endif + +#define DBG_TAG "load.elf" +#ifdef ELF_DEBUG_ENABLE +#define DBG_LVL DBG_LOG +#else +#define DBG_LVL DBG_INFO +#endif +#include <rtdbg.h> + +#define ELF_INVALID_FD -1 +#define ELF_PHDR_NUM_MAX 128 +#define FILE_LENGTH_MAX 0xC0000000 +#define MEM_SIZE_MAX 0xC0000000 +#define ELF_PATH_MAX 256 +#define FLF_PATH_MIN 1 + +#define ELF_PAGESTART(_v) ((_v) & ~(rt_ubase_t)(ARCH_PAGE_SIZE - 1)) +#define ELF_PAGEOFFSET(_v) ((_v) & (ARCH_PAGE_SIZE - 1)) +#define ELF_PAGEALIGN(_v) (((_v) + ARCH_PAGE_SIZE - 1) & ~(ARCH_PAGE_SIZE - 1)) + +#define ELF_EXEC_LOAD_ADDR USER_VADDR_START +#define ELF_INTERP_LOAD_ADDR LDSO_LOAD_VADDR + +#define ELF_AUX_ENT(aux, id, val) \ + do \ + { \ + *aux++ = id; \ + *aux++ = val; \ + } while (0) + +typedef struct +{ + int fd; + char *filename; + rt_size_t file_len; + Elf_Ehdr ehdr; + Elf_Phdr *phdr; + rt_ubase_t map_size; +} elf_info_t; + +typedef struct +{ + struct rt_lwp *lwp; + struct process_aux *aux; + elf_info_t exec_info; + elf_info_t interp_info; + rt_ubase_t load_addr; + rt_ubase_t e_entry; + rt_ubase_t interp_base; +} elf_load_info_t; + +static void elf_user_dump(struct rt_lwp *lwp, void *va, size_t len) +{ +#ifdef ELF_DEBUG_DUMP + uint8_t *k_va; + int ret; + + if (len < 16) + len = 16; + rt_kprintf("\r\n"); + rt_kprintf("%s : user va : %p, len : 0x%x(%d)\n", __func__, va, len, len); + k_va = rt_malloc(len); + if (k_va == RT_NULL) + { + rt_kprintf("%s : malloc failed\n", __func__); + return; + } + rt_memset(k_va, 0, len); + + ret = lwp_data_get(lwp, k_va, va, len); + if (ret != len) + { + rt_kprintf("%s : lwp_get_from_user failed, ret = %d\n", __func__, ret); + return; + } + + rt_kprintf("%s : k_va : %p\n", __func__, k_va); + + for (size_t i = 0; i < len; i += 16) + { + rt_kprintf(" %02x %02x %02x %02x %02x %02x %02x %02x ", k_va[i], k_va[i+1], k_va[i+2], k_va[i+3], + k_va[i+4], k_va[i+5], k_va[i+6], k_va[i+7]); + rt_kprintf(" %02x %02x %02x %02x %02x %02x %02x %02x \n", k_va[i+8], k_va[i+9], k_va[i+10], k_va[i+11], + k_va[i+12], k_va[i+13], k_va[i+14], k_va[i+15]); + } + rt_kprintf("\r\n"); + rt_free(k_va); +#endif +} + +rt_ubase_t elf_random_offset(void) +{ +#ifdef ELF_LOAD_RANDOMIZE + return (rt_tick_get() % 65535) * ARCH_PAGE_SIZE; +#else + return ELF_PAGEALIGN(0); +#endif +} + +static void *file_mmap(struct rt_lwp *lwp, int fd, rt_ubase_t load_addr, + rt_ubase_t map_size, size_t prot, size_t flags, rt_ubase_t offset) +{ + uint8_t *map_va; + + map_va = (uint8_t *)lwp_mmap2(lwp, (void *)load_addr, map_size, prot, flags, fd, offset >> ARCH_PAGE_SHIFT); + if (!map_va || (map_va != (uint8_t *)load_addr)) + { + LOG_E("%s : lwp map user failed!", __func__); + return RT_NULL; + } + LOG_D(" %s : map va = %p load_addr : %p size : 0x%x", __func__, map_va, load_addr, map_size); + + return map_va; +} + +static int elf_file_open(const char *filename) +{ + int fd = -1; + + fd = open(filename, O_BINARY | O_RDONLY, 0); + if (fd < 0) + { + LOG_E("%s : elf file [%s] open failed!", __func__, filename); + } + + return fd; +} + +static int elf_file_close(int fd) +{ + return close(fd); +} + +static int elf_file_length(char *filename, rt_size_t *file_len) +{ + int ret; + struct stat s = { 0 }; + + ret = stat(filename, &s); + if (ret != 0) + { + LOG_E("%s : error", __func__); + return -RT_ERROR; + } + *file_len = (rt_size_t)s.st_size; + + return RT_EOK; +} + +static int elf_file_read(rt_int32_t fd, rt_uint8_t *buffer, size_t size, off_t offset) +{ + ssize_t read_len; + off_t pos; + + if (size > 0) + { + pos = lseek(fd, offset, SEEK_SET); + if (pos != offset) + { + LOG_E("%s : seek file offset: 0x%x failed", __func__, offset); + return -RT_ERROR; + } + + read_len = read(fd, buffer, size); + if (read_len != size) + { + LOG_E("%s : read from offset: 0x%x error", __func__, offset); + return -RT_ERROR; + } + } + + return RT_EOK; +} + +static rt_int32_t elf_check_ehdr(const Elf_Ehdr *ehdr, rt_uint32_t file_len) +{ + if (memcmp(ehdr->e_ident, ELFMAG, SELFMAG) != 0) + { + LOG_E("%s : e_ident error", __func__); + return -RT_ERROR; + } + + if ((ehdr->e_type != ET_EXEC) && (ehdr->e_type != ET_DYN)) + { + LOG_E("%s : e_type error", __func__); + return -RT_ERROR; + } + + if (ehdr->e_machine == EM_NONE) + { + LOG_E("%s : e_machine is EM_NONE", __func__); + return -RT_ERROR; + } + + if (ehdr->e_phnum > ELF_PHDR_NUM_MAX) + { + LOG_E("%s : e_phnum error", __func__); + return -RT_ERROR; + } + + if (ehdr->e_phoff > file_len) + { + LOG_E("%s : e_phoff error", __func__); + return -RT_ERROR; + } + + LOG_D("%s : e_entry : 0x%x", __func__, ehdr->e_entry); + + return RT_EOK; +} + +static int elf_check_phdr(const Elf_Phdr *phdr) +{ + if (phdr->p_filesz > FILE_LENGTH_MAX) + { + LOG_E("%s : phdr p_filesz 0x%x error", __func__, phdr->p_filesz); + return -RT_ERROR; + } + + if (phdr->p_offset > FILE_LENGTH_MAX) + { + LOG_E("%s : phdr p_offset 0x%x error", __func__, phdr->p_offset); + return -RT_ERROR; + } + + if (phdr->p_memsz > MEM_SIZE_MAX) + { + LOG_E("%s[%d], phdr p_memsz 0x%x error", __func__, phdr->p_memsz); + return -RT_ERROR; + } + + LOG_D("%s : phdr p_vaddr : 0x%x", __func__, phdr->p_vaddr); + + return RT_EOK; +} + +static int elf_load_ehdr(elf_info_t *elf_info) +{ + int ret; + + ret = elf_file_open(elf_info->filename); + if (ret < 0) + { + LOG_E("%s : elf_file_open %s failed", __func__, elf_info->filename); + return ret; + } + + elf_info->fd = ret; + + ret = elf_file_length(elf_info->filename, &elf_info->file_len); + if (ret != RT_EOK) + { + return -RT_ERROR; + } + + ret = elf_file_read(elf_info->fd, (rt_uint8_t *)&elf_info->ehdr, sizeof(Elf_Ehdr), 0); + if (ret != RT_EOK) + { + LOG_E("%s : elf_file_read failed, ret : %d", __func__, ret); + return -RT_ERROR; + } + + ret = elf_check_ehdr(&elf_info->ehdr, elf_info->file_len); + if (ret != RT_EOK) + { + LOG_E("%s : elf_check_ehdr failed, ret : %d", __func__, ret); + return -RT_ERROR; + } + + return RT_EOK; +} + +static int elf_load_phdr(elf_info_t *elf_info) +{ + Elf_Ehdr *ehdr = &elf_info->ehdr; + uint32_t size; + int ret; + + if (ehdr->e_phnum < 1) + { + return -RT_ERROR; + } + + if (ehdr->e_phentsize != sizeof(Elf_Phdr)) + { + return -RT_ERROR; + } + + size = sizeof(Elf_Phdr) * ehdr->e_phnum; + if ((ehdr->e_phoff + size) > elf_info->file_len) + { + return -RT_ERROR; + } + + elf_info->phdr = rt_malloc(size); + if (elf_info->phdr == RT_NULL) + { + LOG_E("%s : alloc phdr failed", __func__); + return -RT_ENOMEM; + } + + ret = elf_file_read(elf_info->fd, (rt_uint8_t *)elf_info->phdr, size, ehdr->e_phoff); + if (ret != RT_EOK) + { + rt_free(elf_info->phdr); + elf_info->phdr = RT_NULL; + LOG_E("%s : elf_file_read failed, ret = %d", __func__, ret); + return -RT_ERROR; + } + + return RT_EOK; +} + +static int elf_load_interp(elf_load_info_t *load_info) +{ + Elf_Phdr *phdr = load_info->exec_info.phdr; + int ret; + int i; + + for (i = 0; i < load_info->exec_info.ehdr.e_phnum; ++i, ++phdr) + { + if (phdr->p_type != PT_INTERP) + { + continue; + } + + if (elf_check_phdr(phdr) != RT_EOK) + { + return -RT_ERROR; + } + + if ((phdr->p_filesz > ELF_PATH_MAX) || (phdr->p_filesz < FLF_PATH_MIN)) + { + LOG_E("%s : phdr p_filesz error", __func__, phdr->p_filesz); + return -RT_ERROR; + } + + if (phdr->p_offset + phdr->p_filesz > load_info->exec_info.file_len) + { + LOG_E("%s : phdr p_offset error", __func__, phdr->p_offset); + return -RT_ERROR; + } + + load_info->interp_info.filename = rt_malloc(phdr->p_filesz); + if (load_info->interp_info.filename == RT_NULL) + { + LOG_E("%s : alloc elf interpreter failed", __func__); + return -RT_ENOMEM; + } + + ret = elf_file_read(load_info->exec_info.fd, (rt_uint8_t *)load_info->interp_info.filename, + phdr->p_filesz, phdr->p_offset); + if (ret != RT_EOK) + { + LOG_E("%s : elf_file_read failed, ret = %d", __func__, ret); + ret = -RT_ERROR; + goto error_exit; + } + + if (load_info->interp_info.filename[phdr->p_filesz - 1] != '\0') + { + LOG_E("%s : elf interpreter is invalid", __func__); + ret = -RT_ERROR; + goto error_exit; + } + + LOG_D("%s : elf interpreter : %s", __func__, load_info->interp_info.filename); + + ret = elf_load_ehdr(&load_info->interp_info); + if (ret != RT_EOK) + { + LOG_E("%s : elf_load_ehdr failed, ret = %d", __func__, ret); + goto error_exit; + } + + ret = elf_load_phdr(&load_info->interp_info); + if (ret != RT_EOK) + { + LOG_E("%s : elf_load_phdr failed, ret = %d", __func__, ret); + goto error_exit; + } + break; + } + + return RT_EOK; + +error_exit: + return ret; +} + +static int total_mapping_size(elf_info_t *elf_info) +{ + int i; + int first_idx = -1; + int last_idx = -1; + + for (i = 0; i < elf_info->ehdr.e_phnum; i++) + { + if (elf_info->phdr[i].p_type == PT_LOAD) + { + last_idx = i; + if (first_idx == -1) + first_idx = i; + } + } + + if (first_idx == -1) + return -1; + + elf_info->map_size = elf_info->phdr[last_idx].p_vaddr + elf_info->phdr[last_idx].p_memsz - + ELF_PAGESTART(elf_info->phdr[first_idx].p_vaddr); + + return 0; +} + +static rt_ubase_t elf_map(struct rt_lwp *lwp, const Elf_Phdr *elf_phdr, int fd, rt_ubase_t addr, size_t prot, size_t flags, rt_ubase_t map_size) +{ + rt_ubase_t map_va = 0; + rt_ubase_t va_offset; + addr = ELF_PAGESTART(addr); + va_offset = elf_phdr->p_offset - ELF_PAGEOFFSET(elf_phdr->p_vaddr); + rt_ubase_t size; + + if (map_size != 0) + { + size = map_size; + } + else + { + size = elf_phdr->p_memsz + ELF_PAGEOFFSET(elf_phdr->p_vaddr); + if (size == 0) + { + return addr; + } + } + map_va = (rt_ubase_t)file_mmap(lwp, fd, addr, size, prot, flags, va_offset); + + return map_va; +} + +static int elf_zero_bss(struct rt_lwp *lwp, int fd, const Elf_Phdr *phdr, rt_ubase_t bss_start, + rt_ubase_t bss_end) +{ + lwp_data_set(lwp, (void *)bss_start, 0, bss_end - bss_start); + + return RT_EOK; +} + +static int elf_file_mmap(elf_load_info_t *load_info, elf_info_t *elf_info, rt_ubase_t *elfload_addr, + rt_uint32_t map_size, rt_ubase_t *load_base) +{ + int ret, i; + rt_ubase_t map_va, bss_start, bss_end; + Elf_Ehdr *ehdr = &elf_info->ehdr; + Elf_Phdr *phdr = elf_info->phdr; + const Elf_Phdr *tmp_phdr = phdr; + int fd = elf_info->fd; + rt_ubase_t load_addr; + size_t prot = PROT_READ | PROT_WRITE; + size_t flags = MAP_FIXED | MAP_PRIVATE; + + for (i = 0; i < ehdr->e_phnum; ++i, ++tmp_phdr) + { + if (tmp_phdr->p_type != PT_LOAD) + { + continue; + } + + if (ehdr->e_type == ET_EXEC) + { + if (elf_check_phdr(tmp_phdr) != RT_EOK) + { + LOG_E("%s : elf_check_phdr failed", __func__); + return -RT_ERROR; + } + } + + load_addr = tmp_phdr->p_vaddr + *load_base; + LOG_D("%s : p_vaddr : 0x%x, load_addr : 0x%x", __func__, tmp_phdr->p_vaddr, load_addr); + if ((tmp_phdr->p_vaddr == 0) && (*load_base == 0)) + { + flags &= ~MAP_FIXED; + } + + map_va = elf_map(load_info->lwp, tmp_phdr, fd, load_addr, prot, flags, map_size); + if (!map_va) + { + LOG_E("%s : elf_map failed", __func__); + return -ENOMEM; + } + + map_size = 0; + + elf_user_dump(load_info->lwp, (void *)load_addr, 64); + if ((tmp_phdr->p_memsz > tmp_phdr->p_filesz) && (tmp_phdr->p_flags & PF_W)) + { + bss_start = load_addr + tmp_phdr->p_filesz; + bss_end = load_addr + tmp_phdr->p_memsz; + ret = elf_zero_bss(load_info->lwp, fd, tmp_phdr, bss_start, bss_end); + if (ret) + { + LOG_E("%s : elf_zero_bss error", __func__); + return ret; + } + } + + if (*elfload_addr == 0) + { + *elfload_addr = map_va + ELF_PAGEOFFSET(tmp_phdr->p_vaddr); + LOG_D("%s elf_load_addr : %p, vAddr : %p, load_base : %p, map_va : %p", __func__, + *elfload_addr, tmp_phdr->p_vaddr, *load_base, map_va); + } + + if ((*load_base == 0) && (ehdr->e_type == ET_DYN)) + { + *load_base = map_va; + } + } + + return RT_EOK; +} + +static int load_elf_interp(elf_load_info_t *load_info, rt_ubase_t *interp_base) +{ + int ret; + rt_ubase_t load_base = ELF_INTERP_LOAD_ADDR + elf_random_offset(); + + ret = total_mapping_size(&load_info->interp_info); + if (ret) + { + LOG_E("%s : total_mapping_size failed", __func__); + return -RT_ERROR; + } + LOG_D("%s : total_mapping_size 0x%x", __func__, load_info->interp_info.map_size); + + return elf_file_mmap(load_info, &load_info->interp_info, interp_base, + load_info->interp_info.map_size, &load_base); +} + +static int elf_aux_fill(elf_load_info_t *load_info) +{ + uint8_t *random; + struct process_aux *aux = load_info->aux; + elf_addr_t *aux_info; + uint32_t random_value = rt_tick_get(); + size_t prot = PROT_READ | PROT_WRITE; + size_t flags = MAP_PRIVATE; + void *va; + + if (!aux) + { + LOG_E("%s : aux is null", __func__); + return -1; + } + aux_info = (elf_addr_t *)aux->item; + ELF_AUX_ENT(aux_info, AT_PAGESZ, ARCH_PAGE_SIZE); + + va = lwp_mmap2(load_info->lwp, (void *)(USER_VADDR_TOP - ARCH_PAGE_SIZE * 2), ARCH_PAGE_SIZE, prot, flags, -1, 0); + if (!va) + { + LOG_E("lwp map user failed!"); + return -RT_ERROR; + } + random = (uint8_t *)(USER_VADDR_TOP - ARCH_PAGE_SIZE - sizeof(char[16])); + lwp_data_put(load_info->lwp, random, &random_value, sizeof(random_value)); + ELF_AUX_ENT(aux_info, AT_RANDOM, (size_t)random); + ELF_AUX_ENT(aux_info, AT_PHDR, (size_t)load_info->load_addr + load_info->exec_info.ehdr.e_phoff); + ELF_AUX_ENT(aux_info, AT_PHNUM, (size_t)load_info->exec_info.ehdr.e_phnum); + ELF_AUX_ENT(aux_info, AT_PHENT, sizeof(Elf_Phdr)); + ELF_AUX_ENT(aux_info, AT_BASE, load_info->interp_base); + ELF_AUX_ENT(aux_info, AT_FLAGS, 0); + ELF_AUX_ENT(aux_info, AT_ENTRY, load_info->exec_info.ehdr.e_entry); + ELF_AUX_ENT(aux_info, AT_UID, 0); + ELF_AUX_ENT(aux_info, AT_EUID, 0); + ELF_AUX_ENT(aux_info, AT_GID, 0); + ELF_AUX_ENT(aux_info, AT_EGID, 0); + ELF_AUX_ENT(aux_info, AT_HWCAP, 0); + ELF_AUX_ENT(aux_info, AT_CLKTCK, 0); + ELF_AUX_ENT(aux_info, AT_SECURE, 0); + +#ifdef ARCH_MM_MMU + rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, aux, sizeof(*aux)); +#endif + + return 0; +} + +static int elf_load_segment(elf_load_info_t *load_info) +{ + int ret; + rt_ubase_t app_load_base = 0; + load_info->load_addr = 0; + load_info->interp_base = 0; + load_info->exec_info.map_size = 0; + + if (load_info->exec_info.ehdr.e_type == ET_DYN) + { + ret = total_mapping_size(&load_info->exec_info); + if (ret) + { + LOG_E("%s : total_mapping_size failed", __func__); + return -RT_ERROR; + } + LOG_D("%s : map_size : 0x%x", __func__, load_info->exec_info.map_size); + app_load_base = ELF_EXEC_LOAD_ADDR + elf_random_offset(); + } + + ret = elf_file_mmap(load_info, &load_info->exec_info, &load_info->load_addr, + load_info->exec_info.map_size, &app_load_base); + elf_file_close(load_info->exec_info.fd); + if (ret != RT_EOK) + { + LOG_W("%s : elf_file_close exec failed", __func__); + } + load_info->exec_info.fd = ELF_INVALID_FD; + + if (load_info->interp_info.fd != ELF_INVALID_FD) + { + ret = load_elf_interp(load_info, &load_info->interp_base); + if (ret) + { + LOG_E("%s : load_elf_interp failed, ret = %d", __func__, ret); + return ret; + } + elf_file_close(load_info->interp_info.fd); + if (ret != RT_EOK) + { + LOG_W("%s : elf_file_close interp failed, ret = %d", __func__, ret); + } + load_info->interp_info.fd = ELF_INVALID_FD; + load_info->e_entry = load_info->interp_info.ehdr.e_entry + load_info->interp_base; + load_info->exec_info.ehdr.e_entry = load_info->exec_info.ehdr.e_entry + app_load_base; + } + else + { + load_info->e_entry = load_info->exec_info.ehdr.e_entry; + } + + load_info->lwp->text_entry = (void *)load_info->e_entry; + LOG_D("%s : lwp->text_entry : %p loadaddr : %p", __func__, load_info->lwp->text_entry, app_load_base); + + elf_user_dump(load_info->lwp, load_info->lwp->text_entry, 64); + + ret = elf_aux_fill(load_info); + if (ret) + { + LOG_E("%s : elf_aux_fill failed", __func__); + return ret; + } + + return RT_EOK; +} + +static void elf_load_deinit(elf_load_info_t *load_info) +{ + if (load_info->exec_info.fd != ELF_INVALID_FD) + { + elf_file_close(load_info->exec_info.fd); + } + + if (load_info->interp_info.fd != ELF_INVALID_FD) + { + elf_file_close(load_info->interp_info.fd); + } + + if (load_info->exec_info.phdr != RT_NULL) + { + rt_free(load_info->exec_info.phdr); + } + + if (load_info->exec_info.filename != RT_NULL) + { + rt_free(load_info->exec_info.filename); + } + + if (load_info->interp_info.phdr != RT_NULL) + { + rt_free(load_info->interp_info.phdr); + } + + if (load_info->interp_info.filename != RT_NULL) + { + rt_free(load_info->interp_info.filename); + } +} + +static int elf_load_app(elf_info_t *exec_info) +{ + int ret; + + ret = elf_load_ehdr(exec_info); + if (ret != RT_EOK) + { + return ret; + } + + ret = elf_load_phdr(exec_info); + if (ret != RT_EOK) + { + return ret; + } + + return ret; +} + +static int elf_file_load(elf_load_info_t *load_info) +{ + int ret; + + ret = elf_load_app(&load_info->exec_info); + if (ret != RT_EOK) + { + goto OUT; + } + + ret = elf_load_interp(load_info); + if (ret != RT_EOK) + { + goto OUT; + } + + ret = elf_load_segment(load_info); + if (ret != RT_EOK) + { + goto OUT; + } + +OUT: + elf_load_deinit(load_info); + return ret; +} + +int lwp_load(const char *filename, struct rt_lwp *lwp, uint8_t *load_addr, size_t addr_size, + struct process_aux *aux) +{ + elf_load_info_t load_info = { 0 }; + int len; + int ret; + + if (filename == RT_NULL) + { + LOG_E("%s : file is NULL", __func__); + return -RT_ERROR; + } + + len = rt_strlen(filename); + if (len < FLF_PATH_MIN || len > ELF_PATH_MAX) + { + LOG_E("%s : file length (%d) invalid", __func__, len); + return -RT_ERROR; + } + + load_info.exec_info.filename = rt_malloc(len + 1); + if (!load_info.exec_info.filename) + { + LOG_E("%s : alloc filename failed", __func__, len); + return -RT_ERROR; + } + else + { + rt_memset(load_info.exec_info.filename, 0, len + 1); + rt_strncpy(load_info.exec_info.filename, filename, len); + } + + load_info.lwp = lwp; + load_info.aux = aux; + + load_info.exec_info.fd = ELF_INVALID_FD; + load_info.interp_info.fd = ELF_INVALID_FD; + load_info.load_addr = (rt_ubase_t)load_addr; + + /* copy file name to process name */ + rt_strncpy(lwp->cmd, filename, RT_NAME_MAX); + + ret = elf_file_load(&load_info); + if (ret != RT_EOK) + { + LOG_E("%s : elf_file_load error, ret : %d", __func__, ret); + return ret; + } + + return RT_EOK; +} + +#endif diff --git a/components/lwp/lwp_ipc.c b/components/lwp/lwp_ipc.c index 8410bc2cac..8b7bc47098 100644 --- a/components/lwp/lwp_ipc.c +++ b/components/lwp/lwp_ipc.c @@ -427,7 +427,6 @@ static int _ipc_msg_fd_new(void *file) #endif - return fd; } diff --git a/components/lwp/lwp_pid.c b/components/lwp/lwp_pid.c index d0a9e2cec0..2262a33f0f 100644 --- a/components/lwp/lwp_pid.c +++ b/components/lwp/lwp_pid.c @@ -700,7 +700,7 @@ pid_t waitpid(pid_t pid, int *status, int options) /* delete from sibling list of its parent */ struct rt_lwp **lwp_node; - *status = lwp->lwp_ret; + lwp_data_put(this_lwp, status, &lwp->lwp_ret, sizeof(*status)); lwp_node = &this_lwp->first_child; while (*lwp_node != lwp) { diff --git a/components/lwp/lwp_shm.c b/components/lwp/lwp_shm.c index 7ee7e29d8c..b7399c20d5 100644 --- a/components/lwp/lwp_shm.c +++ b/components/lwp/lwp_shm.c @@ -156,7 +156,6 @@ static int _lwp_shmget(size_t key, size_t size, int create) p->mem_obj.on_varea_open = on_shm_varea_open; p->mem_obj.on_varea_close = on_shm_varea_close; p->mem_obj.hint_free = NULL; - p->mem_obj.on_page_offload = NULL; /* then insert it into the balancing binary tree */ node_key = (struct lwp_avl_struct *)rt_malloc(sizeof(struct lwp_avl_struct) * 2); diff --git a/components/lwp/lwp_syscall.c b/components/lwp/lwp_syscall.c index f638b0bbb1..6bdb786355 100644 --- a/components/lwp/lwp_syscall.c +++ b/components/lwp/lwp_syscall.c @@ -1305,12 +1305,41 @@ rt_base_t sys_brk(void *addr) void *sys_mmap2(void *addr, size_t length, int prot, int flags, int fd, size_t pgoffset) { - return lwp_mmap2(addr, length, prot, flags, fd, pgoffset); + sysret_t rc = 0; + long offset = 0; + + /* aligned for user addr */ + if ((rt_base_t)addr & ARCH_PAGE_MASK) + { + if (flags & MAP_FIXED) + rc = -EINVAL; + else + { + offset = (char *)addr - (char *)RT_ALIGN_DOWN((rt_base_t)addr, ARCH_PAGE_SIZE); + length += offset; + addr = (void *)RT_ALIGN_DOWN((rt_base_t)addr, ARCH_PAGE_SIZE); + } + } + + if (rc == 0) + { + /* fix parameter passing (both along have same effect) */ + if (fd == -1 || flags & MAP_ANONYMOUS) + { + fd = -1; + /* MAP_SHARED has no effect and treated as nothing */ + flags &= ~MAP_SHARED; + flags |= MAP_PRIVATE | MAP_ANONYMOUS; + } + rc = (sysret_t)lwp_mmap2(lwp_self(), addr, length, prot, flags, fd, pgoffset); + } + + return (char *)rc + offset; } sysret_t sys_munmap(void *addr, size_t length) { - return lwp_munmap(addr); + return lwp_munmap(lwp_self(), addr, length); } void *sys_mremap(void *old_address, size_t old_size, @@ -1999,17 +2028,6 @@ rt_weak long sys_clone(void *arg[]) return _sys_clone(arg); } -int lwp_dup_user(rt_varea_t varea, void *arg); - -static int _copy_process(struct rt_lwp *dest_lwp, struct rt_lwp *src_lwp) -{ - int err; - dest_lwp->lwp_obj->source = src_lwp->aspace; - err = rt_aspace_traversal(src_lwp->aspace, lwp_dup_user, dest_lwp); - dest_lwp->lwp_obj->source = NULL; - return err; -} - static void lwp_struct_copy(struct rt_lwp *dst, struct rt_lwp *src) { #ifdef ARCH_MM_MMU @@ -2106,8 +2124,8 @@ sysret_t _sys_fork(void) self_lwp = lwp_self(); - /* copy process */ - if (_copy_process(lwp, self_lwp) != 0) + /* copy address space of process from this proc to forked one */ + if (lwp_fork_aspace(lwp, self_lwp) != 0) { SET_ERRNO(ENOMEM); goto fail; @@ -4222,13 +4240,27 @@ sysret_t sys_getaddrinfo(const char *nodename, SET_ERRNO(EFAULT); goto exit; } -#endif + + k_nodename = (char *)kmem_get(len + 1); + if (!k_nodename) + { + SET_ERRNO(ENOMEM); + goto exit; + } + + if (lwp_get_from_user(k_nodename, (void *)nodename, len + 1) != len + 1) + { + SET_ERRNO(EFAULT); + goto exit; + } +#else k_nodename = rt_strdup(nodename); if (!k_nodename) { SET_ERRNO(ENOMEM); goto exit; } +#endif } if (servname) { @@ -4239,13 +4271,27 @@ sysret_t sys_getaddrinfo(const char *nodename, SET_ERRNO(EFAULT); goto exit; } -#endif + + k_servname = (char *)kmem_get(len + 1); + if (!k_servname) + { + SET_ERRNO(ENOMEM); + goto exit; + } + + if (lwp_get_from_user(k_servname, (void *)servname, len + 1) < 0) + { + SET_ERRNO(EFAULT); + goto exit; + } +#else k_servname = rt_strdup(servname); if (!k_servname) { SET_ERRNO(ENOMEM); goto exit; } +#endif } if (hints) @@ -4300,15 +4346,28 @@ exit: { ret = GET_ERRNO(); } - +#ifdef ARCH_MM_MMU + if (k_nodename) + { + kmem_put(k_nodename); + } +#else if (k_nodename) { rt_free(k_nodename); } +#endif +#ifdef ARCH_MM_MMU + if (k_servname) + { + kmem_put(k_servname); + } +#else if (k_servname) { rt_free(k_servname); } +#endif if (k_hints) { rt_free(k_hints); @@ -4324,7 +4383,7 @@ sysret_t sys_gethostbyname2_r(const char *name, int af, struct hostent *ret, { int ret_val = -1; int sal_ret = -1 , sal_err = -1; - struct hostent sal_he; + struct hostent sal_he, sal_tmp; struct hostent *sal_result = NULL; char *sal_buf = NULL; char *k_name = NULL; @@ -4354,22 +4413,35 @@ sysret_t sys_gethostbyname2_r(const char *name, int af, struct hostent *ret, SET_ERRNO(EFAULT); goto __exit; } -#endif - *result = ret; - sal_buf = (char *)malloc(HOSTENT_BUFSZ); - if (sal_buf == NULL) + k_name = (char *)kmem_get(len + 1); + if (!k_name) { SET_ERRNO(ENOMEM); goto __exit; } + if (lwp_get_from_user(k_name, (void *)name, len + 1) < 0) + { + SET_ERRNO(EFAULT); + goto __exit; + } +#else k_name = rt_strdup(name); if (k_name == NULL) { SET_ERRNO(ENOMEM); goto __exit; } +#endif + + *result = ret; + sal_buf = (char *)malloc(HOSTENT_BUFSZ); + if (sal_buf == NULL) + { + SET_ERRNO(ENOMEM); + goto __exit; + } /* get host by name in SAL */ sal_ret = sal_gethostbyname_r(k_name, &sal_he, sal_buf, HOSTENT_BUFSZ, &sal_result, &sal_err); @@ -4386,6 +4458,28 @@ sysret_t sys_gethostbyname2_r(const char *name, int af, struct hostent *ret, } cnt = index + 1; +#ifdef ARCH_MM_MMU + /* update user space hostent */ + lwp_put_to_user(buf, k_name, buflen - (ptr - buf)); + lwp_memcpy(&sal_tmp, &sal_he, sizeof(sal_he)); + sal_tmp.h_name = ptr; + ptr += rt_strlen(k_name); + + sal_tmp.h_addr_list = (char**)ptr; + ptr += cnt * sizeof(char *); + + index = 0; + while (sal_he.h_addr_list[index] != NULL) + { + sal_tmp.h_addr_list[index] = ptr; + lwp_memcpy(ptr, sal_he.h_addr_list[index], sal_he.h_length); + + ptr += sal_he.h_length; + index++; + } + sal_tmp.h_addr_list[index] = NULL; + lwp_put_to_user(ret, &sal_tmp, sizeof(sal_tmp)); +#else /* update user space hostent */ ret->h_addrtype = sal_he.h_addrtype; ret->h_length = sal_he.h_length; @@ -4407,10 +4501,10 @@ sysret_t sys_gethostbyname2_r(const char *name, int af, struct hostent *ret, index++; } ret->h_addr_list[index] = NULL; +#endif + ret_val = 0; } - ret_val = 0; - __exit: if (ret_val < 0) { @@ -4422,10 +4516,17 @@ __exit: { free(sal_buf); } +#ifdef ARCH_MM_MMU + if (k_name) + { + kmem_put(k_name); + } +#else if (k_name) { free(k_name); } +#endif return ret_val; } diff --git a/components/lwp/lwp_user_mm.c b/components/lwp/lwp_user_mm.c index 19ba3c5c76..eb8b59dd75 100644 --- a/components/lwp/lwp_user_mm.c +++ b/components/lwp/lwp_user_mm.c @@ -11,7 +11,10 @@ * 2021-02-19 lizhirui add riscv64 support for lwp_user_accessable and lwp_get_from_user * 2021-06-07 lizhirui modify user space bound check * 2022-12-25 wangxiaoyao adapt to new mm + * 2023-08-12 Shell Fix parameter passing of lwp_mmap()/lwp_munmap() + * 2023-08-29 Shell Add API accessible()/data_get()/data_set()/data_put() * 2023-09-13 Shell Add lwp_memcpy and support run-time choice of memcpy base on memory attr + * 2023-09-19 Shell add lwp_user_memory_remap_to_kernel */ #include <rtthread.h> @@ -36,28 +39,104 @@ #include "libc_musl.h" #endif -#define DBG_TAG "LwP" -#define DBG_LVL DBG_LOG +#define DBG_TAG "LwP.mman" +#define DBG_LVL DBG_INFO #include <rtdbg.h> +#include <stdlib.h> + +#define STACK_OBJ _null_object + static void _init_lwp_objs(struct rt_lwp_objs *lwp_objs, rt_aspace_t aspace); +static const char *_null_get_name(rt_varea_t varea) +{ + return "null"; +} + +static void _null_page_fault(struct rt_varea *varea, + struct rt_aspace_fault_msg *msg) +{ + static void *null_page; + + if (!null_page) + { + null_page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE); + if (null_page) + memset(null_page, 0, ARCH_PAGE_SIZE); + else + return; + } + + msg->response.status = MM_FAULT_STATUS_OK; + msg->response.size = ARCH_PAGE_SIZE; + msg->response.vaddr = null_page; +} + +static rt_err_t _null_shrink(rt_varea_t varea, void *new_start, rt_size_t size) +{ + return RT_EOK; +} + +static rt_err_t _null_split(struct rt_varea *existed, void *unmap_start, rt_size_t unmap_len, struct rt_varea *subset) +{ + return RT_EOK; +} + +static rt_err_t _null_expand(struct rt_varea *varea, void *new_vaddr, rt_size_t size) +{ + return RT_EOK; +} + +static void _null_page_read(struct rt_varea *varea, struct rt_aspace_io_msg *msg) +{ + void *dest = msg->buffer_vaddr; + memset(dest, 0, ARCH_PAGE_SIZE); + + msg->response.status = MM_FAULT_STATUS_OK; + return ; +} + +static void _null_page_write(struct rt_varea *varea, struct rt_aspace_io_msg *msg) +{ + /* write operation is not allowed */ + msg->response.status = MM_FAULT_STATUS_UNRECOVERABLE; + return ; +} + +static struct rt_mem_obj _null_object = { + .get_name = _null_get_name, + .hint_free = RT_NULL, + .on_page_fault = _null_page_fault, + + .page_read = _null_page_read, + .page_write = _null_page_write, + + .on_varea_expand = _null_expand, + .on_varea_shrink = _null_shrink, + .on_varea_split = _null_split, +}; + int lwp_user_space_init(struct rt_lwp *lwp, rt_bool_t is_fork) { + void *stk_addr; int err = -RT_ENOMEM; + const size_t flags = MMF_MAP_PRIVATE; lwp->lwp_obj = rt_malloc(sizeof(struct rt_lwp_objs)); if (lwp->lwp_obj) { - _init_lwp_objs(lwp->lwp_obj, lwp->aspace); - err = arch_user_space_init(lwp); - if (!is_fork && err == RT_EOK) + if (err == RT_EOK) { - void *addr = (void *)USER_STACK_VSTART; - err = rt_aspace_map(lwp->aspace, &addr, - USER_STACK_VEND - USER_STACK_VSTART, - MMU_MAP_U_RWCB, 0, &lwp->lwp_obj->mem_obj, 0); + _init_lwp_objs(lwp->lwp_obj, lwp->aspace); + if (!is_fork) + { + stk_addr = (void *)USER_STACK_VSTART; + err = rt_aspace_map(lwp->aspace, &stk_addr, + USER_STACK_VEND - USER_STACK_VSTART, + MMU_MAP_U_RWCB, flags, &STACK_OBJ, 0); + } } } @@ -91,7 +170,7 @@ void lwp_unmap_user_space(struct rt_lwp *lwp) rt_free(lwp->lwp_obj); } -static const char *user_get_name(rt_varea_t varea) +static const char *_user_get_name(rt_varea_t varea) { char *name; if (varea->flag & MMF_TEXT) @@ -184,12 +263,9 @@ static void _init_lwp_objs(struct rt_lwp_objs *lwp_objs, rt_aspace_t aspace) * provide identical memory. This is implemented by lwp_objs->source. */ lwp_objs->source = NULL; - lwp_objs->mem_obj.get_name = user_get_name; - lwp_objs->mem_obj.hint_free = NULL; + memcpy(&lwp_objs->mem_obj, &rt_mm_dummy_mapper, sizeof(struct rt_mem_obj)); + lwp_objs->mem_obj.get_name = _user_get_name; lwp_objs->mem_obj.on_page_fault = _user_do_page_fault; - lwp_objs->mem_obj.on_page_offload = rt_mm_dummy_mapper.on_page_offload; - lwp_objs->mem_obj.on_varea_open = rt_mm_dummy_mapper.on_varea_open; - lwp_objs->mem_obj.on_varea_close = rt_mm_dummy_mapper.on_varea_close; } } @@ -198,14 +274,14 @@ static void *_lwp_map_user(struct rt_lwp *lwp, void *map_va, size_t map_size, { void *va = map_va; int ret = 0; - size_t flags = MMF_PREFETCH; + rt_size_t flags = MMF_PREFETCH; + if (text) flags |= MMF_TEXT; + if (va != RT_NULL) + flags |= MMF_MAP_FIXED; - rt_mem_obj_t mem_obj = &lwp->lwp_obj->mem_obj; - - ret = rt_aspace_map(lwp->aspace, &va, map_size, MMU_MAP_U_RWCB, flags, - mem_obj, 0); + ret = rt_aspace_map_private(lwp->aspace, &va, map_size, MMU_MAP_U_RWCB, flags); if (ret != RT_EOK) { va = RT_NULL; @@ -223,98 +299,17 @@ int lwp_unmap_user(struct rt_lwp *lwp, void *va) return err; } -static void _dup_varea(rt_varea_t varea, struct rt_lwp *src_lwp, - rt_aspace_t dst) -{ - char *vaddr = varea->start; - char *vend = vaddr + varea->size; - if (vaddr < (char *)USER_STACK_VSTART || vaddr >= (char *)USER_STACK_VEND) - { - while (vaddr != vend) - { - void *paddr; - paddr = lwp_v2p(src_lwp, vaddr); - if (paddr != ARCH_MAP_FAILED) - { - rt_aspace_load_page(dst, vaddr, 1); - } - vaddr += ARCH_PAGE_SIZE; - } - } - else - { - while (vaddr != vend) - { - vend -= ARCH_PAGE_SIZE; - void *paddr; - paddr = lwp_v2p(src_lwp, vend); - if (paddr != ARCH_MAP_FAILED) - { - rt_aspace_load_page(dst, vend, 1); - } - else - { - break; - } - } - } -} - -int lwp_dup_user(rt_varea_t varea, void *arg) +/** fork the src_lwp->aspace in current */ +int lwp_fork_aspace(struct rt_lwp *dest_lwp, struct rt_lwp *src_lwp) { int err; - struct rt_lwp *self_lwp = lwp_self(); - struct rt_lwp *new_lwp = (struct rt_lwp *)arg; - - void *pa = RT_NULL; - void *va = RT_NULL; - rt_mem_obj_t mem_obj = varea->mem_obj; - - if (!mem_obj) - { - /* duplicate a physical mapping */ - pa = lwp_v2p(self_lwp, (void *)varea->start); - RT_ASSERT(pa != ARCH_MAP_FAILED); - struct rt_mm_va_hint hint = {.flags = MMF_MAP_FIXED, - .limit_range_size = new_lwp->aspace->size, - .limit_start = new_lwp->aspace->start, - .prefer = varea->start, - .map_size = varea->size}; - err = rt_aspace_map_phy(new_lwp->aspace, &hint, varea->attr, - MM_PA_TO_OFF(pa), &va); - if (err != RT_EOK) - { - LOG_W("%s: aspace map failed at %p with size %p", __func__, - varea->start, varea->size); - } - } - else + err = rt_aspace_fork(&src_lwp->aspace, &dest_lwp->aspace); + if (!err) { - /* duplicate a mem_obj backing mapping */ - va = varea->start; - err = rt_aspace_map(new_lwp->aspace, &va, varea->size, varea->attr, - varea->flag, &new_lwp->lwp_obj->mem_obj, - varea->offset); - if (err != RT_EOK) - { - LOG_W("%s: aspace map failed at %p with size %p", __func__, - varea->start, varea->size); - } - else - { - /* loading page frames for !MMF_PREFETCH varea */ - if (!(varea->flag & MMF_PREFETCH)) - { - _dup_varea(varea, self_lwp, new_lwp->aspace); - } - } - } - - if (va != (void *)varea->start) - { - return -1; + /* do a explicit aspace switch if the page table is changed */ + lwp_aspace_switch(rt_thread_self()); } - return 0; + return err; } int lwp_unmap_user_phy(struct rt_lwp *lwp, void *va) @@ -364,6 +359,10 @@ static inline size_t _flags_to_attr(size_t flags) static inline mm_flag_t _flags_to_aspace_flag(size_t flags) { mm_flag_t mm_flag = 0; + if (flags & LWP_MAP_FLAG_MAP_FIXED) + mm_flag |= MMF_MAP_FIXED; + if (flags & LWP_MAP_FLAG_PREFETCH) + mm_flag |= MMF_PREFETCH; return mm_flag; } @@ -372,26 +371,17 @@ static rt_varea_t _lwp_map_user_varea(struct rt_lwp *lwp, void *map_va, size_t m { void *va = map_va; int ret = 0; - rt_mem_obj_t mem_obj = &lwp->lwp_obj->mem_obj; - rt_varea_t varea; + rt_varea_t varea = RT_NULL; mm_flag_t mm_flags; size_t attr; - varea = rt_malloc(sizeof(*varea)); - if (varea) - { - attr = _flags_to_attr(flags); - mm_flags = _flags_to_aspace_flag(flags); - ret = rt_aspace_map_static(lwp->aspace, varea, &va, map_size, - attr, mm_flags, mem_obj, 0); - /* let aspace handle the free of varea */ - varea->flag &= ~MMF_STATIC_ALLOC; - /* don't apply auto fetch on this */ - varea->data = (void *)NO_AUTO_FETCH; - } - else + attr = _flags_to_attr(flags); + mm_flags = _flags_to_aspace_flag(flags); + ret = rt_aspace_map_private(lwp->aspace, &va, map_size, + attr, mm_flags); + if (ret == RT_EOK) { - ret = -RT_ENOMEM; + varea = rt_aspace_query(lwp->aspace, va); } if (ret != RT_EOK) @@ -405,7 +395,6 @@ static rt_varea_t _lwp_map_user_varea(struct rt_lwp *lwp, void *map_va, size_t m static rt_varea_t _map_user_varea_ext(struct rt_lwp *lwp, void *map_va, size_t map_size, size_t flags) { - rt_varea_t varea = RT_NULL; size_t offset = 0; if (!map_size) @@ -417,9 +406,7 @@ static rt_varea_t _map_user_varea_ext(struct rt_lwp *lwp, void *map_va, size_t m map_size &= ~ARCH_PAGE_MASK; map_va = (void *)((size_t)map_va & ~ARCH_PAGE_MASK); - varea = _lwp_map_user_varea(lwp, map_va, map_size, flags); - - return varea; + return _lwp_map_user_varea(lwp, map_va, map_size, flags); } rt_varea_t lwp_map_user_varea_ext(struct rt_lwp *lwp, void *map_va, size_t map_size, size_t flags) @@ -456,11 +443,14 @@ void *lwp_map_user_phy(struct rt_lwp *lwp, void *map_va, void *map_pa, map_size &= ~ARCH_PAGE_MASK; map_pa = (void *)((size_t)map_pa & ~ARCH_PAGE_MASK); - struct rt_mm_va_hint hint = {.flags = MMF_MAP_FIXED, + struct rt_mm_va_hint hint = {.flags = 0, .limit_range_size = lwp->aspace->size, .limit_start = lwp->aspace->start, .prefer = map_va, .map_size = map_size}; + if (map_va != RT_NULL) + hint.flags |= MMF_MAP_FIXED; + rt_size_t attr = cached ? MMU_MAP_U_RWCB : MMU_MAP_U_RW; err = @@ -481,56 +471,160 @@ void *lwp_map_user_phy(struct rt_lwp *lwp, void *map_va, void *map_pa, rt_base_t lwp_brk(void *addr) { rt_base_t ret = -1; + rt_varea_t varea = RT_NULL; struct rt_lwp *lwp = RT_NULL; + size_t size = 0; - rt_mm_lock(); - lwp = rt_thread_self()->lwp; + lwp = lwp_self(); - if ((size_t)addr <= lwp->end_heap) + if ((size_t)addr == RT_NULL) { - ret = (rt_base_t)lwp->end_heap; + addr = (char *)lwp->end_heap + 1; } - else + + if ((size_t)addr <= lwp->end_heap && (size_t)addr > USER_HEAP_VADDR) + { + ret = (size_t)addr; + } + else if ((size_t)addr <= USER_HEAP_VEND) + { + size = RT_ALIGN((size_t)addr - lwp->end_heap, ARCH_PAGE_SIZE); + varea = lwp_map_user_varea_ext(lwp, (void *)lwp->end_heap, size, LWP_MAP_FLAG_PREFETCH); + if (varea) + { + lwp->end_heap = (long)(varea->start + varea->size); + ret = lwp->end_heap; + } + } + + return ret; +} + +rt_inline rt_mem_obj_t _get_mmap_obj(struct rt_lwp *lwp) +{ + return &_null_object; +} + +rt_inline rt_bool_t _memory_threshold_ok(void) +{ + #define GUARDIAN_BITS (10) + size_t total, free; + + rt_page_get_info(&total, &free); + if (free * (0x1000) < 0x100000) + { + LOG_I("%s: low of system memory", __func__); + return RT_FALSE; + } + + return RT_TRUE; +} + +rt_inline long _uflag_to_kernel(long flag) +{ + flag &= ~MMF_MAP_FIXED; + flag &= ~MMF_MAP_PRIVATE; + flag &= ~MMF_MAP_PRIVATE_DONT_SYNC; + return flag; +} + +rt_inline long _uattr_to_kernel(long attr) +{ + /* Warning: be careful with the case if user attribution is unwritable */ + return attr; +} + +static void _prefetch_mmap(rt_aspace_t aspace, void *addr, long size) +{ + struct rt_aspace_fault_msg msg; + + msg.fault_op = MM_FAULT_OP_WRITE; + msg.fault_type = MM_FAULT_TYPE_PAGE_FAULT; + + for (char *base = addr; size > 0; base += ARCH_PAGE_SIZE, size -= ARCH_PAGE_SIZE) { - size_t size = 0; - void *va = RT_NULL; + msg.fault_vaddr = base; + msg.off = (long)base >> MM_PAGE_SHIFT; + rt_aspace_fault_try_fix(aspace, &msg); + } + return ; +} + +void *lwp_user_memory_remap_to_kernel(rt_lwp_t lwp, void *uaddr, size_t length) +{ + long kattr; + long kflag; + long offset_in_mobj; + long offset_in_page; + rt_err_t error; + rt_varea_t uarea; + rt_mem_obj_t mobj; + void *kaddr = 0; - if ((size_t)addr <= USER_HEAP_VEND) + uarea = rt_aspace_query(lwp->aspace, uaddr); + if (uarea) + { + /* setup the identical mapping, and align up for address & length */ + kattr = _uattr_to_kernel(uarea->attr); + kflag = _uflag_to_kernel(uarea->flag); + offset_in_mobj = uarea->offset + ((long)uaddr - (long)uarea->start) / ARCH_PAGE_SIZE; + mobj = uarea->mem_obj; + offset_in_page = (long)uaddr & ARCH_PAGE_MASK; + length = RT_ALIGN(length + offset_in_page, ARCH_PAGE_SIZE); + error = rt_aspace_map(&rt_kernel_space, &kaddr, length, kattr, kflag, mobj, offset_in_mobj); + if (error) { - size = (((size_t)addr - lwp->end_heap) + ARCH_PAGE_SIZE - 1) & - ~ARCH_PAGE_MASK; - va = lwp_map_user(lwp, (void *)lwp->end_heap, size, 0); + LOG_I("%s(length=0x%lx,attr=0x%lx,flags=0x%lx): do map failed", __func__, length, kattr, kflag); + kaddr = 0; } - if (va) + else { - lwp->end_heap += size; - ret = lwp->end_heap; + /* TODO: {make a memory lock?} */ + LOG_D("%s(length=0x%lx,attr=0x%lx,flags=0x%lx,offset=0x%lx) => %p", __func__, length, kattr, kflag, offset_in_mobj, kaddr); + _prefetch_mmap(&rt_kernel_space, kaddr, length); + kaddr += offset_in_page; } } - rt_mm_unlock(); - return ret; + + return kaddr; } -void *lwp_mmap2(void *addr, size_t length, int prot, int flags, int fd, - off_t pgoffset) +void *lwp_mmap2(struct rt_lwp *lwp, void *addr, size_t length, int prot, + int flags, int fd, off_t pgoffset) { - void *ret = (void *)-1; + rt_err_t rc; + rt_size_t k_attr; + rt_size_t k_flags; + rt_size_t k_offset; + rt_aspace_t uspace; + rt_mem_obj_t mem_obj; + void *ret = 0; + LOG_D("%s(addr=0x%lx,length=%ld,fd=%d)", __func__, addr, length, fd); if (fd == -1) { + /** + * todo: add threshold + */ + if (!_memory_threshold_ok()) + return (void *)-ENOMEM; - ret = lwp_map_user(lwp_self(), addr, length, 0); + k_offset = MM_PA_TO_OFF(addr); + k_flags = lwp_user_mm_flag_to_kernel(flags) | MMF_MAP_PRIVATE; + k_attr = lwp_user_mm_attr_to_kernel(prot); - if (ret) + uspace = lwp->aspace; + length = RT_ALIGN(length, ARCH_PAGE_SIZE); + mem_obj = _get_mmap_obj(lwp); + + rc = rt_aspace_map(uspace, &addr, length, k_attr, k_flags, mem_obj, k_offset); + if (rc == RT_EOK) { - if ((flags & MAP_ANONYMOUS) != 0) - { - rt_memset(ret, 0, length); - } + ret = addr; } else { - ret = (void *)-1; + ret = (void *)lwp_errno_to_posix(rc); } } else @@ -538,7 +632,7 @@ void *lwp_mmap2(void *addr, size_t length, int prot, int flags, int fd, struct dfs_file *d; d = fd_get(fd); - if (d && d->vnode->type == FT_DEVICE) + if (d) { struct dfs_mmap2_args mmap2; @@ -548,26 +642,32 @@ void *lwp_mmap2(void *addr, size_t length, int prot, int flags, int fd, mmap2.flags = flags; mmap2.pgoffset = pgoffset; mmap2.ret = (void *)-1; + mmap2.lwp = lwp; - if (dfs_file_mmap2(d, &mmap2) == 0) + rc = dfs_file_mmap2(d, &mmap2); + if (rc == RT_EOK) { ret = mmap2.ret; } + else + { + ret = (void *)lwp_errno_to_posix(rc); + } } } + if ((long)ret <= 0) + LOG_D("%s() => %ld", __func__, ret); return ret; } -int lwp_munmap(void *addr) +int lwp_munmap(struct rt_lwp *lwp, void *addr, size_t length) { - int ret = 0; - - rt_mm_lock(); - ret = lwp_unmap_user(lwp_self(), addr); - rt_mm_unlock(); + int ret; - return ret; + RT_ASSERT(lwp); + ret = rt_aspace_unmap_range(lwp->aspace, addr, length); + return lwp_errno_to_posix(ret); } size_t lwp_get_from_user(void *dst, void *src, size_t size) @@ -725,20 +825,10 @@ int lwp_user_accessible_ext(struct rt_lwp *lwp, void *addr, size_t size) len = size; } tmp_addr = lwp_v2p(lwp, addr_start); - if (tmp_addr == ARCH_MAP_FAILED) + if (tmp_addr == ARCH_MAP_FAILED && + !rt_aspace_query(lwp->aspace, addr_start)) { - if ((rt_ubase_t)addr_start >= USER_STACK_VSTART && (rt_ubase_t)addr_start < USER_STACK_VEND) - { - struct rt_aspace_fault_msg msg = { - .fault_op = MM_FAULT_OP_WRITE, - .fault_type = MM_FAULT_TYPE_PAGE_FAULT, - .fault_vaddr = addr_start, - }; - if (!rt_aspace_fault_try_fix(lwp->aspace, &msg)) - return RT_FALSE; - } - else - return RT_FALSE; + return RT_FALSE; } addr_start = (void *)((char *)addr_start + len); size -= len; @@ -752,135 +842,234 @@ int lwp_user_accessable(void *addr, size_t size) return lwp_user_accessible_ext(lwp_self(), addr, size); } -/* src is in mmu_info space, dst is in current thread space */ +#define ALIGNED(addr) (!((rt_size_t)(addr) & ARCH_PAGE_MASK)) + +/* src is in lwp address space, dst is in current thread space */ size_t lwp_data_get(struct rt_lwp *lwp, void *dst, void *src, size_t size) { size_t copy_len = 0; - void *addr_start = RT_NULL, *addr_end = RT_NULL, *next_page = RT_NULL; - void *tmp_dst = RT_NULL, *tmp_src = RT_NULL; + char *temp_page = 0; + char *dst_iter, *dst_next_page; + char *src_copy_end, *src_iter, *src_iter_aligned; if (!size || !dst) { return 0; } - tmp_dst = dst; - addr_start = src; - addr_end = (void *)((char *)src + size); - next_page = - (void *)(((size_t)addr_start + ARCH_PAGE_SIZE) & ~(ARCH_PAGE_SIZE - 1)); + dst_iter = dst; + src_iter = src; + src_copy_end = src + size; + dst_next_page = + (char *)(((size_t)src_iter + ARCH_PAGE_SIZE) & ~(ARCH_PAGE_SIZE - 1)); do { - size_t len = (char *)next_page - (char *)addr_start; + size_t bytes_to_copy = (char *)dst_next_page - (char *)src_iter; + if (bytes_to_copy > size) + { + bytes_to_copy = size; + } - if (size < len) + if (ALIGNED(src_iter) && bytes_to_copy == ARCH_PAGE_SIZE) { - len = size; + /* get page to kernel buffer */ + if (rt_aspace_page_get(lwp->aspace, src_iter, dst_iter)) + break; } - tmp_src = lwp_v2p(lwp, addr_start); - if (tmp_src == ARCH_MAP_FAILED) + else { - break; + if (!temp_page) + temp_page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE); + if (!temp_page) + break; + + src_iter_aligned = (char *)((long)src_iter & ~ARCH_PAGE_MASK); + if (rt_aspace_page_get(lwp->aspace, src_iter_aligned, temp_page)) + break; + memcpy(dst_iter, temp_page + (src_iter - src_iter_aligned), bytes_to_copy); } - tmp_src = (void *)((char *)tmp_src - PV_OFFSET); - rt_memcpy(tmp_dst, tmp_src, len); - tmp_dst = (void *)((char *)tmp_dst + len); - addr_start = (void *)((char *)addr_start + len); - size -= len; - next_page = (void *)((char *)next_page + ARCH_PAGE_SIZE); - copy_len += len; - } while (addr_start < addr_end); + + dst_iter = dst_iter + bytes_to_copy; + src_iter = src_iter + bytes_to_copy; + size -= bytes_to_copy; + dst_next_page = (void *)((char *)dst_next_page + ARCH_PAGE_SIZE); + copy_len += bytes_to_copy; + } while (src_iter < src_copy_end); + + if (temp_page) + rt_pages_free(temp_page, 0); return copy_len; } -/* dst is in kernel space, src is in current thread space */ +/* dst is in lwp address space, src is in current thread space */ size_t lwp_data_put(struct rt_lwp *lwp, void *dst, void *src, size_t size) { size_t copy_len = 0; - void *addr_start = RT_NULL, *addr_end = RT_NULL, *next_page = RT_NULL; - void *tmp_dst = RT_NULL, *tmp_src = RT_NULL; + char *temp_page = 0; + char *dst_iter, *dst_iter_aligned, *dst_next_page; + char *src_put_end, *src_iter; if (!size || !dst) { return 0; } - tmp_src = src; - addr_start = dst; - addr_end = (void *)((char *)dst + size); - next_page = - (void *)(((size_t)addr_start + ARCH_PAGE_SIZE) & ~(ARCH_PAGE_SIZE - 1)); + + src_iter = src; + dst_iter = dst; + src_put_end = dst + size; + dst_next_page = + (char *)(((size_t)dst_iter + ARCH_PAGE_SIZE) & ~(ARCH_PAGE_SIZE - 1)); do { - size_t len = (char *)next_page - (char *)addr_start; + size_t bytes_to_put = (char *)dst_next_page - (char *)dst_iter; + if (bytes_to_put > size) + { + bytes_to_put = size; + } - if (size < len) + if (ALIGNED(dst_iter) && bytes_to_put == ARCH_PAGE_SIZE) { - len = size; + /* write to page in kernel */ + if (rt_aspace_page_put(lwp->aspace, dst_iter, src_iter)) + break; } - tmp_dst = lwp_v2p(lwp, addr_start); - if (tmp_dst == ARCH_MAP_FAILED) + else { - break; + if (!temp_page) + temp_page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE); + if (!temp_page) + break; + + dst_iter_aligned = (void *)((long)dst_iter & ~ARCH_PAGE_MASK); + if (rt_aspace_page_get(lwp->aspace, dst_iter_aligned, temp_page)) + break; + memcpy(temp_page + (dst_iter - dst_iter_aligned), src_iter, bytes_to_put); + if (rt_aspace_page_put(lwp->aspace, dst_iter_aligned, temp_page)) + break; } - tmp_dst = (void *)((char *)tmp_dst - PV_OFFSET); - rt_memcpy(tmp_dst, tmp_src, len); - tmp_src = (void *)((char *)tmp_src + len); - addr_start = (void *)((char *)addr_start + len); - size -= len; - next_page = (void *)((char *)next_page + ARCH_PAGE_SIZE); - copy_len += len; - } while (addr_start < addr_end); + + src_iter = src_iter + bytes_to_put; + dst_iter = dst_iter + bytes_to_put; + size -= bytes_to_put; + dst_next_page = dst_next_page + ARCH_PAGE_SIZE; + copy_len += bytes_to_put; + } while (dst_iter < src_put_end); + + if (temp_page) + rt_pages_free(temp_page, 0); + return copy_len; +} + +/* Set N bytes of S to C */ +size_t lwp_data_set(struct rt_lwp *lwp, void *dst, int byte, size_t size) +{ + size_t copy_len = 0; + char *temp_page = 0; + char *dst_iter, *dst_iter_aligned, *dst_next_page; + char *dst_put_end; + + if (!size || !dst) + { + return 0; + } + + dst_iter = dst; + dst_put_end = dst + size; + dst_next_page = + (char *)(((size_t)dst_iter + ARCH_PAGE_SIZE) & ~(ARCH_PAGE_SIZE - 1)); + temp_page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE); + if (temp_page) + { + do + { + size_t bytes_to_put = (char *)dst_next_page - (char *)dst_iter; + if (bytes_to_put > size) + { + bytes_to_put = size; + } + + dst_iter_aligned = (void *)((long)dst_iter & ~ARCH_PAGE_MASK); + if (!ALIGNED(dst_iter) || bytes_to_put != ARCH_PAGE_SIZE) + if (rt_aspace_page_get(lwp->aspace, dst_iter_aligned, temp_page)) + break; + + memset(temp_page + (dst_iter - dst_iter_aligned), byte, bytes_to_put); + if (rt_aspace_page_put(lwp->aspace, dst_iter_aligned, temp_page)) + break; + + dst_iter = dst_iter + bytes_to_put; + size -= bytes_to_put; + dst_next_page = dst_next_page + ARCH_PAGE_SIZE; + copy_len += bytes_to_put; + } while (dst_iter < dst_put_end); + + rt_pages_free(temp_page, 0); + } + return copy_len; } size_t lwp_user_strlen_ext(struct rt_lwp *lwp, const char *s) { int len = 0; - char *new_buf = RT_NULL; + char *temp_buf = RT_NULL; void *addr_start = RT_NULL; - int size = 0; - int err = 0; + int get_bytes = 0; + int index = 0; if (s == RT_NULL) return 0; - addr_start = (void *)s; - new_buf = rt_malloc(ARCH_PAGE_SIZE); - if (lwp == RT_NULL) { LOG_W("%s: lwp is NULL", __func__); return -1; } - err = lwp_data_get(lwp, new_buf, addr_start, ARCH_PAGE_SIZE); - if (err == 0) + addr_start = (void *)s; + temp_buf = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE); + if (!temp_buf) + { + LOG_W("%s: No memory", __func__); + return -1; + } + + get_bytes = lwp_data_get(lwp, temp_buf, addr_start, ARCH_PAGE_SIZE); + if (get_bytes == 0) { - rt_free(new_buf); + LOG_I("lwp_data_get(lwp=%p,dst=0x%lx,src=0x%lx,size=0x1000) failed", lwp, temp_buf, addr_start); + rt_pages_free(temp_buf, 0); return -1; } - while (new_buf[size] != '\0') + while (temp_buf[index] != '\0') { - len ++; - if (size == (ARCH_PAGE_SIZE -1)) + len++; + index++; + if (index == get_bytes) { - err = lwp_data_get(lwp, new_buf, addr_start + len, ARCH_PAGE_SIZE); - if (err == 0) + if (get_bytes == ARCH_PAGE_SIZE) + { + get_bytes = lwp_data_get(lwp, temp_buf, addr_start + len, ARCH_PAGE_SIZE); + if (get_bytes == 0) + { + LOG_I("lwp_data_get(lwp=%p,dst=0x%lx,src=0x%lx,size=0x1000): user data unaccessible", + lwp, temp_buf, addr_start); + len = -1; + break; + } + index = 0; + } + else { - rt_free(new_buf); - return -1; + LOG_I("lwp_data_get(lwp=%p,dst=0x%lx,src=0x%lx,size=0x1000): user data unaccessible", + lwp, temp_buf, addr_start); + len = -1; + break; } - - size = 0; - } - else - { - size ++; } } - rt_free(new_buf); - + rt_pages_free(temp_buf, 0); return len; } @@ -894,4 +1083,72 @@ size_t lwp_user_strlen(const char *s) return lwp_user_strlen_ext(lwp, s); } + +char** lwp_get_command_line_args(struct rt_lwp *lwp) +{ + size_t argc = 0; + char** argv = NULL; + int ret; + size_t i; + size_t len; + + if (lwp) + { + ret = lwp_data_get(lwp, &argc, lwp->args, sizeof(argc)); + if (ret == 0) + { + return RT_NULL; + } + argv = (char**)rt_malloc((argc + 1) * sizeof(char*)); + + if (argv) + { + for (i = 0; i < argc; i++) + { + char *argvp = NULL; + ret = lwp_data_get(lwp, &argvp, &((char **)lwp->args)[1 + i], sizeof(argvp)); + if (ret == 0) + { + lwp_free_command_line_args(argv); + return RT_NULL; + } + len = lwp_user_strlen_ext(lwp, argvp); + + if (len > 0) + { + argv[i] = (char*)rt_malloc(len + 1); + ret = lwp_data_get(lwp, argv[i], argvp, len); + if (ret == 0) + { + lwp_free_command_line_args(argv); + return RT_NULL; + } + argv[i][len] = '\0'; + } + else + { + argv[i] = NULL; + } + } + argv[argc] = NULL; + } + } + + return argv; +} + +void lwp_free_command_line_args(char** argv) +{ + size_t i; + + if (argv) + { + for (i = 0; argv[i]; i++) + { + rt_free(argv[i]); + } + rt_free(argv); + } +} + #endif diff --git a/components/lwp/lwp_user_mm.h b/components/lwp/lwp_user_mm.h index ca8845dee0..a63b19f3cd 100644 --- a/components/lwp/lwp_user_mm.h +++ b/components/lwp/lwp_user_mm.h @@ -7,6 +7,7 @@ * Date Author Notes * 2019-10-28 Jesven first version * 2021-02-12 lizhirui add 64-bit support for lwp_brk + * 2023-09-19 Shell add lwp_user_memory_remap_to_kernel */ #ifndef __LWP_USER_MM_H__ #define __LWP_USER_MM_H__ @@ -27,6 +28,8 @@ extern "C" { #define LWP_MAP_FLAG_NONE 0x0000 #define LWP_MAP_FLAG_NOCACHE 0x0001 +#define LWP_MAP_FLAG_MAP_FIXED 0x00010000ul +#define LWP_MAP_FLAG_PREFETCH 0x00020000ul /** * @brief Map files or devices into memory @@ -41,7 +44,7 @@ extern "C" { * @param pgoffset offset to fd in 4096 bytes unit * @return void* the address is successful, otherwise return MAP_FAILED */ -void* lwp_mmap2(void *addr, size_t length, int prot, int flags, int fd, off_t pgoffset); +void* lwp_mmap2(struct rt_lwp *lwp, void *addr, size_t length, int prot, int flags, int fd, off_t pgoffset); /** * @brief Unmap memory region in user space @@ -51,7 +54,7 @@ void* lwp_mmap2(void *addr, size_t length, int prot, int flags, int fd, off_t pg * @param length length in bytes of unmapping * @return int errno */ -int lwp_munmap(void *addr); +int lwp_munmap(struct rt_lwp *lwp, void *addr, size_t length); /** * @brief Test if address from user is accessible address by user @@ -145,8 +148,8 @@ void lwp_unmap_user_space(struct rt_lwp *lwp); int lwp_unmap_user(struct rt_lwp *lwp, void *va); void *lwp_map_user(struct rt_lwp *lwp, void *map_va, size_t map_size, rt_bool_t text); -size_t lwp_user_strlen(const char *s); -size_t lwp_user_strlen_ext(struct rt_lwp *lwp, const char *s); +void lwp_free_command_line_args(char** argv); +char** lwp_get_command_line_args(struct rt_lwp *lwp); rt_varea_t lwp_map_user_varea(struct rt_lwp *lwp, void *map_va, size_t map_size); @@ -158,6 +161,11 @@ int lwp_unmap_user_phy(struct rt_lwp *lwp, void *va); rt_base_t lwp_brk(void *addr); +size_t lwp_user_strlen(const char *s); +size_t lwp_user_strlen_ext(struct rt_lwp *lwp, const char *s); + +int lwp_fork_aspace(struct rt_lwp *dest_lwp, struct rt_lwp *src_lwp); + void lwp_data_cache_flush(struct rt_lwp *lwp, void *vaddr, size_t size); static inline void *_lwp_v2p(struct rt_lwp *lwp, void *vaddr) @@ -173,6 +181,49 @@ static inline void *lwp_v2p(struct rt_lwp *lwp, void *vaddr) return paddr; } +/** + * @brief Remapping user space memory region to kernel + * + * @warning the remapped region in kernel should be unmapped after usage + * + * @param lwp target process + * @param uaddr user space address where the data writes to + * @param length the bytes to redirect + * @return void * the redirection address in kernel space + */ +void *lwp_user_memory_remap_to_kernel(rt_lwp_t lwp, void *uaddr, size_t length); + +rt_inline rt_size_t lwp_user_mm_flag_to_kernel(int flags) +{ + rt_size_t k_flags = 0; + if (flags & MAP_FIXED) + k_flags |= MMF_MAP_FIXED; + if (flags & (MAP_PRIVATE | MAP_ANON | MAP_ANONYMOUS)) + k_flags |= MMF_MAP_PRIVATE; + if (flags & MAP_SHARED) + k_flags |= MMF_MAP_SHARED; + return k_flags; +} + +rt_inline rt_size_t lwp_user_mm_attr_to_kernel(int prot) +{ + rt_size_t k_attr = 0; + +#ifdef IMPL_MPROTECT + if ((prot & PROT_EXEC) || (prot & PROT_WRITE) || + ((prot & PROT_READ) && (prot & PROT_WRITE))) + k_attr = MMU_MAP_U_RWCB; + else if (prot == PROT_NONE) + k_attr = MMU_MAP_K_RWCB; + else + k_attr = MMU_MAP_U_ROCB; +#else + k_attr = MMU_MAP_U_RWCB; +#endif /* IMPL_MPROTECT */ + + return k_attr; +} + #ifdef __cplusplus } #endif diff --git a/components/lwp/syscall_generic.h b/components/lwp/syscall_generic.h index 04b93e5fb8..a589b6fcdd 100644 --- a/components/lwp/syscall_generic.h +++ b/components/lwp/syscall_generic.h @@ -13,6 +13,9 @@ #include <rtthread.h> +#include <errno.h> +#include <stdlib.h> + typedef long sysret_t; struct rt_syscall_def @@ -40,4 +43,36 @@ struct rt_syscall_def #define _SYS_WRAP(func) ({int _ret = func; _ret < 0 ? GET_ERRNO() : _ret;}) +rt_inline sysret_t lwp_errno_to_posix(rt_err_t error) +{ + sysret_t posix_rc; + + switch (labs(error)) + { + case RT_EOK: + posix_rc = 0; + break; + case RT_ETIMEOUT: + posix_rc = -ETIMEDOUT; + break; + case RT_EINVAL: + posix_rc = -EINVAL; + break; + case RT_ENOENT: + posix_rc = -ENOENT; + break; + case RT_ENOSPC: + posix_rc = -ENOSPC; + break; + case RT_EPERM: + posix_rc = -EPERM; + break; + default: + posix_rc = -1; + break; + } + + return posix_rc; +} + #endif /* __SYSCALL_DATA_H__ */ diff --git a/components/mm/SConscript b/components/mm/SConscript index e42fa9f0c8..9c76dd8b41 100644 --- a/components/mm/SConscript +++ b/components/mm/SConscript @@ -1,14 +1,14 @@ import os from building import * -objs = [] -src = [] +objs = [] if GetDepend('ARCH_ARM_CORTEX_A') or GetDepend('ARCH_ARMV8') or GetDepend('ARCH_RISCV64'): cwd = GetCurrentDir() - src += ['avl_adpt.c', 'ioremap.c', 'mm_aspace.c', 'mm_fault.c', 'mm_kmem.c', 'mm_object.c', 'mm_page.c'] - if GetDepend('RT_USING_MEMBLOCK'): - src += ['mm_memblock.c'] + src = Glob('*.c') + Glob('*_gcc.S') + if not GetDepend('RT_USING_MEMBLOCK'): + SrcRemove(src, ['mm_memblock.c']) + CPPPATH = [cwd] group = DefineGroup('mm', src, depend = ['ARCH_MM_MMU'], CPPPATH = CPPPATH) diff --git a/components/mm/mm_anon.c b/components/mm/mm_anon.c new file mode 100644 index 0000000000..e14300a08f --- /dev/null +++ b/components/mm/mm_anon.c @@ -0,0 +1,620 @@ +/* + * Copyright (c) 2006-2023, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2023-08-19 Shell Support PRIVATE mapping and COW + */ + +#define DBG_TAG "mm.anon" +#define DBG_LVL DBG_INFO +#include <rtdbg.h> + +#include <string.h> +#include "mm_private.h" +#include <mmu.h> + +/** + * Anonymous Object directly represent the mappings without backup files in the + * aspace. Their only backup is in the aspace->pgtbl. + */ + +typedef struct rt_private_ctx { + struct rt_mem_obj mem_obj; + rt_aspace_t backup_aspace; + /* both varea and aspace can holds a reference */ + rt_atomic_t reference; + /* readonly `private` is shared object */ + long readonly; +} *rt_private_ctx_t; + +rt_inline rt_aspace_t _anon_obj_get_backup(rt_mem_obj_t mobj) +{ + rt_private_ctx_t pctx; + rt_aspace_t backup; + pctx = rt_container_of(mobj, struct rt_private_ctx, mem_obj); + backup = pctx->backup_aspace; + return backup; +} + +rt_inline rt_atomic_t *_anon_obj_get_reference(rt_mem_obj_t mobj) +{ + rt_private_ctx_t pctx; + pctx = rt_container_of(mobj, struct rt_private_ctx, mem_obj); + return &pctx->reference; +} + +rt_inline rt_private_ctx_t _anon_mobj_to_pctx(rt_mem_obj_t mobj) +{ + return rt_container_of(mobj, struct rt_private_ctx, mem_obj); +} + +static long rt_aspace_anon_ref_inc(rt_mem_obj_t aobj) +{ + long rc; + if (aobj) + { + rc = rt_atomic_add(_anon_obj_get_reference(aobj), 1); + LOG_D("%s(aobj=%p) Cur %ld", __func__, aobj, rc + 1); + } + else + rc = -1; + return rc; +} + +rt_err_t rt_aspace_anon_ref_dec(rt_mem_obj_t aobj) +{ + rt_err_t rc; + rt_aspace_t aspace; + rt_private_ctx_t pctx; + long former_reference; + + if (aobj) + { + pctx = _anon_mobj_to_pctx(aobj); + RT_ASSERT(pctx); + + former_reference = rt_atomic_add(_anon_obj_get_reference(aobj), -1); + LOG_D("%s(aobj=%p) Cur %ld", __func__, aobj, former_reference - 1); + if (pctx->readonly) + { + if (former_reference - 1 <= pctx->readonly) + { + void *pgtbl; + RT_ASSERT(former_reference - 1 == pctx->readonly); + aspace = _anon_obj_get_backup(aobj); + + pctx->readonly = 0; + pgtbl = aspace->page_table; + rt_aspace_delete(aspace); + rt_hw_mmu_pgtbl_delete(pgtbl); + } + } + else if (former_reference < 2) + { + aspace = _anon_obj_get_backup(aobj); + aspace->private_object = RT_NULL; + + rt_free(pctx); + } + rc = RT_EOK; + } + else + { + rc = -RT_EINVAL; + } + + return rc; +} + +static const char *_anon_get_name(rt_varea_t varea) +{ + return varea->aspace == _anon_obj_get_backup(varea->mem_obj) ? "anonymous" : "reference"; +} + +static void _anon_varea_open(struct rt_varea *varea) +{ + rt_aspace_anon_ref_inc(varea->mem_obj); + + if (varea->aspace == _anon_obj_get_backup(varea->mem_obj)) + varea->offset = MM_PA_TO_OFF(varea->start); + + varea->data = NULL; +} + +static void _anon_varea_close(struct rt_varea *varea) +{ + rt_aspace_anon_ref_dec(varea->mem_obj); +} + +static rt_err_t _anon_varea_expand(struct rt_varea *varea, void *new_vaddr, rt_size_t size) +{ + return RT_EOK; +} + +static rt_err_t _anon_varea_shrink(rt_varea_t varea, void *new_start, rt_size_t size) +{ + return rt_mm_dummy_mapper.on_varea_shrink(varea, new_start, size); +} + +static rt_err_t _anon_varea_split(struct rt_varea *existed, void *unmap_start, rt_size_t unmap_len, struct rt_varea *subset) +{ + _anon_varea_open(subset); + return rt_mm_dummy_mapper.on_varea_split(existed, unmap_start, unmap_len, subset); +} + +static rt_err_t _anon_varea_merge(struct rt_varea *merge_to, struct rt_varea *merge_from) +{ + _anon_varea_close(merge_from); + return rt_mm_dummy_mapper.on_varea_merge(merge_to, merge_from); +} + +rt_inline void _map_page_in_varea(rt_aspace_t asapce, rt_varea_t varea, + struct rt_aspace_fault_msg *msg, char *fault_addr) +{ + if (rt_varea_map_page(varea, fault_addr, msg->response.vaddr) == RT_EOK) + { + msg->response.status = MM_FAULT_STATUS_OK_MAPPED; + } + else + { + /* revoke the allocated page */ + rt_varea_pgmgr_pop(varea, msg->response.vaddr, ARCH_PAGE_SIZE); + msg->response.status = MM_FAULT_STATUS_UNRECOVERABLE; + LOG_W("%s: failed to map page into varea", __func__); + } +} + +static void *_get_page_from_backup(rt_aspace_t backup, rt_base_t offset_in_mobj) +{ + void *frame_pa; + char *backup_addr; + rt_varea_t backup_varea; + void *rc = RT_NULL; + + backup_addr = (char *)(offset_in_mobj << MM_PAGE_SHIFT); + backup_varea = rt_aspace_query(backup, backup_addr); + + if (backup_varea) + { + /* synchronize between multiple request by aspace lock of backup */ + WR_LOCK(backup); + + frame_pa = rt_hw_mmu_v2p(backup, backup_addr); + if (frame_pa == ARCH_MAP_FAILED) + { + /* provide the page in backup varea */ + struct rt_aspace_fault_msg msg; + msg.fault_op = MM_FAULT_OP_WRITE; + msg.fault_type = MM_FAULT_TYPE_PAGE_FAULT; + msg.fault_vaddr = backup_addr; + msg.off = offset_in_mobj; + rt_mm_fault_res_init(&msg.response); + + rt_mm_dummy_mapper.on_page_fault(backup_varea, &msg); + if (msg.response.status != MM_FAULT_STATUS_UNRECOVERABLE) + { + _map_page_in_varea(backup, backup_varea, &msg, backup_addr); + if (msg.response.status == MM_FAULT_STATUS_OK_MAPPED) + { + rc = msg.response.vaddr; + } + } + } + else + { + rc = rt_kmem_p2v(frame_pa); + if (!rc) + RT_ASSERT(0 && "No kernel address of target page frame"); + } + WR_UNLOCK(backup); + } + else + { + /* out of range error */ + LOG_E("(backup_addr=%p): Page request out of range", backup_addr); + } + + return rc; +} + +/* get the backup page in kernel for the address in user space */ +static void _anon_page_fault(struct rt_varea *varea, struct rt_aspace_fault_msg *msg) +{ + void *paddr; + char *frame_ka; + rt_aspace_t from_aspace = varea->aspace; + rt_aspace_t backup = _anon_obj_get_backup(varea->mem_obj); + + RDWR_LOCK(from_aspace); + + /** + * if the page is already mapped(this may caused by data race while other + * thread success to take the lock and mapped the page before this), return okay + */ + paddr = rt_hw_mmu_v2p(from_aspace, msg->fault_vaddr); + if (paddr == ARCH_MAP_FAILED) + { + if (backup == from_aspace) + { + rt_mm_dummy_mapper.on_page_fault(varea, msg); + if (msg->response.status != MM_FAULT_STATUS_UNRECOVERABLE) + { + _map_page_in_varea(backup, varea, msg, msg->fault_vaddr); + } + } + else + { + frame_ka = _get_page_from_backup(backup, msg->off); + if (frame_ka) + { + msg->response.vaddr = frame_ka; + msg->response.size = ARCH_PAGE_SIZE; + _map_page_in_varea(from_aspace, varea, msg, msg->fault_vaddr); + } + } + } + else + { + msg->response.status = MM_FAULT_STATUS_OK_MAPPED; + } + RDWR_UNLOCK(from_aspace); +} + +static void read_by_mte(rt_aspace_t aspace, struct rt_aspace_io_msg *iomsg) +{ + if (rt_aspace_page_get_phy(aspace, iomsg->fault_vaddr, iomsg->buffer_vaddr) == RT_EOK) + iomsg->response.status = MM_FAULT_STATUS_OK; +} + +static void _anon_page_read(struct rt_varea *varea, struct rt_aspace_io_msg *iomsg) +{ + rt_aspace_t from_aspace = varea->aspace; + + if (rt_hw_mmu_v2p(from_aspace, iomsg->fault_vaddr) == ARCH_MAP_FAILED) + { + struct rt_aspace_fault_msg msg; + msg.fault_op = MM_FAULT_OP_READ; + msg.fault_type = MM_FAULT_TYPE_PAGE_FAULT; + msg.fault_vaddr = iomsg->fault_vaddr; + msg.off = iomsg->off; + rt_mm_fault_res_init(&msg.response); + + _anon_page_fault(varea, &msg); + if (msg.response.status != MM_FAULT_STATUS_UNRECOVERABLE) + { + read_by_mte(from_aspace, iomsg); + } + } + else + { + read_by_mte(from_aspace, iomsg); + } +} + +static void write_by_mte(rt_aspace_t aspace, struct rt_aspace_io_msg *iomsg) +{ + if (rt_aspace_page_put_phy(aspace, iomsg->fault_vaddr, iomsg->buffer_vaddr) == RT_EOK) + iomsg->response.status = MM_FAULT_STATUS_OK; +} + +static void _anon_page_write(struct rt_varea *varea, struct rt_aspace_io_msg *iomsg) +{ + rt_aspace_t from_aspace = varea->aspace; + + if (rt_hw_mmu_v2p(from_aspace, iomsg->fault_vaddr) == ARCH_MAP_FAILED) + { + struct rt_aspace_fault_msg msg; + msg.fault_op = MM_FAULT_OP_WRITE; + msg.fault_type = MM_FAULT_TYPE_PAGE_FAULT; + msg.fault_vaddr = iomsg->fault_vaddr; + msg.off = iomsg->off; + rt_mm_fault_res_init(&msg.response); + + _anon_page_fault(varea, &msg); + if (msg.response.status != MM_FAULT_STATUS_UNRECOVERABLE) + { + write_by_mte(from_aspace, iomsg); + } + } + else + { + write_by_mte(from_aspace, iomsg); + } +} + +static struct rt_private_ctx _priv_obj = { + .mem_obj.get_name = _anon_get_name, + .mem_obj.on_page_fault = _anon_page_fault, + .mem_obj.hint_free = NULL, + .mem_obj.on_varea_open = _anon_varea_open, + .mem_obj.on_varea_close = _anon_varea_close, + .mem_obj.on_varea_shrink = _anon_varea_shrink, + .mem_obj.on_varea_split = _anon_varea_split, + .mem_obj.on_varea_expand = _anon_varea_expand, + .mem_obj.on_varea_merge = _anon_varea_merge, + .mem_obj.page_read = _anon_page_read, + .mem_obj.page_write = _anon_page_write, +}; + +rt_inline rt_private_ctx_t rt_private_obj_create_n_bind(rt_aspace_t aspace) +{ + rt_private_ctx_t private_object; + private_object = rt_malloc(sizeof(struct rt_private_ctx)); + if (private_object) + { + memcpy(&private_object->mem_obj, &_priv_obj, sizeof(_priv_obj)); + + /* hold a init ref from backup aspace */ + rt_atomic_store(&private_object->reference, 1); + + private_object->readonly = RT_FALSE; + private_object->backup_aspace = aspace; + aspace->private_object = &private_object->mem_obj; + } + + return private_object; +} + +rt_inline rt_mem_obj_t _get_private_obj(rt_aspace_t aspace) +{ + rt_private_ctx_t priv; + rt_mem_obj_t rc; + rc = aspace->private_object; + if (!aspace->private_object) + { + priv = rt_private_obj_create_n_bind(aspace); + if (priv) + { + rc = &priv->mem_obj; + aspace->private_object = rc; + } + } + return rc; +} + +static int _override_map(rt_varea_t varea, rt_aspace_t aspace, void *fault_vaddr, struct rt_aspace_fault_msg *msg, void *page) +{ + int rc = MM_FAULT_FIXABLE_FALSE; + rt_mem_obj_t private_object; + rt_varea_t map_varea = RT_NULL; + rt_err_t error; + rt_size_t flags; + rt_size_t attr; + + LOG_D("%s", __func__); + + private_object = _get_private_obj(aspace); + + if (private_object) + { + flags = varea->flag | MMF_MAP_FIXED; + /* don't prefetch and do it latter */ + flags &= ~MMF_PREFETCH; + attr = rt_hw_mmu_attr_add_perm(varea->attr, RT_HW_MMU_PROT_USER | RT_HW_MMU_PROT_WRITE); + + /* override existing mapping at fault_vaddr */ + error = _mm_aspace_map( + aspace, &map_varea, &fault_vaddr, ARCH_PAGE_SIZE, attr, + flags, private_object, MM_PA_TO_OFF(fault_vaddr)); + + if (error == RT_EOK) + { + msg->response.status = MM_FAULT_STATUS_OK; + msg->response.vaddr = page; + msg->response.size = ARCH_PAGE_SIZE; + if (rt_varea_map_with_msg(map_varea, msg) != RT_EOK) + { + LOG_E("%s: fault_va=%p,(priv_va=%p,priv_sz=0x%lx) at %s", __func__, msg->fault_vaddr, map_varea->start, map_varea->size, VAREA_NAME(map_varea)); + RT_ASSERT(0 && "should never failed"); + } + RT_ASSERT(rt_hw_mmu_v2p(aspace, msg->fault_vaddr) == (page + PV_OFFSET)); + rc = MM_FAULT_FIXABLE_TRUE; + rt_varea_pgmgr_insert(map_varea, page); + } + else + { + /* private object will be release on destruction of aspace */ + rt_free(map_varea); + } + } + else + { + LOG_I("%s: out of memory", __func__); + rc = MM_FAULT_FIXABLE_FALSE; + } + + return rc; +} + +/** + * replace an existing mapping to a private one, this is identical to: + * => aspace_unmap(ex_varea, ) + * => aspace_map() + */ +int rt_varea_fix_private_locked(rt_varea_t ex_varea, void *pa, + struct rt_aspace_fault_msg *msg, + rt_bool_t dont_copy) +{ + /** + * todo: READ -> WRITE lock here + */ + void *page; + void *fault_vaddr; + rt_aspace_t aspace; + rt_mem_obj_t ex_obj; + int rc = MM_FAULT_FIXABLE_FALSE; + ex_obj = ex_varea->mem_obj; + + if (ex_obj) + { + fault_vaddr = msg->fault_vaddr; + aspace = ex_varea->aspace; + RT_ASSERT(!!aspace); + + /** + * todo: what if multiple pages are required? + */ + if (aspace->private_object == ex_obj) + { + RT_ASSERT(0 && "recursion"); + } + else if (ex_obj->page_read) + { + page = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE); + if (page) + { + /** setup message & fetch the data from source object */ + if (!dont_copy) + { + struct rt_aspace_io_msg io_msg; + rt_mm_io_msg_init(&io_msg, msg->off, msg->fault_vaddr, page); + ex_obj->page_read(ex_varea, &io_msg); + /** + * Note: if ex_obj have mapped into varea, it's still okay since + * we will override it latter + */ + if (io_msg.response.status != MM_FAULT_STATUS_UNRECOVERABLE) + { + rc = _override_map(ex_varea, aspace, fault_vaddr, msg, page); + } + else + { + rt_pages_free(page, 0); + LOG_I("%s: page read(va=%p) fault from %s(start=%p,size=%p)", __func__, + msg->fault_vaddr, VAREA_NAME(ex_varea), ex_varea->start, ex_varea->size); + } + } + else + { + rc = _override_map(ex_varea, aspace, fault_vaddr, msg, page); + } + } + else + { + LOG_I("%s: pages allocation failed", __func__); + } + } + else + { + LOG_I("%s: no page read method provided from %s", __func__, VAREA_NAME(ex_varea)); + } + } + else + { + LOG_I("%s: unavailable memory object", __func__); + } + + return rc; +} + +int rt_aspace_map_private(rt_aspace_t aspace, void **addr, rt_size_t length, + rt_size_t attr, mm_flag_t flags) +{ + int rc; + rt_mem_obj_t priv_obj; + + if (flags & MMF_STATIC_ALLOC) + { + rc = -RT_EINVAL; + } + else + { + priv_obj = _get_private_obj(aspace); + if (priv_obj) + { + flags |= MMF_MAP_PRIVATE; + flags &= ~MMF_PREFETCH; + rc = rt_aspace_map(aspace, addr, length, attr, flags, priv_obj, 0); + } + else + { + rc = -RT_ENOMEM; + } + } + return rc; +} + +static int _release_shared(rt_varea_t varea, void *arg) +{ + rt_aspace_t src = varea->aspace; + rt_mem_obj_t mem_obj = varea->mem_obj; + + if (mem_obj != _get_private_obj(src)) + { + _varea_uninstall_locked(varea); + if (VAREA_NOT_STATIC(varea)) + { + rt_free(varea); + } + } + + return 0; +} + +static rt_err_t _convert_readonly(rt_aspace_t aspace, long base_reference) +{ + rt_mem_obj_t aobj; + rt_private_ctx_t pctx; + aobj = _get_private_obj(aspace); + pctx = _anon_mobj_to_pctx(aobj); + + LOG_D("Ref(cur=%d,base=%d)", pctx->reference, base_reference); + rt_aspace_traversal(aspace, _release_shared, 0); + pctx->readonly = base_reference; + return 0; +} + +rt_inline void _switch_aspace(rt_aspace_t *pa, rt_aspace_t *pb) +{ + rt_aspace_t temp; + temp = *pa; + *pa = *pb; + *pb = temp; +} + +rt_err_t rt_aspace_fork(rt_aspace_t *psrc, rt_aspace_t *pdst) +{ + rt_err_t rc; + void *pgtbl; + rt_aspace_t backup; + rt_aspace_t src = *psrc; + rt_aspace_t dst = *pdst; + long base_reference; + + pgtbl = rt_hw_mmu_pgtbl_create(); + if (pgtbl) + { + backup = rt_aspace_create(src->start, src->size, pgtbl); + if (backup) + { + WR_LOCK(src); + base_reference = rt_atomic_load(_anon_obj_get_reference(src->private_object)); + rc = rt_aspace_duplicate_locked(src, dst); + WR_UNLOCK(src); + + if (!rc) + { + /* WR_LOCK(dst) is not necessary since dst is not available currently */ + rc = rt_aspace_duplicate_locked(dst, backup); + if (!rc) + { + _switch_aspace(psrc, &backup); + _convert_readonly(backup, base_reference); + } + } + } + else + { + rc = -RT_ENOMEM; + } + } + else + { + rc = -RT_ENOMEM; + } + + return rc; +} diff --git a/components/mm/mm_aspace.c b/components/mm/mm_aspace.c index 6249e44ac3..42278e684f 100644 --- a/components/mm/mm_aspace.c +++ b/components/mm/mm_aspace.c @@ -1,40 +1,41 @@ /* - * Copyright (c) 2006-2022, RT-Thread Development Team + * Copyright (c) 2006-2023, RT-Thread Development Team * * SPDX-License-Identifier: Apache-2.0 * * Change Logs: * Date Author Notes * 2022-11-14 WangXiaoyao the first version + * 2023-08-17 Shell Add unmap_range for MAP_PRIVATE + * Support MAP_FIXED in aspace_map(), and + * Add better support of permission in mmap */ /** * @brief Virtual Address Space */ -#include <rtdef.h> #include <rtthread.h> -#include <stddef.h> -#include <stdint.h> #define DBG_TAG "mm.aspace" #define DBG_LVL DBG_INFO #include <rtdbg.h> #include "avl_adpt.h" -#include "mm_aspace.h" -#include "mm_fault.h" -#include "mm_flag.h" -#include "mm_page.h" #include "mm_private.h" #include <mmu.h> #include <tlb.h> +#include <stddef.h> +#include <stdint.h> + +#define ALIGNED(addr) (!((rt_size_t)(addr) & ARCH_PAGE_MASK)) + static void *_find_free(rt_aspace_t aspace, void *prefer, rt_size_t req_size, void *limit_start, rt_size_t limit_size, mm_flag_t flags); -static void _varea_uninstall(rt_varea_t varea); +static int _unmap_range_locked(rt_aspace_t aspace, void *addr, size_t length); struct rt_aspace rt_kernel_space; @@ -56,6 +57,7 @@ rt_err_t rt_aspace_init(rt_aspace_t aspace, void *start, rt_size_t length, void aspace->page_table = pgtbl; aspace->start = start; aspace->size = length; + aspace->private_object = RT_NULL; err = _aspace_bst_init(aspace); if (err == RT_EOK) @@ -102,21 +104,25 @@ rt_aspace_t rt_aspace_create(void *start, rt_size_t length, void *pgtbl) void rt_aspace_detach(rt_aspace_t aspace) { + rt_varea_t varea; + WR_LOCK(aspace); - rt_varea_t varea = ASPACE_VAREA_FIRST(aspace); + varea = ASPACE_VAREA_FIRST(aspace); while (varea) { rt_varea_t prev = varea; - _varea_uninstall(varea); - varea = ASPACE_VAREA_NEXT(varea); - if (!(prev->flag & MMF_STATIC_ALLOC)) + + _varea_uninstall_locked(prev); + if (VAREA_NOT_STATIC(prev)) { rt_free(prev); } } WR_UNLOCK(aspace); + rt_aspace_anon_ref_dec(aspace->private_object); + rt_mutex_detach(&aspace->bst_lock); } @@ -127,16 +133,29 @@ void rt_aspace_delete(rt_aspace_t aspace) rt_free(aspace); } -static int _do_named_map(rt_aspace_t aspace, void *vaddr, rt_size_t length, - rt_size_t offset, rt_size_t attr) +rt_inline rt_size_t _get_effect_attr(rt_aspace_t aspace, rt_varea_t varea) +{ + rt_size_t attr = varea->attr; + + /* not write permission for user on private mapping */ + if (rt_varea_is_private_locked(varea)) + attr = rt_hw_mmu_attr_rm_perm(attr, RT_HW_MMU_PROT_USER | RT_HW_MMU_PROT_WRITE); + + return attr; +} + +static int _do_named_map(rt_aspace_t aspace, rt_varea_t varea, void *vaddr, + rt_size_t length, rt_size_t offset, rt_size_t attr) { LOG_D("%s: va %p length %p", __func__, vaddr, length); int err = RT_EOK; /* it's ensured by caller that (void*)end will not overflow */ void *phyaddr = (void *)(offset << MM_PAGE_SHIFT); + void *ret; - void *ret = rt_hw_mmu_map(aspace, vaddr, phyaddr, length, attr); + attr = _get_effect_attr(aspace, varea); + ret = rt_hw_mmu_map(aspace, vaddr, phyaddr, length, attr); if (ret == RT_NULL) { err = -RT_ERROR; @@ -160,11 +179,13 @@ rt_inline void _do_page_fault(struct rt_aspace_fault_msg *msg, rt_size_t off, msg->response.vaddr = 0; msg->response.size = 0; + RT_ASSERT(mem_obj->on_page_fault); mem_obj->on_page_fault(varea, msg); } -int _varea_map_with_msg(rt_varea_t varea, struct rt_aspace_fault_msg *msg) +int rt_varea_map_with_msg(rt_varea_t varea, struct rt_aspace_fault_msg *msg) { + rt_aspace_t aspace; int err = -RT_ERROR; if (msg->response.status == MM_FAULT_STATUS_OK) { @@ -182,9 +203,15 @@ int _varea_map_with_msg(rt_varea_t varea, struct rt_aspace_fault_msg *msg) else { void *map; + rt_size_t attr; void *v_addr = msg->fault_vaddr; void *p_addr = store + PV_OFFSET; - map = rt_hw_mmu_map(varea->aspace, v_addr, p_addr, store_sz, varea->attr); + + aspace = varea->aspace; + RT_ASSERT(aspace); + + attr = _get_effect_attr(aspace, varea); + map = rt_hw_mmu_map(aspace, v_addr, p_addr, store_sz, attr); if (!map) { @@ -202,12 +229,9 @@ int _varea_map_with_msg(rt_varea_t varea, struct rt_aspace_fault_msg *msg) { if (rt_hw_mmu_v2p(varea->aspace, msg->fault_vaddr) == ARCH_MAP_FAILED) { - LOG_W("%s: no page is mapped on %p", __func__, msg->fault_vaddr); - } - else - { - err = RT_EOK; + LOG_D("%s: no page is mapped on %p", __func__, msg->fault_vaddr); } + err = RT_EOK; } else { @@ -234,7 +258,7 @@ static int _do_prefetch(rt_aspace_t aspace, rt_varea_t varea, void *start, struct rt_aspace_fault_msg msg; _do_page_fault(&msg, off, vaddr, varea->mem_obj, varea); - if (_varea_map_with_msg(varea, &msg)) + if (rt_varea_map_with_msg(varea, &msg)) { err = -RT_ENOMEM; break; @@ -253,30 +277,210 @@ static int _do_prefetch(rt_aspace_t aspace, rt_varea_t varea, void *start, return err; } -/* caller must hold the aspace lock */ -static int _varea_install(rt_aspace_t aspace, rt_varea_t varea, rt_mm_va_hint_t hint) +static rt_err_t _expand_varea(rt_varea_t varea, void *new_va, rt_size_t size) { - void *alloc_va; - int err = RT_EOK; + rt_err_t error; + rt_aspace_t aspace; + void *old_va; - /** - * find a suitable va range. - * even though this is sleepable, it's still ok for startup routine - */ - alloc_va = - _find_free(aspace, hint->prefer, hint->map_size, hint->limit_start, - hint->limit_range_size, hint->flags); + if (varea->mem_obj && varea->mem_obj->on_varea_expand) + error = varea->mem_obj->on_varea_expand(varea, new_va, size); + else + error = -RT_EPERM; - /* TODO try merge surrounding regions to optimize memory footprint */ + if (error == RT_EOK) + { + aspace = varea->aspace; + old_va = varea->start; + varea->size = size; - if (alloc_va != RT_NULL) + if (old_va != new_va) + { + varea->start = new_va; + varea->offset += ((long)new_va - (long)old_va) >> MM_PAGE_SHIFT; + _aspace_bst_remove(aspace, varea); + _aspace_bst_insert(aspace, varea); + } + } + return error; +} + +struct _mapping_property { + rt_size_t attr; + rt_size_t flags; + rt_size_t offset; + struct rt_mem_obj *mem_obj; +}; + +#define INIT_PROP(obj,off,fl,attr) \ + {.mem_obj = (obj), \ + .offset = (off), \ + .flags = (fl), \ + .attr = (attr),} + +static rt_bool_t _contiguous_offset(rt_varea_t neighbour, rt_size_t map_size, + struct _mapping_property *prop) +{ + rt_size_t n_off = neighbour->offset; + rt_size_t map_off = prop->offset; + return n_off < map_off ? + n_off + (neighbour->size >> MM_PAGE_SHIFT) == map_off : + map_off + (map_size >> MM_PAGE_SHIFT) == n_off; +} + +static rt_bool_t _compatible(rt_varea_t neighbour, rt_size_t map_size, + struct _mapping_property *prop) +{ + return (prop->attr == neighbour->attr && prop->flags == neighbour->flag && + prop->mem_obj == neighbour->mem_obj && + _contiguous_offset(neighbour, map_size, prop)); +} + +rt_inline rt_err_t _migrate_and_release_varea(rt_aspace_t aspace, rt_varea_t to, rt_varea_t from, + rt_err_t (*on_varea_merge)(struct rt_varea *to, struct rt_varea *from)) +{ + rt_err_t error; + error = on_varea_merge(to, from); + if (error == RT_EOK) + { + /* uninstall operand & release the varea */ + _aspace_bst_remove(aspace, from); + if (!(from->flag & MMF_STATIC_ALLOC)) + rt_free(from); + + to->size += from->size; + } + return error; +} + +static rt_varea_t _merge_surrounding(rt_aspace_t aspace, rt_varea_t operand, + struct _mapping_property *prop) +{ + int again; + rt_err_t error; + int can_merge_fw; + int can_merge_bw; + rt_varea_t neighbour; + char *operand_start; + size_t operand_size; + rt_err_t (*on_varea_merge)(struct rt_varea *to, struct rt_varea *from); + + if (operand->mem_obj && operand->mem_obj->on_varea_merge) + { + on_varea_merge = operand->mem_obj->on_varea_merge; + do { + operand_start = operand->start; + operand_size = operand->size; + LOG_D("search op_start=%p,op_size=0x%lx", operand_start, operand_size); + + /* find a compatible neighbour if any and setup direction */ + can_merge_fw = can_merge_bw = 0; + neighbour = _aspace_bst_search(aspace, operand_start - 1); + if (!neighbour || !_compatible(neighbour, operand_size, prop)) + { + neighbour = _aspace_bst_search(aspace, operand_start + operand_size); + if (neighbour && _compatible(neighbour, operand_size, prop)) + can_merge_bw = 1; + } + else + can_merge_fw = 1; + + if (can_merge_fw || can_merge_bw) + { + /* merge operand with its predecessor or successor */ + if (can_merge_fw) + { + error = _migrate_and_release_varea(aspace, neighbour, operand, on_varea_merge); + operand = neighbour; + } + else + error = _migrate_and_release_varea(aspace, operand, neighbour, on_varea_merge); + + if (error == RT_EOK) + again = 1; + } + else + again = 0; + + } while (again); + } + return operand; +} + +/** + * Brief: expand and merge surrounding until not possible and + * setup the pvarea if new virt address region is installed + */ +static rt_bool_t _try_expand_and_merge_okay(rt_aspace_t aspace, rt_varea_t *pvarea, + void *alloc_va, rt_mm_va_hint_t hint, + struct _mapping_property *prop) +{ + int can_expand_fw; + int can_expand_bw; + rt_varea_t neighbour; + rt_varea_t new_region_at = RT_NULL; + rt_bool_t install_ok = RT_FALSE; + char *operand_start = alloc_va; + size_t operand_size = hint->map_size; + + /* find a compatible neighbour if any and setup direction */ + LOG_D("search op_start=%p,op_size=0x%lx", operand_start, operand_size); + can_expand_fw = can_expand_bw = 0; + neighbour = _aspace_bst_search(aspace, operand_start - 1); + if (!neighbour || !_compatible(neighbour, operand_size, prop)) + { + neighbour = _aspace_bst_search(aspace, operand_start + operand_size); + if (neighbour && _compatible(neighbour, operand_size, prop)) + can_expand_bw = 1; + } + else + can_expand_fw = 1; + + if (can_expand_fw || can_expand_bw) + { + /* expand varea at head or tailing */ + if (can_expand_fw) + operand_start = neighbour->start; + operand_size += neighbour->size; + + LOG_D("expand op_start=%p,op_size=0x%lx", operand_start, operand_size); + + if (_expand_varea(neighbour, operand_start, operand_size) == RT_EOK) + { + new_region_at = _merge_surrounding(aspace, neighbour, prop); + *pvarea = new_region_at; + install_ok = RT_TRUE; + } + } + + return install_ok; +} + +static rt_varea_t _varea_create(void *start, rt_size_t size); + +static int _insert_new_varea(rt_aspace_t aspace, rt_varea_t *pvarea, + void *alloc_va, rt_mm_va_hint_t hint) +{ + int err; + rt_varea_t varea = *pvarea; + if (varea == RT_NULL) + { + /* no preallocate buffer is provided, then create one */ + varea = _varea_create(hint->prefer, hint->map_size); + hint->flags &= ~MMF_STATIC_ALLOC; + *pvarea = varea; + } + + if (varea) { varea->start = alloc_va; _aspace_bst_insert(aspace, varea); + err = RT_EOK; } else { - err = -RT_ENOSPC; + LOG_W("%s: Out of memory", __func__); + err = -RT_ENOMEM; } return err; @@ -298,10 +502,64 @@ static inline void _varea_post_install(rt_varea_t varea, rt_aspace_t aspace, } /** + * Install new virtual address region into address space + * The pvarea will be set to the varea where new virt memory installed which can + * be a newly created or existed one. + * + * Note: caller must hold the aspace lock + */ +static int _varea_install(rt_aspace_t aspace, rt_varea_t *pvarea, + rt_mm_va_hint_t hint, struct _mapping_property *prop, + void **pva) +{ + void *alloc_va; + int err = RT_EOK; + + if (hint->flags & MMF_MAP_FIXED) + { + alloc_va = hint->prefer; + err = _unmap_range_locked(aspace, alloc_va, hint->map_size); + if (err != RT_EOK) + { + /* Note: MAP_FIXED must failed if unable to unmap existing mapping */ + LOG_I("%s: unmap range failed in %p with size 0x%lx, error=%d", __func__, alloc_va, hint->map_size, err); + } + } + else + { + alloc_va = + _find_free(aspace, hint->prefer, hint->map_size, hint->limit_start, + hint->limit_range_size, hint->flags); + if (alloc_va == RT_NULL) + err = -RT_ENOSPC; + } + + if (alloc_va != RT_NULL) + { + /* TODO: fix to private mapping directly */ + if (!_try_expand_and_merge_okay(aspace, pvarea, alloc_va, hint, prop)) + { + err = _insert_new_varea(aspace, pvarea, alloc_va, hint); + + if (err == RT_EOK) + _varea_post_install(*pvarea, aspace, prop->attr, prop->flags, + prop->mem_obj, prop->offset); + } + + if (err == RT_EOK) + { + RT_ASSERT(*pvarea); + *pva = alloc_va; + } + } + + return err; +} + +/** * restore context modified by varea install - * caller must NOT hold the aspace lock */ -static void _varea_uninstall(rt_varea_t varea) +void _varea_uninstall_locked(rt_varea_t varea) { rt_aspace_t aspace = varea->aspace; @@ -313,16 +571,16 @@ static void _varea_uninstall(rt_varea_t varea) rt_varea_pgmgr_pop_all(varea); - WR_LOCK(aspace); _aspace_bst_remove(aspace, varea); - WR_UNLOCK(aspace); } -static int _mm_aspace_map(rt_aspace_t aspace, rt_varea_t varea, rt_size_t attr, - mm_flag_t flags, rt_mem_obj_t mem_obj, - rt_size_t offset) +int _mm_aspace_map(rt_aspace_t aspace, rt_varea_t *pvarea, void **addr, + rt_size_t length, rt_size_t attr, mm_flag_t flags, + rt_mem_obj_t mem_obj, rt_size_t offset) { int err = RT_EOK; + rt_varea_t varea; + struct _mapping_property prop = INIT_PROP(mem_obj, offset, flags, attr); WR_LOCK(aspace); @@ -331,8 +589,8 @@ static int _mm_aspace_map(rt_aspace_t aspace, rt_varea_t varea, rt_size_t attr, * .limit_start & .limit_range_size have default to be in range of aspace * .flags is from parameter, and will be fill in varea if install successfully */ - struct rt_mm_va_hint hint = {.prefer = varea->start, - .map_size = varea->size, + struct rt_mm_va_hint hint = {.prefer = *addr, + .map_size = length, .limit_start = aspace->start, .limit_range_size = aspace->size, .flags = flags}; @@ -344,30 +602,34 @@ static int _mm_aspace_map(rt_aspace_t aspace, rt_varea_t varea, rt_size_t attr, } /* try to allocate a virtual address region for varea */ - err = _varea_install(aspace, varea, &hint); - WR_UNLOCK(aspace); + err = _varea_install(aspace, pvarea, &hint, &prop, addr); if (err == RT_EOK) { - /* fill in varea data */ - _varea_post_install(varea, aspace, attr, flags, mem_obj, offset); - + varea = *pvarea; if (MMF_TEST_CNTL(flags, MMF_PREFETCH)) { /* do the MMU & TLB business */ err = _do_prefetch(aspace, varea, varea->start, varea->size); if (err) { + LOG_I("%s: failed to prefetch page into varea(%s)", + __func__, VAREA_NAME(varea)); + /* restore data structure and MMU */ - _varea_uninstall(varea); + _varea_uninstall_locked(varea); + if (!(varea->flag & MMF_STATIC_ALLOC)) + rt_free(varea); } } } + WR_UNLOCK(aspace); + return err; } -rt_varea_t _varea_create(void *start, rt_size_t size) +static rt_varea_t _varea_create(void *start, rt_size_t size) { rt_varea_t varea; varea = (rt_varea_t)rt_malloc(sizeof(struct rt_varea)); @@ -385,13 +647,14 @@ rt_varea_t _varea_create(void *start, rt_size_t size) static inline int _not_in_range(void *start, rt_size_t length, void *limit_start, rt_size_t limit_size) { - if (start != RT_NULL) - LOG_D("%s: [%p : %p] [%p : %p]", __func__, start, length, limit_start, limit_size); /* assuming (base + length) will not overflow except (0) */ - return start != RT_NULL + int rc = start != RT_NULL ? (_IS_OVERFLOW(start, length) || start < limit_start || _IS_OVERSIZE(start, length, limit_start, limit_size)) : length > limit_size; + if (rc) + LOG_D("%s: [%p : %p] [%p : %p]", __func__, start, length, limit_start, limit_size); + return rc; } static inline int _not_align(void *start, rt_size_t length, rt_size_t mask) @@ -411,9 +674,10 @@ int rt_aspace_map(rt_aspace_t aspace, void **addr, rt_size_t length, rt_size_t attr, mm_flag_t flags, rt_mem_obj_t mem_obj, rt_size_t offset) { - /* TODO check not in atomic context: irq, spinlock, local intr disable... */ int err; - rt_varea_t varea; + rt_varea_t varea = RT_NULL; + + RT_DEBUG_SCHEDULER_AVAILABLE(1); if (!aspace || !addr || !mem_obj || length == 0) { @@ -433,26 +697,9 @@ int rt_aspace_map(rt_aspace_t aspace, void **addr, rt_size_t length, } else { - /* allocate the varea and fill in start and size */ - varea = _varea_create(*addr, length); - - if (varea) - { - err = _mm_aspace_map(aspace, varea, attr, flags, mem_obj, offset); - if (err != RT_EOK) - { - rt_free(varea); - } - else - { - *addr = varea->start; - } - } - else - { - LOG_W("%s: memory allocation failed", __func__); - err = -RT_ENOMEM; - } + RT_ASSERT((length & ARCH_PAGE_MASK) == 0); + RT_ASSERT(((long)*addr & ARCH_PAGE_MASK) == 0); + err = _mm_aspace_map(aspace, &varea, addr, length, attr, flags, mem_obj, offset); } if (err != RT_EOK) @@ -485,7 +732,11 @@ int rt_aspace_map_static(rt_aspace_t aspace, rt_varea_t varea, void **addr, varea->size = length; varea->start = *addr; flags |= MMF_STATIC_ALLOC; - err = _mm_aspace_map(aspace, varea, attr, flags, mem_obj, offset); + + /** + * todo: fix if mapping expand, the static varea is not used at all + */ + err = _mm_aspace_map(aspace, &varea, addr, length, attr, flags, mem_obj, offset); } if (err != RT_EOK) @@ -526,24 +777,22 @@ int _mm_aspace_map_phy(rt_aspace_t aspace, rt_varea_t varea, } else { - WR_LOCK(aspace); - err = _varea_install(aspace, varea, hint); - WR_UNLOCK(aspace); + struct _mapping_property prop = INIT_PROP(0, pa_off, hint->flags, attr); + WR_LOCK(aspace); + err = _varea_install(aspace, &varea, hint, &prop, &vaddr); if (err == RT_EOK) { - _varea_post_install(varea, aspace, attr, hint->flags, NULL, pa_off); - - vaddr = varea->start; - - err = _do_named_map(aspace, varea->start, varea->size, + err = _do_named_map(aspace, varea, varea->start, varea->size, (rt_size_t)pa_off, attr); if (err != RT_EOK) { - _varea_uninstall(varea); + _varea_uninstall_locked(varea); } } + + WR_UNLOCK(aspace); } if (ret_va) @@ -567,6 +816,7 @@ int rt_aspace_map_phy(rt_aspace_t aspace, rt_mm_va_hint_t hint, rt_size_t attr, rt_varea_t varea = _varea_create(hint->prefer, hint->map_size); if (varea) { + hint->flags &= ~MMF_STATIC_ALLOC; err = _mm_aspace_map_phy(aspace, varea, hint, attr, pa_off, ret_va); if (err != RT_EOK) { @@ -596,7 +846,7 @@ int rt_aspace_map_phy_static(rt_aspace_t aspace, rt_varea_t varea, { varea->start = hint->prefer; varea->size = hint->map_size; - hint->flags |= (MMF_MAP_FIXED | MMF_STATIC_ALLOC); + hint->flags |= (MMF_STATIC_ALLOC); LOG_D("%s: start %p size %p phy at %p", __func__, varea->start, varea->size, pa_off << MM_PAGE_SHIFT); err = _mm_aspace_map_phy(aspace, varea, hint, attr, pa_off, ret_va); } @@ -608,42 +858,216 @@ int rt_aspace_map_phy_static(rt_aspace_t aspace, rt_varea_t varea, return err; } -void _aspace_unmap(rt_aspace_t aspace, void *addr) +int _aspace_unmap(rt_aspace_t aspace, void *addr) { + int error; + rt_varea_t varea; + WR_LOCK(aspace); - rt_varea_t varea = _aspace_bst_search(aspace, addr); - WR_UNLOCK(aspace); + varea = _aspace_bst_search(aspace, addr); if (varea == RT_NULL) { - LOG_I("%s: No such entry found at %p\n", __func__, addr); + LOG_D("%s: No such entry found at %p\n", __func__, addr); + error = -RT_ENOENT; } - - _varea_uninstall(varea); - if (!(varea->flag & MMF_STATIC_ALLOC)) + else { - rt_free(varea); + _varea_uninstall_locked(varea); + if (!(varea->flag & MMF_STATIC_ALLOC)) + { + rt_free(varea); + } + error = RT_EOK; } + + WR_UNLOCK(aspace); + return error; } int rt_aspace_unmap(rt_aspace_t aspace, void *addr) { + int error; if (!aspace) { LOG_I("%s: Invalid input", __func__); - return -RT_EINVAL; + error = -RT_EINVAL; + } + else if (_not_in_range(addr, 1, aspace->start, aspace->size)) + { + LOG_I("%s: %lx not in range of aspace[%lx:%lx]", __func__, addr, + aspace->start, (char *)aspace->start + aspace->size); + error = -RT_EINVAL; + } + else + { + error = _aspace_unmap(aspace, addr); } - if (_not_in_range(addr, 1, aspace->start, aspace->size)) + return error; +} + +/** + * modify the property of existed varea by shrink its size. Mem_obj is + * notified to released the resource. + */ +static rt_err_t _shrink_varea(rt_varea_t varea, void *new_va, rt_size_t size) +{ + rt_err_t error; + rt_aspace_t aspace; + void *old_va; + + if (varea->mem_obj && varea->mem_obj->on_varea_shrink) + error = varea->mem_obj->on_varea_shrink(varea, new_va, size); + else + error = -RT_EPERM; + + if (error == RT_EOK) + { + aspace = varea->aspace; + old_va = varea->start; + varea->size = size; + + if (old_va != new_va) + { + varea->start = new_va; + varea->offset += ((long)new_va - (long)old_va) >> MM_PAGE_SHIFT; + _aspace_bst_remove(aspace, varea); + _aspace_bst_insert(aspace, varea); + } + } + return error; +} + +static rt_err_t _split_varea(rt_varea_t existed, char *ex_end, char *unmap_start, char *unmap_end, rt_size_t unmap_len) +{ + int error; + size_t rela_offset; + rt_varea_t subset; + char *subset_start; + size_t subset_size; + + if (existed->mem_obj && existed->mem_obj->on_varea_split) + { + subset_start = unmap_end; + subset_size = ex_end - subset_start; + subset = _varea_create(subset_start, subset_size); + if (subset) + { + rela_offset = MM_PA_TO_OFF(subset_start - (char *)existed->start); + subset->aspace = existed->aspace; + subset->attr = existed->attr; + subset->mem_obj = existed->mem_obj; + subset->flag = existed->flag & ~MMF_STATIC_ALLOC; + subset->offset = existed->offset + rela_offset; + subset->frames = NULL; + + error = existed->mem_obj->on_varea_split(existed, unmap_start, unmap_len, subset); + if (error == RT_EOK) + { + existed->size = unmap_start - (char *)existed->start; + _aspace_bst_insert(existed->aspace, subset); + } + + if (error != RT_EOK) + rt_free(subset); + } + else + error = -RT_ENOMEM; + } + else + error = -RT_EPERM; + + return error; +} + +/* remove overlapped pages from varea */ +static int _remove_overlapped_varea(rt_varea_t existed, char *unmap_start, rt_size_t unmap_len) +{ + int error; + char *ex_start = existed->start; + char *ex_end = ex_start + existed->size; + char *unmap_end = unmap_start + unmap_len; + + if (ex_start < unmap_start) + { + if (ex_end > unmap_end) + error = _split_varea(existed, ex_end, unmap_start, unmap_end, unmap_len); + else + error = _shrink_varea(existed, ex_start, unmap_start - ex_start); + } + else if (ex_end > unmap_end) + error = _shrink_varea(existed, unmap_end, ex_end - unmap_end); + else + { + _varea_uninstall_locked(existed); + if (VAREA_NOT_STATIC(existed)) + { + rt_free(existed); + } + error = RT_EOK; + } + + return error; +} + +static int _unmap_range_locked(rt_aspace_t aspace, void *addr, size_t length) +{ + int error = RT_EOK; + rt_varea_t existed; + struct _mm_range unmap_range; + + unmap_range.start = addr; + unmap_range.end = addr + length - 1; + + existed = _aspace_bst_search_overlap(aspace, unmap_range); + while (existed) + { + error = _remove_overlapped_varea(existed, addr, length); + + if (error == RT_EOK) + existed = _aspace_bst_search_overlap(aspace, unmap_range); + else + break; + } + + return error; +} + +int rt_aspace_unmap_range(rt_aspace_t aspace, void *addr, size_t length) +{ + int error; + + if (!aspace) + { + LOG_I("%s: Invalid input", __func__); + error = -RT_EINVAL; + } + else if (_not_in_range(addr, length, aspace->start, aspace->size)) { LOG_I("%s: %lx not in range of aspace[%lx:%lx]", __func__, addr, aspace->start, (char *)aspace->start + aspace->size); - return -RT_EINVAL; + error = -RT_EINVAL; } + else if (!ALIGNED(addr)) + { + LOG_I("%s(addr=%p): Unaligned address", __func__, addr); + error = -RT_EINVAL; + } + else + { + /** + * Brief: re-arrange the address space to remove existing pages mapping + * in [unmap_start, unmap_start + unmap_len) + */ + length = RT_ALIGN(length, ARCH_PAGE_SIZE); - _aspace_unmap(aspace, addr); + WR_LOCK(aspace); + error = _unmap_range_locked(aspace, addr, length); + WR_UNLOCK(aspace); + } - return RT_EOK; + return error; } static inline void *_lower(void *a, void *b) @@ -843,6 +1267,7 @@ int rt_varea_map_page(rt_varea_t varea, void *vaddr, void *page) { err = _do_named_map( varea->aspace, + varea, vaddr, ARCH_PAGE_SIZE, MM_PA_TO_OFF(page_pa), @@ -853,8 +1278,6 @@ int rt_varea_map_page(rt_varea_t varea, void *vaddr, void *page) return err; } -#define ALIGNED(addr) (!((rt_size_t)(addr) & ARCH_PAGE_MASK)) - int rt_varea_unmap_page(rt_varea_t varea, void *vaddr) { void *va_aligned = (void *)RT_ALIGN_DOWN((rt_base_t)vaddr, ARCH_PAGE_SIZE); @@ -883,6 +1306,7 @@ int rt_varea_map_range(rt_varea_t varea, void *vaddr, void *paddr, rt_size_t len { err = _do_named_map( varea->aspace, + varea, vaddr, length, MM_PA_TO_OFF(paddr), @@ -986,3 +1410,353 @@ void rt_aspace_print_all(rt_aspace_t aspace) { rt_aspace_traversal(aspace, _dump, NULL); } + +static int _count_vsz(rt_varea_t varea, void *arg) +{ + rt_base_t *pvsz = arg; + RT_ASSERT(varea); + *pvsz = *pvsz + varea->size; + return 0; +} + +rt_base_t rt_aspace_count_vsz(rt_aspace_t aspace) +{ + rt_base_t vsz = 0; + rt_aspace_traversal(aspace, _count_vsz, &vsz); + return vsz; +} + +static int _dup_varea(rt_varea_t src_varea, void *arg) +{ + int err; + rt_aspace_t dst = arg; + rt_aspace_t src = src_varea->aspace; + + void *pa = RT_NULL; + void *va = RT_NULL; + rt_mem_obj_t mem_obj = src_varea->mem_obj; + + if (!mem_obj) + { + /* duplicate a physical mapping */ + pa = rt_hw_mmu_v2p(src, (void *)src_varea->start); + RT_ASSERT(pa != ARCH_MAP_FAILED); + struct rt_mm_va_hint hint = {.flags = src_varea->flag, + .limit_range_size = dst->size, + .limit_start = dst->start, + .prefer = src_varea->start, + .map_size = src_varea->size}; + err = rt_aspace_map_phy(dst, &hint, src_varea->attr, + MM_PA_TO_OFF(pa), &va); + if (err != RT_EOK) + { + LOG_W("%s: aspace map failed at %p with size %p", __func__, + src_varea->start, src_varea->size); + } + } + else + { + /* duplicate a mem_obj backing mapping */ + rt_base_t flags = src_varea->flag | MMF_MAP_FIXED; + flags &= ~MMF_STATIC_ALLOC; + flags &= ~MMF_PREFETCH; + va = src_varea->start; + + err = rt_aspace_map(dst, &va, src_varea->size, src_varea->attr, + flags, mem_obj, src_varea->offset); + if (err != RT_EOK) + { + LOG_W("%s: aspace map failed at %p with size %p", __func__, + src_varea->start, src_varea->size); + } + } + + if (va != (void *)src_varea->start) + { + return -1; + } + return 0; +} + +struct _compare_param { + rt_aspace_t dst; + int rc; +}; + +rt_err_t rt_aspace_duplicate_locked(rt_aspace_t src, rt_aspace_t dst) +{ + return rt_aspace_traversal(src, _dup_varea, dst); +} + +rt_inline int _varea_same(rt_varea_t a, rt_varea_t b) +{ + return a->attr == b->attr && a->flag == b->flag && a->mem_obj == b->mem_obj; +} + +rt_inline void _dump_varea(rt_varea_t varea) +{ + LOG_W("%s(attr=0x%lx, flags=0x%lx, start=0x%lx, size=0x%lx, mem_obj=%p)", VAREA_NAME(varea), varea->attr, varea->flag, varea->start, varea->size, varea->mem_obj); +} + +static int _compare_varea(rt_varea_t src_varea, void *arg) +{ + struct _compare_param *param = arg; + rt_varea_t dst_varea; + rt_aspace_t dst = param->dst; + rt_aspace_t src = src_varea->aspace; + + dst_varea = _aspace_bst_search(dst, src_varea->start); + if (dst_varea) + { + char *buf1 = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE); + char *buf2 = rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE); + char *vend = src_varea->start + src_varea->size; + for (char *i = src_varea->start; i < vend; i += ARCH_PAGE_SIZE) + { + int rc; + rt_aspace_page_get(src, i, buf1); + rt_aspace_page_get(dst, i, buf2); + rc = memcmp(buf1, buf2, ARCH_PAGE_SIZE); + if (rc) + { + if (param->rc == 0) + param->rc = rc; + LOG_E("%s(a_varea=%s, b_varea=%s)", __func__, VAREA_NAME(src_varea), VAREA_NAME(dst_varea)); + _dump_varea(src_varea); + _dump_varea(dst_varea); + RT_ASSERT(0); + } + } + + rt_pages_free(buf1, 0); + rt_pages_free(buf2, 0); + } + else + { + param->rc = -RT_ENOENT; + } + + return 0; +} + +rt_err_t rt_aspace_compare(rt_aspace_t src, rt_aspace_t dst) +{ + struct _compare_param param = {.rc = 0, .dst = dst}; + rt_aspace_traversal(src, _compare_varea, ¶m); + return param.rc; +} + +/* dst are page aligned */ +rt_inline rt_err_t _page_put(rt_varea_t varea, void *page_va, void *buffer) +{ + struct rt_aspace_io_msg iomsg; + rt_err_t rc; + + rt_mm_io_msg_init(&iomsg, VAREA_VA_TO_OFFSET(varea, page_va), page_va, buffer); + varea->mem_obj->page_write(varea, &iomsg); + + if (iomsg.response.status == MM_FAULT_STATUS_UNRECOVERABLE) + rc = -RT_ERROR; + else + rc = RT_EOK; + return rc; +} + +/* dst are page aligned */ +rt_inline rt_err_t _page_get(rt_varea_t varea, void *page_va, void *buffer) +{ + struct rt_aspace_io_msg iomsg; + rt_err_t rc; + + rt_mm_io_msg_init(&iomsg, VAREA_VA_TO_OFFSET(varea, page_va), page_va, buffer); + varea->mem_obj->page_read(varea, &iomsg); + + if (iomsg.response.status == MM_FAULT_STATUS_UNRECOVERABLE) + rc = -RT_ERROR; + else + rc = RT_EOK; + return rc; +} + +#ifdef RT_USING_SMART +#include "lwp.h" +rt_inline rt_aspace_t _current_uspace(void) +{ + rt_lwp_t this_proc = lwp_self(); + return this_proc ? this_proc->aspace : RT_NULL; +} +#else +rt_inline rt_aspace_t _current_uspace(void) +{ + return RT_NULL; +} +#endif + +rt_err_t rt_aspace_page_get_phy(rt_aspace_t aspace, void *page_va, void *buffer) +{ + rt_err_t rc = -RT_ERROR; + + char *frame_ka = rt_hw_mmu_v2p(aspace, page_va); + if (frame_ka != ARCH_MAP_FAILED) + { + frame_ka = rt_kmem_p2v(frame_ka); + if (frame_ka) + { + rt_memcpy(buffer, frame_ka, ARCH_PAGE_SIZE); + rc = RT_EOK; + } + else if (aspace == _current_uspace() || aspace == &rt_kernel_space) + { + /* direct IO */ + rt_memcpy(buffer, page_va, ARCH_PAGE_SIZE); + rc = RT_EOK; + } + else + { + /* user memory region remap ? */ + LOG_W("%s(aspace=0x%lx,va=%p): Operation not support", + __func__, aspace, page_va); + rc = -RT_ENOSYS; + } + } + else + { + LOG_W("%s(aspace=0x%lx,va=%p): PTE not existed", + __func__, aspace, page_va); + rc = -RT_ENOENT; + } + return rc; +} + +rt_err_t rt_aspace_page_put_phy(rt_aspace_t aspace, void *page_va, void *buffer) +{ + rt_err_t rc = -RT_ERROR; + + char *frame_ka = rt_hw_mmu_v2p(aspace, page_va); + if (frame_ka != ARCH_MAP_FAILED) + { + frame_ka = rt_kmem_p2v(frame_ka); + if (frame_ka) + { + rt_memcpy(frame_ka, buffer, ARCH_PAGE_SIZE); + rc = RT_EOK; + } + else if (aspace == _current_uspace() || aspace == &rt_kernel_space) + { + /* direct IO */ + rt_memcpy(page_va, buffer, ARCH_PAGE_SIZE); + rc = RT_EOK; + } + else + { + /* user memory region remap ? */ + LOG_W("%s(aspace=0x%lx,va=%p): Operation not support", + __func__, aspace, page_va); + rc = -RT_ENOSYS; + } + } + else + { + LOG_W("%s(aspace=0x%lx,va=%p): PTE not existed", + __func__, aspace, page_va); + rc = -RT_ENOENT; + } + + return rc; +} + +rt_err_t rt_aspace_page_put(rt_aspace_t aspace, void *page_va, void *buffer) +{ + rt_err_t rc = -RT_ERROR; + rt_varea_t varea; + + RT_ASSERT(aspace); + RD_LOCK(aspace); + varea = _aspace_bst_search(aspace, page_va); + if (varea && ALIGNED(page_va)) + { + if (varea->mem_obj) + { + if (varea->mem_obj->page_write) + { + if (rt_varea_is_private_locked(varea)) + { + RDWR_LOCK(aspace); + struct rt_aspace_fault_msg msg; + msg.fault_op = MM_FAULT_OP_WRITE; + msg.fault_type = MM_FAULT_TYPE_ACCESS_FAULT; + msg.fault_vaddr = page_va; + rc = rt_varea_fix_private_locked(varea, rt_hw_mmu_v2p(aspace, page_va), + &msg, RT_TRUE); + RDWR_UNLOCK(aspace); + if (rc == MM_FAULT_FIXABLE_TRUE) + { + varea = _aspace_bst_search(aspace, page_va); + rc = _page_put(varea, page_va, buffer); + } + else + rc = -RT_ERROR; + } + else + rc = _page_put(varea, page_va, buffer); + } + else + { + rc = -RT_EINVAL; + LOG_I("%s: Operation not allowed", __func__); + } + } + else + { + rc = rt_aspace_page_put_phy(aspace, page_va, buffer); + } + } + else + rc = -RT_EINVAL; + RD_UNLOCK(aspace); + + return rc; +} + +rt_err_t rt_aspace_page_get(rt_aspace_t aspace, void *page_va, void *buffer) +{ + rt_err_t rc = -RT_ERROR; + rt_varea_t varea; + + /* TODO: cache the last search item */ + RT_ASSERT(aspace); + RD_LOCK(aspace); + varea = _aspace_bst_search(aspace, page_va); + if (varea && ALIGNED(page_va)) + { + if (varea->mem_obj) + { + if (varea->mem_obj->page_read) + { + rc = _page_get(varea, page_va, buffer); + } + else + { + LOG_I("%s: Operation not allowed", __func__); + } + } + else + { + rc = rt_aspace_page_get_phy(aspace, page_va, buffer); + } + } + else + { + rc = -RT_EINVAL; + LOG_D("%s(va=%p,varea=0x%lx): Invalid address", + __func__, page_va, varea); + } + RD_UNLOCK(aspace); + + return rc; +} + +rt_varea_t rt_aspace_query(rt_aspace_t aspace, void *vaddr) +{ + return _aspace_bst_search(aspace, vaddr); +} diff --git a/components/mm/mm_aspace.h b/components/mm/mm_aspace.h index 6c24040368..5748647903 100644 --- a/components/mm/mm_aspace.h +++ b/components/mm/mm_aspace.h @@ -1,23 +1,26 @@ /* - * Copyright (c) 2006-2022, RT-Thread Development Team + * Copyright (c) 2006-2023, RT-Thread Development Team * * SPDX-License-Identifier: Apache-2.0 * * Change Logs: * Date Author Notes * 2022-11-14 WangXiaoyao the first version + * 2023-08-17 Shell Add unmap_range for MAP_PRIVATE */ #ifndef __MM_ASPACE_H__ #define __MM_ASPACE_H__ #include <rthw.h> #include <rtthread.h> -#include <stddef.h> #include "avl_adpt.h" #include "mm_fault.h" #include "mm_flag.h" +#include <stddef.h> +#include <string.h> + #define MM_PAGE_SHIFT 12 #define MM_PA_TO_OFF(pa) ((uintptr_t)(pa) >> MM_PAGE_SHIFT) #define PV_OFFSET (rt_kmem_pvoff()) @@ -55,6 +58,7 @@ typedef struct rt_aspace struct _aspace_tree tree; struct rt_mutex bst_lock; + struct rt_mem_obj *private_object; rt_uint64_t asid; } *rt_aspace_t; @@ -97,7 +101,25 @@ typedef struct rt_mem_obj /* do post close bushiness like def a ref */ void (*on_varea_close)(struct rt_varea *varea); - void (*on_page_offload)(struct rt_varea *varea, void *vaddr, rt_size_t size); + /* do preparation for address space modification of varea */ + rt_err_t (*on_varea_shrink)(struct rt_varea *varea, void *new_vaddr, rt_size_t size); + /* do preparation for address space modification of varea */ + rt_err_t (*on_varea_expand)(struct rt_varea *varea, void *new_vaddr, rt_size_t size); + /** + * this is like an on_varea_open() to `subset`, and an on_varea_shrink() to `existed` + * while resource can migrate from `existed` to `subset` at the same time + */ + rt_err_t (*on_varea_split)(struct rt_varea *existed, void *unmap_start, + rt_size_t unmap_len, struct rt_varea *subset); + /** + * this is like a on_varea_expand() to `merge_to` and on_varea_close() to `merge_from` + * while resource can migrate from `merge_from` to `merge_to` at the same time + */ + rt_err_t (*on_varea_merge)(struct rt_varea *merge_to, struct rt_varea *merge_from); + + /* dynamic mem_obj API */ + void (*page_read)(struct rt_varea *varea, struct rt_aspace_io_msg *msg); + void (*page_write)(struct rt_varea *varea, struct rt_aspace_io_msg *msg); const char *(*get_name)(rt_varea_t varea); } *rt_mem_obj_t; @@ -110,6 +132,8 @@ enum rt_mmu_cntl MMU_CNTL_CACHE, MMU_CNTL_READONLY, MMU_CNTL_READWRITE, + MMU_CNTL_OFFLOAD, + MMU_CNTL_INSTALL, MMU_CNTL_DUMMY_END, }; @@ -122,8 +146,11 @@ enum rt_mmu_cntl #define WR_UNLOCK(aspace) \ rt_thread_self() ? rt_mutex_release(&(aspace)->bst_lock) : 0 +/* FIXME: fix rd_lock */ #define RD_LOCK(aspace) WR_LOCK(aspace) #define RD_UNLOCK(aspace) WR_UNLOCK(aspace) +#define RDWR_LOCK(aspace) ((void)aspace) +#define RDWR_UNLOCK(aspace) ((void)aspace) rt_aspace_t rt_aspace_create(void *start, rt_size_t length, void *pgtbl); @@ -176,14 +203,50 @@ int rt_aspace_map_phy_static(rt_aspace_t aspace, rt_varea_t varea, rt_mm_va_hint_t hint, rt_size_t attr, rt_size_t pa_off, void **ret_va); +/** map a private memory region to aspace */ +int rt_aspace_map_private(rt_aspace_t aspace, void **addr, rt_size_t length, + rt_size_t attr, mm_flag_t flags); + /** - * @brief Remove any mappings overlap the range [addr, addr + bytes) + * @brief Remove mappings containing address specified by addr * - * @param aspace + * @param aspace target virtual address space + * @param addr addresses that mapping to be removed contains + * @return int rt errno + */ +int rt_aspace_unmap(rt_aspace_t aspace, void *addr); + +/** + * @brief Remove pages of existed mappings in the range [addr, addr+length) + * Length is automatically rounded up to the next multiple of the page size. + * + * @param aspace target virtual address space + * @param addr the beginning of the range of pages to be unmapped + * @param length length of range in bytes + * @return int rt errno + */ +int rt_aspace_unmap_range(rt_aspace_t aspace, void *addr, size_t length); + +/** + * @brief Remove pages of existed mappings in the range [addr, addr+length) + * Length is automatically rounded up to the next multiple of the page size. + * + * @param aspace target virtual address space + * @param addr the beginning of the range of pages to be unmapped + * @param length length of range in bytes + * @return int rt errno + */ +int rt_aspace_unmap_range(rt_aspace_t aspace, void *addr, size_t length); + +/** + * @brief Remove pages of existed mappings in the range [addr, addr+length) + * Length is automatically rounded up to the next multiple of the page size. + * + * @param aspace target virtual address space * @param addr * @return int */ -int rt_aspace_unmap(rt_aspace_t aspace, void *addr); +int rt_aspace_unmap_range(rt_aspace_t aspace, void *addr, size_t length); int rt_aspace_control(rt_aspace_t aspace, void *addr, enum rt_mmu_cntl cmd); @@ -191,14 +254,28 @@ int rt_aspace_load_page(rt_aspace_t aspace, void *addr, rt_size_t npage); int rt_aspace_offload_page(rt_aspace_t aspace, void *addr, rt_size_t npage); +rt_err_t rt_aspace_page_put(rt_aspace_t aspace, void *page_va, void *buffer); + +rt_err_t rt_aspace_page_get(rt_aspace_t aspace, void *page_va, void *buffer); + int rt_aspace_traversal(rt_aspace_t aspace, int (*fn)(rt_varea_t varea, void *arg), void *arg); void rt_aspace_print_all(rt_aspace_t aspace); +rt_base_t rt_aspace_count_vsz(rt_aspace_t aspace); + +rt_varea_t rt_aspace_query(rt_aspace_t aspace, void *vaddr); + +rt_err_t rt_aspace_duplicate_locked(rt_aspace_t src, rt_aspace_t dst); +rt_err_t rt_aspace_fork(rt_aspace_t *psrc, rt_aspace_t *pdst); +rt_err_t rt_aspace_compare(rt_aspace_t src, rt_aspace_t dst); + /** * @brief Map one page to varea * + * @note caller should take the read/write lock + * * @param varea target varea * @param addr user address * @param page the page frame to be mapped @@ -209,6 +286,8 @@ int rt_varea_map_page(rt_varea_t varea, void *vaddr, void *page); /** * @brief Unmap one page in varea * + * @note caller should take the read/write lock + * * @param varea target varea * @param addr user address * @param page the page frame to be mapped @@ -252,7 +331,16 @@ int rt_varea_unmap_range(rt_varea_t varea, void *vaddr, rt_size_t length); */ void rt_varea_pgmgr_insert(rt_varea_t varea, void *page_addr); -rt_ubase_t rt_kmem_pvoff(void); +rt_inline rt_mem_obj_t rt_mem_obj_create(rt_mem_obj_t source) +{ + rt_mem_obj_t target; + target = rt_malloc(sizeof(*target)); + if (target) + memcpy(target, source, sizeof(*target)); + return target; +} + +const rt_ubase_t rt_kmem_pvoff(void); void rt_kmem_pvoff_set(rt_ubase_t pvoff); @@ -260,6 +348,8 @@ int rt_kmem_map_phy(void *va, void *pa, rt_size_t length, rt_size_t attr); void *rt_kmem_v2p(void *vaddr); +void *rt_kmem_p2v(void *paddr); + void rt_kmem_list(void); #endif /* __MM_ASPACE_H__ */ diff --git a/components/mm/mm_fault.c b/components/mm/mm_fault.c index a73c173acf..4dcef168a5 100644 --- a/components/mm/mm_fault.c +++ b/components/mm/mm_fault.c @@ -6,6 +6,7 @@ * Change Logs: * Date Author Notes * 2022-12-06 WangXiaoyao the first version + * 2023-08-19 Shell Support PRIVATE mapping and COW */ #include <rtthread.h> @@ -23,27 +24,21 @@ #include <mmu.h> #include <tlb.h> -#define UNRECOVERABLE 0 -#define RECOVERABLE 1 - static int _fetch_page(rt_varea_t varea, struct rt_aspace_fault_msg *msg) { - int err = UNRECOVERABLE; - msg->response.status = MM_FAULT_STATUS_UNRECOVERABLE; - msg->response.vaddr = 0; - msg->response.size = 0; + int err = MM_FAULT_FIXABLE_FALSE; if (varea->mem_obj && varea->mem_obj->on_page_fault) { varea->mem_obj->on_page_fault(varea, msg); - err = _varea_map_with_msg(varea, msg); - err = (err == RT_EOK ? RECOVERABLE : UNRECOVERABLE); + err = rt_varea_map_with_msg(varea, msg); + err = (err == RT_EOK ? MM_FAULT_FIXABLE_TRUE : MM_FAULT_FIXABLE_FALSE); } return err; } static int _read_fault(rt_varea_t varea, void *pa, struct rt_aspace_fault_msg *msg) { - int err = UNRECOVERABLE; + int err = MM_FAULT_FIXABLE_FALSE; if (msg->fault_type == MM_FAULT_TYPE_PAGE_FAULT) { RT_ASSERT(pa == ARCH_MAP_FAILED); @@ -59,19 +54,37 @@ static int _read_fault(rt_varea_t varea, void *pa, struct rt_aspace_fault_msg *m static int _write_fault(rt_varea_t varea, void *pa, struct rt_aspace_fault_msg *msg) { - int err = UNRECOVERABLE; - if (msg->fault_type == MM_FAULT_TYPE_PAGE_FAULT) + rt_aspace_t aspace = varea->aspace; + int err = MM_FAULT_FIXABLE_FALSE; + + if (rt_varea_is_private_locked(varea)) + { + if (VAREA_IS_WRITABLE(varea) && ( + msg->fault_type == MM_FAULT_TYPE_ACCESS_FAULT || + msg->fault_type == MM_FAULT_TYPE_PAGE_FAULT)) + { + RDWR_LOCK(aspace); + err = rt_varea_fix_private_locked(varea, pa, msg, RT_FALSE); + RDWR_UNLOCK(aspace); + if (err == MM_FAULT_FIXABLE_FALSE) + LOG_I("%s: fix private failure", __func__); + } + else + { + LOG_I("%s: No permission on %s(attr=0x%lx)", __func__, VAREA_NAME(varea), varea->attr); + } + } + else if (msg->fault_type == MM_FAULT_TYPE_PAGE_FAULT) { RT_ASSERT(pa == ARCH_MAP_FAILED); RT_ASSERT(!(varea->flag & MMF_PREFETCH)); err = _fetch_page(varea, msg); - } - else if (msg->fault_type == MM_FAULT_TYPE_ACCESS_FAULT && - varea->flag & MMF_COW) - { + if (err == MM_FAULT_FIXABLE_FALSE) + LOG_I("%s: page fault failure", __func__); } else { + LOG_D("%s: can not fix", __func__); /* signal a fault to user? */ } return err; @@ -79,7 +92,7 @@ static int _write_fault(rt_varea_t varea, void *pa, struct rt_aspace_fault_msg * static int _exec_fault(rt_varea_t varea, void *pa, struct rt_aspace_fault_msg *msg) { - int err = UNRECOVERABLE; + int err = MM_FAULT_FIXABLE_FALSE; if (msg->fault_type == MM_FAULT_TYPE_PAGE_FAULT) { RT_ASSERT(pa == ARCH_MAP_FAILED); @@ -91,10 +104,13 @@ static int _exec_fault(rt_varea_t varea, void *pa, struct rt_aspace_fault_msg *m int rt_aspace_fault_try_fix(rt_aspace_t aspace, struct rt_aspace_fault_msg *msg) { - int err = UNRECOVERABLE; + int err = MM_FAULT_FIXABLE_FALSE; uintptr_t va = (uintptr_t)msg->fault_vaddr; va &= ~ARCH_PAGE_MASK; msg->fault_vaddr = (void *)va; + rt_mm_fault_res_init(&msg->response); + + RT_DEBUG_SCHEDULER_AVAILABLE(1); if (aspace) { @@ -105,21 +121,34 @@ int rt_aspace_fault_try_fix(rt_aspace_t aspace, struct rt_aspace_fault_msg *msg) if (varea) { void *pa = rt_hw_mmu_v2p(aspace, msg->fault_vaddr); - msg->off = ((char *)msg->fault_vaddr - (char *)varea->start) >> ARCH_PAGE_SHIFT; - - /* permission checked by fault op */ - switch (msg->fault_op) + if (pa != ARCH_MAP_FAILED && msg->fault_type == MM_FAULT_TYPE_PAGE_FAULT) { - case MM_FAULT_OP_READ: - err = _read_fault(varea, pa, msg); - break; - case MM_FAULT_OP_WRITE: - err = _write_fault(varea, pa, msg); - break; - case MM_FAULT_OP_EXECUTE: - err = _exec_fault(varea, pa, msg); - break; + LOG_D("%s(fault=%p) has already fixed", __func__, msg->fault_vaddr); + err = MM_FAULT_FIXABLE_TRUE; } + else + { + LOG_D("%s(varea=%s,fault=%p,fault_op=%d,phy=%p)", __func__, VAREA_NAME(varea), msg->fault_vaddr, msg->fault_op, pa); + msg->off = varea->offset + ((long)msg->fault_vaddr - (long)varea->start) / ARCH_PAGE_SIZE; + + /* permission checked by fault op */ + switch (msg->fault_op) + { + case MM_FAULT_OP_READ: + err = _read_fault(varea, pa, msg); + break; + case MM_FAULT_OP_WRITE: + err = _write_fault(varea, pa, msg); + break; + case MM_FAULT_OP_EXECUTE: + err = _exec_fault(varea, pa, msg); + break; + } + } + } + else + { + LOG_I("%s: varea not found at 0x%lx", __func__, msg->fault_vaddr); } RD_UNLOCK(aspace); } diff --git a/components/mm/mm_fault.h b/components/mm/mm_fault.h index eeb413025f..28671aef74 100644 --- a/components/mm/mm_fault.h +++ b/components/mm/mm_fault.h @@ -20,12 +20,8 @@ #define MM_FAULT_STATUS_OK_MAPPED 1 #define MM_FAULT_STATUS_UNRECOVERABLE 4 -struct rt_mm_fault_res -{ - void *vaddr; - rt_size_t size; - int status; -}; +#define MM_FAULT_FIXABLE_FALSE 0 +#define MM_FAULT_FIXABLE_TRUE 1 enum rt_mm_fault_op { @@ -36,10 +32,41 @@ enum rt_mm_fault_op enum rt_mm_fault_type { + /** + * Occurs when an instruction attempts to access a memory address that it + * does not have permission to access + */ MM_FAULT_TYPE_ACCESS_FAULT, + + /** + * Occurs when a load or store instruction accesses a virtual memory + * address that is not currently mapped to a physical memory page + */ MM_FAULT_TYPE_PAGE_FAULT, + + /** + * Occurs like a SIGBUS + */ MM_FAULT_TYPE_BUS_ERROR, + MM_FAULT_TYPE_GENERIC, + __PRIVATE_PAGE_INSERT, +}; + +enum rt_mm_hint_prefetch +{ + MM_FAULT_HINT_PREFETCH_NONE, + MM_FAULT_HINT_PREFETCH_READY, +}; + +struct rt_mm_fault_res +{ + void *vaddr; + rt_size_t size; + int status; + + /* hint for prefetch strategy */ + enum rt_mm_hint_prefetch hint; }; struct rt_aspace_fault_msg @@ -52,8 +79,36 @@ struct rt_aspace_fault_msg struct rt_mm_fault_res response; }; +struct rt_aspace_io_msg +{ + /* offset in varea */ + rt_size_t off; + /* fault address in target address space */ + void *fault_vaddr; + /* read/write buffer in kernel space */ + void *buffer_vaddr; + + struct rt_mm_fault_res response; +}; + +rt_inline void rt_mm_fault_res_init(struct rt_mm_fault_res *res) +{ + res->vaddr = RT_NULL; + res->size = 0; + res->hint = MM_FAULT_HINT_PREFETCH_NONE; + res->status = MM_FAULT_STATUS_UNRECOVERABLE; +} + +rt_inline void rt_mm_io_msg_init(struct rt_aspace_io_msg *io, rt_size_t off, void *fault_vaddr, void *buffer_vaddr) +{ + io->off = off; + io->fault_vaddr = fault_vaddr; + io->buffer_vaddr = buffer_vaddr; + rt_mm_fault_res_init(&io->response); +} + struct rt_aspace; -/* MMU base page fault handler, return 1 is fixable */ +/* MMU base page fault handler, MM_FAULT_FIXABLE_TRUE/MM_FAULT_FIXABLE_FALSE will be returned */ int rt_aspace_fault_try_fix(struct rt_aspace *aspace, struct rt_aspace_fault_msg *msg); #endif /* __MM_FAULT_H__ */ diff --git a/components/mm/mm_flag.h b/components/mm/mm_flag.h index 4104a60b8f..6d58027b36 100644 --- a/components/mm/mm_flag.h +++ b/components/mm/mm_flag.h @@ -10,6 +10,8 @@ #ifndef __MM_FLAG_H__ #define __MM_FLAG_H__ +#include <rtthread.h> + /** * @brief mm_flag_t * |max ------- 7|6 ----- 0| @@ -27,10 +29,19 @@ typedef unsigned long mm_flag_t; enum mm_flag_cntl { /** - * @brief Indicate a possible COW mapping + * @brief Modifications to the mapped data shall be visible only to the + * aspace only and shall not change the underlying object. It is + * unspecified whether modifications to the underlying object done after + * the MAP_PRIVATE mapping is established are visible through the + * MAP_PRIVATE mapping. */ MMF_MAP_PRIVATE = _DEF_FLAG(0), - MMF_COW = _DEF_FLAG(1), + + /** + * @brief Same as MMF_MAP_PRIVATE, except the modification after mapping is + * invisible to the varea + */ + MMF_MAP_PRIVATE_DONT_SYNC = _DEF_FLAG(1), /** * @brief [POSIX MAP_FIXED] When MAP_FIXED is set in the flags argument, the diff --git a/components/mm/mm_kmem.c b/components/mm/mm_kmem.c index f981b36a83..7299435cb3 100644 --- a/components/mm/mm_kmem.c +++ b/components/mm/mm_kmem.c @@ -27,7 +27,7 @@ void rt_kmem_list(void) __attribute__((alias("list_kmem"))); static rt_ubase_t rt_pv_offset; -rt_ubase_t rt_kmem_pvoff(void) +const rt_ubase_t rt_kmem_pvoff(void) { return rt_pv_offset; } @@ -78,3 +78,29 @@ void *rt_kmem_v2p(void *vaddr) { return rt_hw_mmu_v2p(&rt_kernel_space, vaddr); } + +void *rt_kmem_p2v(void *paddr) +{ + char *rc; + char *linear_va; + char *linear_pa; + if (paddr != ARCH_MAP_FAILED) + { + linear_va = (char *)paddr - PV_OFFSET; + linear_pa = rt_kmem_v2p(linear_va); + if (linear_pa != paddr) + { + rc = RT_NULL; + } + else + { + rc = linear_va; + } + } + else + { + rc = RT_NULL; + } + + return rc; +} diff --git a/components/mm/mm_object.c b/components/mm/mm_object.c index 3b4ccddc89..f26b3173f5 100644 --- a/components/mm/mm_object.c +++ b/components/mm/mm_object.c @@ -6,8 +6,13 @@ * Change Logs: * Date Author Notes * 2022-11-30 WangXiaoyao the first version + * 2023-08-19 Shell Support varea modification handler */ +#define DBG_TAG "mm.object" +#define DBG_LVL DBG_INFO +#include "rtdbg.h" + #include <rtthread.h> #include "mm_aspace.h" @@ -15,15 +20,38 @@ #include "mm_page.h" #include <mmu.h> -#define DBG_TAG "mm.object" -#define DBG_LVL DBG_INFO -#include "rtdbg.h" +#include <string.h> +#include <stdlib.h> + +/** varea based dummy memory object whose data comes directly from page frame */ static const char *get_name(rt_varea_t varea) { return "dummy-mapper"; } +static rt_bool_t _varea_pgmgr_frame_is_member(rt_varea_t varea, rt_page_t frame) +{ + rt_page_t iter; + rt_bool_t rc = RT_FALSE; + + if (varea->frames) + { + iter = varea->frames; + do + { + if (iter == frame) + { + rc = RT_TRUE; + break; + } + iter = iter->next; + } while (iter); + } + + return rc; +} + void rt_varea_pgmgr_insert(rt_varea_t varea, void *page_addr) { rt_page_t page = rt_page_addr2page(page_addr); @@ -31,10 +59,12 @@ void rt_varea_pgmgr_insert(rt_varea_t varea, void *page_addr) if (varea->frames == NULL) { varea->frames = page; - page->next = NULL; + page->pre = RT_NULL; + page->next = RT_NULL; } else { + page->pre = RT_NULL; varea->frames->pre = page; page->next = varea->frames; varea->frames = page; @@ -52,6 +82,8 @@ void rt_varea_pgmgr_pop_all(rt_varea_t varea) rt_pages_free(pg_va, 0); page = next; } + + varea->frames = RT_NULL; } void rt_varea_pgmgr_pop(rt_varea_t varea, void *vaddr, rt_size_t size) @@ -60,9 +92,16 @@ void rt_varea_pgmgr_pop(rt_varea_t varea, void *vaddr, rt_size_t size) while (vaddr != vend) { rt_page_t page = rt_page_addr2page(vaddr); - page->pre->next = page->next; - page->next->pre = page->pre; - rt_pages_free(vaddr, 0); + if (_varea_pgmgr_frame_is_member(varea, page)) + { + if (page->pre) + page->pre->next = page->next; + if (page->next) + page->next->pre = page->pre; + if (varea->frames == page) + varea->frames = page->next; + rt_pages_free(vaddr, 0); + } vaddr = (char *)vaddr + ARCH_PAGE_SIZE; } } @@ -94,9 +133,149 @@ static void on_varea_close(struct rt_varea *varea) { } -static void on_page_offload(rt_varea_t varea, void *vaddr, rt_size_t size) +static rt_err_t on_varea_expand(struct rt_varea *varea, void *new_vaddr, rt_size_t size) +{ + return RT_EOK; +} + +static void _remove_pages(rt_varea_t varea, void *rm_start, void *rm_end) { - rt_varea_pgmgr_pop(varea, vaddr, size); + void *page_va; + + RT_ASSERT(!((rt_ubase_t)rm_start & ARCH_PAGE_MASK)); + RT_ASSERT(!((rt_ubase_t)rm_end & ARCH_PAGE_MASK)); + while (rm_start != rm_end) + { + page_va = rt_hw_mmu_v2p(varea->aspace, rm_start); + + if (page_va != ARCH_MAP_FAILED) + { + page_va -= PV_OFFSET; + LOG_D("%s: free page %p", __func__, page_va); + rt_varea_unmap_page(varea, rm_start); + rt_varea_pgmgr_pop(varea, page_va, ARCH_PAGE_SIZE); + } + rm_start += ARCH_PAGE_SIZE; + } +} + +static rt_err_t on_varea_shrink(rt_varea_t varea, void *new_start, rt_size_t size) +{ + char *varea_start = varea->start; + void *rm_start; + void *rm_end; + + if (varea_start == (char *)new_start) + { + rm_start = varea_start + size; + rm_end = varea_start + varea->size; + } + else /* if (varea_start < (char *)new_start) */ + { + RT_ASSERT(varea_start < (char *)new_start); + rm_start = varea_start; + rm_end = new_start; + } + + _remove_pages(varea, rm_start, rm_end); + return RT_EOK; +} + +static rt_err_t on_varea_split(struct rt_varea *existed, void *unmap_start, rt_size_t unmap_len, struct rt_varea *subset) +{ + void *sub_start = subset->start; + void *sub_end = sub_start + subset->size; + void *page_va; + + _remove_pages(existed, unmap_start, (char *)unmap_start + unmap_len); + + RT_ASSERT(!((rt_ubase_t)sub_start & ARCH_PAGE_MASK)); + RT_ASSERT(!((rt_ubase_t)sub_end & ARCH_PAGE_MASK)); + while (sub_start != sub_end) + { + page_va = rt_hw_mmu_v2p(existed->aspace, sub_start); + + if (page_va != ARCH_MAP_FAILED) + { + rt_page_t frame; + page_va = rt_kmem_p2v(page_va); + if (page_va) + { + frame = rt_page_addr2page(page_va); + if (frame && _varea_pgmgr_frame_is_member(existed, frame)) + { + LOG_D("%s: free page %p", __func__, page_va); + rt_page_ref_inc(page_va, 0); + rt_varea_pgmgr_pop(existed, page_va, ARCH_PAGE_SIZE); + rt_varea_pgmgr_insert(subset, page_va); + } + } + } + sub_start += ARCH_PAGE_SIZE; + } + + return RT_EOK; +} + +static rt_err_t on_varea_merge(struct rt_varea *merge_to, struct rt_varea *merge_from) +{ + /* transport page */ + void *mr_start = merge_from->start; + void *mr_end = mr_start + merge_from->size; + void *page_va; + + RT_ASSERT(!((rt_ubase_t)mr_start & ARCH_PAGE_MASK)); + RT_ASSERT(!((rt_ubase_t)mr_end & ARCH_PAGE_MASK)); + while (mr_start != mr_end) + { + page_va = rt_hw_mmu_v2p(merge_from->aspace, mr_start); + if (page_va != ARCH_MAP_FAILED) + { + rt_page_t frame; + page_va = rt_kmem_p2v(page_va); + if (page_va) + { + frame = rt_page_addr2page(page_va); + if (frame && _varea_pgmgr_frame_is_member(merge_from, frame)) + { + LOG_D("%s: free page %p", __func__, page_va); + rt_page_ref_inc(page_va, 0); + rt_varea_pgmgr_pop(merge_from, page_va, ARCH_PAGE_SIZE); + rt_varea_pgmgr_insert(merge_to, page_va); + } + } + } + mr_start += ARCH_PAGE_SIZE; + } + return RT_EOK; +} + +static void page_read(struct rt_varea *varea, struct rt_aspace_io_msg *msg) +{ + char *dst_k; + rt_aspace_t aspace = varea->aspace; + dst_k = rt_hw_mmu_v2p(aspace, msg->fault_vaddr); + if (dst_k != ARCH_MAP_FAILED) + { + RT_ASSERT(!((long)dst_k & ARCH_PAGE_MASK)); + dst_k = (void *)((char *)dst_k - PV_OFFSET); + memcpy(msg->buffer_vaddr, dst_k, ARCH_PAGE_SIZE); + msg->response.status = MM_FAULT_STATUS_OK; + } +} + +static void page_write(struct rt_varea *varea, struct rt_aspace_io_msg *msg) +{ + void *dst_k; + rt_aspace_t aspace = varea->aspace; + dst_k = rt_hw_mmu_v2p(aspace, msg->fault_vaddr); + if (dst_k != ARCH_MAP_FAILED) + { + RT_ASSERT(!((long)dst_k & ARCH_PAGE_MASK)); + dst_k = (void *)((char *)dst_k - PV_OFFSET); + memcpy(dst_k, msg->buffer_vaddr, ARCH_PAGE_SIZE); + msg->response.status = MM_FAULT_STATUS_OK; + } } struct rt_mem_obj rt_mm_dummy_mapper = { @@ -105,5 +284,12 @@ struct rt_mem_obj rt_mm_dummy_mapper = { .hint_free = NULL, .on_varea_open = on_varea_open, .on_varea_close = on_varea_close, - .on_page_offload = on_page_offload, + + .on_varea_shrink = on_varea_shrink, + .on_varea_split = on_varea_split, + .on_varea_expand = on_varea_expand, + .on_varea_merge = on_varea_merge, + + .page_write = page_write, + .page_read = page_read, }; diff --git a/components/mm/mm_page.c b/components/mm/mm_page.c index 99110ab435..0ddb392b63 100644 --- a/components/mm/mm_page.c +++ b/components/mm/mm_page.c @@ -91,10 +91,13 @@ static rt_page_t _trace_head; #define TRACE_ALLOC(pg, size) _trace_alloc(pg, __builtin_return_address(0), size) #define TRACE_FREE(pgaddr, size) _trace_free(pgaddr, __builtin_return_address(0), size) +static long _alloc_cnt; + void rt_page_leak_trace_start() { // TODO multicore safety _trace_head = NULL; + _alloc_cnt = 0; enable = 1; } MSH_CMD_EXPORT(rt_page_leak_trace_start, start page leak tracer); @@ -104,16 +107,18 @@ static void _collect() rt_page_t page = _trace_head; if (!page) { - rt_kputs("ok!\n"); + LOG_RAW("ok! ALLOC CNT %ld\n", _alloc_cnt); } - - while (page) + else { - rt_page_t next = page->next; - void *pg_va = rt_page_page2addr(page); - LOG_W("LEAK: %p, allocator: %p, size bits: %lx", pg_va, page->caller, page->trace_size); - rt_pages_free(pg_va, page->trace_size); - page = next; + while (page) + { + rt_page_t next = page->tl_next; + void *pg_va = rt_page_page2addr(page); + LOG_W("LEAK: %p, allocator: %p, size bits: %lx", pg_va, page->caller, page->trace_size); + rt_pages_free(pg_va, page->trace_size); + page = next; + } } } @@ -129,11 +134,13 @@ static void _trace_alloc(rt_page_t page, void *caller, size_t size_bits) { if (enable) { + char *page_va = rt_page_page2addr(page); page->caller = caller; page->trace_size = size_bits; page->tl_prev = NULL; page->tl_next = NULL; + _alloc_cnt++; if (_trace_head == NULL) { _trace_head = page; @@ -147,12 +154,12 @@ static void _trace_alloc(rt_page_t page, void *caller, size_t size_bits) } } -void _report(rt_page_t page, size_bits, char *msg) +void _report(rt_page_t page, size_t size_bits, char *msg) { void *pg_va = rt_page_page2addr(page); LOG_W("%s: %p, allocator: %p, size bits: %lx", msg, pg_va, page->caller, page->trace_size); - rt_kputs("backtrace\n"); - rt_hw_backtrace(0, 0); + LOG_RAW("backtrace\n"); + rt_backtrace(); } static void _trace_free(rt_page_t page, void *caller, size_t size_bits) @@ -162,25 +169,26 @@ static void _trace_free(rt_page_t page, void *caller, size_t size_bits) /* free after free */ if (page->trace_size == 0xabadcafe) { - _report("free after free") + _report(page, size_bits, "free after free"); return ; } else if (page->trace_size != size_bits) { - rt_kprintf("free with size bits %lx\n", size_bits); - _report("incompatible size bits parameter"); + LOG_RAW("free with size bits %lx\n", size_bits); + _report(page, size_bits, "incompatible size bits parameter"); return ; } - if (page->ref_cnt == 1) + if (page->ref_cnt == 0) { + _alloc_cnt--; if (page->tl_prev) page->tl_prev->tl_next = page->tl_next; if (page->tl_next) page->tl_next->tl_prev = page->tl_prev; if (page == _trace_head) - _trace_head = page->next; + _trace_head = page->tl_next; page->tl_prev = NULL; page->tl_next = NULL; @@ -531,7 +539,7 @@ static rt_page_t *_flag_to_page_list(size_t flags) return page_list; } -static void *_do_pages_alloc(rt_uint32_t size_bits, size_t flags) +rt_inline void *_do_pages_alloc(rt_uint32_t size_bits, size_t flags) { void *alloc_buf = RT_NULL; struct rt_page *p; @@ -613,27 +621,27 @@ void list_page(void) struct rt_page *lp = page_list_low[i]; struct rt_page *hp = page_list_high[i]; - rt_kprintf("level %d ", i); + LOG_RAW("level %d ", i); while (lp) { free += (1UL << i); - rt_kprintf("[0x%08p]", rt_page_page2addr(lp)); + LOG_RAW("[0x%08p]", rt_page_page2addr(lp)); lp = lp->next; } while (hp) { free += (1UL << i); - rt_kprintf("[0x%08p]", rt_page_page2addr(hp)); + LOG_RAW("[0x%08p]", rt_page_page2addr(hp)); hp = hp->next; } - rt_kprintf("\n"); + LOG_RAW("\n"); } rt_hw_interrupt_enable(level); - rt_kprintf("-------------------------------\n"); - rt_kprintf("Page Summary:\n => free/installed: 0x%lx/0x%lx (%ld/%ld KB)\n", free, installed, PGNR2SIZE(free), PGNR2SIZE(installed)); - rt_kprintf("-------------------------------\n"); + LOG_RAW("-------------------------------\n"); + LOG_RAW("Page Summary:\n => free/installed: 0x%lx/0x%lx (%ld/%ld KB)\n", free, installed, PGNR2SIZE(free), PGNR2SIZE(installed)); + LOG_RAW("-------------------------------\n"); } MSH_CMD_EXPORT(list_page, show page info); diff --git a/components/mm/mm_page.h b/components/mm/mm_page.h index a0171d6672..e1557b923d 100644 --- a/components/mm/mm_page.h +++ b/components/mm/mm_page.h @@ -35,7 +35,7 @@ #ifdef RT_DEBUGING_PAGE_LEAK -#define DEBUG_FIELD { \ +#define DEBUG_FIELD struct { \ /* trace list */ \ struct rt_page *tl_next; \ struct rt_page *tl_prev; \ @@ -50,7 +50,7 @@ DEF_PAGE_T( struct rt_page *next; /* same level next */ struct rt_page *pre; /* same level pre */ - DEBUG_FIELD + DEBUG_FIELD; rt_uint32_t size_bits; /* if is ARCH_ADDRESS_WIDTH_BITS, means not free */ rt_uint32_t ref_cnt; /* page group ref count */ diff --git a/components/mm/mm_private.h b/components/mm/mm_private.h index 364ddab717..3fd3b88f86 100644 --- a/components/mm/mm_private.h +++ b/components/mm/mm_private.h @@ -11,6 +11,10 @@ #define __MM_PRIVATE_H__ #include "mm_aspace.h" +#include "mm_fault.h" +#include "mm_flag.h" +#include "mm_page.h" + #include <rtdef.h> #include <stddef.h> @@ -29,6 +33,17 @@ * where both start and end are inclusive. */ +#define VAREA_NOT_STATIC(varea) (!((varea)->flag & MMF_STATIC_ALLOC)) +#define VAREA_NAME(varea) \ + ((!varea->mem_obj || !varea->mem_obj->get_name) \ + ? "unknow" \ + : varea->mem_obj->get_name(varea)) +#define VAREA_IS_WRITABLE(varea) \ + (rt_hw_mmu_attr_test_perm(varea->attr, \ + RT_HW_MMU_PROT_USER | RT_HW_MMU_PROT_WRITE)) +#define VAREA_VA_TO_OFFSET(varea, va) \ + ((varea)->offset + MM_PA_TO_OFF((long)(va) - (long)(varea)->start)) + struct _mm_range { void *start; @@ -94,6 +109,29 @@ void rt_varea_pgmgr_pop(rt_varea_t varea, void *vaddr, rt_size_t size); void rt_varea_pgmgr_pop_all(rt_varea_t varea); -int _varea_map_with_msg(rt_varea_t varea, struct rt_aspace_fault_msg *msg); +int rt_varea_fix_private_locked(rt_varea_t ex_varea, void *pa, + struct rt_aspace_fault_msg *msg, + rt_bool_t dont_copy); + +int rt_varea_map_with_msg(rt_varea_t varea, struct rt_aspace_fault_msg *msg); + +void _varea_uninstall_locked(rt_varea_t varea); + +int _mm_aspace_map(rt_aspace_t aspace, rt_varea_t *pvarea, void **addr, + rt_size_t length, rt_size_t attr, mm_flag_t flags, + rt_mem_obj_t mem_obj, rt_size_t offset); + +rt_inline rt_bool_t rt_varea_is_private_locked(rt_varea_t varea) +{ + rt_base_t flags = varea->flag; + return !!( + (flags & (MMF_MAP_PRIVATE | MMF_MAP_PRIVATE_DONT_SYNC)) + && (varea->aspace->private_object != varea->mem_obj) + ); +} + +rt_err_t rt_aspace_anon_ref_dec(rt_mem_obj_t aobj); +rt_err_t rt_aspace_page_get_phy(rt_aspace_t aspace, void *page_va, void *buffer); +rt_err_t rt_aspace_page_put_phy(rt_aspace_t aspace, void *page_va, void *buffer); #endif /* __MM_PRIVATE_H__ */ |