diff --git a/config b/config index 14e75bfff2..872220d176 100644 --- a/config +++ b/config @@ -303,6 +303,7 @@ HTTP_LUA_SRCS=" \ $ngx_addon_dir/src/ngx_http_lua_input_filters.c \ $ngx_addon_dir/src/ngx_http_lua_pipe.c \ $ngx_addon_dir/src/ngx_http_lua_worker_thread.c \ + $ngx_addon_dir/src/ngx_http_lua_async_fs.c \ " HTTP_LUA_DEPS=" \ @@ -369,6 +370,7 @@ HTTP_LUA_DEPS=" \ $ngx_addon_dir/src/ngx_http_lua_input_filters.h \ $ngx_addon_dir/src/ngx_http_lua_pipe.h \ $ngx_addon_dir/src/ngx_http_lua_worker_thread.h \ + $ngx_addon_dir/src/ngx_http_lua_async_fs.h \ " # ---------------------------------------- diff --git a/src/ngx_http_lua_async_fs.c b/src/ngx_http_lua_async_fs.c new file mode 100644 index 0000000000..687d5f0656 --- /dev/null +++ b/src/ngx_http_lua_async_fs.c @@ -0,0 +1,1002 @@ + +/* + * Copyright (C) Yichun Zhang (agentzh) + */ + + +#ifndef DDEBUG +#define DDEBUG 0 +#endif +#include "ddebug.h" + + +#include "ngx_http_lua_async_fs.h" +#include "ngx_http_lua_util.h" +#include "ngx_http_lua_contentby.h" + +#include + + +#if (NGX_THREADS) + +#include +#include + + +#define NGX_HTTP_LUA_FS_OP_OPEN 0 +#define NGX_HTTP_LUA_FS_OP_READ 1 +#define NGX_HTTP_LUA_FS_OP_WRITE 2 +#define NGX_HTTP_LUA_FS_OP_STAT 3 + +#define NGX_HTTP_LUA_FS_FILE_MT "ngx_http_lua_fs_file" +#define NGX_HTTP_LUA_FS_POOL_NAME_MAX 64 + + +typedef struct { + ngx_fd_t fd; + ngx_uint_t ninflight; + unsigned closed:1; + unsigned orphaned:1; +} ngx_http_lua_fs_fd_ref_t; + + +typedef struct { + ngx_http_lua_fs_fd_ref_t *ref; + u_char pool_name[NGX_HTTP_LUA_FS_POOL_NAME_MAX]; + size_t pool_name_len; +} ngx_http_lua_fs_file_t; + + +typedef struct { + ngx_uint_t op; + + /* open params */ + u_char *path; + int flags; + int create_mode; + + /* read/write params */ + ngx_fd_t fd; + u_char *buf; + size_t size; + off_t offset; + + ngx_http_lua_fs_fd_ref_t *ref; + + ssize_t nbytes; + ngx_fd_t result_fd; + ngx_err_t err; + struct stat fi; + + u_char pool_name[NGX_HTTP_LUA_FS_POOL_NAME_MAX]; + size_t pool_name_len; + + ngx_http_lua_co_ctx_t *wait_co_ctx; + + unsigned is_abort:1; +} ngx_http_lua_fs_ctx_t; + + +static int ngx_http_lua_fs_open(lua_State *L); +static int ngx_http_lua_fs_stat(lua_State *L); + +static int ngx_http_lua_fs_file_read(lua_State *L); +static int ngx_http_lua_fs_file_write(lua_State *L); +static int ngx_http_lua_fs_file_close(lua_State *L); +static int ngx_http_lua_fs_file_gc(lua_State *L); +static int ngx_http_lua_fs_file_tostring(lua_State *L); + +static void ngx_http_lua_fs_thread_handler(void *data, ngx_log_t *log); +static void ngx_http_lua_fs_event_handler(ngx_event_t *ev); +static void ngx_http_lua_fs_cleanup(void *data); +static ngx_int_t ngx_http_lua_fs_resume(ngx_http_request_t *r); + + +static void +ngx_http_lua_fs_fd_ref_release(ngx_http_lua_fs_fd_ref_t *ref) +{ + if (!ref->closed) { + ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0, + "lua async fs: auto-closing fd %d%s", + ref->fd, + ref->orphaned ? " (orphaned)" : ""); + + close(ref->fd); + ref->closed = 1; + } + + ngx_free(ref); +} + + +static ngx_thread_task_t * +ngx_http_lua_fs_task_alloc(size_t size) +{ + ngx_thread_task_t *task; + + task = ngx_calloc(sizeof(ngx_thread_task_t) + size, ngx_cycle->log); + if (task == NULL) { + return NULL; + } + + task->ctx = task + 1; + + return task; +} + + +static void +ngx_http_lua_fs_task_free(ngx_http_lua_fs_ctx_t *fs_ctx) +{ + ngx_thread_task_t *task; + + task = (ngx_thread_task_t *) fs_ctx - 1; + ngx_free(task); +} + + +static void +ngx_http_lua_fs_free_bufs(ngx_http_lua_fs_ctx_t *fs_ctx) +{ + if (fs_ctx->path != NULL) { + ngx_free(fs_ctx->path); + fs_ctx->path = NULL; + } + + if (fs_ctx->buf != NULL) { + ngx_free(fs_ctx->buf); + fs_ctx->buf = NULL; + } +} + + +static void +ngx_http_lua_fs_thread_handler(void *data, ngx_log_t *log) +{ + ngx_http_lua_fs_ctx_t *fs_ctx = data; + + switch (fs_ctx->op) { + + case NGX_HTTP_LUA_FS_OP_OPEN: + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, log, 0, + "lua async fs: open \"%s\"", fs_ctx->path); + + fs_ctx->result_fd = open((const char *) fs_ctx->path, + fs_ctx->flags, fs_ctx->create_mode); + + if (fs_ctx->result_fd == -1) { + fs_ctx->err = ngx_errno; + } + + break; + + case NGX_HTTP_LUA_FS_OP_READ: + ngx_log_debug3(NGX_LOG_DEBUG_HTTP, log, 0, + "lua async fs: pread fd=%d size=%uz offset=%O", + fs_ctx->fd, fs_ctx->size, fs_ctx->offset); + + fs_ctx->nbytes = pread(fs_ctx->fd, fs_ctx->buf, + fs_ctx->size, fs_ctx->offset); + + if (fs_ctx->nbytes == -1) { + fs_ctx->err = ngx_errno; + } + + break; + + case NGX_HTTP_LUA_FS_OP_WRITE: + ngx_log_debug3(NGX_LOG_DEBUG_HTTP, log, 0, + "lua async fs: pwrite fd=%d size=%uz offset=%O", + fs_ctx->fd, fs_ctx->size, fs_ctx->offset); + + fs_ctx->nbytes = pwrite(fs_ctx->fd, fs_ctx->buf, + fs_ctx->size, fs_ctx->offset); + + if (fs_ctx->nbytes == -1) { + fs_ctx->err = ngx_errno; + } + + break; + + case NGX_HTTP_LUA_FS_OP_STAT: + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, log, 0, + "lua async fs: lstat \"%s\"", fs_ctx->path); + + if (lstat((const char *) fs_ctx->path, &fs_ctx->fi) == -1) { + fs_ctx->err = ngx_errno; + } + + break; + + default: + ngx_log_error(NGX_LOG_ALERT, log, 0, + "lua async fs: unknown op %ui", fs_ctx->op); + break; + } +} + + +static ngx_int_t +ngx_http_lua_fs_resume(ngx_http_request_t *r) +{ + lua_State *vm; + ngx_connection_t *c; + ngx_int_t rc; + ngx_uint_t nreqs; + ngx_http_lua_ctx_t *ctx; + + ctx = ngx_http_get_module_ctx(r, ngx_http_lua_module); + if (ctx == NULL) { + return NGX_ERROR; + } + + ctx->resume_handler = ngx_http_lua_wev_handler; + + c = r->connection; + vm = ngx_http_lua_get_lua_vm(r, ctx); + nreqs = c->requests; + + rc = ngx_http_lua_run_thread(vm, r, ctx, + ctx->cur_co_ctx + ->nresults_from_worker_thread); + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "lua async fs run thread returned %d", rc); + + if (rc == NGX_AGAIN) { + return ngx_http_lua_run_posted_threads(c, vm, r, ctx, nreqs); + } + + if (rc == NGX_DONE) { + ngx_http_lua_finalize_request(r, NGX_DONE); + return ngx_http_lua_run_posted_threads(c, vm, r, ctx, nreqs); + } + + if (ctx->entered_content_phase) { + ngx_http_lua_finalize_request(r, rc); + return NGX_DONE; + } + + return rc; +} + + +static void +ngx_http_lua_fs_push_stat_table(lua_State *L, struct stat *fi) +{ + lua_createtable(L, 0, 12); + + lua_pushinteger(L, (lua_Integer) fi->st_size); + lua_setfield(L, -2, "size"); + + lua_pushinteger(L, (lua_Integer) fi->st_mtime); + lua_setfield(L, -2, "mtime"); + + lua_pushinteger(L, (lua_Integer) fi->st_atime); + lua_setfield(L, -2, "atime"); + + lua_pushinteger(L, (lua_Integer) fi->st_ctime); + lua_setfield(L, -2, "ctime"); + + lua_pushinteger(L, (lua_Integer) fi->st_mode); + lua_setfield(L, -2, "mode"); + + lua_pushinteger(L, (lua_Integer) fi->st_ino); + lua_setfield(L, -2, "ino"); + + lua_pushinteger(L, (lua_Integer) fi->st_nlink); + lua_setfield(L, -2, "nlink"); + + lua_pushinteger(L, (lua_Integer) fi->st_uid); + lua_setfield(L, -2, "uid"); + + lua_pushinteger(L, (lua_Integer) fi->st_gid); + lua_setfield(L, -2, "gid"); + + lua_pushboolean(L, S_ISDIR(fi->st_mode)); + lua_setfield(L, -2, "is_dir"); + + lua_pushboolean(L, S_ISREG(fi->st_mode)); + lua_setfield(L, -2, "is_file"); + + lua_pushboolean(L, S_ISLNK(fi->st_mode)); + lua_setfield(L, -2, "is_link"); +} + + +static void +ngx_http_lua_fs_abort_cleanup(ngx_http_lua_fs_ctx_t *fs_ctx) +{ + ngx_http_lua_fs_fd_ref_t *ref; + + ref = fs_ctx->ref; + + if (ref != NULL) { + ref->ninflight--; + + if (ref->orphaned && ref->ninflight == 0) { + ngx_http_lua_fs_fd_ref_release(ref); + } + + fs_ctx->ref = NULL; + } + + if (fs_ctx->op == NGX_HTTP_LUA_FS_OP_OPEN + && fs_ctx->err == 0 + && fs_ctx->result_fd != -1) + { + ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0, + "lua async fs: closing leaked fd %d " + "from aborted open", + fs_ctx->result_fd); + + if (close(fs_ctx->result_fd) == -1) { + ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, ngx_errno, + "lua async fs: close(%d) failed", + fs_ctx->result_fd); + } + } +} + + +static void +ngx_http_lua_fs_event_handler(ngx_event_t *ev) +{ + ngx_http_lua_fs_ctx_t *fs_ctx; + ngx_http_lua_fs_file_t *file; + ngx_http_lua_fs_fd_ref_t *ref; + lua_State *L; + ngx_http_request_t *r; + ngx_connection_t *c; + ngx_http_lua_ctx_t *ctx; + int nresults; + + fs_ctx = ev->data; + + if (fs_ctx->is_abort) { + ngx_http_lua_fs_abort_cleanup(fs_ctx); + + ngx_http_lua_fs_free_bufs(fs_ctx); + ngx_http_lua_fs_task_free(fs_ctx); + return; + } + + L = fs_ctx->wait_co_ctx->co; + + r = ngx_http_lua_get_req(L); + if (r == NULL) { + goto failed; + } + + c = r->connection; + + ctx = ngx_http_get_module_ctx(r, ngx_http_lua_module); + if (ctx == NULL) { + goto failed; + } + + if (fs_ctx->ref != NULL) { + fs_ctx->ref->ninflight--; + fs_ctx->ref = NULL; + } + + switch (fs_ctx->op) { + + case NGX_HTTP_LUA_FS_OP_OPEN: + if (fs_ctx->err) { + lua_pushnil(L); + lua_pushfstring(L, "open() \"%s\" failed (%d: %s)", + fs_ctx->path, (int) fs_ctx->err, + strerror(fs_ctx->err)); + nresults = 2; + + } else { + ref = ngx_calloc(sizeof(ngx_http_lua_fs_fd_ref_t), + ngx_cycle->log); + + if (ref == NULL) { + close(fs_ctx->result_fd); + + lua_pushnil(L); + lua_pushliteral(L, "no memory for fd ref"); + nresults = 2; + break; + } + + ref->fd = fs_ctx->result_fd; + + file = (ngx_http_lua_fs_file_t *) + lua_newuserdata(L, sizeof(ngx_http_lua_fs_file_t)); + + file->ref = ref; + file->pool_name_len = fs_ctx->pool_name_len; + ngx_memcpy(file->pool_name, fs_ctx->pool_name, + fs_ctx->pool_name_len); + + luaL_getmetatable(L, NGX_HTTP_LUA_FS_FILE_MT); + lua_setmetatable(L, -2); + + nresults = 1; + } + + break; + + case NGX_HTTP_LUA_FS_OP_READ: + if (fs_ctx->err) { + lua_pushnil(L); + lua_pushfstring(L, "pread() failed (%d: %s)", + (int) fs_ctx->err, strerror(fs_ctx->err)); + nresults = 2; + + } else if (fs_ctx->nbytes == 0) { + lua_pushliteral(L, ""); + nresults = 1; + + } else { + lua_pushlstring(L, (const char *) fs_ctx->buf, + (size_t) fs_ctx->nbytes); + nresults = 1; + } + + break; + + case NGX_HTTP_LUA_FS_OP_WRITE: + if (fs_ctx->err) { + lua_pushnil(L); + lua_pushfstring(L, "pwrite() failed (%d: %s)", + (int) fs_ctx->err, strerror(fs_ctx->err)); + nresults = 2; + + } else { + lua_pushinteger(L, (lua_Integer) fs_ctx->nbytes); + nresults = 1; + } + + break; + + case NGX_HTTP_LUA_FS_OP_STAT: + if (fs_ctx->err) { + lua_pushnil(L); + lua_pushfstring(L, "stat() \"%s\" failed (%d: %s)", + fs_ctx->path, (int) fs_ctx->err, + strerror(fs_ctx->err)); + nresults = 2; + + } else { + ngx_http_lua_fs_push_stat_table(L, &fs_ctx->fi); + nresults = 1; + } + + break; + + default: + lua_pushnil(L); + lua_pushliteral(L, "unknown fs operation"); + nresults = 2; + break; + } + + ctx->cur_co_ctx = fs_ctx->wait_co_ctx; + ctx->cur_co_ctx->nresults_from_worker_thread = nresults; + ctx->cur_co_ctx->cleanup = NULL; + + ngx_http_lua_fs_free_bufs(fs_ctx); + ngx_http_lua_fs_task_free(fs_ctx); + + if (ctx->entered_content_phase) { + (void) ngx_http_lua_fs_resume(r); + + } else { + ctx->resume_handler = ngx_http_lua_fs_resume; + ngx_http_core_run_phases(r); + } + + ngx_http_run_posted_requests(c); + return; + +failed: + + ngx_http_lua_fs_abort_cleanup(fs_ctx); + + ngx_http_lua_fs_free_bufs(fs_ctx); + ngx_http_lua_fs_task_free(fs_ctx); +} + + +static void +ngx_http_lua_fs_cleanup(void *data) +{ + ngx_http_lua_co_ctx_t *coctx = data; + ngx_http_lua_fs_ctx_t *fs_ctx; + + fs_ctx = coctx->data; + fs_ctx->is_abort = 1; +} + + +static int +ngx_http_lua_fs_post_task(lua_State *L, ngx_http_request_t *r, + ngx_http_lua_fs_ctx_t *fs_ctx, ngx_str_t *pool_name) +{ + ngx_thread_pool_t *tp; + ngx_thread_task_t *task; + ngx_http_lua_ctx_t *ctx; + ngx_http_lua_co_ctx_t *coctx; + + ctx = ngx_http_get_module_ctx(r, ngx_http_lua_module); + + tp = ngx_thread_pool_get((ngx_cycle_t *) ngx_cycle, pool_name); + + if (tp == NULL) { + if (fs_ctx->ref != NULL) { + fs_ctx->ref->ninflight--; + } + + ngx_http_lua_fs_free_bufs(fs_ctx); + ngx_http_lua_fs_task_free(fs_ctx); + + lua_pushnil(L); + lua_pushfstring(L, "thread pool \"%s\" not found, " + "add \"thread_pool %s ...;\" to nginx.conf " + "and build nginx with --with-threads", + pool_name->data, pool_name->data); + return 2; + } + + task = (ngx_thread_task_t *) fs_ctx - 1; + + coctx = ctx->cur_co_ctx; + + fs_ctx->wait_co_ctx = coctx; + fs_ctx->is_abort = 0; + + ngx_http_lua_cleanup_pending_operation(coctx); + coctx->cleanup = ngx_http_lua_fs_cleanup; + coctx->data = fs_ctx; + + task->handler = ngx_http_lua_fs_thread_handler; + task->event.handler = ngx_http_lua_fs_event_handler; + task->event.data = fs_ctx; + + if (ngx_thread_task_post(tp, task) != NGX_OK) { + coctx->cleanup = NULL; + coctx->data = NULL; + + if (fs_ctx->ref != NULL) { + fs_ctx->ref->ninflight--; + } + + ngx_http_lua_fs_free_bufs(fs_ctx); + ngx_http_lua_fs_task_free(fs_ctx); + + lua_pushnil(L); + lua_pushliteral(L, "thread pool task post failed"); + return 2; + } + + return lua_yield(L, 0); +} + + +static int +ngx_http_lua_fs_open(lua_State *L) +{ + ngx_http_request_t *r; + ngx_http_lua_ctx_t *ctx; + ngx_thread_task_t *task; + ngx_http_lua_fs_ctx_t *fs_ctx; + const char *path; + const char *mode; + size_t path_len; + int flags; + int create_mode; + ngx_str_t pool_name; + + r = ngx_http_lua_get_req(L); + if (r == NULL) { + return luaL_error(L, "no request found"); + } + + ctx = ngx_http_get_module_ctx(r, ngx_http_lua_module); + if (ctx == NULL) { + return luaL_error(L, "no request ctx found"); + } + + ngx_http_lua_check_context(L, ctx, NGX_HTTP_LUA_CONTEXT_YIELDABLE); + + if (lua_gettop(L) < 1) { + return luaL_error(L, "expecting at least 1 argument (path)"); + } + + path = luaL_checklstring(L, 1, &path_len); + mode = luaL_optstring(L, 2, "r"); + + pool_name.data = (u_char *) luaL_optlstring(L, 3, "default_lua_io", + &pool_name.len); + + if (pool_name.len >= NGX_HTTP_LUA_FS_POOL_NAME_MAX) { + return luaL_error(L, "thread pool name too long"); + } + + create_mode = 0644; + + if (mode[0] == 'r' && mode[1] == '\0') { + flags = O_RDONLY; + + } else if (mode[0] == 'w' && mode[1] == '\0') { + flags = O_WRONLY | O_CREAT | O_TRUNC; + + } else if (mode[0] == 'a' && mode[1] == '\0') { + flags = O_WRONLY | O_CREAT | O_APPEND; + + } else if (mode[0] == 'r' && mode[1] == '+' && mode[2] == '\0') { + flags = O_RDWR; + + } else if (mode[0] == 'w' && mode[1] == '+' && mode[2] == '\0') { + flags = O_RDWR | O_CREAT | O_TRUNC; + + } else { + return luaL_error(L, "invalid mode \"%s\" " + "(expected \"r\", \"w\", \"a\", \"r+\", or \"w+\")", + mode); + } + + task = ngx_http_lua_fs_task_alloc(sizeof(ngx_http_lua_fs_ctx_t)); + if (task == NULL) { + return luaL_error(L, "no memory"); + } + + fs_ctx = task->ctx; + ngx_memzero(fs_ctx, sizeof(ngx_http_lua_fs_ctx_t)); + + fs_ctx->op = NGX_HTTP_LUA_FS_OP_OPEN; + fs_ctx->flags = flags; + fs_ctx->create_mode = create_mode; + fs_ctx->result_fd = -1; + + fs_ctx->path = ngx_alloc(path_len + 1, ngx_cycle->log); + if (fs_ctx->path == NULL) { + ngx_http_lua_fs_task_free(fs_ctx); + return luaL_error(L, "no memory"); + } + + ngx_memcpy(fs_ctx->path, path, path_len); + fs_ctx->path[path_len] = '\0'; + + ngx_memcpy(fs_ctx->pool_name, pool_name.data, pool_name.len); + fs_ctx->pool_name_len = pool_name.len; + + return ngx_http_lua_fs_post_task(L, r, fs_ctx, &pool_name); +} + + +static int +ngx_http_lua_fs_file_read(lua_State *L) +{ + ngx_http_request_t *r; + ngx_http_lua_ctx_t *ctx; + ngx_thread_task_t *task; + ngx_http_lua_fs_ctx_t *fs_ctx; + ngx_http_lua_fs_file_t *file; + lua_Integer size; + lua_Integer offset; + ngx_str_t pool_name; + + r = ngx_http_lua_get_req(L); + if (r == NULL) { + return luaL_error(L, "no request found"); + } + + ctx = ngx_http_get_module_ctx(r, ngx_http_lua_module); + if (ctx == NULL) { + return luaL_error(L, "no request ctx found"); + } + + ngx_http_lua_check_context(L, ctx, NGX_HTTP_LUA_CONTEXT_YIELDABLE); + + file = (ngx_http_lua_fs_file_t *) + luaL_checkudata(L, 1, NGX_HTTP_LUA_FS_FILE_MT); + + if (file->ref->closed) { + return luaL_error(L, "attempt to read from a closed file"); + } + + size = luaL_checkinteger(L, 2); + offset = luaL_optinteger(L, 3, 0); + + if (size <= 0) { + return luaL_error(L, "invalid size: %d", (int) size); + } + + if (offset < 0) { + return luaL_error(L, "invalid offset: %d", (int) offset); + } + + pool_name.data = file->pool_name; + pool_name.len = file->pool_name_len; + + task = ngx_http_lua_fs_task_alloc(sizeof(ngx_http_lua_fs_ctx_t)); + if (task == NULL) { + return luaL_error(L, "no memory"); + } + + fs_ctx = task->ctx; + ngx_memzero(fs_ctx, sizeof(ngx_http_lua_fs_ctx_t)); + + fs_ctx->buf = ngx_alloc((size_t) size, ngx_cycle->log); + if (fs_ctx->buf == NULL) { + ngx_http_lua_fs_task_free(fs_ctx); + return luaL_error(L, "no memory"); + } + + fs_ctx->op = NGX_HTTP_LUA_FS_OP_READ; + fs_ctx->fd = file->ref->fd; + fs_ctx->size = (size_t) size; + fs_ctx->offset = (off_t) offset; + fs_ctx->ref = file->ref; + + file->ref->ninflight++; + + return ngx_http_lua_fs_post_task(L, r, fs_ctx, &pool_name); +} + + +static int +ngx_http_lua_fs_file_write(lua_State *L) +{ + ngx_http_request_t *r; + ngx_http_lua_ctx_t *ctx; + ngx_thread_task_t *task; + ngx_http_lua_fs_ctx_t *fs_ctx; + ngx_http_lua_fs_file_t *file; + const char *data; + size_t data_len; + lua_Integer offset; + ngx_str_t pool_name; + + r = ngx_http_lua_get_req(L); + if (r == NULL) { + return luaL_error(L, "no request found"); + } + + ctx = ngx_http_get_module_ctx(r, ngx_http_lua_module); + if (ctx == NULL) { + return luaL_error(L, "no request ctx found"); + } + + ngx_http_lua_check_context(L, ctx, NGX_HTTP_LUA_CONTEXT_YIELDABLE); + + file = (ngx_http_lua_fs_file_t *) + luaL_checkudata(L, 1, NGX_HTTP_LUA_FS_FILE_MT); + + if (file->ref->closed) { + return luaL_error(L, "attempt to write to a closed file"); + } + + data = luaL_checklstring(L, 2, &data_len); + offset = luaL_optinteger(L, 3, 0); + + if (data_len == 0) { + lua_pushinteger(L, 0); + return 1; + } + + if (offset < 0) { + return luaL_error(L, "invalid offset: %d", (int) offset); + } + + pool_name.data = file->pool_name; + pool_name.len = file->pool_name_len; + + task = ngx_http_lua_fs_task_alloc(sizeof(ngx_http_lua_fs_ctx_t)); + if (task == NULL) { + return luaL_error(L, "no memory"); + } + + fs_ctx = task->ctx; + ngx_memzero(fs_ctx, sizeof(ngx_http_lua_fs_ctx_t)); + + fs_ctx->buf = ngx_alloc(data_len, ngx_cycle->log); + if (fs_ctx->buf == NULL) { + ngx_http_lua_fs_task_free(fs_ctx); + return luaL_error(L, "no memory"); + } + + ngx_memcpy(fs_ctx->buf, data, data_len); + + fs_ctx->op = NGX_HTTP_LUA_FS_OP_WRITE; + fs_ctx->fd = file->ref->fd; + fs_ctx->size = data_len; + fs_ctx->offset = (off_t) offset; + fs_ctx->ref = file->ref; + + file->ref->ninflight++; + + return ngx_http_lua_fs_post_task(L, r, fs_ctx, &pool_name); +} + + +static int +ngx_http_lua_fs_file_close(lua_State *L) +{ + ngx_http_lua_fs_file_t *file; + + file = (ngx_http_lua_fs_file_t *) + luaL_checkudata(L, 1, NGX_HTTP_LUA_FS_FILE_MT); + + if (file->ref->closed) { + lua_pushboolean(L, 0); + lua_pushliteral(L, "file already closed"); + return 2; + } + + if (file->ref->ninflight > 0) { + lua_pushboolean(L, 0); + lua_pushfstring(L, "file has %d in-flight operation(s)", + (int) file->ref->ninflight); + return 2; + } + + if (close(file->ref->fd) == -1) { + lua_pushboolean(L, 0); + lua_pushfstring(L, "close() failed (%d: %s)", + (int) ngx_errno, strerror(ngx_errno)); + return 2; + } + + file->ref->closed = 1; + + lua_pushboolean(L, 1); + return 1; +} + + +static int +ngx_http_lua_fs_stat(lua_State *L) +{ + ngx_http_request_t *r; + ngx_http_lua_ctx_t *ctx; + ngx_thread_task_t *task; + ngx_http_lua_fs_ctx_t *fs_ctx; + const char *path; + size_t path_len; + ngx_str_t pool_name; + + r = ngx_http_lua_get_req(L); + if (r == NULL) { + return luaL_error(L, "no request found"); + } + + ctx = ngx_http_get_module_ctx(r, ngx_http_lua_module); + if (ctx == NULL) { + return luaL_error(L, "no request ctx found"); + } + + ngx_http_lua_check_context(L, ctx, NGX_HTTP_LUA_CONTEXT_YIELDABLE); + + if (lua_gettop(L) < 1) { + return luaL_error(L, "expecting at least 1 argument: path"); + } + + path = luaL_checklstring(L, 1, &path_len); + + pool_name.data = (u_char *) luaL_optlstring(L, 2, "default_lua_io", + &pool_name.len); + + task = ngx_http_lua_fs_task_alloc(sizeof(ngx_http_lua_fs_ctx_t)); + if (task == NULL) { + return luaL_error(L, "no memory"); + } + + fs_ctx = task->ctx; + ngx_memzero(fs_ctx, sizeof(ngx_http_lua_fs_ctx_t)); + + fs_ctx->op = NGX_HTTP_LUA_FS_OP_STAT; + + fs_ctx->path = ngx_alloc(path_len + 1, ngx_cycle->log); + if (fs_ctx->path == NULL) { + ngx_http_lua_fs_task_free(fs_ctx); + return luaL_error(L, "no memory"); + } + + ngx_memcpy(fs_ctx->path, path, path_len); + fs_ctx->path[path_len] = '\0'; + + return ngx_http_lua_fs_post_task(L, r, fs_ctx, &pool_name); +} + + +static int +ngx_http_lua_fs_file_gc(lua_State *L) +{ + ngx_http_lua_fs_file_t *file; + + file = (ngx_http_lua_fs_file_t *) + luaL_checkudata(L, 1, NGX_HTTP_LUA_FS_FILE_MT); + + if (file->ref == NULL) { + return 0; + } + + if (file->ref->ninflight == 0) { + ngx_http_lua_fs_fd_ref_release(file->ref); + + } else { + file->ref->orphaned = 1; + } + + file->ref = NULL; + + return 0; +} + + +static int +ngx_http_lua_fs_file_tostring(lua_State *L) +{ + ngx_http_lua_fs_file_t *file; + + file = (ngx_http_lua_fs_file_t *) + luaL_checkudata(L, 1, NGX_HTTP_LUA_FS_FILE_MT); + + if (file->ref == NULL || file->ref->closed) { + lua_pushliteral(L, "file (closed)"); + + } else { + lua_pushfstring(L, "file (fd=%d)", file->ref->fd); + } + + return 1; +} + + +void +ngx_http_lua_inject_async_fs_api(lua_State *L) +{ + luaL_newmetatable(L, NGX_HTTP_LUA_FS_FILE_MT); + + lua_pushcfunction(L, ngx_http_lua_fs_file_gc); + lua_setfield(L, -2, "__gc"); + + lua_pushcfunction(L, ngx_http_lua_fs_file_tostring); + lua_setfield(L, -2, "__tostring"); + + lua_createtable(L, 0, 3); + + lua_pushcfunction(L, ngx_http_lua_fs_file_read); + lua_setfield(L, -2, "read"); + + lua_pushcfunction(L, ngx_http_lua_fs_file_write); + lua_setfield(L, -2, "write"); + + lua_pushcfunction(L, ngx_http_lua_fs_file_close); + lua_setfield(L, -2, "close"); + + lua_setfield(L, -2, "__index"); + + lua_pop(L, 1); + + lua_createtable(L, 0, 2); + + lua_pushcfunction(L, ngx_http_lua_fs_open); + lua_setfield(L, -2, "open"); + + lua_pushcfunction(L, ngx_http_lua_fs_stat); + lua_setfield(L, -2, "stat"); + + lua_setfield(L, -2, "fs"); +} + + +#else /* !NGX_THREADS */ + + +void +ngx_http_lua_inject_async_fs_api(lua_State *L) +{ +} + + +#endif /* NGX_THREADS */ + +/* vi:set ft=c ts=4 sw=4 et fdm=marker: */ diff --git a/src/ngx_http_lua_async_fs.h b/src/ngx_http_lua_async_fs.h new file mode 100644 index 0000000000..1d24c69543 --- /dev/null +++ b/src/ngx_http_lua_async_fs.h @@ -0,0 +1,13 @@ +#ifndef _NGX_HTTP_LUA_ASYNC_FS_H_INCLUDED_ +#define _NGX_HTTP_LUA_ASYNC_FS_H_INCLUDED_ + + +#include "ngx_http_lua_common.h" + + +void ngx_http_lua_inject_async_fs_api(lua_State *L); + + +#endif /* _NGX_HTTP_LUA_ASYNC_FS_H_INCLUDED_ */ + +/* vi:set ft=c ts=4 sw=4 et fdm=marker: */ diff --git a/src/ngx_http_lua_util.c b/src/ngx_http_lua_util.c index 01c36440e5..6d44c0cd62 100644 --- a/src/ngx_http_lua_util.c +++ b/src/ngx_http_lua_util.c @@ -48,6 +48,7 @@ #include "ngx_http_lua_log_ringbuf.h" #if (NGX_THREADS) #include "ngx_http_lua_worker_thread.h" +#include "ngx_http_lua_async_fs.h" #endif #if (NGX_HTTP_V3) #include @@ -863,6 +864,7 @@ ngx_http_lua_inject_ngx_api(lua_State *L, ngx_http_lua_main_conf_t *lmcf, ngx_http_lua_inject_config_api(L); #if (NGX_THREADS) ngx_http_lua_inject_worker_thread_api(log, L); + ngx_http_lua_inject_async_fs_api(L); #endif lua_getglobal(L, "package"); /* ngx package */ diff --git a/t/062-count.t b/t/062-count.t index b3d3ef1c50..62973aa15b 100644 --- a/t/062-count.t +++ b/t/062-count.t @@ -34,7 +34,7 @@ __DATA__ --- request GET /test --- response_body -ngx: 117 +ngx: 118 --- no_error_log [error] @@ -55,7 +55,7 @@ ngx: 117 --- request GET /test --- response_body -117 +118 --- no_error_log [error] @@ -83,7 +83,7 @@ GET /test --- request GET /test --- response_body -n = 117 +n = 118 --- no_error_log [error] @@ -306,7 +306,7 @@ GET /t --- response_body_like: 404 Not Found --- error_code: 404 --- error_log -ngx. entry count: 117 +ngx. entry count: 118 diff --git a/t/193-async-fs.t b/t/193-async-fs.t new file mode 100644 index 0000000000..c0e0d6e8c2 --- /dev/null +++ b/t/193-async-fs.t @@ -0,0 +1,1484 @@ +# vim:set ft= ts=4 sw=4 et fdm=marker: + +our $SkipReason; + +BEGIN { + if ($ENV{TEST_NGINX_EVENT_TYPE} + && $ENV{TEST_NGINX_EVENT_TYPE} !~ /^kqueue|epoll|eventport$/) + { + $SkipReason = "unavailable for the event type '$ENV{TEST_NGINX_EVENT_TYPE}'"; + } +} + +use Test::Nginx::Socket::Lua $SkipReason ? (skip_all => $SkipReason) : (); + +repeat_each(1); + +plan tests => repeat_each() * (blocks() * 2 + 7); + +our $HtmlDir = html_dir; + +$ENV{TEST_NGINX_HTML_DIR} = $HtmlDir; + +#no_diff(); +#no_long_string(); +run_tests(); + +__DATA__ + +=== TEST 1: open and close +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local f, err = ngx.fs.open("$TEST_NGINX_HTML_DIR/test.txt") + if not f then + ngx.say("open failed: ", err) + return + end + + ngx.say("type: ", type(f)) + ngx.say("tostring: ", tostring(f):find("^file %(fd=") ~= nil) + + local ok, err = f:close() + ngx.say("close: ", ok) + } +} +--- user_files +>>> test.txt +hello world +--- request +GET /t +--- response_body +type: userdata +tostring: true +close: true + + + +=== TEST 2: open + read +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local f, err = ngx.fs.open("$TEST_NGINX_HTML_DIR/test.txt", "r") + if not f then + ngx.say("open failed: ", err) + return + end + + local data, err = f:read(4096) + if not data then + ngx.say("read failed: ", err) + f:close() + return + end + + ngx.say("data: [", data, "]") + f:close() + } +} +--- user_files +>>> test.txt +hello world +--- request +GET /t +--- response_body +data: [hello world +] + + + +=== TEST 3: open + write + readback +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local path = "$TEST_NGINX_HTML_DIR/out.txt" + + local f, err = ngx.fs.open(path, "w") + if not f then + ngx.say("open failed: ", err) + return + end + + local nbytes, err = f:write("hello async fs") + if not nbytes then + ngx.say("write failed: ", err) + f:close() + return + end + + ngx.say("wrote ", nbytes, " bytes") + f:close() + + local f2 = ngx.fs.open(path, "r") + local data = f2:read(4096) + ngx.say("readback: [", data, "]") + f2:close() + } +} +--- request +GET /t +--- response_body +wrote 14 bytes +readback: [hello async fs] + + + +=== TEST 4: open nonexistent file +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local f, err = ngx.fs.open("$TEST_NGINX_HTML_DIR/no_such_file.txt", "r") + if not f then + ngx.say("open failed: true") + ngx.say("err type: ", type(err)) + return + end + + ngx.say("should not reach here") + } +} +--- request +GET /t +--- response_body +open failed: true +err type: string + + + +=== TEST 5: invalid mode +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local ok, err = pcall(ngx.fs.open, "$TEST_NGINX_HTML_DIR/test.txt", "x") + ngx.say("ok: ", ok) + ngx.say("err match: ", string.find(err, "invalid mode") ~= nil) + } +} +--- user_files +>>> test.txt +hello +--- request +GET /t +--- response_body +ok: false +err match: true + + + +=== TEST 6: read with offset +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local f = ngx.fs.open("$TEST_NGINX_HTML_DIR/test.txt", "r") + + local data = f:read(5, 6) + ngx.say("data: [", data, "]") + f:close() + } +} +--- user_files +>>> test.txt +hello world +--- request +GET /t +--- response_body +data: [world] + + + +=== TEST 7: read EOF returns empty string +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local f = ngx.fs.open("$TEST_NGINX_HTML_DIR/test.txt", "r") + + local data = f:read(4096, 99999) + ngx.say("data len: ", #data) + ngx.say("is empty: ", data == "") + f:close() + } +} +--- user_files +>>> test.txt +hello +--- request +GET /t +--- response_body +data len: 0 +is empty: true + + + +=== TEST 8: write with offset (pwrite) +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local path = "$TEST_NGINX_HTML_DIR/out.txt" + + local f = ngx.fs.open(path, "w+") + + local n1 = f:write("hello world") + ngx.say("write1: ", n1) + + local n2 = f:write("nginx", 6) + ngx.say("write2: ", n2) + + local data = f:read(4096) + ngx.say("data: [", data, "]") + f:close() + } +} +--- request +GET /t +--- response_body +write1: 11 +write2: 5 +data: [hello nginx] + + + +=== TEST 9: write empty data returns 0 +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local f = ngx.fs.open("$TEST_NGINX_HTML_DIR/out.txt", "w") + + local nbytes, err = f:write("") + ngx.say("nbytes: ", nbytes) + ngx.say("err: ", err) + f:close() + } +} +--- request +GET /t +--- response_body +nbytes: 0 +err: nil + + + +=== TEST 10: append mode +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local path = "$TEST_NGINX_HTML_DIR/out.txt" + + local f = ngx.fs.open(path, "w") + f:write("hello") + f:close() + + f = ngx.fs.open(path, "a") + local nbytes = f:write(" world") + ngx.say("append wrote: ", nbytes) + f:close() + + f = ngx.fs.open(path, "r") + local data = f:read(4096) + ngx.say("data: [", data, "]") + f:close() + } +} +--- request +GET /t +--- response_body +append wrote: 6 +data: [hello world] + + + +=== TEST 11: w mode truncates +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local path = "$TEST_NGINX_HTML_DIR/test.txt" + + local f = ngx.fs.open(path, "w") + f:write("new") + f:close() + + f = ngx.fs.open(path, "r") + local data = f:read(4096) + ngx.say("data: [", data, "]") + f:close() + } +} +--- user_files +>>> test.txt +this is old content that should be truncated +--- request +GET /t +--- response_body +data: [new] + + + +=== TEST 12: r+ mode (read/write without truncation) +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local path = "$TEST_NGINX_HTML_DIR/test.txt" + + local f = ngx.fs.open(path, "r+") + + local nbytes = f:write("HELLO") + ngx.say("wrote: ", nbytes) + + local data = f:read(4096) + ngx.say("data: [", data, "]") + f:close() + } +} +--- user_files +>>> test.txt +hello world +--- request +GET /t +--- response_body +wrote: 5 +data: [HELLO world +] + + + +=== TEST 13: stat - regular file +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local info, err = ngx.fs.stat("$TEST_NGINX_HTML_DIR/test.txt") + if not info then + ngx.say("stat failed: ", err) + return + end + + ngx.say("is_file: ", info.is_file) + ngx.say("is_dir: ", info.is_dir) + ngx.say("is_link: ", info.is_link) + ngx.say("size: ", info.size) + ngx.say("has mtime: ", type(info.mtime) == "number") + ngx.say("has atime: ", type(info.atime) == "number") + ngx.say("has ctime: ", type(info.ctime) == "number") + ngx.say("has mode: ", type(info.mode) == "number") + ngx.say("has ino: ", type(info.ino) == "number") + ngx.say("has uid: ", type(info.uid) == "number") + ngx.say("has gid: ", type(info.gid) == "number") + ngx.say("has nlink: ", type(info.nlink) == "number") + } +} +--- user_files +>>> test.txt +hello +--- request +GET /t +--- response_body +is_file: true +is_dir: false +is_link: false +size: 6 +has mtime: true +has atime: true +has ctime: true +has mode: true +has ino: true +has uid: true +has gid: true +has nlink: true + + + +=== TEST 14: stat - directory +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local info = ngx.fs.stat("$TEST_NGINX_HTML_DIR") + + ngx.say("is_file: ", info.is_file) + ngx.say("is_dir: ", info.is_dir) + } +} +--- request +GET /t +--- response_body +is_file: false +is_dir: true + + + +=== TEST 15: stat - nonexistent path +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local info, err = ngx.fs.stat("$TEST_NGINX_HTML_DIR/no_such_file") + if not info then + ngx.say("stat failed: true") + ngx.say("err type: ", type(err)) + return + end + } +} +--- request +GET /t +--- response_body +stat failed: true +err type: string + + + +=== TEST 16: double close +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local f = ngx.fs.open("$TEST_NGINX_HTML_DIR/test.txt", "r") + + local ok, err = f:close() + ngx.say("first close: ", ok) + + ok, err = f:close() + ngx.say("second close ok: ", ok) + ngx.say("second close err: ", err) + + ngx.say("tostring: ", tostring(f)) + } +} +--- user_files +>>> test.txt +hello +--- request +GET /t +--- response_body +first close: true +second close ok: false +second close err: file already closed +tostring: file (closed) + + + +=== TEST 17: open - missing path argument +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local ok, err = pcall(ngx.fs.open) + ngx.say("ok: ", ok) + ngx.say("err match: ", string.find(err, "expecting") ~= nil or string.find(err, "bad argument") ~= nil) + } +} +--- request +GET /t +--- response_body +ok: false +err match: true + + + +=== TEST 18: read - invalid size +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local f = ngx.fs.open("$TEST_NGINX_HTML_DIR/test.txt", "r") + + local ok, err = pcall(f.read, f, 0) + ngx.say("ok: ", ok) + ngx.say("err match: ", string.find(err, "invalid size") ~= nil) + f:close() + } +} +--- user_files +>>> test.txt +hello +--- request +GET /t +--- response_body +ok: false +err match: true + + + +=== TEST 19: read - negative offset +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local f = ngx.fs.open("$TEST_NGINX_HTML_DIR/test.txt", "r") + + local ok, err = pcall(f.read, f, 10, -1) + ngx.say("ok: ", ok) + ngx.say("err match: ", string.find(err, "invalid offset") ~= nil) + f:close() + } +} +--- user_files +>>> test.txt +hello +--- request +GET /t +--- response_body +ok: false +err match: true + + + +=== TEST 20: write - negative offset +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local f = ngx.fs.open("$TEST_NGINX_HTML_DIR/out.txt", "w") + + local ok, err = pcall(f.write, f, "data", -1) + ngx.say("ok: ", ok) + ngx.say("err match: ", string.find(err, "invalid offset") ~= nil) + f:close() + } +} +--- request +GET /t +--- response_body +ok: false +err match: true + + + +=== TEST 21: thread pool not found +--- config +location /t { + content_by_lua_block { + local f, err = ngx.fs.open("$TEST_NGINX_HTML_DIR/test.txt", "r") + if not f then + ngx.say("open failed: true") + ngx.say("has pool name: ", string.find(err, '"default_lua_io"', 1, true) ~= nil) + ngx.say("has hint: ", string.find(err, "thread_pool") ~= nil) + return + end + } +} +--- user_files +>>> test.txt +hello +--- request +GET /t +--- response_body +open failed: true +has pool name: true +has hint: true + + + +=== TEST 22: multiple sequential operations +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local path = "$TEST_NGINX_HTML_DIR/seq.txt" + + local f = ngx.fs.open(path, "w") + f:write("line1\n") + f:close() + + f = ngx.fs.open(path, "a") + f:write("line2\n") + f:close() + + local info = ngx.fs.stat(path) + ngx.say("size: ", info.size) + + f = ngx.fs.open(path, "r") + local data = f:read(4096) + ngx.say("data: [", data, "]") + f:close() + } +} +--- request +GET /t +--- response_body +size: 12 +data: [line1 +line2 +] + + + +=== TEST 23: large file write and read +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local path = "$TEST_NGINX_HTML_DIR/large.txt" + local chunk = string.rep("A", 1024) + local total = 1024 + + local f = ngx.fs.open(path, "w") + local written = 0 + for i = 0, total - 1 do + local nbytes = f:write(chunk, i * 1024) + written = written + nbytes + end + f:close() + ngx.say("wrote: ", written) + + local info = ngx.fs.stat(path) + ngx.say("file size: ", info.size) + + f = ngx.fs.open(path, "r") + local data = f:read(1024 * 1024) + ngx.say("read back: ", #data) + ngx.say("match: ", data == string.rep("A", 1024 * 1024)) + f:close() + } +} +--- request +GET /t +--- response_body +wrote: 1048576 +file size: 1048576 +read back: 1048576 +match: true +--- timeout: 10 + + + +=== TEST 24: partial read +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local f = ngx.fs.open("$TEST_NGINX_HTML_DIR/test.txt", "r") + + local d1 = f:read(5) + ngx.say("part1: [", d1, "]") + + local d2 = f:read(5, 5) + ngx.say("part2: [", d2, "]") + f:close() + } +} +--- user_files +>>> test.txt +helloworld +--- request +GET /t +--- response_body +part1: [hello] +part2: [world] + + + +=== TEST 25: in access_by_lua_block +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + access_by_lua_block { + local info = ngx.fs.stat("$TEST_NGINX_HTML_DIR/test.txt") + ngx.ctx.file_size = info.size + } + + content_by_lua_block { + ngx.say("size: ", ngx.ctx.file_size) + } +} +--- user_files +>>> test.txt +hello +--- request +GET /t +--- response_body +size: 6 + + + +=== TEST 26: stat size matches written bytes +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local path = "$TEST_NGINX_HTML_DIR/out.txt" + local content = "exactly 30 bytes of content!!\n" + + local f = ngx.fs.open(path, "w") + local nbytes = f:write(content) + f:close() + + local info = ngx.fs.stat(path) + ngx.say("written: ", nbytes) + ngx.say("stat size: ", info.size) + ngx.say("match: ", nbytes == info.size) + } +} +--- request +GET /t +--- response_body +written: 30 +stat size: 30 +match: true + + + +=== TEST 27: read - missing size argument +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local f = ngx.fs.open("$TEST_NGINX_HTML_DIR/test.txt", "r") + local ok, err = pcall(f.read, f) + ngx.say("ok: ", ok) + ngx.say("err match: ", string.find(err, "bad argument") ~= nil + or string.find(err, "number expected") ~= nil) + f:close() + } +} +--- user_files +>>> test.txt +hello +--- request +GET /t +--- response_body +ok: false +err match: true + + + +=== TEST 28: write - missing data argument +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local f = ngx.fs.open("$TEST_NGINX_HTML_DIR/out.txt", "w") + local ok, err = pcall(f.write, f) + ngx.say("ok: ", ok) + ngx.say("err match: ", string.find(err, "bad argument") ~= nil + or string.find(err, "string expected") ~= nil) + f:close() + } +} +--- request +GET /t +--- response_body +ok: false +err match: true + + + +=== TEST 29: stat - missing path argument +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local ok, err = pcall(ngx.fs.stat) + ngx.say("ok: ", ok) + ngx.say("err match: ", string.find(err, "bad argument") ~= nil + or string.find(err, "expecting") ~= nil) + } +} +--- request +GET /t +--- response_body +ok: false +err match: true + + + +=== TEST 30: w+ mode +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local f = ngx.fs.open("$TEST_NGINX_HTML_DIR/test.txt", "w+") + + f:write("new content") + local data = f:read(4096) + ngx.say("data: [", data, "]") + f:close() + } +} +--- user_files +>>> test.txt +old content that should be gone +--- request +GET /t +--- response_body +data: [new content] + + + +=== TEST 31: open error message includes path +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local f, err = ngx.fs.open("/no/such/path/file.txt", "r") + if not f then + ngx.say("has path: ", string.find(err, "/no/such/path/file.txt") ~= nil) + return + end + } +} +--- request +GET /t +--- response_body +has path: true + + + +=== TEST 32: stat error message includes path +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local info, err = ngx.fs.stat("/no/such/path/file.txt") + if not info then + ngx.say("has path: ", string.find(err, "/no/such/path/file.txt") ~= nil) + return + end + } +} +--- request +GET /t +--- response_body +has path: true + + + +=== TEST 33: concurrent reads from uthreads +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local results = {} + + local function read_file(idx, path) + local f = ngx.fs.open(path, "r") + if not f then + results[idx] = "open failed" + return + end + + local data = f:read(4096) + f:close() + results[idx] = data + end + + local t1 = ngx.thread.spawn(read_file, 1, "$TEST_NGINX_HTML_DIR/a.txt") + local t2 = ngx.thread.spawn(read_file, 2, "$TEST_NGINX_HTML_DIR/b.txt") + ngx.thread.wait(t1) + ngx.thread.wait(t2) + + ngx.say("a: [", results[1], "]") + ngx.say("b: [", results[2], "]") + } +} +--- user_files +>>> a.txt +content_a +>>> b.txt +content_b +--- request +GET /t +--- response_body +a: [content_a +] +b: [content_b +] + + + +=== TEST 34: all 5 modes work +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local path = "$TEST_NGINX_HTML_DIR/mode_test.txt" + + local modes = {"r", "w", "a", "r+", "w+"} + for _, m in ipairs(modes) do + if m == "r" or m == "r+" then + local wf = ngx.fs.open(path, "w") + wf:write("test") + wf:close() + end + + local f, err = ngx.fs.open(path, m) + if not f then + ngx.say("mode ", m, ": FAIL - ", err) + else + ngx.say("mode ", m, ": OK") + f:close() + end + end + } +} +--- request +GET /t +--- response_body +mode r: OK +mode w: OK +mode a: OK +mode r+: OK +mode w+: OK + + + +=== TEST 35: write binary data +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local path = "$TEST_NGINX_HTML_DIR/binary.bin" + local bin = "\x00\x01\x02\xff\xfe\xfd\x00\x00" + + local f = ngx.fs.open(path, "w") + local nbytes = f:write(bin) + f:close() + ngx.say("wrote: ", nbytes) + + f = ngx.fs.open(path, "r") + local data = f:read(4096) + f:close() + ngx.say("read len: ", #data) + ngx.say("match: ", data == bin) + } +} +--- request +GET /t +--- response_body +wrote: 8 +read len: 8 +match: true + + + +=== TEST 36: in rewrite_by_lua_block +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + rewrite_by_lua_block { + local info = ngx.fs.stat("$TEST_NGINX_HTML_DIR/test.txt") + ngx.ctx.is_file = info.is_file + } + + content_by_lua_block { + ngx.say("is_file: ", ngx.ctx.is_file) + } +} +--- user_files +>>> test.txt +hello +--- request +GET /t +--- response_body +is_file: true + + + +=== TEST 37: GC auto-closes unclosed file, fd is reusable afterward +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + do + local f = ngx.fs.open("$TEST_NGINX_HTML_DIR/test.txt", "r") + local data = f:read(4096) + ngx.say("data: [", data, "]") + end + + collectgarbage() + collectgarbage() + + local f2, err = ngx.fs.open("$TEST_NGINX_HTML_DIR/test.txt", "r") + if not f2 then + ngx.say("reopen failed: ", err) + return + end + + local data2 = f2:read(4096) + ngx.say("reopen data: [", data2, "]") + f2:close() + } +} +--- user_files +>>> test.txt +hello +--- request +GET /t +--- response_body +data: [hello +] +reopen data: [hello +] +--- error_log +auto-closing fd +--- no_error_log +[error] +[alert] + + + +=== TEST 38: custom thread pool +--- main_config + thread_pool mypool threads=4; +--- config +location /t { + content_by_lua_block { + local f = ngx.fs.open("$TEST_NGINX_HTML_DIR/test.txt", "r", "mypool") + if not f then + ngx.say("open failed") + return + end + + ngx.say("type: ", type(f)) + f:close() + } +} +--- user_files +>>> test.txt +hello +--- request +GET /t +--- response_body +type: userdata + + + +=== TEST 39: full cycle with custom pool, pool auto-inherited +--- main_config + thread_pool fspool threads=8; +--- config +location /t { + content_by_lua_block { + local path = "$TEST_NGINX_HTML_DIR/out.txt" + + local f = ngx.fs.open(path, "w+", "fspool") + + local nbytes = f:write("custom pool test") + ngx.say("wrote: ", nbytes) + + local data = f:read(4096) + ngx.say("data: [", data, "]") + f:close() + + local info = ngx.fs.stat(path, "fspool") + ngx.say("size: ", info.size) + } +} +--- request +GET /t +--- response_body +wrote: 16 +data: [custom pool test] +size: 16 + + + +=== TEST 40: custom pool not found +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local f, err = ngx.fs.open("$TEST_NGINX_HTML_DIR/test.txt", "r", "nopool") + if not f then + ngx.say("open failed: true") + ngx.say("has pool name: ", string.find(err, '"nopool"', 1, true) ~= nil) + return + end + } +} +--- user_files +>>> test.txt +hello +--- request +GET /t +--- response_body +open failed: true +has pool name: true + + + +=== TEST 41: stat custom pool not found +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local info, err = ngx.fs.stat("$TEST_NGINX_HTML_DIR/test.txt", "badpool") + if not info then + ngx.say("stat failed: true") + ngx.say("has pool name: ", string.find(err, '"badpool"', 1, true) ~= nil) + return + end + } +} +--- user_files +>>> test.txt +hello +--- request +GET /t +--- response_body +stat failed: true +has pool name: true + + + +=== TEST 42: mix default and custom pools +--- main_config + thread_pool default_lua_io threads=4; + thread_pool iopool threads=8; +--- config +location /t { + content_by_lua_block { + local path = "$TEST_NGINX_HTML_DIR/test.txt" + + local f1 = ngx.fs.open(path, "r") + local data1 = f1:read(4096) + f1:close() + + local f2 = ngx.fs.open(path, "r", "iopool") + local data2 = f2:read(4096) + f2:close() + + ngx.say("default: [", data1, "]") + ngx.say("iopool: [", data2, "]") + ngx.say("match: ", data1 == data2) + } +} +--- user_files +>>> test.txt +hello from both pools +--- request +GET /t +--- response_body +default: [hello from both pools +] +iopool: [hello from both pools +] +match: true + + + +=== TEST 43: read on closed file +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local f = ngx.fs.open("$TEST_NGINX_HTML_DIR/test.txt", "r") + f:close() + + local ok, err = pcall(f.read, f, 4096) + ngx.say("ok: ", ok) + ngx.say("err match: ", string.find(err, "closed") ~= nil) + } +} +--- user_files +>>> test.txt +hello +--- request +GET /t +--- response_body +ok: false +err match: true + + + +=== TEST 44: write on closed file +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local f = ngx.fs.open("$TEST_NGINX_HTML_DIR/out.txt", "w") + f:close() + + local ok, err = pcall(f.write, f, "data") + ngx.say("ok: ", ok) + ngx.say("err match: ", string.find(err, "closed") ~= nil) + } +} +--- request +GET /t +--- response_body +ok: false +err match: true + + + +=== TEST 45: write then stat (size consistency loop) +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local path = "$TEST_NGINX_HTML_DIR/sized.txt" + + local sizes = {0, 1, 100, 1024, 4096} + for _, sz in ipairs(sizes) do + local f = ngx.fs.open(path, "w") + f:write(string.rep("x", sz)) + f:close() + + local info = ngx.fs.stat(path) + ngx.say("wrote ", sz, " -> stat size: ", info.size) + end + } +} +--- request +GET /t +--- response_body +wrote 0 -> stat size: 0 +wrote 1 -> stat size: 1 +wrote 100 -> stat size: 100 +wrote 1024 -> stat size: 1024 +wrote 4096 -> stat size: 4096 + + + +=== TEST 46: stat - is_link with symlink (lstat) +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local ffi = require "ffi" + ffi.cdef[[int symlink(const char *target, const char *linkpath); + int unlink(const char *pathname);]] + + local target = "$TEST_NGINX_HTML_DIR/real.txt" + local link = "$TEST_NGINX_HTML_DIR/link.txt" + + local f = ngx.fs.open(target, "w") + f:write("target content") + f:close() + + ffi.C.unlink(link) + local rc = ffi.C.symlink(target, link) + if rc ~= 0 then + ngx.say("symlink failed") + return + end + + local info_link = ngx.fs.stat(link) + ngx.say("link is_link: ", info_link.is_link) + ngx.say("link is_file: ", info_link.is_file) + + local info_real = ngx.fs.stat(target) + ngx.say("real is_link: ", info_real.is_link) + ngx.say("real is_file: ", info_real.is_file) + } +} +--- request +GET /t +--- response_body +link is_link: true +link is_file: false +real is_link: false +real is_file: true + + + +=== TEST 47: close refused while read in flight (two uthreads) +--- no_http2 +--- quic_max_idle_timeout: 5 +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local f = ngx.fs.open("$TEST_NGINX_HTML_DIR/test.txt", "r") + + local function reader() + local data = f:read(4096) + ngx.say("read: [", data, "]") + end + + local t1 = ngx.thread.spawn(reader) + local ok, err = f:close() + ngx.say("close ok: ", ok) + ngx.say("close err match: ", string.find(err, "in%-flight") ~= nil) + + ngx.thread.wait(t1) + ok, err = f:close() + ngx.say("final close: ", ok) + } +} +--- user_files +>>> test.txt +hello +--- request +GET /t +--- response_body +close ok: false +close err match: true +read: [hello +] +final close: true + + + +=== TEST 48: kill uthread during open, aborted fd is closed +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local function do_open() + local f, err = ngx.fs.open("$TEST_NGINX_HTML_DIR/test.txt", "r") + if f then + ngx.say("child opened") + f:close() + end + end + + local t1 = ngx.thread.spawn(do_open) + ngx.thread.kill(t1) + ngx.say("killed") + + ngx.sleep(0.5) + + local f2, err = ngx.fs.open("$TEST_NGINX_HTML_DIR/test.txt", "r") + if not f2 then + ngx.say("reopen failed: ", err) + return + end + + local data = f2:read(4096) + ngx.say("reopen ok: ", data ~= nil) + f2:close() + } +} +--- user_files +>>> test.txt +hello +--- request +GET /t +--- response_body +killed +reopen ok: true +--- no_error_log +[error] +[alert] + + + +=== TEST 49: fd not leaked after repeated kill-during-open cycles +--- main_config + thread_pool default_lua_io threads=32; +--- config +location /t { + content_by_lua_block { + local function count_fds() + local pid = ngx.worker.pid() + local n = 0 + local p = io.popen("ls /proc/" .. pid .. "/fd 2>/dev/null | wc -l") + if p then + n = tonumber(p:read("*a")) or -1 + p:close() + end + return n + end + + local before = count_fds() + + local N = 200 + for i = 1, N do + local function do_open() + local f = ngx.fs.open("$TEST_NGINX_HTML_DIR/test.txt", "r") + if f then + local _ = f:read(4096) + f:close() + end + end + + local t = ngx.thread.spawn(do_open) + ngx.thread.kill(t) + t = nil + end + + collectgarbage() + collectgarbage() + + ngx.sleep(2) + + collectgarbage() + collectgarbage() + + local after = count_fds() + + local delta = after - before + ngx.say("iterations: ", N) + ngx.say("fd delta acceptable: ", delta < 10) + + local f = ngx.fs.open("$TEST_NGINX_HTML_DIR/test.txt", "r") + if not f then + ngx.say("final open failed - fd leak!") + return + end + + local data = f:read(4096) + ngx.say("final read ok: ", data ~= nil) + f:close() + } +} +--- user_files +>>> test.txt +hello +--- request +GET /t +--- response_body +iterations: 200 +fd delta acceptable: true +final read ok: true +--- no_error_log +[error] +[alert] +--- timeout: 15