feat: script distribute and run (#1982)

Supporting distribution and execution scripts, we can implement many new features, 
such as plug-in orchestration.

apisix/admin/routes.lua

@@ -124,6 +124,18 @@ local function check_conf(id, conf, need_id)
         end
     end
 
+    if conf.script then
+        local obj, err = loadstring(conf.script)
+        if not obj then
+            return nil, {error_msg = "failed to load 'script' string: "
+                                     .. err}
+        end
+
+        if type(obj()) ~= "table" then
+            return nil, {error_msg = "'script' should be a Lua object"}
+        end
+    end
+
     return need_id and id or true
 end
 

apisix/admin/services.lua

@@ -21,6 +21,7 @@ local upstreams = require("apisix.admin.upstreams")
 local tostring = tostring
 local ipairs = ipairs
 local type = type
+local loadstring = loadstring
 
 
 local _M = {
@@ -91,6 +92,18 @@ local function check_conf(id, conf, need_id)
         end
     end
 
+    if conf.script then
+        local obj, err = loadstring(conf.script)
+        if not obj then
+            return nil, {error_msg = "failed to load 'script' string: "
+                                     .. err}
+        end
+
+        if type(obj()) ~= "table" then
+            return nil, {error_msg = "'script' should be a Lua object"}
+        end
+    end
+
     return need_id and id or true
 end
 

apisix/init.lua

@@ -18,6 +18,7 @@ local require       = require
 local core          = require("apisix.core")
 local config_util   = require("apisix.core.config_util")
 local plugin        = require("apisix.plugin")
+local script        = require("apisix.script")
 local service_fetch = require("apisix.http.service").get
 local admin_init    = require("apisix.admin.init")
 local get_var       = require("resty.ngxvar").fetch
@@ -452,19 +453,24 @@ function _M.http_access_phase()
         api_ctx.var.upstream_connection = api_ctx.var.http_connection
     end
 
-    local plugins = plugin.filter(route)
-    api_ctx.plugins = plugins
-
-    run_plugin("rewrite", plugins, api_ctx)
-    if api_ctx.consumer then
-        local changed
-        route, changed = plugin.merge_consumer_route(route, api_ctx.consumer)
-        if changed then
-            core.table.clear(api_ctx.plugins)
-            api_ctx.plugins = plugin.filter(route, api_ctx.plugins)
+    if route.value.script then
+        script.load(route, api_ctx)
+        script.run("access", api_ctx)
+    else
+        local plugins = plugin.filter(route)
+        api_ctx.plugins = plugins
+
+        run_plugin("rewrite", plugins, api_ctx)
+        if api_ctx.consumer then
+            local changed
+            route, changed = plugin.merge_consumer_route(route, api_ctx.consumer)
+            if changed then
+                core.table.clear(api_ctx.plugins)
+                api_ctx.plugins = plugin.filter(route, api_ctx.plugins)
+            end
         end
+        run_plugin("access", plugins, api_ctx)
     end
-    run_plugin("access", plugins, api_ctx)
 
     local ok, err = set_upstream(route, api_ctx)
     if not ok then
@@ -553,7 +559,12 @@ local function common_phase(phase_name)
         core.tablepool.release("plugins", plugins)
     end
 
-    run_plugin(phase_name, nil, api_ctx)
+    if api_ctx.script_obj then
+        script.run(phase_name, api_ctx)
+    else
+        run_plugin(phase_name, nil, api_ctx)
+    end
+
     return api_ctx
 end
 

apisix/schema_def.lua

@@ -422,6 +422,8 @@ _M.route = {
             pattern = [[^function]],
         },
 
+        script = {type = "string", minLength = 10, maxLength = 102400},
+
         plugins = plugins_schema,
         upstream = upstream_schema,
 
@@ -441,6 +443,13 @@ _M.route = {
         {required = {"upstream", "uris"}},
         {required = {"upstream_id", "uris"}},
         {required = {"service_id", "uris"}},
+        {required = {"script", "uri"}},
+        {required = {"script", "uris"}},
+    },
+    ["not"] = {
+        anyOf = {
+            {required = {"script", "plugins"}}
+        }
     },
     additionalProperties = false,
 }
@@ -455,11 +464,13 @@ _M.service = {
         upstream_id = id_schema,
         name = {type = "string", maxLength = 50},
         desc = {type = "string", maxLength = 256},
+        script = {type = "string", minLength = 10, maxLength = 102400},
     },
     anyOf = {
         {required = {"upstream"}},
         {required = {"upstream_id"}},
         {required = {"plugins"}},
+        {required = {"script"}},
     },
     additionalProperties = false,
 }

apisix/script.lua

+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local require    = require
+local core       = require("apisix.core")
+local loadstring = loadstring
+local error      = error
+
+
+local _M = {}
+
+
+function _M.load(route, api_ctx)
+    local script = route.value.script
+    if script == nil or script == "" then
+        error("missing valid script")
+    end
+
+    local loadfun, err = loadstring(script, "route#" .. route.value.id)
+    if not loadfun then
+        error("failed to load script: " .. err .. " script: " .. script)
+        return nil
+    end
+    api_ctx.script_obj = loadfun()
+end
+
+
+function _M.run(phase, api_ctx)
+    local obj = api_ctx and api_ctx.script_obj
+    if not obj then
+        core.log.error("missing loaded script object")
+        return api_ctx
+    end
+
+    core.log.info("loaded script_obj: ", core.json.delay_encode(obj, true))
+
+    local phase_fun = obj[phase]
+    if phase_fun then
+        phase_fun(api_ctx)
+    end
+
+    return api_ctx
+end
+
+
+return _M

doc/admin-api.md

@@ -64,6 +64,7 @@
 |vars       |False  |Match Rules |A list of one or more `{var, operator, val}` elements, like this: `{{var, operator, val}, {var, operator, val}, ...}}`. For example: `{"arg_name", "==", "json"}` means that the current request parameter `name` is `json`. The `var` here is consistent with the internal variable name of Nginx, so you can also use `request_uri`, `host`, etc. For the operator part, the currently supported operators are `==`, `~=`,`>`, `<`, and `~~`. For the `>` and `<` operators, the result is first converted to `number` and then compared. See a list of [supported operators](#available-operators) |{{"arg_name", "==", "json"}, {"arg_age", ">", 18}}|
 |filter_func|False|Match Rules|User-defined filtering function. You can use it to achieve matching requirements for special scenarios. This function accepts an input parameter named `vars` by default, which you can use to get Nginx variables.|function(vars) return vars["arg_name"] == "json" end|
 |plugins  |False |Plugin|See [Plugin](architecture-design.md#plugin) for more ||
+|script  |False |Script|See [Script](architecture-design.md#script) for more ||
 |upstream |False |Upstream|Enabled Upstream configuration, see [Upstream](architecture-design.md#upstream) for more||
 |upstream_id|False |Upstream|Enabled upstream id, see [Upstream](architecture-design.md#upstream) for more ||
 |service_id|False |Service|Binded Service configuration, see [Service](architecture-design.md#service) for more ||

doc/architecture-design.md

@@ -26,6 +26,7 @@
 - [**Route**](#route)
 - [**Service**](#service)
 - [**Plugin**](#plugin)
+- [**Script**](#script)
 - [**Upstream**](#upstream)
 - [**Router**](#router)
 - [**Consumer**](#consumer)
@@ -216,6 +217,25 @@ Not all plugins have specific configuration items. For example, there is no spec
 
 [Back to top](#Table-of-contents)
 
+## Script
+
+`Script` represents a script that will be executed during the `HTTP` request/response life cycle.
+
+The `Script` configuration can be directly bound to the `Route`.
+
+`Script` and `Plugin` are mutually exclusive, and `Script` is executed first. This means that after configuring `Script`, the `Plugin` configured on `Route` will not be executed.
+
+In theory, you can write arbitrary Lua code in `Script`, or you can directly call existing plugins to reuse existing code.
+
+`Script` also has the concept of execution phase, supporting `access`, `header_filer`, `body_filter` and `log` phase. The system will automatically execute the code of the corresponding phase in the `Script` script in the corresponding phase.
+
+```json
+{
+    ...
+    "script": "local _M = {} \n function _M.access(api_ctx) \n ngx.log(ngx.INFO,\"hit access phase\") \n end \nreturn _M"
+}
+```
+
 ## Upstream
 
 Upstream is a virtual host abstraction that performs load balancing on a given set of service nodes according to configuration rules. Upstream address information can be directly configured to `Route` (or `Service`). When Upstream has duplicates, you need to use "reference" to avoid duplication.

doc/zh-cn/admin-api.md

@@ -57,10 +57,11 @@
 |---------|---------|----|-----------|----|
 |uri      |与 `uris` 二选一 |匹配规则|除了如 `/foo/bar`、`/foo/gloo` 这种全量匹配外，使用不同 [Router](architecture-design.md#router) 还允许更高级匹配，更多见 [Router](architecture-design.md#router)。|"/hello"|
 |uris     |与 `uri` 二选一 |匹配规则|数组形式，可以匹配多个 `uri`|["/hello", "/world"]|
-|plugins  |`plugins`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Plugin|详见 [Plugin](architecture-design.md#plugin) ||
-|upstream |`plugins`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Upstream|启用的 Upstream 配置，详见 [Upstream](architecture-design.md#upstream)||
-|upstream_id|`plugins`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Upstream|启用的 upstream id，详见 [Upstream](architecture-design.md#upstream)||
-|service_id|`plugins`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Service|绑定的 Service 配置，详见 [Service](architecture-design.md#service)||
+|plugins  |`plugins`、`script`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Plugin|详见 [Plugin](architecture-design.md#plugin) ||
+|script  |`plugins`、`script`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Script|详见 [Script](architecture-design.md#script) ||
+|upstream |`plugins`、`script`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Upstream|启用的 Upstream 配置，详见 [Upstream](architecture-design.md#upstream)||
+|upstream_id|`plugins`、`script`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Upstream|启用的 upstream id，详见 [Upstream](architecture-design.md#upstream)||
+|service_id|`plugins`、`script`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Service|绑定的 Service 配置，详见 [Service](architecture-design.md#service)||
 |service_protocol|可选|上游协议类型|只可以是 "grpc", "http" 二选一。|默认 "http"，使用gRPC proxy 或gRPC transcode 时，必须用"grpc"|
 |name     |可选 |辅助   |标识路由名称|route-xxxx|
 |desc     |可选 |辅助   |标识描述、使用场景等。|客户 xxxx|
@@ -75,7 +76,7 @@
 
 有两点需要特别注意：
 
-* 除了 `uri`/`uris` 是必选的之外，`plugins`、`upstream`/`upstream_id`、`service_id` 这三类必须选择其中至少一个。
+* 除了 `uri`/`uris` 是必选的之外，`plugins`、`script`、`upstream`/`upstream_id`、`service_id` 这三类必须选择其中至少一个。
 * 对于同一类参数比如 `uri`与 `uris`，`upstream` 与 `upstream_id`，`host` 与 `hosts`，`remote_addr` 与 `remote_addrs` 等，是不能同时存在，二者只能选择其一。如果同时启用，接口会报错。
 
 route 对象 json 配置内容：

doc/zh-cn/architecture-design.md

@@ -24,6 +24,7 @@
 - [**Route**](#route)
 - [**Service**](#service)
 - [**Plugin**](#plugin)
+- [**Script**](#script)
 - [**Upstream**](#upstream)
 - [**Router**](#router)
 - [**Consumer**](#consumer)
@@ -220,6 +221,27 @@ curl http://127.0.0.1:9080/apisix/admin/routes/102 -H 'X-API-KEY: edd1c9f034335f
 
 [返回目录](#目录)
 
+## Script
+
+`Script` 表示将在 `HTTP` 请求/响应生命周期期间执行的脚本。
+
+`Script` 配置可直接绑定在 `Route` 上。
+
+`Script` 与 `Plugin` 互斥，且优先执行 `Script` ，这意味着配置 `Script` 后，`Route` 上配置的 `Plugin` 将不被执行。
+
+理论上，在 `Script` 中可以写任意 lua 代码，也可以直接调用已有插件以重用已有的代码。
+
+`Script` 也有执行阶段概念，支持 `access`、`header_filer`、`body_filter` 和 `log` 阶段。系统会在相应阶段中自动执行 `Script` 脚本中对应阶段的代码。
+
+```json
+{
+    ...
+    "script": "local _M = {} \n function _M.access(api_ctx) \n ngx.log(ngx.INFO,\"hit access phase\") \n end \nreturn _M"
+}
+```
+
+[返回目录](#目录)
+
 ## Upstream
 
 Upstream 是虚拟主机抽象，对给定的多个服务节点按照配置规则进行负载均衡。Upstream 的地址信息可以直接配置到 `Route`（或 `Service`) 上，当 Upstream 有重复时，就需要用“引用”方式避免重复了。

t/admin/routes.t

@@ -2108,3 +2108,42 @@ GET /t
 {"error_msg":"invalid request body: request size 1678025 is greater than the maximum size 1572864 allowed"}
 --- error_log
 failed to read request body: request size 1678025 is greater than the maximum size 1572864 allowed
+
+
+
+=== TEST 58: uri + plugins + script  failed
+--- config
+    location /t {
+        content_by_lua_block {
+            local core = require("apisix.core")
+            local t = require("lib.test_admin").test
+            local code, message, res = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "plugins": {
+                            "limit-count": {
+                                "count": 2,
+                                "time_window": 60,
+                                "rejected_code": 503,
+                                "key": "remote_addr"
+                            }
+                        },
+                        "script": "local _M = {} \n function _M.access(api_ctx) \n ngx.log(ngx.INFO,\"hit access phase\") \n end \nreturn _M",
+                        "uri": "/index.html"
+                }]]
+                )
+
+            if code ~= 200 then
+                ngx.status = code
+                ngx.say(message)
+                return
+            end
+        }
+    }
+--- request
+GET /t
+--- error_code: 400
+--- response_body_like
+{"error_msg":"invalid configuration: value wasn't supposed to match schema"}
+--- no_error_log
+[error]

t/script/script_distribute.t

+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+no_root_location();
+no_shuffle();
+
+
+run_tests;
+
+__DATA__
+
+=== TEST 1: set route(host + uri)
+--- config
+    location /t {
+        content_by_lua_block {
+            local core = require("apisix.core")
+            local t = require("lib.test_admin")
+
+            local script = t.read_file("t/script/script_test.lua")
+            local data = {
+                script = script,
+                uri = "/hello",
+                upstream = {
+                    nodes = {
+                        ["127.0.0.1:1980"] = 1
+                    },
+                    type = "roundrobin"
+                }
+            }
+
+            local code, body = t.test('/apisix/admin/routes/1',
+                ngx.HTTP_PUT,
+                core.json.encode(data))
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- yaml_config eval: $::yaml_config
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 2: hit routes
+--- request
+GET /hello
+--- yaml_config eval: $::yaml_config
+--- response_body
+hello world
+--- no_error_log
+[error]
+--- error_log
+string "route#1"
+phase_fun(): hit access phase
+phase_fun(): hit header_filter phase
+phase_fun(): hit body_filter phase
+phase_fun(): hit body_filter phase
+phase_fun(): hit log phase while
+
+
+
+=== TEST 3: invalid script in route
+--- config
+    location /t {
+        content_by_lua_block {
+            local core = require("apisix.core")
+            local t = require("lib.test_admin")
+
+            local data = {
+                script = "invalid script",
+                uri = "/hello",
+                upstream = {
+                    nodes = {
+                        ["127.0.0.1:1980"] = 1
+                    },
+                    type = "roundrobin"
+                }
+            }
+
+            local code, body = t.test('/apisix/admin/routes/1',
+                ngx.HTTP_PUT,
+                core.json.encode(data))
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.print(body)
+        }
+    }
+--- yaml_config eval: $::yaml_config
+--- request
+GET /t
+--- error_code: 400
+--- response_body
+{"error_msg":"failed to load 'script' string: [string \"invalid script\"]:1: '=' expected near 'script'"}
+--- no_error_log
+[error]
+
+
+
+=== TEST 4: invalid script in service
+--- config
+    location /t {
+        content_by_lua_block {
+            local core = require("apisix.core")
+            local t = require("lib.test_admin")
+
+            local data = {
+                script = "invalid script",
+                upstream = {
+                    nodes = {
+                        ["127.0.0.1:1980"] = 1
+                    },
+                    type = "roundrobin"
+                }
+            }
+
+            local code, body = t.test('/apisix/admin/services/1',
+                ngx.HTTP_PUT,
+                core.json.encode(data))
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.print(body)
+        }
+    }
+--- yaml_config eval: $::yaml_config
+--- request
+GET /t
+--- error_code: 400
+--- response_body
+{"error_msg":"failed to load 'script' string: [string \"invalid script\"]:1: '=' expected near 'script'"}
+--- no_error_log
+[error]

t/script/script_test.lua

+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local core   = require("apisix.core")
+
+
+local _M = {}
+
+
+function _M.access(api_ctx)
+    core.log.warn("hit access phase")
+end
+
+
+function _M.header_filter(ctx)
+    core.log.warn("hit header_filter phase")
+end
+
+
+function _M.body_filter(ctx)
+    core.log.warn("hit body_filter phase")
+end
+
+
+function _M.log(ctx)
+    core.log.warn("hit log phase")
+end
+
+
+return _M