Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(response-rewrite): support brotli when use filters.regex option #10733

Merged
merged 4 commits into from
Jan 4, 2024
Merged

feat(response-rewrite): support brotli when use filters.regex option #10733

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7551eb9
demo
yuweizzz Dec 25, 2023
b90d036
split func
yuweizzz Dec 27, 2023
9a42115
t
yuweizzz Dec 28, 2023
d055ff6
vars rename
yuweizzz Jan 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions apisix/plugins/response-rewrite.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ local type = type
local pcall = pcall
local zlib = require("ffi-zlib")
local str_buffer = require("string.buffer")
local is_br_libs_loaded, brotli = pcall(require, "brotli")


local lrucache = core.lrucache.new({
Expand Down Expand Up @@ -227,6 +228,58 @@ local function inflate_gzip(data)
end


local function brotli_stream_decode(read_inputs, write_outputs)
-- read 64k data per times
local read_size = 64 * 1024
local decompressor = brotli.decompressor:new()

local chunk, ok, res
repeat
chunk = read_inputs(read_size)
if chunk then
ok, res = pcall(function()
return decompressor:decompress(chunk)
end)
else
ok, res = pcall(function()
return decompressor:finish()
end)
end
if not ok then
return false, res
end
write_outputs(res)
until not chunk

return true, nil
end


local function brotli_decode(data)
local inputs = str_buffer.new():set(data)
local outputs = str_buffer.new()

local read_inputs = function(size)
local data = inputs:get(size)
if data == "" then
return nil
end
return data
end

local write_outputs = function(data)
return outputs:put(data)
end

local ok, err = brotli_stream_decode(read_inputs, write_outputs)
if not ok then
return nil, err
end

return outputs:get()
end


function _M.check_schema(conf)
local ok, err = core.schema.check(schema, conf)
if not ok then
Expand Down Expand Up @@ -294,6 +347,12 @@ function _M.body_filter(conf, ctx)
core.log.error("filters may not work as expected, inflate gzip err: ", err)
return
end
elseif ctx.response_encoding == "br" and is_br_libs_loaded then
body, err = brotli_decode(body)
if err ~= nil then
core.log.error("filters may not work as expected, brotli decode err: ", err)
return
end
elseif ctx.response_encoding ~= nil then
core.log.error("filters may not work as expected ",
"due to unsupported compression encoding type: ",
Expand Down
246 changes: 244 additions & 2 deletions t/plugin/response-rewrite3.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,45 @@ add_block_preprocessor(sub {
}
}
}

server {
listen 11452;
location /brotli_hello {
content_by_lua_block {
ngx.req.read_body()
local s = "hello world hello world hello world"
ngx.header['Content-Length'] = #s + 1
ngx.say(s)
}
header_filter_by_lua_block {
local conf = {
comp_level = 6,
http_version = 1.1,
lgblock = 0,
lgwin = 19,
min_length = 1,
mode = 0,
types = "*",
}
local brotli = require("apisix.plugins.brotli")
brotli.header_filter(conf, ngx.ctx)
}
body_filter_by_lua_block {
local conf = {
comp_level = 6,
http_version = 1.1,
lgblock = 0,
lgwin = 19,
min_length = 1,
mode = 0,
types = "*",
}
local brotli = require("apisix.plugins.brotli")
brotli.body_filter(conf, ngx.ctx)
}
}
}

_EOC_

$block->set_value("http_config", $http_config);
Expand Down Expand Up @@ -238,7 +277,6 @@ Content-Encoding:

if code >= 300 then
ngx.status = code
return
end
ngx.say(body)
}
Expand Down Expand Up @@ -296,7 +334,6 @@ Content-Encoding:

if code >= 300 then
ngx.status = code
return
end
ngx.say(body)
}
Expand Down Expand Up @@ -433,3 +470,208 @@ Content-Encoding: deflate
X-Server-id: 3
X-Server-status: on
Content-Type:



=== TEST 15: set route use brotli upstream
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/routes/1',
ngx.HTTP_PUT,
[[{
"uri": "/brotli_hello",
"upstream": {
"nodes": {
"127.0.0.1:11452": 1
},
"type": "roundrobin"
}
}]]
)

if code >= 300 then
ngx.status = code
end
ngx.say(body)
}
}
--- request
GET /t
--- response_body
passed



=== TEST 16: should return brotli body
--- request
GET /brotli_hello
--- more_headers
Accept-Encoding: br
--- response_headers
Content-Encoding: br



=== TEST 17: set route use brotli upstream and response-rewrite body conf
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/routes/1',
ngx.HTTP_PUT,
[[{
"uri": "/brotli_hello",
"upstream": {
"nodes": {
"127.0.0.1:11452": 1
},
"type": "roundrobin"
},
"plugins": {
"response-rewrite": {
"vars": [
["status","==",200]
],
"body": "new body\n"
}
}
}]]
)

if code >= 300 then
ngx.status = code
end
ngx.say(body)
}
}
--- request
GET /t
--- response_body
passed



=== TEST 18: should rewrite body and clear Content-Encoding header
--- request
GET /brotli_hello
--- more_headers
Accept-Encoding: br
--- response_body
new body
--- response_headers
Content-Encoding:



=== TEST 19: set route use brotli upstream and response-rewrite filter conf
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/routes/1',
ngx.HTTP_PUT,
[[{
"uri": "/brotli_hello",
"upstream": {
"nodes": {
"127.0.0.1:11452": 1
},
"type": "roundrobin"
},
"plugins": {
"response-rewrite": {
"vars": [
["status","==",200]
],
"filters": [
{
"regex": "hello",
"replace": "test"
}
]
}
}
}]]
)

if code >= 300 then
ngx.status = code
end
ngx.say(body)
}
}
--- request
GET /t
--- response_body
passed



=== TEST 20: brotli decode support, should rewrite body and clear Content-Encoding header
--- request
GET /brotli_hello
--- more_headers
Accept-Encoding: br
--- response_body
test world hello world hello world
--- response_headers
Content-Encoding:



=== TEST 21: set route use response-write plugin but not use filter conf or body conf
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/routes/1',
ngx.HTTP_PUT,
[[{
"uri": "/brotli_hello",
"upstream": {
"nodes": {
"127.0.0.1:11452": 1
},
"type": "roundrobin"
},
"plugins": {
"response-rewrite": {
"vars": [
["status","==",200]
],
"headers": {
"set": {
"X-Server-id": 3,
"X-Server-status": "on",
"Content-Type": ""
}
}
}
}
}]]
)
if code >= 300 then
ngx.status = code
end
ngx.say(body)
}
}
--- request
GET /t
--- response_body
passed



=== TEST 22: should keep Content-Encoding
--- request
GET /brotli_hello
--- more_headers
Accept-Encoding: br
--- response_headers
Content-Encoding: br
X-Server-id: 3
X-Server-status: on
Content-Type:
Loading