tarantool / tarantool/tarantool

Port patches for next/pairs recording

Open
#6,475 1 comment 0 reactions 1 assignee View on GitHub

@Buristan is already working on this.

Since Aug 19, 2026.

feature luajit
Dominant language
Lua
Stars
3.7k
Forks
419
Avg merge
1d 23h
Merged PRs (30d)
88

Description

It would be great to port https://github.com/LuaJIT/LuaJIT/commit/bb0f24101565d34ea8b70fdec4dd3f3b35a70e7b (and following bugfixes as well) to our LuaJIT fork.

Motivation: it affect performance. In my case I decided to test TDG codegenreation for flattening objects to tuples.
I ported a code that TDG internally generate to pure LuaJIT. I put it at the bottom.

So, let's compare performance:

Before patch (8ff09d9f5ad5b037926be2a50dc32b681c5e7597):
../sandbox/LuaJIT/src/luajit code.lua 
4.05692

After patch (cb0f124f8f8717eb235989f9c86168140fbf96fd):
../sandbox/LuaJIT/src/luajit code.lua 
2.041649

So we see x2 performance improvements. I think it's quite significant reason for porting this patchset :)

local ffi = require('ffi')
local format = string.format
local floor = math.floor
local NULL = ffi.cast('void*', 0)
local json = {
    encode = tostring,
}


local function is_int_impl(data)
    if type(data) ~= 'number' and
        (not ffi.istype('int64_t', data)) and
        (not ffi.istype('uint64_t', data)) then
        return false
    end

    if data < -2147483648 or data > 2147483647 or floor(tonumber(data)) ~= data then
        return false
    end
    return true
end

local function to_double_impl(data)
    local xtype = type(data)
    if xtype == "number" then
        return true
    else
        if xtype == "cdata" then
            local xdata = tonumber(data)
            if xdata == nil then
                return false
            else
                return true
            end
        end
    end
    return false
end

local function to_long_impl(data)
    if type(data) ~= 'number' and
        (not ffi.istype('int64_t', data)) and
        (not ffi.istype('uint64_t', data)) then
        return false
    end

    local n = tonumber(data)
    -- note: if it's not a number or cdata(numbertype),
    --       the expression below will raise
    -- note: boundaries were carefully picked to avoid
    --       rounding errors, they are INT64_MIN and INT64_MAX+1,
    --       respectively (both 2**k)
    if n < -9223372036854775808 or n >= 9223372036854775808 or
       floor(n) ~= n then
        -- due to rounding errors, INT64_MAX-1023..INT64_MAX
        -- fails the range check above, check explicitly for this
        -- case; in number > cdata(uint64_t) expression, number
        -- is implicitly coerced to uint64_t
        if n ~= 9223372036854775808 or n > 9223372036854775807ULL then
            return false
        end
    end
    return true
end

local function to_long(data, path)
    if not to_long_impl(data) then
        error(format('%s is not a "long": %s', path, json.encode(data)), 0)
    end
    return data
end

local function to_string(data, path)
    if type(data) ~= 'string' then
        error(format('%s is not a "string": %s', path, json.encode(data)), 0)
    end
    return data
end

local function to_string_var(data, path, ...)
    if type(data) ~= 'string' then
        local fullpath = format(path, ...)
        error(format('%s is not a "string": %s', fullpath, json.encode(data)), 0)
    end
    return data
end

local function is_array(data, path)
    if type(data) ~= 'table' then
        error(format('%s is not an array: %s', path, json.encode(data)), 0)
    end

    for k in pairs(data) do
        if type(k) ~= 'number' then
            error(format('%s contains non-number keys: %s', path, json.encode(k)), 0)
        end
    end

    return data
end

local function is_array_var(data, path, ...)
    if type(data) ~= 'table' then
        local fullpath = format(path, ...)
        error(format('%s is not an array: %s', fullpath, json.encode(data)), 0)
    end

    for k in pairs(data) do
        if type(k) ~= 'number' then
            local fullpath = format(path, ...)
            error(format('%s contains non-number keys: %s', fullpath, json.encode(k)), 0)
        end
    end

    return data
end

local fieldmap = {
    this_is_really_big_field_name = 4,
    id = 1,
    fieldA = 2,
    array = 5,
    fieldB = 3,
}
local is_record_Entity = function(data, path)
    if type(data) ~= 'table' then
        error(format('%s is not a record %s: %s', path, 'Entity', json.encode(data)), 0)
    end

    for k, _ in pairs(data) do
        if fieldmap[k] == nil then
            error(format('%s/%s: Unknown field', path, k), 0)
        end
    end
end

local fieldmap = {
    entity3 = 5,
    value_object3 = 8,
    id = 1,
    entity = 3,
    number = 2,
    value_object = 6,
    entity2 = 4,
    value_object2 = 7,
    record_array = 11,
    array = 10,
    this_is_really_big_field_name = 9,
}
local is_record_Object = function(data, path)
    if type(data) ~= 'table' then
        error(format('%s is not a record %s: %s', path, 'Object', json.encode(data)), 0)
    end

    for k, _ in pairs(data) do
        if fieldmap[k] == nil then
            error(format('%s/%s: Unknown field', path, k), 0)
        end
    end
end

local fieldmap = {
    this_is_really_big_field_name = 5,
    id = 1,
    date = 3,
    time = 4,
    array = 6,
    datetime = 2,
}

local is_record_ValueObject = function(data, path)
    if type(data) ~= 'table' then
        error(format('%s is not a record %s: %s', path, 'ValueObject', json.encode(data)), 0)
    end

    for k, _ in pairs(data) do
        if fieldmap[k] == nil then
            error(format('%s/%s: Unknown field', path, k), 0)
        end
    end
end

local fieldmap = {
    payload = 2,
    array = 3,
    id = 1,
}
local is_record_EntityInArray = function(data, path)
    if type(data) ~= 'table' then
        error(format('%s is not a record %s: %s', path, 'EntityInArray', json.encode(data)), 0)
    end

    for k, _ in pairs(data) do
        if fieldmap[k] == nil then
            error(format('%s/%s: Unknown field', path, k), 0)
        end
    end
end

local serialize = function(object)
    local result = {NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,}
    is_record_Object(object, "Object")
    result[1] = to_string(object["id"], "Object/id")
    result[23] = to_long(object["number"], "Object/number")
    if object["entity"] ~= nil then
        is_record_Entity(object["entity"], "Object/entity")
        result[5] = true
        local tmp_1 = object["entity"]
        result[9] = to_string(tmp_1["id"], "Object/entity/id")
        result[7] = to_string(tmp_1["fieldA"], "Object/entity/fieldA")
        result[8] = to_string(tmp_1["fieldB"], "Object/entity/fieldB")
        result[10] = to_string(tmp_1["this_is_really_big_field_name"], "Object/entity/this_is_really_big_field_name")
        is_array(tmp_1["array"], "Object/entity/array")
        result[6] = {}
        for i1, value in ipairs(tmp_1["array"]) do
            result[6][i1] = to_string_var(value, "Object/entity/array/%d", i1)
        end
    else
        result[5] = NULL
    end
    if object["entity2"] ~= nil then
        is_record_Entity(object["entity2"], "Object/entity2")
        result[11] = true
        local tmp_1 = object["entity2"]
        result[15] = to_string(tmp_1["id"], "Object/entity2/id")
        result[13] = to_string(tmp_1["fieldA"], "Object/entity2/fieldA")
        result[14] = to_string(tmp_1["fieldB"], "Object/entity2/fieldB")
        result[16] = to_string(tmp_1["this_is_really_big_field_name"], "Object/entity2/this_is_really_big_field_name")
        is_array(tmp_1["array"], "Object/entity2/array")
        result[12] = {}
        for i1, value in ipairs(tmp_1["array"]) do
            result[12][i1] = to_string_var(value, "Object/entity2/array/%d", i1)
        end
    else
        result[11] = NULL
    end
    if object["entity3"] ~= nil then
        is_record_Entity(object["entity3"], "Object/entity3")
        result[17] = true
        local tmp_1 = object["entity3"]
        result[21] = to_string(tmp_1["id"], "Object/entity3/id")
        result[19] = to_string(tmp_1["fieldA"], "Object/entity3/fieldA")
        result[20] = to_string(tmp_1["fieldB"], "Object/entity3/fieldB")
        result[22] = to_string(tmp_1["this_is_really_big_field_name"], "Object/entity3/this_is_really_big_field_name")
        is_array(tmp_1["array"], "Object/entity3/array")
        result[18] = {}
        for i1, value in ipairs(tmp_1["array"]) do
            result[18][i1] = to_string_var(value, "Object/entity3/array/%d", i1)
        end
    else
        result[17] = NULL
    end
    if object["value_object"] ~= nil then
        is_record_ValueObject(object["value_object"], "Object/value_object")
        result[29] = true
        local tmp_1 = object["value_object"]
        result[33] = to_string(tmp_1["id"], "Object/value_object/id")
        result[32] = to_string(tmp_1["datetime"], "Object/value_object/datetime")
        result[31] = to_string(tmp_1["date"], "Object/value_object/date")
        result[35] = to_string(tmp_1["time"], "Object/value_object/time")
        result[34] = to_string(tmp_1["this_is_really_big_field_name"], "Object/value_object/this_is_really_big_field_name")
        is_array(tmp_1["array"], "Object/value_object/array")
        result[30] = {}
        for i1, value in ipairs(tmp_1["array"]) do
            result[30][i1] = to_string_var(value, "Object/value_object/array/%d", i1)
        end
    else
        result[29] = NULL
    end
    if object["value_object2"] ~= nil then
        is_record_ValueObject(object["value_object2"], "Object/value_object2")
        result[36] = true
        local tmp_1 = object["value_object2"]
        result[40] = to_string(tmp_1["id"], "Object/value_object2/id")
        result[39] = to_string(tmp_1["datetime"], "Object/value_object2/datetime")
        result[38] = to_string(tmp_1["date"], "Object/value_object2/date")
        result[42] = to_string(tmp_1["time"], "Object/value_object2/time")
        result[41] = to_string(tmp_1["this_is_really_big_field_name"], "Object/value_object2/this_is_really_big_field_name")
        is_array(tmp_1["array"], "Object/value_object2/array")
        result[37] = {}
        for i1, value in ipairs(tmp_1["array"]) do
            result[37][i1] = to_string_var(value, "Object/value_object2/array/%d", i1)
        end
    else
        result[36] = NULL
    end
    if object["value_object3"] ~= nil then
        is_record_ValueObject(object["value_object3"], "Object/value_object3")
        result[43] = true
        local tmp_1 = object["value_object3"]
        result[47] = to_string(tmp_1["id"], "Object/value_object3/id")
        result[46] = to_string(tmp_1["datetime"], "Object/value_object3/datetime")
        result[45] = to_string(tmp_1["date"], "Object/value_object3/date")
        result[49] = to_string(tmp_1["time"], "Object/value_object3/time")
        result[48] = to_string(tmp_1["this_is_really_big_field_name"], "Object/value_object3/this_is_really_big_field_name")
        is_array(tmp_1["array"], "Object/value_object3/array")
        result[44] = {}
        for i1, value in ipairs(tmp_1["array"]) do
            result[44][i1] = to_string_var(value, "Object/value_object3/array/%d", i1)
        end
    else
        result[43] = NULL
    end
    result[28] = to_string(object["this_is_really_big_field_name"], "Object/this_is_really_big_field_name")
    is_array(object["array"], "Object/array")
    result[4] = {}
    for i1, value in ipairs(object["array"]) do
        result[4][i1] = to_string_var(value, "Object/array/%d", i1)
    end
    is_array(object["record_array"], "Object/record_array")
    result[24] = {}
    result[25] = {}
    result[26] = {}
    result[27] = {}
    for i1, value in ipairs(object["record_array"]) do
        is_record_EntityInArray(value, "Object/record_array/%d", i1)
        result[24][i1] = true
        local tmp_2 = value
        result[26][i1] = to_string_var(tmp_2["id"], "Object/record_array/%d/id", i1)
        result[27][i1] = to_string_var(tmp_2["payload"], "Object/record_array/%d/payload", i1)
        is_array_var(tmp_2["array"], "Object/record_array/%d/array", i1)
        result[25][i1] = tmp_2["array"]
        for i2, value in ipairs(tmp_2["array"]) do
            is_record_Entity(value, "Object/record_array/%d/array/%d", i1, i2)
            to_string_var(value["id"], "Object/record_array/%d/array/%d/id", i1, i2)
            to_string_var(value["fieldA"], "Object/record_array/%d/array/%d/fieldA", i1, i2)
            to_string_var(value["fieldB"], "Object/record_array/%d/array/%d/fieldB", i1, i2)
            to_string_var(value["this_is_really_big_field_name"], "Object/record_array/%d/array/%d/this_is_really_big_field_name", i1, i2)
            is_array_var(value["array"], "Object/record_array/%d/array/%d/array", i1, i2)
            for i3, value in ipairs(value["array"]) do
                to_string_var(value, "Object/record_array/%d/array/%d/array/%d", i1, i2, i3)
            end
        end
    end
    return result
end

local obj = {
    array = {"first", "second", "third"},
    entity = {
        array = {"first", "second", "third"},
        fieldA = "fieldA",
        fieldB = "fieldB",
        id = "1",
        this_is_really_big_field_name = "123",
    },
    entity2 = {
        array = {"first", "second", "third"},
        fieldA = "fieldA",
        fieldB = "fieldB",
        id = "1",
        this_is_really_big_field_name = "123",
    },
    entity3 = {
        array = {"first", "second", "third"},
        fieldA = "fieldA",
        fieldB = "fieldB",
        id = "1",
        this_is_really_big_field_name = "123",
    },
    id = "1",
    number = 1,
    record_array = {
        {
            payload = "123",
            array = {
                {
                    array = {"first", "second", "third"},
                    fieldA = "fieldA",
                    fieldB = "fieldB",
                    id = "1",
                    this_is_really_big_field_name = "123",
                }
            },
            id = "1"
        },
    },
    value_object = {
        array = {"first", "second", "third"},
        date = "2020-12-12",
        datetime = "2020-06-13T07:38:45+00:00",
        id = "1",
        time = "12:34:56",
        this_is_really_big_field_name = "123",
    },
    value_object2 = {
        array = {"first", "second", "third"},
        date = "2020-12-12",
        datetime = "2020-06-13T07:38:45+00:00",
        id = "1",
        time = "12:34:56",
        this_is_really_big_field_name = "123",
    },
    value_object3 = {
        array = {"first", "second", "third"},
        date = "2020-12-12",
        datetime = "2020-06-13T07:38:45+00:00",
        id = "1", time = "12:34:56",
        this_is_really_big_field_name = "123",
    },
    this_is_really_big_field_name = "123",
}

local t = os.clock()

for _ = 1, 1e6 do
    local _ = serialize(obj)
end
print(os.clock() - t)

Contributor guide

Open the contributing guide

First steps

  1. Read the whole issue, then the project's contributing guide.
  2. Comment on the issue to say you are picking it up — it saves two people doing the same work.
  3. Fork the repository and make your change on a branch.
  4. Open a pull request that references the issue number.

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.