facebook / facebook/rocksdb

WAL sequence number not always in order with log number

Open
#10,516 1 comment 0 reactions 0 assignees View on GitHub
question up-for-grabs
Dominant language
C++
Stars
32.1k
Forks
6.9k
Avg merge
32m
Merged PRs (30d)
1

Description

I've noticed that somewhat frequently the result from `GetSortedWalFiles` return WAL files that contain weird results where some wal files sequence number are not in order with log number.

In particular it's very common to see WAL files that start over from sequence 1.

Here is the result from my local computer from 4 rocks databases.

What could cause this?

Should rocks detect and repair this when opening?

```json
{
"db1": [
{
"pathName": "/archive/000004.log",
"logNumber": 4,
"type": 0,
"startSequence": 1,
"sizeFileBytes": 12865
},
{
"pathName": "/archive/000012.log",
"logNumber": 12,
"type": 0,
"startSequence": 149,
"sizeFileBytes": 964
},
{
"pathName": "/archive/000021.log",
"logNumber": 21,
"type": 0,
"startSequence": 157,
"sizeFileBytes": 964
},
{
"pathName": "/archive/000031.log",
"logNumber": 31,
"type": 0,
"startSequence": 165,
"sizeFileBytes": 470772
},
{
"pathName": "/archive/000037.log",
"logNumber": 37,
"type": 0,
"startSequence": 3330,
"sizeFileBytes": 80725
},
{
"pathName": "/archive/000047.log",
"logNumber": 47,
"type": 0,
"startSequence": 1,
"sizeFileBytes": 11
},
{
"pathName": "/archive/000051.log",
"logNumber": 51,
"type": 0,
"startSequence": 1,
"sizeFileBytes": 11
},
{
"pathName": "/archive/000055.log",
"logNumber": 55,
"type": 0,
"startSequence": 3872,
"sizeFileBytes": 1562
},
{
"pathName": "/archive/000061.log",
"logNumber": 61,
"type": 0,
"startSequence": 3882,
"sizeFileBytes": 1554
},
{
"pathName": "/archive/000070.log",
"logNumber": 70,
"type": 0,
"startSequence": 3892,
"sizeFileBytes": 1230
},
{
"pathName": "/archive/000076.log",
"logNumber": 76,
"type": 0,
"startSequence": 3901,
"sizeFileBytes": 365
},
{
"pathName": "/archive/000086.log",
"logNumber": 86,
"type": 0,
"startSequence": 3903,
"sizeFileBytes": 2255
},
{
"pathName": "/archive/000092.log",
"logNumber": 92,
"type": 0,
"startSequence": 3918,
"sizeFileBytes": 472
},
{
"pathName": "/archive/000101.log",
"logNumber": 101,
"type": 0,
"startSequence": 3921,
"sizeFileBytes": 1653
},
{
"pathName": "/000107.log",
"logNumber": 107,
"type": 1,
"startSequence": 1,
"sizeFileBytes": 480
}
],
"db2": [
{
"pathName": "/archive/000004.log",
"logNumber": 4,
"type": 0,
"startSequence": 1,
"sizeFileBytes": 8913
},
{
"pathName": "/archive/000012.log",
"logNumber": 12,
"type": 0,
"startSequence": 57,
"sizeFileBytes": 353
},
{
"pathName": "/archive/000021.log",
"logNumber": 21,
"type": 0,
"startSequence": 59,
"sizeFileBytes": 354
},
{
"pathName": "/archive/000031.log",
"logNumber": 31,
"type": 0,
"startSequence": 61,
"sizeFileBytes": 609607
},
{
"pathName": "/archive/000037.log",
"logNumber": 37,
"type": 0,
"startSequence": 4213,
"sizeFileBytes": 103223
},
{
"pathName": "/archive/000047.log",
"logNumber": 47,
"type": 0,
"startSequence": 1,
"sizeFileBytes": 11
},
{
"pathName": "/archive/000051.log",
"logNumber": 51,
"type": 0,
"startSequence": 1,
"sizeFileBytes": 11
},
{
"pathName": "/archive/000055.log",
"logNumber": 55,
"type": 0,
"startSequence": 4900,
"sizeFileBytes": 2117
},
{
"pathName": "/archive/000061.log",
"logNumber": 61,
"type": 0,
"startSequence": 4908,
"sizeFileBytes": 530
},
{
"pathName": "/archive/000070.log",
"logNumber": 70,
"type": 0,
"startSequence": 4911,
"sizeFileBytes": 529
},
{
"pathName": "/archive/000076.log",
"logNumber": 76,
"type": 0,
"startSequence": 4914,
"sizeFileBytes": 529
},
{
"pathName": "/archive/000086.log",
"logNumber": 86,
"type": 0,
"startSequence": 4917,
"sizeFileBytes": 1125
},
{
"pathName": "/archive/000092.log",
"logNumber": 92,
"type": 0,
"startSequence": 4924,
"sizeFileBytes": 585
},
{
"pathName": "/archive/000101.log",
"logNumber": 101,
"type": 0,
"startSequence": 4927,
"sizeFileBytes": 1762
},
{
"pathName": "/000107.log",
"logNumber": 107,
"type": 1,
"startSequence": 1,
"sizeFileBytes": 1198
}
],
"db3": [
{
"pathName": "/archive/000004.log",
"logNumber": 4,
"type": 0,
"startSequence": 1,
"sizeFileBytes": 13083
},
{
"pathName": "/archive/000012.log",
"logNumber": 12,
"type": 0,
"startSequence": 135,
"sizeFileBytes": 1118
},
{
"pathName": "/archive/000021.log",
"logNumber": 21,
"type": 0,
"startSequence": 148,
"sizeFileBytes": 1118
},
{
"pathName": "/archive/000031.log",
"logNumber": 31,
"type": 0,
"startSequence": 161,
"sizeFileBytes": 123106
},
{
"pathName": "/archive/000037.log",
"logNumber": 37,
"type": 0,
"startSequence": 1054,
"sizeFileBytes": 36587
},
{
"pathName": "/archive/000047.log",
"logNumber": 47,
"type": 0,
"startSequence": 1,
"sizeFileBytes": 11
},
{
"pathName": "/archive/000051.log",
"logNumber": 51,
"type": 0,
"startSequence": 1,
"sizeFileBytes": 11
},
{
"pathName": "/archive/000055.log",
"logNumber": 55,
"type": 0,
"startSequence": 1298,
"sizeFileBytes": 2479
},
{
"pathName": "/archive/000061.log",
"logNumber": 61,
"type": 0,
"startSequence": 1307,
"sizeFileBytes": 320
},
{
"pathName": "/archive/000070.log",
"logNumber": 70,
"type": 0,
"startSequence": 1309,
"sizeFileBytes": 320
},
{
"pathName": "/archive/000076.log",
"logNumber": 76,
"type": 0,
"startSequence": 1311,
"sizeFileBytes": 320
},
{
"pathName": "/archive/000086.log",
"logNumber": 86,
"type": 0,
"startSequence": 1313,
"sizeFileBytes": 909
},
{
"pathName": "/archive/000092.log",
"logNumber": 92,
"type": 0,
"startSequence": 1319,
"sizeFileBytes": 365
},
{
"pathName": "/archive/000101.log",
"logNumber": 101,
"type": 0,
"startSequence": 1321,
"sizeFileBytes": 365
},
{
"pathName": "/000107.log",
"logNumber": 107,
"type": 1,
"startSequence": 1,
"sizeFileBytes": 911
}
],
"db4": [
{
"pathName": "/archive/000004.log",
"logNumber": 4,
"type": 0,
"startSequence": 1,
"sizeFileBytes": 8652
},
{
"pathName": "/archive/000012.log",
"logNumber": 12,
"type": 0,
"startSequence": 73,
"sizeFileBytes": 439
},
{
"pathName": "/archive/000022.log",
"logNumber": 22,
"type": 0,
"startSequence": 76,
"sizeFileBytes": 439
},
{
"pathName": "/archive/000031.log",
"logNumber": 31,
"type": 0,
"startSequence": 79,
"sizeFileBytes": 10919
},
{
"pathName": "/archive/000037.log",
"logNumber": 37,
"type": 0,
"startSequence": 169,
"sizeFileBytes": 10785
},
{
"pathName": "/archive/000047.log",
"logNumber": 47,
"type": 0,
"startSequence": 1,
"sizeFileBytes": 11
},
{
"pathName": "/archive/000051.log",
"logNumber": 51,
"type": 0,
"startSequence": 1,
"sizeFileBytes": 11
},
{
"pathName": "/archive/000055.log",
"logNumber": 55,
"type": 0,
"startSequence": 235,
"sizeFileBytes": 662
},
{
"pathName": "/000061.log",
"logNumber": 61,
"type": 1,
"startSequence": 1,
"sizeFileBytes": 11
},
{
"pathName": "/000068.log",
"logNumber": 68,
"type": 1,
"startSequence": 1,
"sizeFileBytes": 11
},
{
"pathName": "/000072.log",
"logNumber": 72,
"type": 1,
"startSequence": 1,
"sizeFileBytes": 11
},
{
"pathName": "/000076.log",
"logNumber": 76,
"type": 1,
"startSequence": 1,
"sizeFileBytes": 11
},
{
"pathName": "/000080.log",
"logNumber": 80,
"type": 1,
"startSequence": 1,
"sizeFileBytes": 11
},
{
"pathName": "/000084.log",
"logNumber": 84,
"type": 1,
"startSequence": 1,
"sizeFileBytes": 11
},
{
"pathName": "/000088.log",
"logNumber": 88,
"type": 1,
"startSequence": 1,
"sizeFileBytes": 11
}
]
}
```

Refs: https://github.com/facebook/rocksdb/issues/10476

Contributor guide

Open the contributing guide

Research direction

Start with the GetSortedWalFiles entry point and read the linked issue #10476 for related context. Compare the archived and live WAL records shown in the report, then determine whether the sequence resets are expected and what opening the database should do. Done means a confirmed cause and a decided detection or repair behavior.

Written by the indexing model from the issue text.

Assessment

Tech stack
cpp
Domain
databases
Issue type
Bug
Difficulty
5/5
Estimated time
Over a week
Activity status
Stale
Clarity
Needs clarification
Newbie friendliness
25/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.