apache / apache/iotdb

IoTDB升级,从0.13.4升级到1.2.2版本,使用load迁移数据报错

Đang mở
#11,659 3 bình luận 0 reaction 0 người được giao Xem trên GitHub
Ngôn ngữ chính
Java
Star
6.4k
Fork
1.2k
Merge trung bình
1 ngày 23 giờ
Pull request đã merge (30 ngày)
115

Mô tả

### update method

![iotdb-update-method](https://github.com/apache/iotdb/assets/39052598/4364b751-570c-4ded-a462-0f4c99c19d0a)

IoTDB> load '/opt/server/eos-iotdb-0.13.4/data.20231204' verify=false onSuccess=none;
Msg: 701: Parse file /opt/server/eos-iotdb-0.13.4/data.20231204/data/data/sequence/root.ops/0/0/1685351762938-9-0-581.tsfile to resource error

查看日志:**vim log_datanode_warn.log**
~~~

2023-12-05 08:40:07,571 [main] WARN o.a.r.m.MetricRegistriesLoader:81 - Found multiple MetricRegistries implementations: class org.apache.iotdb.consensus.ratis.metrics.MetricRegistryManager, class org.apache.ratis.metrics.impl.MetricRegistriesImpl. Using first found implementation: org.apache.iotdb.consensus.ratis.metrics.MetricRegistryManager@623dcf2a
2023-12-05 08:45:34,200 [pool-30-IoTDB-ClientRPC-Processor-2$20231205_004317_00001_1] WARN o.a.i.d.q.p.a.LoadTsfileAnalyzer$SchemaAutoCreatorAndVerifier:331 - Auto create or verify schema error.
java.lang.UnsupportedOperationException: This node isn't instance of SchemaEntityNode.
at org.apache.iotdb.db.queryengine.common.schematree.node.SchemaNode.getAsEntityNode(SchemaNode.java:71)
at org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree.searchDeviceSchemaInfo(ClusterSchemaTree.java:160)
at org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher.checkMissingMeasurementsAfterSchemaFetch(ClusterSchemaFetcher.java:360)
at org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher.fetchSchemaListWithAutoCreate(ClusterSchemaFetcher.java:283)
at org.apache.iotdb.db.queryengine.plan.analyze.schema.SchemaValidator.validate(SchemaValidator.java:66)
at org.apache.iotdb.db.queryengine.plan.analyze.LoadTsfileAnalyzer$SchemaAutoCreatorAndVerifier.autoCreateSchema(LoadTsfileAnalyzer.java:444)
at org.apache.iotdb.db.queryengine.plan.analyze.LoadTsfileAnalyzer$SchemaAutoCreatorAndVerifier.doAutoCreateAndVerify(LoadTsfileAnalyzer.java:325)
at org.apache.iotdb.db.queryengine.plan.analyze.LoadTsfileAnalyzer$SchemaAutoCreatorAndVerifier.flush(LoadTsfileAnalyzer.java:303)
at org.apache.iotdb.db.queryengine.plan.analyze.LoadTsfileAnalyzer$SchemaAutoCreatorAndVerifier.autoCreateAndVerify(LoadTsfileAnalyzer.java:285)
at org.apache.iotdb.db.queryengine.plan.analyze.LoadTsfileAnalyzer.analyzeSingleTsFile(LoadTsfileAnalyzer.java:155)
at org.apache.iotdb.db.queryengine.plan.analyze.LoadTsfileAnalyzer.analyzeFileByFile(LoadTsfileAnalyzer.java:112)
at org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.visitLoadFile(AnalyzeVisitor.java:2568)
at org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.visitLoadFile(AnalyzeVisitor.java:199)
at org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement.accept(LoadTsFileStatement.java:161)
at org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor.process(StatementVisitor.java:130)
at org.apache.iotdb.db.queryengine.plan.analyze.Analyzer.analyze(Analyzer.java:47)
at org.apache.iotdb.db.queryengine.plan.execution.QueryExecution.analyze(QueryExecution.java:310)
at org.apache.iotdb.db.queryengine.plan.execution.QueryExecution.(QueryExecution.java:170)
at org.apache.iotdb.db.queryengine.plan.Coordinator.createQueryExecution(Coordinator.java:113)
at org.apache.iotdb.db.queryengine.plan.Coordinator.execute(Coordinator.java:147)
at org.apache.iotdb.db.protocol.thrift.impl.ClientRPCServiceImpl.executeStatementInternal(ClientRPCServiceImpl.java:297)
at org.apache.iotdb.db.protocol.thrift.impl.ClientRPCServiceImpl.executeStatementV2(ClientRPCServiceImpl.java:738)
at org.apache.iotdb.service.rpc.thrift.IClientRPCService$Processor$executeStatementV2.getResult(IClientRPCService.java:3869)
at org.apache.iotdb.service.rpc.thrift.IClientRPCService$Processor$executeStatementV2.getResult(IClientRPCService.java:3849)
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:38)
at org.apache.iotdb.db.protocol.thrift.ProcessorWithMetrics.process(ProcessorWithMetrics.java:64)
at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:248)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:829)
2023-12-05 08:45:34,202 [pool-30-IoTDB-ClientRPC-Processor-2$20231205_004317_00001_1] WARN o.a.i.d.q.p.a.LoadTsfileAnalyzer:126 - Parse file /opt/server/eos-iotdb-0.13.4/data.20231204/data/data/sequence/root.ops/0/0/1685351762938-9-0-581.tsfile to resource error.
org.apache.iotdb.db.exception.sql.SemanticException: Auto create or verify schema error when executing statement LoadTsFileStatement{file=/opt/server/eos-iotdb-0.13.4/data.20231204, deleteAfterLoad=false, databaseLevel=1, verifySchema=false, tsFiles Size=261}.
at org.apache.iotdb.db.queryengine.plan.analyze.LoadTsfileAnalyzer$SchemaAutoCreatorAndVerifier.doAutoCreateAndVerify(LoadTsfileAnalyzer.java:333)
at org.apache.iotdb.db.queryengine.plan.analyze.LoadTsfileAnalyzer$SchemaAutoCreatorAndVerifier.flush(LoadTsfileAnalyzer.java:303)
at org.apache.iotdb.db.queryengine.plan.analyze.LoadTsfileAnalyzer$SchemaAutoCreatorAndVerifier.autoCreateAndVerify(LoadTsfileAnalyzer.java:285)
at org.apache.iotdb.db.queryengine.plan.analyze.LoadTsfileAnalyzer.analyzeSingleTsFile(LoadTsfileAnalyzer.java:155)
at org.apache.iotdb.db.queryengine.plan.analyze.LoadTsfileAnalyzer.analyzeFileByFile(LoadTsfileAnalyzer.java:112)
at org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.visitLoadFile(AnalyzeVisitor.java:2568)
at org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.visitLoadFile(AnalyzeVisitor.java:199)
at org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement.accept(LoadTsFileStatement.java:161)
at org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor.process(StatementVisitor.java:130)
at org.apache.iotdb.db.queryengine.plan.analyze.Analyzer.analyze(Analyzer.java:47)
at org.apache.iotdb.db.queryengine.plan.execution.QueryExecution.analyze(QueryExecution.java:310)
at org.apache.iotdb.db.queryengine.plan.execution.QueryExecution.(QueryExecution.java:170)
at org.apache.iotdb.db.queryengine.plan.Coordinator.createQueryExecution(Coordinator.java:113)
at org.apache.iotdb.db.queryengine.plan.Coordinator.execute(Coordinator.java:147)
at org.apache.iotdb.db.protocol.thrift.impl.ClientRPCServiceImpl.executeStatementInternal(ClientRPCServiceImpl.java:297)
at org.apache.iotdb.db.protocol.thrift.impl.ClientRPCServiceImpl.executeStatementV2(ClientRPCServiceImpl.java:738)
at org.apache.iotdb.service.rpc.thrift.IClientRPCService$Processor$executeStatementV2.getResult(IClientRPCService.java:3869)
at org.apache.iotdb.service.rpc.thrift.IClientRPCService$Processor$executeStatementV2.getResult(IClientRPCService.java:3849)
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:38)
at org.apache.iotdb.db.protocol.thrift.ProcessorWithMetrics.process(ProcessorWithMetrics.java:64)
at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:248)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:829)
2023-12-05 08:45:34,203 [pool-30-IoTDB-ClientRPC-Processor-2] WARN o.a.i.d.u.ErrorHandlingUtils:91 - Status code: 701, Query Statement: "load '/opt/server/eos-iotdb-0.13.4/data.20231204' verify=false onSuccess=none". executeStatement failed
org.apache.iotdb.db.exception.sql.SemanticException: Parse file /opt/server/eos-iotdb-0.13.4/data.20231204/data/data/sequence/root.ops/0/0/1685351762938-9-0-581.tsfile to resource error
at org.apache.iotdb.db.queryengine.plan.analyze.LoadTsfileAnalyzer.analyzeFileByFile(LoadTsfileAnalyzer.java:128)
at org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.visitLoadFile(AnalyzeVisitor.java:2568)
at org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.visitLoadFile(AnalyzeVisitor.java:199)
at org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement.accept(LoadTsFileStatement.java:161)
at org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor.process(StatementVisitor.java:130)
at org.apache.iotdb.db.queryengine.plan.analyze.Analyzer.analyze(Analyzer.java:47)
at org.apache.iotdb.db.queryengine.plan.execution.QueryExecution.analyze(QueryExecution.java:310)
at org.apache.iotdb.db.queryengine.plan.execution.QueryExecution.(QueryExecution.java:170)
at org.apache.iotdb.db.queryengine.plan.Coordinator.createQueryExecution(Coordinator.java:113)
at org.apache.iotdb.db.queryengine.plan.Coordinator.execute(Coordinator.java:147)
at org.apache.iotdb.db.protocol.thrift.impl.ClientRPCServiceImpl.executeStatementInternal(ClientRPCServiceImpl.java:297)
at org.apache.iotdb.db.protocol.thrift.impl.ClientRPCServiceImpl.executeStatementV2(ClientRPCServiceImpl.java:738)
at org.apache.iotdb.service.rpc.thrift.IClientRPCService$Processor$executeStatementV2.getResult(IClientRPCService.java:3869)
at org.apache.iotdb.service.rpc.thrift.IClientRPCService$Processor$executeStatementV2.getResult(IClientRPCService.java:3849)
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:38)
at org.apache.iotdb.db.protocol.thrift.ProcessorWithMetrics.process(ProcessorWithMetrics.java:64)
at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:248)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:829)

~~~

Hướng dẫn đóng góp

Mở hướng dẫn đóng góp

Hướng nghiên cứu

Tái hiện quá trình migration bằng lệnh `load` đã được báo cáo và kiểm tra `log_datanode_warn.log`. Bắt đầu với `LoadTsfileAnalyzer`, `ClusterSchemaFetcher` và `ClusterSchemaTree`, theo dõi exception `SchemaEntityNode` đã được báo cáo trong quá trình xác minh schema. Được xem là hoàn tất khi thư mục dữ liệu của 0.13.4 được tải vào IoTDB 1.2.2 mà không có lỗi parse hoặc lỗi schema.

Do mô hình lập chỉ mục viết ra từ nội dung của issue.

Đánh giá

Công nghệ
java
Lĩnh vực
databases
Loại issue
Lỗi
Độ khó
4/5
Thời gian dự kiến
3-5 ngày
Mức độ hoạt động
Đình trệ
Độ rõ ràng
Cần làm rõ
Mức phù hợp với người mới
25/100

Nhận issue mới trong hộp thư của bạn

Bản tóm tắt ngắn những issue GitHub phù hợp với người mới.