Folder corrupted when using `ipfs get`
- Dominant language
- Go
- Stars
- 17.1k
- Forks
- 3.2k
- Avg merge
- 3d 18h
- Merged PRs (30d)
- 11
Description
### Checklist
- [X] This is a bug report, not a question. Ask questions on [discuss.ipfs.io](https://discuss.ipfs.io).
- [X] I have searched on the [issue tracker](https://github.com/ipfs/go-ipfs/issues?q=is%3Aissue) for my bug.
- [X] I am running the latest [go-ipfs version](https://dist.ipfs.io/#go-ipfs) or have an issue updating.
### Installation method
ipfs-desktop
### Version
```Text
go-ipfs version: 0.10.0
Repo version: 11
System version: amd64/windows
Golang version: go1.16.8
--------------------------------------------------------------------------------
go-ipfs version: 0.10.0-64b532f
Repo version: 11
System version: amd64/linux
Golang version: go1.16.7
```
### Config
```json
{
"API": {
"HTTPHeaders": {}
},
"Addresses": {
"API": "/ip4/127.0.0.1/tcp/5001",
"Announce": [],
"Gateway": "/ip4/127.0.0.1/tcp/8080",
"NoAnnounce": [],
"Swarm": [
"/ip4/0.0.0.0/tcp/4001",
"/ip6/::/tcp/4001",
"/ip4/0.0.0.0/udp/4001/quic",
"/ip6/::/udp/4001/quic"
]
},
"AutoNAT": {},
"Bootstrap": [
"/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
"/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
"/ip4/132.226.12.0/tcp/4001/p2p/12D3KooWLQSvjfzZaBjGvNs99YaG9ojSMkKGAVB2PeRNBt6omtot",
"/ip4/104.131.131.82/udp/4001/quic/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN"
],
"Datastore": {
"BloomFilterSize": 0,
"GCPeriod": "1h",
"HashOnRead": false,
"Spec": {
"mounts": [
{
"child": {
"path": "blocks",
"shardFunc": "/repo/flatfs/shard/v1/next-to-last/2",
"sync": true,
"type": "flatfs"
},
"mountpoint": "/blocks",
"prefix": "flatfs.datastore",
"type": "measure"
},
{
"child": {
"compression": "none",
"path": "datastore",
"type": "levelds"
},
"mountpoint": "/",
"prefix": "leveldb.datastore",
"type": "measure"
}
],
"type": "mount"
},
"StorageGCWatermark": 90,
"StorageMax": "250GB"
},
"Discovery": {
"MDNS": {
"Enabled": true,
"Interval": 10
}
},
"Experimental": {
"FilestoreEnabled": false,
"GraphsyncEnabled": false,
"Libp2pStreamMounting": false,
"P2pHttpProxy": false,
"ShardingEnabled": false,
"StrategicProviding": false,
"UrlstoreEnabled": false
},
"Gateway": {
"APICommands": [],
"HTTPHeaders": {
"Access-Control-Allow-Headers": [
"X-Requested-With",
"Range",
"User-Agent"
],
"Access-Control-Allow-Methods": [
"GET"
],
"Access-Control-Allow-Origin": [
"*"
]
},
"NoDNSLink": false,
"NoFetch": false,
"PathPrefixes": [],
"PublicGateways": null,
"RootRedirect": "",
"Writable": false
},
"Identity": {
"PeerID": "12D3KooWPdaeapEAhnnvaBiCag4zganrvGycw3N87axfTgaFduVx"
},
"Ipns": {
"RecordLifetime": "",
"RepublishPeriod": "",
"ResolveCacheSize": 128
},
"Mounts": {
"FuseAllowOther": false,
"IPFS": "/ipfs",
"IPNS": "/ipns"
},
"Peering": {
"Peers": null
},
"Pinning": {
"RemoteServices": {
"Pinata": {
"API": {
"Endpoint": "https://api.pinata.cloud/psa"
},
"Policies": {
"MFS": {
"Enable": false,
"PinName": "",
"RepinInterval": ""
}
}
}
}
},
"Plugins": {
"Plugins": null
},
"Provider": {
"Strategy": ""
},
"Pubsub": {
"DisableSigning": false,
"Router": ""
},
"Reprovider": {
"Interval": "12h",
"Strategy": "all"
},
"Routing": {
"Type": "dht"
},
"Swarm": {
"AddrFilters": null,
"ConnMgr": {
"GracePeriod": "300s",
"HighWater": 300,
"LowWater": 50,
"Type": "basic"
},
"DisableBandwidthMetrics": false,
"DisableNatPortMap": false,
"EnableAutoRelay": false,
"EnableRelayHop": false,
"Transports": {
"Multiplexers": {},
"Network": {},
"Security": {}
}
}
}
--------------------------------------------------------------------------------
{
"API": {
"HTTPHeaders": {}
},
"Addresses": {
"API": "/ip4/0.0.0.0/tcp/5002",
"Announce": [],
"Gateway": "/ip4/0.0.0.0/tcp/8081",
"NoAnnounce": [],
"Swarm": [
"/ip4/0.0.0.0/tcp/4001",
"/ip6/::/tcp/4001",
"/ip4/0.0.0.0/udp/4001/quic",
"/ip6/::/udp/4001/quic"
]
},
"AutoNAT": {},
"Bootstrap": [
"/ip4/132.226.12.0/tcp/4001/p2p/12D3KooWLQSvjfzZaBjGvNs99YaG9ojSMkKGAVB2PeRNBt6omtot",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb",
"/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
"/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
"/ip4/104.131.131.82/udp/4001/quic/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ"
],
"DNS": {
"Resolvers": {}
},
"Datastore": {
"BloomFilterSize": 0,
"GCPeriod": "1h",
"HashOnRead": false,
"Spec": {
"mounts": [
{
"child": {
"path": "blocks",
"shardFunc": "/repo/flatfs/shard/v1/next-to-last/2",
"sync": true,
"type": "flatfs"
},
"mountpoint": "/blocks",
"prefix": "flatfs.datastore",
"type": "measure"
},
{
"child": {
"compression": "none",
"path": "datastore",
"type": "levelds"
},
"mountpoint": "/",
"prefix": "leveldb.datastore",
"type": "measure"
}
],
"type": "mount"
},
"StorageGCWatermark": 80,
"StorageMax": "500GB"
},
"Discovery": {
"MDNS": {
"Enabled": true,
"Interval": 10
}
},
"Experimental": {
"AcceleratedDHTClient": false,
"FilestoreEnabled": false,
"GraphsyncEnabled": false,
"Libp2pStreamMounting": false,
"P2pHttpProxy": false,
"ShardingEnabled": false,
"StrategicProviding": false,
"UrlstoreEnabled": false
},
"Gateway": {
"APICommands": [],
"HTTPHeaders": {
"Access-Control-Allow-Headers": [
"X-Requested-With",
"Range",
"User-Agent"
],
"Access-Control-Allow-Methods": [
"GET"
],
"Access-Control-Allow-Origin": [
"*"
]
},
"NoDNSLink": false,
"NoFetch": false,
"PathPrefixes": [],
"PublicGateways": null,
"RootRedirect": "",
"Writable": false
},
"Identity": {
"PeerID": "12D3KooWLb8jTr854zvpc5kHeKnfeGa1ZgfXwLUtq599yCxLvDn4"
},
"Internal": {},
"Ipns": {
"RecordLifetime": "",
"RepublishPeriod": "",
"ResolveCacheSize": 128
},
"Migration": {
"DownloadSources": [],
"Keep": ""
},
"Mounts": {
"FuseAllowOther": false,
"IPFS": "/ipfs",
"IPNS": "/ipns"
},
"Peering": {
"Peers": null
},
"Pinning": {
"RemoteServices": {}
},
"Plugins": {
"Plugins": null
},
"Provider": {
"Strategy": ""
},
"Pubsub": {
"DisableSigning": false,
"Router": ""
},
"Reprovider": {
"Interval": "12h",
"Strategy": "all"
},
"Routing": {
"Type": "dht"
},
"Swarm": {
"AddrFilters": null,
"ConnMgr": {
"GracePeriod": "20s",
"HighWater": 900,
"LowWater": 600,
"Type": "basic"
},
"DisableBandwidthMetrics": false,
"DisableNatPortMap": true,
"EnableAutoRelay": true,
"EnableRelayHop": true,
"Transports": {
"Multiplexers": {},
"Network": {},
"Security": {}
}
}
}
```
### Description
Hi,
I noticed the folder structure cannot be recreated when using `ipfs get`. The CID is `QmXSPqQpNcbZ9tW1te16xWQfA3a1R9Yt86JzTP6XjtvdVe`.
I add those files into IPFS on Node `12D3KooWPdaeapEAhnnvaBiCag4zganrvGycw3N87axfTgaFduVx`, this is the original copy, then I pinned it to my server, which is node `12D3KooWLQSvjfzZaBjGvNs99YaG9ojSMkKGAVB2PeRNBt6omtot`. Then I use `ipfs get` to download this on my friend's server, not just holding those dag nodes, but reconstructing them back into files.
I use `ipfs get -o "[Airota&VCB-Studio] Liz to Aoi Tori [Ma10p_1080p]" QmXSPqQpNcbZ9tW1te16xWQfA3a1R9Yt86JzTP6XjtvdVe`, this command can recreate folder structure correctly, but then I get this error:
```
Error: contains more than one root or the root directory is not the first entry : tar has invalid root
```
Only ~2GB of 14GB can be downloaded by this method, the rest of them are canceled because of that error.
Mentioned in #8293 , I tried to get the archive file using this command `ipfs get -a -o "[Airota&VCB-Studio] Liz to Aoi Tori [Ma10p_1080p].tar" QmXSPqQpNcbZ9tW1te16xWQfA3a1R9Yt86JzTP6XjtvdVe`, I got this with no error, but when I exam the result, I noticed data in the `CD` folder are corrupted.


The folder name is not correct and the data in the folder cannot be found, however, the total size is correct. So technically speaking, no data is missing, but some of them are not addressable by the tar file.
If you check the CID using some gateway, like mines: https://ampere.skyblond.info/ipfs/QmXSPqQpNcbZ9tW1te16xWQfA3a1R9Yt86JzTP6XjtvdVe/CDs/%5B180425%5D%20%EF%BD%A2girls,dance,staircase%EF%BD%A3%EF%BC%8F%E7%89%9B%E5%B0%BE%E6%86%B2%E8%BC%94%20%5B24bit_96kHz%5D%20%28flac+webp%29 , you can see it's perfectly fine and I download some of them, it works perfectly fine, I confirmed this with my friend, which the data is distributed to his machine, but cannot be reconstructed back into files using `ipfs get`.
Note: those corrupted folders can do `ipfs get` properly without the archive flag, also they are the only folder gets recovered before encountering the `tar has invalid root` error. And looks like they are the only folders corrupted in the archive file.
I *inferred* that it's the issue with the CLI interface since the file itself contains non-ASCII chars. The HTTP interface is working properly, both the gateway page and my Java program can confirm it.
Contributor guide
Assessment
This issue has not been assessed yet.