google / google/tensorstore

Writing local files fails on Windows 11

Open
#160 6 comments 0 reactions 0 assignees View on GitHub
Dominant language
C++
Stars
1.5k
Forks
145
PR merge metrics
No merged PRs in 30d

Description

Related to #123.

Writing to filesystem fails on Windows 11.

It is interesting that this spec does not throw the exception, but unfortunately it does not work since no files are created on the filesystem:
```
spec = {
'driver': 'zarr',
'kvstore': {
'driver': 'file',
'path': "/tmp_zarr_new",
}
}
```

I also tried with different configurations, but this is a minimal code block that showcases the issue. Please let me know if you need more info.

I am attaching the script with further details:
```{python}
import tensorstore as ts
from importlib.metadata import version

def run_test_1():
"""Fails!"""
spec = {
'driver': 'zarr',
'kvstore': {
'driver': 'file',
'path': "tmp_zarr",
}
}
ts.open(
spec,
delete_existing=True,
create=True,
dtype=ts.float32,
shape=[10, 2]
).result()

def run_test_2():
"""Similar called the second time without create - works since file was created."""
spec = {
'driver': 'zarr',
'kvstore': {
'driver': 'file',
'path': "tmp_zarr",
}
}
ts.open(
spec,
dtype=ts.float32,
shape=[10, 2]
).result()

def run_test_3():
"""This somehow works, but nothing is created on filesystem."""
spec = {
'driver': 'zarr',
'kvstore': {
'driver': 'file',
'path': "/tmp_zarr_new",
}
}
ts.open(
spec,
delete_existing=True,
create=True,
dtype=ts.float32,
shape=[10, 2]
).result()

def run_test_4():
"""Same issue with n5 => code from tutorial."""
dataset = ts.open({
'driver': 'n5',
'kvstore': {
'driver': 'file',
'path': 'tmp_n5/',
},
'metadata': {
'compression': {
'type': 'gzip'
},
'dataType': 'uint32',
'dimensions': [1000, 1000],
'blockSize': [100, 100],
},
'create': True,
'delete_existing': True,
}).result()

if __name__ == '__main__':
print(version('tensorstore'))
try:
run_test_1()
print("Test 1 passed!")
except Exception as e:
print(e)

try:
run_test_2()
print("Test 2 passed")
except Exception as e:
print(e)

try:
run_test_3()
print("Test 3 passed")
except Exception as e:
print(e)

try:
run_test_4()
print("Test 4 passed")
except Exception as e:
print(e)
```
Output:
```
0.1.59
NOT_FOUND: Error opening "zarr" driver: Error writing local file "tmp_zarr/.zarray": Error getting file info: tmp_zarr/.zarray.__lock [OS error: No such file or directory] [source locations='tensorstore/kvstore/file/file_key_value_store.cc:339\ntensorstore/kvstore/kvstore.cc:373\ntensorstore/driver/driver.cc:117'] [tensorstore_spec='{\"context\":{\"cache_pool\":{},\"data_copy_concurrency\":{},\"file_io_concurrency\":{},\"file_io_sync\":true},\"create\":true,\"delete_existing\":true,\"driver\":\"zarr\",\"dtype\":\"float32\",\"kvstore\":{\"driver\":\"file\",\"path\":\"tmp_zarr/\"},\"schema\":{\"domain\":{\"exclusive_max\":[10,2],\"inclusive_min\":[0,0]}},\"transform\":{\"input_exclusive_max\":[[10],[2]],\"input_inclusive_min\":[0,0]}}']
Test 2 passed
Test 3 passed
NOT_FOUND: Error opening "n5" driver: Error writing local file "tmp_n5/attributes.json": Error getting file info: tmp_n5/attributes.json.__lock [OS error: No such file or directory] [source locations='tensorstore/kvstore/file/file_key_value_store.cc:339\ntensorstore/kvstore/kvstore.cc:373\ntensorstore/driver/driver.cc:117'] [tensorstore_spec='{\"context\":{\"cache_pool\":{},\"data_copy_concurrency\":{},\"file_io_concurrency\":{},\"file_io_sync\":true},\"create\":true,\"delete_existing\":true,\"driver\":\"n5\",\"dtype\":\"uint32\",\"kvstore\":{\"driver\":\"file\",\"path\":\"tmp_n5/\"},\"metadata\":{\"blockSize\":[100,100],\"compression\":{\"level\":-1,\"type\":\"gzip\",\"useZlib\":false},\"dataType\":\"uint32\",\"dimensions\":[1000,1000]},\"transform\":{\"input_exclusive_max\":[[1000],[1000]],\"input_inclusive_min\":[0,0]}}']
```

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.