developmentseed / developmentseed/obstore

`FsspecStore.rm()` does not work for AzureStore directory

Open
#556 1 comment 0 reactions 0 assignees View on GitHub
fsspec
Dominant language
Python
Stars
810
Forks
42
Avg merge
1d 15h
Merged PRs (30d)
6

Description

Hi,

I ran into the issue where `FsspecStore.rm()` fails when executing on directory.

[fsspec docs is a little bit imprecise here:](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.rm)
Image
The main description indicates that the method is meant to use files, but presence and description of `recurse` and `maxdepth` imply that it should also work for directories. (I created an issue for it as well https://github.com/fsspec/filesystem_spec/issues/1914)
When using orignal fsspec on local file system, it does work for directories.

However, obstores `FsspecStore` is different:
```python
# path points to directory that is not empty (containing sub directories and files)
fsspec.rm("abfss://sbx@«storage_account».dfs.core.windows.net/mbe/tmp/obstore/lorem", recursive=True)
```
```python
---------------------------------------------------------------------------
GenericError Traceback (most recent call last)
Cell In[6], line 1
----> 1 fsspec.rm("abfss://sbx@«storage_account».dfs.core.windows.net/mbe/tmp/obstore/lorem", recursive=True)

File c:\...\.venv\lib\site-packages\fsspec\asyn.py:118, in sync_wrapper..wrapper(*args, **kwargs)
115 @functools.wraps(func)
116 def wrapper(*args, **kwargs):
117 self = obj or args[0]
--> 118 return sync(self.loop, func, *args, **kwargs)

File c:\...\.venv\lib\site-packages\fsspec\asyn.py:103, in sync(loop, func, timeout, *args, **kwargs)
101 raise FSTimeoutError from return_result
102 elif isinstance(return_result, BaseException):
--> 103 raise return_result
104 else:
105 return return_result

File c:\...\.venv\lib\site-packages\fsspec\asyn.py:56, in _runner(event, coro, result, timeout)
54 coro = asyncio.wait_for(coro, timeout=timeout)
55 try:
---> 56 result[0] = await coro
57 except Exception as ex:
58 result[0] = ex

File c:\...\.venv\lib\site-packages\fsspec\asyn.py:337, in AsyncFileSystem._rm(self, path, recursive, batch_size, **kwargs)
335 batch_size = batch_size or self.batch_size
336 path = await self._expand_path(path, recursive=recursive)
--> 337 return await _run_coros_in_chunks(
338 [self._rm_file(p, **kwargs) for p in reversed(path)],
339 batch_size=batch_size,
340 nofiles=True,
341 )

File c:\...\.venv\lib\site-packages\fsspec\asyn.py:267, in _run_coros_in_chunks(coros, batch_size, callback, timeout, return_exceptions, nofiles)
265 done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
266 while done:
--> 267 result, k = await done.pop()
268 results[k] = result
270 return results

File c:\...\.venv\lib\site-packages\fsspec\asyn.py:244, in _run_coros_in_chunks.._run_coro(coro, i)
242 async def _run_coro(coro, i):
243 try:
--> 244 return await asyncio.wait_for(coro, timeout=timeout), i
245 except Exception as e:
246 if not return_exceptions:

File C:\Program Files\Python310\lib\asyncio\tasks.py:408, in wait_for(fut, timeout)
405 loop = events.get_running_loop()
407 if timeout is None:
--> 408 return await fut
410 if timeout <= 0:
411 fut = ensure_future(fut, loop=loop)

File c:\...\.venv\lib\site-packages\obstore\fsspec.py:338, in FsspecStore._rm_file(self, path, **_kwargs)
336 bucket, path = self._split_path(path)
337 store = self._construct_store(bucket)
--> 338 return await obs.delete_async(store, path)

GenericError: Generic MicrosoftAzure error: Error performing DELETE https://«storage_account».blob.core.windows.net/sbx/mbe/tmp/obstore/lorem? in 164.9255ms - Server returned non-2xx status code: 400 Bad Request: OperationNotSupportedOnDirectoryThe requested operation is not supported on a directory.
RequestId:fe35354c-b01e-003f-206d-1f2cc1000000
Time:2025-09-06T20:31:07.9542239Z

Debug source:
Generic {
store: "MicrosoftAzure",
source: RetryError(
RetryErrorImpl {
method: DELETE,
uri: Some(
https://«storage_account».blob.core.windows.net/sbx/mbe/tmp/obstore/lorem?,
),
retries: 0,
max_retries: 10,
elapsed: 164.9255ms,
retry_timeout: 180s,
inner: Status {
status: 400,
body: Some(
"\u{feff}OperationNotSupportedOnDirectoryThe requested operation is not supported on a directory.\nRequestId:fe35354c-b01e-003f-206d-1f2cc1000000\nTime:2025-09-06T20:31:07.9542239Z",
),
},
},
),
}
```

```python
# path points to directory that is empty
fsspec.rm("abfss://sbx@«storage_account».dfs.core.windows.net/mbe/tmp/obstore/lorem/ipsum2")
```
```python
---------------------------------------------------------------------------
FileNotFoundError Traceback (most recent call last)
Cell In[14], line 1
----> 1 fsspec.rm("abfss://sbx@«storage_account».dfs.core.windows.net/mbe/tmp/obstore/lorem/ipsum2")

File c:\...\.venv\lib\site-packages\fsspec\asyn.py:118, in sync_wrapper..wrapper(*args, **kwargs)
115 @functools.wraps(func)
116 def wrapper(*args, **kwargs):
117 self = obj or args[0]
--> 118 return sync(self.loop, func, *args, **kwargs)

File c:\...\.venv\lib\site-packages\fsspec\asyn.py:103, in sync(loop, func, timeout, *args, **kwargs)
101 raise FSTimeoutError from return_result
102 elif isinstance(return_result, BaseException):
--> 103 raise return_result
104 else:
105 return return_result

File c:\...\.venv\lib\site-packages\fsspec\asyn.py:56, in _runner(event, coro, result, timeout)
54 coro = asyncio.wait_for(coro, timeout=timeout)
55 try:
---> 56 result[0] = await coro
57 except Exception as ex:
58 result[0] = ex

File c:\...\.venv\lib\site-packages\fsspec\asyn.py:337, in AsyncFileSystem._rm(self, path, recursive, batch_size, **kwargs)
335 batch_size = batch_size or self.batch_size
336 path = await self._expand_path(path, recursive=recursive)
--> 337 return await _run_coros_in_chunks(
338 [self._rm_file(p, **kwargs) for p in reversed(path)],
339 batch_size=batch_size,
340 nofiles=True,
341 )

File c:\...\.venv\lib\site-packages\fsspec\asyn.py:267, in _run_coros_in_chunks(coros, batch_size, callback, timeout, return_exceptions, nofiles)
265 done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
266 while done:
--> 267 result, k = await done.pop()
268 results[k] = result
270 return results

File c:\...\.venv\lib\site-packages\fsspec\asyn.py:244, in _run_coros_in_chunks.._run_coro(coro, i)
242 async def _run_coro(coro, i):
243 try:
--> 244 return await asyncio.wait_for(coro, timeout=timeout), i
245 except Exception as e:
246 if not return_exceptions:

File C:\Program Files\Python310\lib\asyncio\tasks.py:408, in wait_for(fut, timeout)
405 loop = events.get_running_loop()
407 if timeout is None:
--> 408 return await fut
410 if timeout <= 0:
411 fut = ensure_future(fut, loop=loop)

File c:\...\.venv\lib\site-packages\obstore\fsspec.py:338, in FsspecStore._rm_file(self, path, **_kwargs)
336 bucket, path = self._split_path(path)
337 store = self._construct_store(bucket)
--> 338 return await obs.delete_async(store, path)

FileNotFoundError: Object at location mbe/tmp/obstore/lorem/ipsum2 not found: Error performing DELETE https://«storage_account».blob.core.windows.net/sbx/mbe/tmp/obstore/lorem/ipsum2? in 249.8147ms - Server returned non-2xx status code: 404 Not Found: BlobNotFoundThe specified blob does not exist.
RequestId:ceb97508-501e-0027-0ee7-1ff3a6000000
Time:2025-09-07T11:08:08.4175540Z

Debug source:
NotFound {
path: "mbe/tmp/obstore/lorem/ipsum2",
source: RetryError(
RetryErrorImpl {
method: DELETE,
uri: Some(
https://«storage_account».blob.core.windows.net/sbx/mbe/tmp/obstore/lorem/ipsum2?,
),
retries: 0,
max_retries: 10,
elapsed: 249.8147ms,
retry_timeout: 180s,
inner: Status {
status: 404,
body: Some(
"\u{feff}BlobNotFoundThe specified blob does not exist.\nRequestId:ceb97508-501e-0027-0ee7-1ff3a6000000\nTime:2025-09-07T11:08:08.4175540Z",
),
},
},
),
}
```

Contributor guide

No contributing guide indexed for this repository

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.