io.grpc.StatusRuntimeException: INVALID_ARGUMENT: etcdserver: revision of auth store is old - client doesn't retry.
- Dominant language
- Java
- Stars
- 1.2k
- Forks
- 312
- Avg merge
- 5d 4h
- Merged PRs (30d)
- 2
Description
**Versions**
- etcd: `3.5.9`
- jetcd: `main`
- java: `openjdk 20.0.2 2023-07-18`
**Describe the bug**
When spinning up `etcd` with `--auth-token=jwt,pub-key=jwt_RS256.pub,priv-key=jwt_RS256,sign-method=RS256 --auth-token-ttl=1`, run into this `io.grpc.StatusRuntimeException: INVALID_ARGUMENT: etcdserver: revision of auth store is old` and the client fails to retry.
**To Reproduce**
- Create a `tmp` directory
- cd `tmp`
- Create a `Dockerfile`
```
FROM gcr.io/etcd-development/etcd:v3.5.9
COPY jwt_RS256 /usr/local/bin/jwt_RS256
COPY jwt_RS256 /var/etcd/jwt_RS256
COPY jwt_RS256 /var/lib/etcd/jwt_RS256
COPY jwt_RS256 /jwt_RS256
COPY jwt_RS256.pub /usr/local/bin/jwt_RS256.pub
COPY jwt_RS256.pub /var/etcd/jwt_RS256.pub
COPY jwt_RS256.pub /var/lib/etcd/jwt_RS256.pub
COPY jwt_RS256.pub /jwt_RS256.pub
ENV ETCD_UNSUPPORTED_ARCH=arm64
EXPOSE 2379 2380
# Define default command.
CMD ["/usr/local/bin/etcd"]
```
- Create random keys
```# gen key pair for RS256
openssl genrsa -out jwt_RS256 4096
openssl rsa -in jwt_RS256 -pubout > jwt_RS256.pub
```
- `docker build -t etcd-custom .`
- Open `jetcd` project, and create `AuthClientTTLTest.java`
```
package io.etcd.jetcd.impl;
import io.etcd.jetcd.Auth;
import io.etcd.jetcd.ByteSequence;
import io.etcd.jetcd.Client;
import io.etcd.jetcd.KV;
import io.etcd.jetcd.auth.AuthRoleListResponse;
import io.etcd.jetcd.auth.Permission;
import io.etcd.jetcd.kv.DeleteResponse;
import io.etcd.jetcd.kv.GetResponse;
import io.etcd.jetcd.test.EtcdClusterExtension;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.api.extension.RegisterExtension;
import java.util.List;
import java.util.concurrent.TimeUnit;
import static io.etcd.jetcd.impl.TestUtil.bytesOf;
import static org.assertj.core.api.Assertions.assertThat;
@Timeout(value = 30, unit = TimeUnit.SECONDS)
public class AuthClientTTLTest {
@RegisterExtension
public static final EtcdClusterExtension cluster = EtcdClusterExtension.builder()
.withNodes(1)
.withAdditionalArgs(List.of("--auth-token=jwt,pub-key=jwt_RS256.pub,priv-key=jwt_RS256,sign-method=RS256", "--auth-token-ttl=1"))
.withImage("etcd-custom")
.build();
private static final String rootString = "root";
private static final ByteSequence rootPass = bytesOf("123");
private static final String rootRoleString = "root";
private static final String userString = "user";
private static final String userRoleString = "userRole";
private static Auth authDisabledAuthClient;
private static KV authDisabledKVClient;
private final ByteSequence rootRoleKey = bytesOf("root");
private final ByteSequence rootRoleValue = bytesOf("b");
private final ByteSequence rootRoleKeyRangeBegin = bytesOf("root");
private final ByteSequence rootRoleKeyRangeEnd = bytesOf("root1");
private final ByteSequence userRoleKey = bytesOf("foo");
private final ByteSequence userRoleValue = bytesOf("bar");
private final ByteSequence userRoleKeyRangeBegin = bytesOf("foo");
private final ByteSequence userRoleKeyRangeEnd = bytesOf("foo1");
private final ByteSequence root = bytesOf(rootString);
private final ByteSequence rootRole = bytesOf(rootRoleString);
private final ByteSequence user = bytesOf(userString);
private final ByteSequence userPass = bytesOf("userPass");
private final ByteSequence userNewPass = bytesOf("newUserPass");
private final ByteSequence userRole = bytesOf(userRoleString);
private final ByteSequence testKey = bytesOf("test_key");
private final ByteSequence testValue = bytesOf("test_value");
/**
* Build etcd client to create role, permission.
*/
@BeforeAll
public static void setupEnv() {
Client client = TestUtil.client(cluster).build();
authDisabledAuthClient = client.getAuthClient();
authDisabledKVClient = client.getKVClient();
}
@Test
public void testAuth() throws Exception {
// add root auth role, list auth role, and verify
authDisabledAuthClient.roleAdd(rootRole).get();
final AuthRoleListResponse response = authDisabledAuthClient.roleList().get();
assertThat(response.getRoles()).containsOnly(rootRoleString);
// role grant permission
authDisabledAuthClient
.roleGrantPermission(rootRole, rootRoleKeyRangeBegin, rootRoleKeyRangeEnd, Permission.Type.READWRITE).get();
// add root user, list users and verify
authDisabledAuthClient.userAdd(root, rootPass).get();
List users = authDisabledAuthClient.userList().get().getUsers();
assertThat(users).containsOnly(rootString);
// role granted to user and verify
authDisabledAuthClient.userGrantRole(root, rootRole).get();
assertThat(authDisabledAuthClient.userGet(root).get().getRoles()).containsOnly(rootRoleString);
// enable auth
authDisabledAuthClient.authEnable().get();
// create an auth enabled root client
final Client authEnabledRootClient = TestUtil.client(cluster).user(root).password(rootPass).build();
final Auth authEnabledAuthClient = authEnabledRootClient.getAuthClient();
final KV authEnabledKVClient = authEnabledRootClient.getKVClient();
authEnabledKVClient.put(testKey, testValue).get();
final GetResponse getResponse = authEnabledKVClient.get(testKey).get();
authEnabledAuthClient.roleAdd(userRole).get();
authEnabledAuthClient.roleGrantPermission(userRole, userRoleKeyRangeBegin, userRoleKeyRangeEnd, Permission.Type.READWRITE).get();
authEnabledAuthClient.userAdd(user, userPass).get();
authEnabledAuthClient.userGrantRole(user, userRole);
Thread.sleep(5000);
DeleteResponse deleteResponse = authEnabledKVClient.delete(testKey).get();
}
}
```
**Expected behavior**
Client should retry.
**Additional context**
I see there was a fix for this: https://github.com/etcd-io/jetcd/pull/1103, but maybe I'm missing context here.
If it's easier to recreate this issue, I've made a standalone app so that you can hit the `etcd-server`
```
/*
* This Java source file was generated by the Gradle 'init' task.
*/
package jetcd.sample;
import io.etcd.jetcd.Auth;
import io.etcd.jetcd.ByteSequence;
import io.etcd.jetcd.Client;
import io.etcd.jetcd.KV;
import io.etcd.jetcd.auth.Permission;
import io.etcd.jetcd.kv.GetResponse;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
public class App {
public String getGreeting() {
return "Hello World!";
}
public static void main(String[] args) throws ExecutionException, InterruptedException {
System.out.println(new App().getGreeting());
final ByteSequence user = ByteSequence.from("root".getBytes());
final ByteSequence pass = ByteSequence.from("root".getBytes());
// create client using endpoints
final Client client1 = Client.builder()
.user(user)
.password(user)
.endpoints("http://localhost:2379")
.build();
// everything works fine
final KV kvClient = client1.getKVClient();
final ByteSequence key = ByteSequence.from("test_key".getBytes());
final ByteSequence value = ByteSequence.from("test_value".getBytes());
// put the key-value
kvClient.put(key, value).get();
// get the CompletableFuture
CompletableFuture getFuture = kvClient.get(key);
// get the value from CompletableFuture
GetResponse response = getFuture.get();
System.out.println(response.toString());
// create client using endpoints
final Client client2 = Client.builder()
.user(user)
.password(user)
.endpoints("http://localhost:2379")
.build();
final Auth authClient = client2.getAuthClient();
final ByteSequence role0 = ByteSequence.from("role0".getBytes());
final ByteSequence user0 = ByteSequence.from("user0".getBytes());
authClient.roleAdd(role0);
authClient.roleGrantPermission(role0, key, key, Permission.Type.READWRITE);
authClient.userAdd(user0, pass);
authClient.userGrantRole(user0, role0);
Thread.sleep(11000);
// should fail
kvClient.delete(key).get();
// close
client1.close();
System.out.println(new App().getGreeting());
}
}
```
Contributor guide
Assessment
This issue has not been assessed yet.