NoisyNets implementation issues
- Dominant language
- Jupyter Notebook
- Stars
- 10.9k
- Forks
- 1.4k
- PR merge metrics
- No merged PRs in 30d
Description
I'm implementing my own RL framework in Jax to better understand RL algorithms and found your code very helpful
Looking at the NoisyNets implementation, on line 316 and 317 (https://github.com/google/dopamine/blob/master/dopamine/jax/networks.py)
The same rng_key is used each time noise is generated meaning that no 'new' noise is generated each time an input is passed to the layer. In effect, the layer just applies a linear transform I think
This is a short testing example
```
import jax
import numpy as np
from dopamine.jax.networks import NoisyNetwork
if __name__ == '__main__':
rng = jax.random.PRNGKey(1)
rng, rng_net_def, rng_net_param = jax.random.split(rng, num=3)
net_def = NoisyNetwork(rng_key=rng_net_def, eval_mode=False)
net_params = net_def.init(rng_net_param, x=np.zeros(10), features=3)
state = np.random.random(10)
print(net_def.apply(net_params, x=state, features=3))
print(net_def.apply(net_params, x=state, features=3))
```
If this is an issue, then I implemented the following code for my framework
```
from typing import Sequence
import jax
import numpy as onp
import jax.numpy as jnp
from flax import linen as nn
class NoisyDense(nn.Module):
features: int
use_bias: bool = True
@staticmethod
@jax.jit
def _f(x: jnp.ndarray) -> jnp.ndarray:
# See (10) and (11) in Fortunato et al. (2018).
return jnp.multiply(jnp.sign(x), jnp.power(jnp.abs(x), 0.5))
@nn.compact
def __call__(self, inputs: onp.ndarray, eval_mode: bool = True, rng: jnp.DeviceArray = None) -> jnp.ndarray:
if eval_mode: # Turn off noise during evaluation
w_epsilon = jnp.zeros(shape=(inputs.shape[0], self.features), dtype=onp.float32)
b_epsilon = jnp.zeros(shape=(self.features,), dtype=onp.float32)
else: # Factored gaussian noise in (10) and (11) in Fortunato et al. (2018).
p_key, q_key = jax.random.split(rng)
p, q = jax.random.normal(p_key, [inputs.shape[0], 1]), jax.random.normal(q_key, [1, self.features])
f_p, f_q = self._f(p), self._f(q)
w_epsilon, b_epsilon = f_p * f_p, jnp.squeeze(f_q)
def _mu_init(key: jnp.DeviceArray, shape: Sequence[int]):
# Initialization of mean noise parameters (Section 3.2)
mean = 1 / jnp.power(inputs.shape[0], 0.5)
return jax.random.uniform(key, minval=-mean, maxval=mean, shape=shape)
def _sigma_init(_key: jnp.DeviceArray, shape: Sequence[int], dtype=jnp.float32):
# Initialization of sigma noise parameters (Section 3.2)
return jnp.ones(shape, dtype) * (0.1 / onp.sqrt(inputs.shape[0]))
# See (8) and (9) in Fortunato et al. (2018) for output computation.
w_mu = self.param('kernel_mu', _mu_init, (inputs.shape[0], self.features))
w_sigma = self.param('kernel_sigma', _sigma_init, (inputs.shape[0], self.features))
out = jnp.matmul(inputs, w_mu + jnp.multiply(w_sigma, w_epsilon))
if self.use_bias:
b_mu = self.param('bias_mu', _mu_init, (self.features,))
b_sigma = self.param('bias_sigma', _sigma_init, (self.features,))
out = out + b_mu + jnp.multiply(b_sigma, b_epsilon)
return out
```
Here is some similar testing code
```
if __name__ == '__main__':
rng = jax.random.PRNGKey(1)
rng, rng_net_def, rng_net_param = jax.random.split(rng, num=3)
net_def = NoisyDense(features=2)
net_params = net_def.init(rng_net_param, np.zeros(10))
state = np.random.random(10)
print(net_def.apply(net_params, inputs=state))
print(net_def.apply(net_params, inputs=state, eval_mode=False, rng=rng_net_def))
print(net_def.apply(net_params, inputs=state, eval_mode=False, rng=rng))
```
I would have submitted this as a pull request but noticed that you are not accepting merges
Contributor guide
Assessment
This issue has not been assessed yet.