NVIDIA-Merlin / NVIDIA-Merlin/Transformers4Rec
[BUG] Flakky test error in A100
Open
@sararb is already working on this.
Since Apr 5, 2022.
bug
P2
status/needs-triage
- Dominant language
- Python
- Stars
- 1.3k
- Forks
- 165
- Avg merge
- 1m
- Merged PRs (30d)
- 2
Description
Getting some tests errors in A100 but not V100
From https://github.com/NVIDIA-Merlin/Transformers4Rec
* branch main -> FETCH_HEAD
Already up to date.
============================= test session starts ==============================
platform linux -- Python 3.8.10, pytest-7.1.1, pluggy-1.0.0
rootdir: /transformers4rec
plugins: typeguard-2.13.3
collected 199 items / 1 skipped
tests/tf/test_masking.py ................... [ 9%]
tests/tf/test_public_api.py ....... [ 13%]
tests/tf/block/test_base.py .. [ 14%]
tests/tf/block/test_dlrm.py . [ 14%]
tests/tf/block/test_mlp.py ........................ [ 26%]
tests/tf/block/test_transformer.py .................. [ 35%]
tests/tf/features/test_continuous.py ..... [ 38%]
tests/tf/features/test_embedding.py ........ [ 42%]
tests/tf/features/test_sequence.py ............ [ 48%]
tests/tf/features/test_tabular.py ....... [ 51%]
tests/tf/model/test_head.py ................................ [ 67%]
tests/tf/model/test_model.py ............................... [ 83%]
tests/tf/tabular/test_aggregation.py ......... [ 87%]
tests/tf/tabular/test_tabular.py ......... [ 92%]
tests/tf/tabular/test_transformations.py .FFF.....FFFF [ 98%]
tests/tf/utils/test_schema_utils.py .. [100%]
See full log
_______________________ test_stochastic_swap_noise[0.3] ________________________
replacement_prob = 0.3
@pytest.mark.parametrize("replacement_prob", [0.1, 0.3, 0.5, 0.7])
def test_stochastic_swap_noise(replacement_prob):
NUM_SEQS = 100
SEQ_LENGTH = 80
PAD_TOKEN = 0
# Creating some input sequences with padding in the end
# (to emulate sessions with different lengths)
seq_inputs = {
"categ_seq_feat": tf.experimental.numpy.tril(
tf.random.uniform((NUM_SEQS, SEQ_LENGTH), minval=1, maxval=100, dtype=tf.int32), 1
),
"cont_seq_feat": tf.experimental.numpy.tril(tf.random.uniform((NUM_SEQS, SEQ_LENGTH)), 1),
"categ_feat": tf.random.uniform((NUM_SEQS,), minval=1, maxval=100, dtype=tf.int32),
}
tf.random.set_seed(0)
ssn = tr.StochasticSwapNoise(pad_token=PAD_TOKEN, replacement_prob=replacement_prob)
mask = seq_inputs["categ_seq_feat"] != PAD_TOKEN
out_features_ssn = ssn(seq_inputs, input_mask=mask)
for fname in seq_inputs:
replaced_mask = out_features_ssn[fname] != seq_inputs[fname]
replaced_mask_non_padded = tf.boolean_mask(replaced_mask, seq_inputs[fname] != PAD_TOKEN)
replacement_rate = tf.reduce_mean(
tf.cast(replaced_mask_non_padded, dtype=tf.float32)
).numpy()
> assert replacement_rate == pytest.approx(replacement_prob, abs=0.15)
E assert 0.0 == 0.3 ± 1.5e-01
E comparison failed
E Obtained: 0.0
E Expected: 0.3 ± 1.5e-01
tests/tf/tabular/test_transformations.py:52: AssertionError
_______________________ test_stochastic_swap_noise[0.5] ________________________
replacement_prob = 0.5
@pytest.mark.parametrize("replacement_prob", [0.1, 0.3, 0.5, 0.7])
def test_stochastic_swap_noise(replacement_prob):
NUM_SEQS = 100
SEQ_LENGTH = 80
PAD_TOKEN = 0
# Creating some input sequences with padding in the end
# (to emulate sessions with different lengths)
seq_inputs = {
"categ_seq_feat": tf.experimental.numpy.tril(
tf.random.uniform((NUM_SEQS, SEQ_LENGTH), minval=1, maxval=100, dtype=tf.int32), 1
),
"cont_seq_feat": tf.experimental.numpy.tril(tf.random.uniform((NUM_SEQS, SEQ_LENGTH)), 1),
"categ_feat": tf.random.uniform((NUM_SEQS,), minval=1, maxval=100, dtype=tf.int32),
}
tf.random.set_seed(0)
ssn = tr.StochasticSwapNoise(pad_token=PAD_TOKEN, replacement_prob=replacement_prob)
mask = seq_inputs["categ_seq_feat"] != PAD_TOKEN
out_features_ssn = ssn(seq_inputs, input_mask=mask)
for fname in seq_inputs:
replaced_mask = out_features_ssn[fname] != seq_inputs[fname]
replaced_mask_non_padded = tf.boolean_mask(replaced_mask, seq_inputs[fname] != PAD_TOKEN)
replacement_rate = tf.reduce_mean(
tf.cast(replaced_mask_non_padded, dtype=tf.float32)
).numpy()
> assert replacement_rate == pytest.approx(replacement_prob, abs=0.15)
E assert 0.0 == 0.5 ± 1.5e-01
E comparison failed
E Obtained: 0.0
E Expected: 0.5 ± 1.5e-01
tests/tf/tabular/test_transformations.py:52: AssertionError
_______________________ test_stochastic_swap_noise[0.7] ________________________
replacement_prob = 0.7
@pytest.mark.parametrize("replacement_prob", [0.1, 0.3, 0.5, 0.7])
def test_stochastic_swap_noise(replacement_prob):
NUM_SEQS = 100
SEQ_LENGTH = 80
PAD_TOKEN = 0
# Creating some input sequences with padding in the end
# (to emulate sessions with different lengths)
seq_inputs = {
"categ_seq_feat": tf.experimental.numpy.tril(
tf.random.uniform((NUM_SEQS, SEQ_LENGTH), minval=1, maxval=100, dtype=tf.int32), 1
),
"cont_seq_feat": tf.experimental.numpy.tril(tf.random.uniform((NUM_SEQS, SEQ_LENGTH)), 1),
"categ_feat": tf.random.uniform((NUM_SEQS,), minval=1, maxval=100, dtype=tf.int32),
}
tf.random.set_seed(0)
ssn = tr.StochasticSwapNoise(pad_token=PAD_TOKEN, replacement_prob=replacement_prob)
mask = seq_inputs["categ_seq_feat"] != PAD_TOKEN
out_features_ssn = ssn(seq_inputs, input_mask=mask)
for fname in seq_inputs:
replaced_mask = out_features_ssn[fname] != seq_inputs[fname]
replaced_mask_non_padded = tf.boolean_mask(replaced_mask, seq_inputs[fname] != PAD_TOKEN)
replacement_rate = tf.reduce_mean(
tf.cast(replaced_mask_non_padded, dtype=tf.float32)
).numpy()
> assert replacement_rate == pytest.approx(replacement_prob, abs=0.15)
E assert 0.0 == 0.7 ± 1.5e-01
E comparison failed
E Obtained: 0.0
E Expected: 0.7 ± 1.5e-01
tests/tf/tabular/test_transformations.py:52: AssertionError
____________ test_stochastic_swap_noise_with_tabular_features[0.3] _____________
yoochoose_schema = [{'name': 'session_id', 'type': 'INT', 'int_domain': {'name': 'session_id', 'min': '1', 'max': '11562158'}, 'annotatio...ype': 'FLOAT', 'float_domain': {'name': 'user_age', 'max': 0.4079650044441223}, 'annotation': {'tag': ['continuous']}}]
tf_yoochoose_like = {'category/list': <tf.Tensor: shape=(100, 20), dtype=int32, numpy=
array([[ 75, 10, 247, ..., 2, 0, 0],
..., 9450385, 1274920, 3624043, 1992476, 11443349,
1244836, 7307614, 4965882, 11263996], dtype=int32)>, ...}
replacement_prob = 0.3
@pytest.mark.parametrize("replacement_prob", [0.1, 0.3, 0.5, 0.7])
def test_stochastic_swap_noise_with_tabular_features(
yoochoose_schema, tf_yoochoose_like, replacement_prob
):
inputs = tf_yoochoose_like
tab_module = tr.TabularSequenceFeatures.from_schema(yoochoose_schema)
out_features = tab_module(inputs)
PAD_TOKEN = 0
tf.random.set_seed(0)
ssn = tr.StochasticSwapNoise(
pad_token=PAD_TOKEN, replacement_prob=replacement_prob, schema=yoochoose_schema
)
out_features_ssn = tab_module(inputs, pre=ssn, training=True)
for fname in out_features_ssn:
replaced_mask = out_features[fname] != out_features_ssn[fname]
# Ignoring padding items to compute the mean replacement rate
feat_non_padding_mask = inputs[fname] != PAD_TOKEN
replaced_mask_non_padded = tf.boolean_mask(replaced_mask, feat_non_padding_mask)
replacement_rate = tf.reduce_mean(
tf.cast(replaced_mask_non_padded, dtype=tf.float32)
).numpy()
> assert replacement_rate == pytest.approx(replacement_prob, abs=0.15)
E assert 0.0 == 0.3 ± 1.5e-01
E comparison failed
E Obtained: 0.0
E Expected: 0.3 ± 1.5e-01
tests/tf/tabular/test_transformations.py:106: AssertionError
____________ test_stochastic_swap_noise_with_tabular_features[0.5] _____________
yoochoose_schema = [{'name': 'session_id', 'type': 'INT', 'int_domain': {'name': 'session_id', 'min': '1', 'max': '11562158'}, 'annotatio...ype': 'FLOAT', 'float_domain': {'name': 'user_age', 'max': 0.4079650044441223}, 'annotation': {'tag': ['continuous']}}]
tf_yoochoose_like = {'category/list': <tf.Tensor: shape=(100, 20), dtype=int32, numpy=
array([[ 75, 10, 247, ..., 0, 0, 0],
..., 9450385, 1274920, 3624043, 1992476, 11443349,
1244836, 7307614, 4965882, 11263996], dtype=int32)>, ...}
replacement_prob = 0.5
@pytest.mark.parametrize("replacement_prob", [0.1, 0.3, 0.5, 0.7])
def test_stochastic_swap_noise_with_tabular_features(
yoochoose_schema, tf_yoochoose_like, replacement_prob
):
inputs = tf_yoochoose_like
tab_module = tr.TabularSequenceFeatures.from_schema(yoochoose_schema)
out_features = tab_module(inputs)
PAD_TOKEN = 0
tf.random.set_seed(0)
ssn = tr.StochasticSwapNoise(
pad_token=PAD_TOKEN, replacement_prob=replacement_prob, schema=yoochoose_schema
)
out_features_ssn = tab_module(inputs, pre=ssn, training=True)
for fname in out_features_ssn:
replaced_mask = out_features[fname] != out_features_ssn[fname]
# Ignoring padding items to compute the mean replacement rate
feat_non_padding_mask = inputs[fname] != PAD_TOKEN
replaced_mask_non_padded = tf.boolean_mask(replaced_mask, feat_non_padding_mask)
replacement_rate = tf.reduce_mean(
tf.cast(replaced_mask_non_padded, dtype=tf.float32)
).numpy()
> assert replacement_rate == pytest.approx(replacement_prob, abs=0.15)
E assert 0.0 == 0.5 ± 1.5e-01
E comparison failed
E Obtained: 0.0
E Expected: 0.5 ± 1.5e-01
tests/tf/tabular/test_transformations.py:106: AssertionError
____________ test_stochastic_swap_noise_with_tabular_features[0.7] _____________
yoochoose_schema = [{'name': 'session_id', 'type': 'INT', 'int_domain': {'name': 'session_id', 'min': '1', 'max': '11562158'}, 'annotatio...ype': 'FLOAT', 'float_domain': {'name': 'user_age', 'max': 0.4079650044441223}, 'annotation': {'tag': ['continuous']}}]
tf_yoochoose_like = {'category/list': <tf.Tensor: shape=(100, 20), dtype=int32, numpy=
array([[ 75, 10, 247, ..., 0, 0, 0],
..., 9450385, 1274920, 3624043, 1992476, 11443349,
1244836, 7307614, 4965882, 11263996], dtype=int32)>, ...}
replacement_prob = 0.7
@pytest.mark.parametrize("replacement_prob", [0.1, 0.3, 0.5, 0.7])
def test_stochastic_swap_noise_with_tabular_features(
yoochoose_schema, tf_yoochoose_like, replacement_prob
):
inputs = tf_yoochoose_like
tab_module = tr.TabularSequenceFeatures.from_schema(yoochoose_schema)
out_features = tab_module(inputs)
PAD_TOKEN = 0
tf.random.set_seed(0)
ssn = tr.StochasticSwapNoise(
pad_token=PAD_TOKEN, replacement_prob=replacement_prob, schema=yoochoose_schema
)
out_features_ssn = tab_module(inputs, pre=ssn, training=True)
for fname in out_features_ssn:
replaced_mask = out_features[fname] != out_features_ssn[fname]
# Ignoring padding items to compute the mean replacement rate
feat_non_padding_mask = inputs[fname] != PAD_TOKEN
replaced_mask_non_padded = tf.boolean_mask(replaced_mask, feat_non_padding_mask)
replacement_rate = tf.reduce_mean(
tf.cast(replaced_mask_non_padded, dtype=tf.float32)
).numpy()
> assert replacement_rate == pytest.approx(replacement_prob, abs=0.15)
E assert 0.0 == 0.7 ± 1.5e-01
E comparison failed
E Obtained: 0.0
E Expected: 0.7 ± 1.5e-01
tests/tf/tabular/test_transformations.py:106: AssertionError
__________ test_stochastic_swap_noise_raise_exception_not_2d_item_id ___________
def test_stochastic_swap_noise_raise_exception_not_2d_item_id():
s = schema.Schema(
[
schema.ColumnSchema.create_categorical(
"item_id_feat", num_items=1000, tags=[Tag.ITEM_ID]
),
]
)
NUM_SEQS = 100
SEQ_LENGTH = 80
PAD_TOKEN = 0
seq_inputs = {
"item_id_feat": tf.experimental.numpy.tril(
tf.random.uniform((NUM_SEQS, SEQ_LENGTH, 64), minval=1, maxval=100, dtype=tf.int32), 1
),
}
ssn = tr.StochasticSwapNoise(pad_token=PAD_TOKEN, replacement_prob=0.3, schema=s)
with pytest.raises(ValueError) as excinfo:
> ssn(seq_inputs)
E Failed: DID NOT RAISE <class 'ValueError'>
tests/tf/tabular/test_transformations.py:132: Failed
=========================== short test summary info ============================
FAILED tests/tf/tabular/test_transformations.py::test_stochastic_swap_noise[0.3]
FAILED tests/tf/tabular/test_transformations.py::test_stochastic_swap_noise[0.5]
FAILED tests/tf/tabular/test_transformations.py::test_stochastic_swap_noise[0.7]
FAILED tests/tf/tabular/test_transformations.py::test_stochastic_swap_noise_with_tabular_features[0.3]
FAILED tests/tf/tabular/test_transformations.py::test_stochastic_swap_noise_with_tabular_features[0.5]
FAILED tests/tf/tabular/test_transformations.py::test_stochastic_swap_noise_with_tabular_features[0.7]
FAILED tests/tf/tabular/test_transformations.py::test_stochastic_swap_noise_raise_exception_not_2d_item_id
Contributor guide
First steps
- Read the whole issue, then the project's contributing guide.
- Comment on the issue to say you are picking it up — it saves two people doing the same work.
- Fork the repository and make your change on a branch.
- Open a pull request that references the issue number.
Assessment
This issue has not been assessed yet.