Training job not executing

everytime i run

`estimator.fit({'training':'s3://xxx/xxx/xxx/xxx/train.csv','validation':'s3://xxx/xxx/xxx/xxx/val.csv'})`

it returns **UnexpectedStatusException: Error for Training job xyz-2020-05-22-11-02-50-985: Failed. Reason: AlgorithmError: Exit Code: 1**
----------------------------------------
DOCKERFILE for my image
```
FROM python:3.6

RUN pip install numpy
RUN pip install pandas
RUN pip install torch
RUN pip install transformers
RUN pip install sklearn
RUN pip install boto3

COPY train.py /opt/ml/code/train.py

ENTRYPOINT ["python", "/opt/ml/code/train.py"]

```
--------------------------------------------
train.py file

```
import pandas as pd
import numpy as np
import torch
import torch.nn as nn
import transformers
from sklearn import model_selection
from sklearn import metrics
from transformers import AdamW
from transformers import get_linear_schedule_with_warmup
import torch
import torch.nn as nn
from tqdm import tqdm
import boto3

s3client = boto3.client('s3')
s3client.download_file('xxxx', 'yyyy/biobert_v1.1_pubmed', 'embed_pubmed')

class BERTConfig:
MAX_LEN = 40
TRAIN_BATCH_SIZE = 8
VALID_BATCH_SIZE = 4
EPOCHS = 10
BERT_PATH = '/opt/ml/input/data/emb/embed_pubmed'
#MODEL_PATH = CURRENT_DIR / "artifacts" / "bert_large.bin"
#TRAINING_FILE = CURRENT_DIR / "artifacts" / "dataset.csv"
TOKENIZER = transformers.BertTokenizer.from_pretrained(str('/opt/ml/input/data/emb/embed_pubmed'), do_lower_case=True)

class BERTDataset:
def __init__(self, sentence, target):
self.sentence = sentence
self.target = target
self.tokenizer = BERTConfig.TOKENIZER
self.max_len = BERTConfig.MAX_LEN

def __len__(self):
return len(self.sentence)

def __getitem__(self, item):
sentence = str(self.sentence[item])
sentence = " ".join(sentence.split())

inputs = self.tokenizer.encode_plus(
sentence,
None,
add_special_tokens=True,
max_length=self.max_len,
pad_to_max_length=True,
)

ids = inputs["input_ids"]
mask = inputs["attention_mask"]
token_type_ids = inputs["token_type_ids"]

return {
"ids": torch.tensor(ids, dtype=torch.long),
"mask": torch.tensor(mask, dtype=torch.long),
"token_type_ids": torch.tensor(token_type_ids, dtype=torch.long),
"targets": torch.tensor(self.target[item], dtype=torch.float),
}



class BERTBaseUncased(nn.Module):
def __init__(self):
super(BERTBaseUncased, self).__init__()
self.bert = transformers.BertModel.from_pretrained('/opt/ml/input/data/emb/embed_pubmed')
self.bert_drop = nn.Dropout(0.3)
self.out = nn.Linear(768, 1)

def forward(self, ids, mask, token_type_ids):
_, o2 = self.bert(ids, attention_mask=mask, token_type_ids=token_type_ids)
bo = self.bert_drop(o2)
output = self.out(bo)
return output

def loss_fn(outputs, targets):
return nn.BCEWithLogitsLoss()(outputs, targets.view(-1, 1))

def train_fn(data_loader, model, optimizer, device, scheduler):
model.train()

for bi, d in tqdm(enumerate(data_loader), total=len(data_loader)):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
targets = d["targets"]

ids = ids.to(device, dtype=torch.long)
token_type_ids = token_type_ids.to(device, dtype=torch.long)
mask = mask.to(device, dtype=torch.long)
targets = targets.to(device, dtype=torch.float)

optimizer.zero_grad()
outputs = model(ids=ids, mask=mask, token_type_ids=token_type_ids)

loss = loss_fn(outputs, targets)
loss.backward()
optimizer.step()
scheduler.step()

def eval_fn(data_loader, model, device):
model.eval()
fin_targets = []
fin_outputs = []
with torch.no_grad():
for bi, d in tqdm(enumerate(data_loader), total=len(data_loader)):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
targets = d["targets"]

ids = ids.to(device, dtype=torch.long)
token_type_ids = token_type_ids.to(device, dtype=torch.long)
mask = mask.to(device, dtype=torch.long)
targets = targets.to(device, dtype=torch.float)

outputs = model(ids=ids, mask=mask, token_type_ids=token_type_ids)
fin_targets.extend(targets.cpu().detach().numpy())
outputs = torch.sigmoid(outputs).cpu().detach().numpy()
fin_outputs.extend(outputs)
return fin_outputs, fin_targets

df_train = pd.read_csv('/opt/ml/input/data/training/train.csv')
df_valid = pd.read_csv('/opt/ml/input/data/validation/val.csv')

train_dataset = BERTDataset(sentence=df_train.sent.values, target=df_train.L3_correct_label.values)

train_data_loader = torch.utils.data.DataLoader(
train_dataset, batch_size=BERTConfig.TRAIN_BATCH_SIZE, num_workers=4
)

valid_dataset = BERTDataset(
sentence=df_valid.sent.values, target=df_valid.L3_correct_label.values
)

valid_data_loader = torch.utils.data.DataLoader(
valid_dataset, batch_size=BERTConfig.VALID_BATCH_SIZE, num_workers=1
)

device = torch.device("cuda")
model = BERTBaseUncased()
model.to(device)

optimizer = AdamW(model.parameters(), lr=2e-5, correct_bias=False)
total_steps = len(train_data_loader) * 10
scheduler = get_linear_schedule_with_warmup(optimizer,num_warmup_steps=0,num_training_steps=total_steps)

best_accuracy = 0
for epoch in range(10):
train_fn(train_data_loader, model, optimizer, device, scheduler)
outputs, targets = eval_fn(valid_data_loader, model, device)
#torch.save(model.state_dict(), '_bert_bm.bin')
finalout=[]
#outputs = np.array(outputs) >= 0.5
for x in range(0,len(outputs)):
if outputs[x][0]>=0.5:
finalout.append(1.0)
else:
finalout.append(0.0)

accuracy = metrics.accuracy_score(targets, finalout)
print(f"Accuracy Score = {accuracy}")
if accuracy > best_accuracy:
torch.save(model.state_dict(), '/opt/ml/model/bert_bm.bin')
#withoutN_bert_bm.bin
best_accuracy = accuracy

```
-----------------
sagemaker.ipynb

```
import sagemaker
role = 'abc'
session = sagemaker.Session()
account = session.boto_session.client('sts').get_caller_identity()['Account']
region = session.boto_session.region_name
image = "{}.dkr.ecr.{}.amazonaws.com/ccccccccc".format(account, region)

output_path='abc/BERT_experiment'
estimator = sagemaker.estimator.Estimator(image,
role,
train_instance_count=1,
train_use_spot_instances=True,
train_max_run = (2 * 60 * 60),
train_max_wait = (2 * 60 * 60),
train_instance_type='ml.p2.xlarge',
output_path=output_path,
base_job_name='sagetrial',
enable_sagemaker_metrics=True,
#hyperparameters=hyperparameters,
sagemaker_session=session
)

`estimator.fit({'training':'s3://xxx/xxx/xxx/xxx/train.csv','validation':'s3://xxx/xxx/xxx/xxx/val.csv'})`

```

Will really appreciate if someone can help me out with this

Start with the SageMaker training-job logs for the reported failed job, then inspect the Dockerfile, train.py entry point, and sagemaker.ipynb configuration together. Trace the first reported failure through the container and training paths; done means the same estimator.fit call completes successfully and produces the expected model output.

Written by the indexing model from the issue text.