Auth token not successfully propagated...

cc @Kami

Context: ST2 2.2.0 without $ST2_AUTH_TOKEN environment variable set, but after successfully running st2 login.

root@7b0ec35f2fe8:~# cat ~/.st2/token-st2admin 
{"expire_timestamp": 1490134508, "token": "ef38f59b20c84cbda03e19d4da1c56a9"}
root@7b0ec35f2fe8:~# cat ~/.st2/config
[credentials]
username = st2admin

st2-self-check fails execution. It appears the auth token is not propagated successfully.

Here's the output from the first failed command run by st2-self-check (run manually: st2 --debug run tests.test_packs_pack):

root@7b0ec35f2fe8:~# st2 --debug run tests.test_key_triggers
2017-03-21 08:01:31,647  DEBUG - Using cached token from file "/root/.st2/token-st2admin"
# -------- begin 140054699284944 request ----------
curl -X GET -H  'Connection: keep-alive' -H  'Accept-Encoding: gzip, deflate' -H  'Accept: */*' -H  'User-Agent: python-requests/2.11.1' -H  'X-Auth-Token: ef38f59b20c84cbda03e19d4da1c56a9' http://127.0.0.1:9101/v1/actions/tests.test_key_triggers
# -------- begin 140054699284944 response ----------
{"name": "test_key_triggers", "parameters": {"token": {"default": "", "type": "string", "description": "st2 auth token"}, "hostname": {"default": "127.0.0.1", "type": "string", "description": "St2 host to run tests against"}, "protocol": {"default": "http", "type": "string", "description": "http/https"}}, "tags": [], "description": "Workflow tests key triggers functionality", "enabled": true, "entry_point": "chains/test_key_triggers.yaml", "notify": {}, "uid": "action:tests:test_key_triggers", "pack": "tests", "ref": "tests.test_key_triggers", "id": "58d0178550484300eb5228ca", "runner_type": "action-chain"}
# -------- end 140054699284944 response ------------

# -------- begin 140054699284816 request ----------
curl -X GET -H  'Connection: keep-alive' -H  'Accept-Encoding: gzip, deflate' -H  'Accept: */*' -H  'User-Agent: python-requests/2.11.1' -H  'X-Auth-Token: ef38f59b20c84cbda03e19d4da1c56a9' 'http://127.0.0.1:9101/v1/runnertypes/?name=action-chain'
# -------- begin 140054699284816 response ----------
[{"runner_module": "action_chain_runner", "uid": "runner_type:action-chain", "description": "A runner for launching linear action chains.", "enabled": true, "runner_parameters": {"display_published": {"default": false, "type": "boolean", "description": "Intermediate published variables will be stored and displayed."}, "skip_notify": {"default": [], "type": "array", "description": "List of tasks to skip notifications for."}}, "id": "58d0178550484300eb5228af", "name": "action-chain"}]
# -------- end 140054699284816 response ------------

# -------- begin 140054699285712 request ----------
curl -X POST -H  'Connection: keep-alive' -H  'Accept-Encoding: gzip, deflate' -H  'Accept: */*' -H  'User-Agent: python-requests/2.11.1' -H  'content-type: application/json' -H  'X-Auth-Token: ef38f59b20c84cbda03e19d4da1c56a9' -H  'Content-Length: 69' --data-binary '{"action": "tests.test_key_triggers", "user": null, "parameters": {}}' http://127.0.0.1:9101/v1/executions
# -------- begin 140054699285712 response ----------
{"status": "requested", "start_timestamp": "2017-03-21T08:01:31.719729Z", "log": [{"status": "requested", "timestamp": "2017-03-21T08:01:31.729699Z"}], "runner": {"runner_module": "action_chain_runner", "uid": "runner_type:action-chain", "description": "A runner for launching linear action chains.", "enabled": true, "runner_parameters": {"display_published": {"default": false, "type": "boolean", "description": "Intermediate published variables will be stored and displayed."}, "skip_notify": {"default": [], "type": "array", "description": "List of tasks to skip notifications for."}}, "id": "58d0178550484300eb5228af", "name": "action-chain"}, "web_url": "https://7b0ec35f2fe8/#/history/58d0dddb50484300b2a734ec/general", "context": {"user": "st2admin"}, "action": {"description": "Workflow tests key triggers functionality", "runner_type": "action-chain", "tags": [], "enabled": true, "pack": "tests", "entry_point": "chains/test_key_triggers.yaml", "notify": {}, "uid": "action:tests:test_key_triggers", "parameters": {"token": {"default": "", "type": "string", "description": "st2 auth token"}, "hostname": {"default": "127.0.0.1", "type": "string", "description": "St2 host to run tests against"}, "protocol": {"default": "http", "type": "string", "description": "http/https"}}, "ref": "tests.test_key_triggers", "id": "58d0178550484300eb5228ca", "name": "test_key_triggers"}, "liveaction": {"runner_info": {}, "parameters": {}, "action_is_workflow": true, "callback": {}, "action": "tests.test_key_triggers", "id": "58d0dddb50484300b2a734eb"}, "id": "58d0dddb50484300b2a734ec"}
# -------- end 140054699285712 response ------------

.# -------- begin 140054699285712 request ----------
curl -X GET -H  'Connection: keep-alive' -H  'Accept-Encoding: gzip, deflate' -H  'Accept: */*' -H  'User-Agent: python-requests/2.11.1' -H  'X-Auth-Token: ef38f59b20c84cbda03e19d4da1c56a9' http://127.0.0.1:9101/v1/executions/58d0dddb50484300b2a734ec
# -------- begin 140054699285712 response ----------
{"status": "running", "start_timestamp": "2017-03-21T08:01:31.719729Z", "log": [{"status": "requested", "timestamp": "2017-03-21T08:01:31.729000Z"}, {"status": "scheduled", "timestamp": "2017-03-21T08:01:31.792000Z"}, {"status": "running", "timestamp": "2017-03-21T08:01:31.847000Z"}], "runner": {"runner_module": "action_chain_runner", "uid": "runner_type:action-chain", "enabled": true, "name": "action-chain", "runner_parameters": {"display_published": {"default": false, "type": "boolean", "description": "Intermediate published variables will be stored and displayed."}, "skip_notify": {"default": [], "type": "array", "description": "List of tasks to skip notifications for."}}, "id": "58d0178550484300eb5228af", "description": "A runner for launching linear action chains."}, "id": "58d0dddb50484300b2a734ec", "web_url": "https://7b0ec35f2fe8/#/history/58d0dddb50484300b2a734ec/general", "context": {"user": "st2admin"}, "action": {"runner_type": "action-chain", "name": "test_key_triggers", "parameters": {"token": {"default": "", "type": "string", "description": "st2 auth token"}, "hostname": {"default": "127.0.0.1", "type": "string", "description": "St2 host to run tests against"}, "protocol": {"default": "http", "type": "string", "description": "http/https"}}, "tags": [], "enabled": true, "entry_point": "chains/test_key_triggers.yaml", "notify": {}, "uid": "action:tests:test_key_triggers", "pack": "tests", "ref": "tests.test_key_triggers", "id": "58d0178550484300eb5228ca", "description": "Workflow tests key triggers functionality"}, "liveaction": {"runner_info": {"hostname": "7b0ec35f2fe8", "pid": 95}, "parameters": {}, "action_is_workflow": true, "callback": {}, "action": "tests.test_key_triggers", "id": "58d0dddb50484300b2a734eb"}, "children": ["58d0dddc504843005fcaa98c", "58d0dddd504843005fcaa98e"]}
# -------- end 140054699285712 response ------------

.# -------- begin 140054699285712 request ----------
curl -X GET -H  'Connection: keep-alive' -H  'Accept-Encoding: gzip, deflate' -H  'Accept: */*' -H  'User-Agent: python-requests/2.11.1' -H  'X-Auth-Token: ef38f59b20c84cbda03e19d4da1c56a9' http://127.0.0.1:9101/v1/executions/58d0dddb50484300b2a734ec
# -------- begin 140054699285712 response ----------
{"status": "failed", "start_timestamp": "2017-03-21T08:01:31.719729Z", "log": [{"status": "requested", "timestamp": "2017-03-21T08:01:31.729000Z"}, {"status": "scheduled", "timestamp": "2017-03-21T08:01:31.792000Z"}, {"status": "running", "timestamp": "2017-03-21T08:01:31.847000Z"}, {"status": "failed", "timestamp": "2017-03-21T08:01:34.300000Z"}], "runner": {"runner_module": "action_chain_runner", "uid": "runner_type:action-chain", "enabled": true, "name": "action-chain", "runner_parameters": {"display_published": {"default": false, "type": "boolean", "description": "Intermediate published variables will be stored and displayed."}, "skip_notify": {"default": [], "type": "array", "description": "List of tasks to skip notifications for."}}, "id": "58d0178550484300eb5228af", "description": "A runner for launching linear action chains."}, "id": "58d0dddb50484300b2a734ec", "elapsed_seconds": 2.562138, "web_url": "https://7b0ec35f2fe8/#/history/58d0dddb50484300b2a734ec/general", "result": {"tasks": [{"name": "cleanup_environment", "workflow": null, "created_at": "2017-03-21T08:01:31.982818+00:00", "updated_at": "2017-03-21T08:01:33.118877+00:00", "state": "succeeded", "result": {"failed": false, "stderr": "Traceback (most recent call last):\n  File \"/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2client/commands/resource.py\", line 162, in get_resource_by_pk\n    instance = self.manager.get_by_id(pk, **kwargs)\n  File \"/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2client/models/core.py\", line 39, in decorate\n    return func(*args, **kwargs)\n  File \"/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2client/models/core.py\", line 204, in get_by_id\n    self.handle_error(response)\n  File \"/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2client/models/core.py\", line 165, in handle_error\n    response.raise_for_status()\n  File \"/opt/stackstorm/st2/local/lib/python2.7/site-packages/requests/models.py\", line 862, in raise_for_status\n    raise HTTPError(http_error_msg, response=self)\nHTTPError: 401 Client Error: Unauthorized\nMESSAGE: Unauthorized - One of Token or API key required. for url: http://127.0.0.1:9101/keys/a?scope=system", "return_code": 0, "succeeded": true, "stdout": "ERROR: 401 Client Error: Unauthorized\nMESSAGE: Unauthorized - One of Token or API key required. for url: http://127.0.0.1:9101/keys/a?scope=system\n"}, "id": "cleanup_environment", "execution_id": "58d0dddc504843005fcaa98c"}, {"name": "check_if_key_create_trigger_is_registered", "workflow": null, "created_at": "2017-03-21T08:01:33.127298+00:00", "updated_at": "2017-03-21T08:01:34.270785+00:00", "state": "failed", "result": {"failed": true, "stderr": "Traceback (most recent call last):\n  File \"/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2client/commands/resource.py\", line 162, in get_resource_by_pk\n    instance = self.manager.get_by_id(pk, **kwargs)\n  File \"/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2client/models/core.py\", line 39, in decorate\n    return func(*args, **kwargs)\n  File \"/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2client/models/core.py\", line 204, in get_by_id\n    self.handle_error(response)\n  File \"/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2client/models/core.py\", line 165, in handle_error\n    response.raise_for_status()\n  File \"/opt/stackstorm/st2/local/lib/python2.7/site-packages/requests/models.py\", line 862, in raise_for_status\n    raise HTTPError(http_error_msg, response=self)\nHTTPError: 401 Client Error: Unauthorized\nMESSAGE: Unauthorized - One of Token or API key required. for url: http://127.0.0.1:9101/triggertypes/core.st2.key_value_pair.create", "return_code": 1, "succeeded": false, "stdout": "ERROR: 401 Client Error: Unauthorized\nMESSAGE: Unauthorized - One of Token or API key required. for url: http://127.0.0.1:9101/triggertypes/core.st2.key_value_pair.create\n"}, "id": "check_if_key_create_trigger_is_registered", "execution_id": "58d0dddd504843005fcaa98e"}]}, "context": {"user": "st2admin"}, "action": {"runner_type": "action-chain", "name": "test_key_triggers", "parameters": {"token": {"default": "", "type": "string", "description": "st2 auth token"}, "hostname": {"default": "127.0.0.1", "type": "string", "description": "St2 host to run tests against"}, "protocol": {"default": "http", "type": "string", "description": "http/https"}}, "tags": [], "enabled": true, "entry_point": "chains/test_key_triggers.yaml", "notify": {}, "uid": "action:tests:test_key_triggers", "pack": "tests", "ref": "tests.test_key_triggers", "id": "58d0178550484300eb5228ca", "description": "Workflow tests key triggers functionality"}, "liveaction": {"runner_info": {"hostname": "7b0ec35f2fe8", "pid": 95}, "parameters": {}, "action_is_workflow": true, "callback": {}, "action": "tests.test_key_triggers", "id": "58d0dddb50484300b2a734eb"}, "children": ["58d0dddc504843005fcaa98c", "58d0dddd504843005fcaa98e"], "end_timestamp": "2017-03-21T08:01:34.281867Z"}
# -------- end 140054699285712 response ------------


# -------- begin 140054699285712 request ----------
curl -X GET -H  'Connection: keep-alive' -H  'Accept-Encoding: gzip, deflate' -H  'Accept: */*' -H  'User-Agent: python-requests/2.11.1' -H  'X-Auth-Token: ef38f59b20c84cbda03e19d4da1c56a9' 'http://127.0.0.1:9101/v1/executions/58d0dddb50484300b2a734ec/children/?depth=-1'
# -------- begin 140054699285712 response ----------
[{"status": "succeeded", "start_timestamp": "2017-03-21T08:01:32.009663Z", "log": [{"status": "requested", "timestamp": "2017-03-21T08:01:32.029000Z"}, {"status": "scheduled", "timestamp": "2017-03-21T08:01:32.102000Z"}, {"status": "running", "timestamp": "2017-03-21T08:01:32.166000Z"}, {"status": "succeeded", "timestamp": "2017-03-21T08:01:32.832000Z"}], "parameters": {"cmd": "st2 key delete a || true", "env": {"ST2_AUTH_URL": "http://127.0.0.1:9100", "ST2_BASE_URL": "http://127.0.0.1", "ST2_AUTH_TOKEN": "", "ST2_API_URL": "http://127.0.0.1:9101"}}, "runner": {"runner_module": "local_runner", "uid": "runner_type:local-shell-cmd", "enabled": true, "name": "local-shell-cmd", "runner_parameters": {"sudo": {"default": false, "type": "boolean", "description": "The command will be executed with sudo."}, "timeout": {"default": 60, "type": "integer", "description": "Action timeout in seconds. Action will get killed if it doesn't finish in timeout seconds."}, "cmd": {"type": "string", "description": "Arbitrary Linux command to be executed on the host."}, "kwarg_op": {"default": "--", "type": "string", "description": "Operator to use in front of keyword args i.e. \"--\" or \"-\"."}, "env": {"type": "object", "description": "Environment variables which will be available to the command(e.g. key1=val1,key2=val2)"}, "cwd": {"type": "string", "description": "Working directory where the command will be executed in"}}, "id": "58d0178550484300eb5228ba", "description": "A runner to execute local actions as a fixed user."}, "elapsed_seconds": 0.806438, "web_url": "https://7b0ec35f2fe8/#/history/58d0dddc504843005fcaa98c/general", "parent": "58d0dddb50484300b2a734ec", "result": {"failed": false, "stderr": "Traceback (most recent call last):\n  File \"/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2client/commands/resource.py\", line 162, in get_resource_by_pk\n    instance = self.manager.get_by_id(pk, **kwargs)\n  File \"/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2client/models/core.py\", line 39, in decorate\n    return func(*args, **kwargs)\n  File \"/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2client/models/core.py\", line 204, in get_by_id\n    self.handle_error(response)\n  File \"/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2client/models/core.py\", line 165, in handle_error\n    response.raise_for_status()\n  File \"/opt/stackstorm/st2/local/lib/python2.7/site-packages/requests/models.py\", line 862, in raise_for_status\n    raise HTTPError(http_error_msg, response=self)\nHTTPError: 401 Client Error: Unauthorized\nMESSAGE: Unauthorized - One of Token or API key required. for url: http://127.0.0.1:9101/keys/a?scope=system", "return_code": 0, "succeeded": true, "stdout": "ERROR: 401 Client Error: Unauthorized\nMESSAGE: Unauthorized - One of Token or API key required. for url: http://127.0.0.1:9101/keys/a?scope=system\n"}, "context": {"parent": {"user": "st2admin", "execution_id": "58d0dddb50484300b2a734ec"}, "chain": {"name": "cleanup_environment", "parameters": null, "publish": null, "params": {"cmd": "st2 key delete a || true", "env": {"ST2_AUTH_URL": "{{protocol}}://{{hostname}}:9100", "ST2_BASE_URL": "{{protocol}}://{{hostname}}", "ST2_AUTH_TOKEN": "{{token}}", "ST2_API_URL": "{{protocol}}://{{hostname}}:9101"}}, "notify": null, "on_failure": null, "ref": "core.local", "on_success": "check_if_key_create_trigger_is_registered"}, "user": "st2admin"}, "action": {"runner_type": "local-shell-cmd", "name": "local", "parameters": {"cmd": {"required": true, "type": "string", "description": "Arbitrary Linux command to be executed on the local host."}, "sudo": {"immutable": true}}, "tags": [], "enabled": true, "entry_point": "", "notify": {}, "uid": "action:core:local", "pack": "core", "ref": "core.local", "id": "58d0178550484300eb5228c1", "description": "Action that executes an arbitrary Linux command on the localhost."}, "liveaction": {"runner_info": {"hostname": "7b0ec35f2fe8", "pid": 89}, "parameters": {"cmd": "st2 key delete a || true", "env": {"ST2_AUTH_URL": "http://127.0.0.1:9100", "ST2_BASE_URL": "http://127.0.0.1", "ST2_AUTH_TOKEN": "", "ST2_API_URL": "http://127.0.0.1:9101"}}, "action_is_workflow": false, "callback": {}, "action": "core.local", "id": "58d0dddc504843005fcaa98b"}, "id": "58d0dddc504843005fcaa98c", "end_timestamp": "2017-03-21T08:01:32.816101Z"}, {"status": "failed", "start_timestamp": "2017-03-21T08:01:33.153541Z", "log": [{"status": "requested", "timestamp": "2017-03-21T08:01:33.173000Z"}, {"status": "scheduled", "timestamp": "2017-03-21T08:01:33.255000Z"}, {"status": "running", "timestamp": "2017-03-21T08:01:33.323000Z"}, {"status": "failed", "timestamp": "2017-03-21T08:01:34.008000Z"}], "parameters": {"cmd": "st2 trigger get core.st2.key_value_pair.create", "env": {"ST2_AUTH_URL": "http://127.0.0.1:9100", "ST2_BASE_URL": "http://127.0.0.1", "ST2_AUTH_TOKEN": "", "ST2_API_URL": "http://127.0.0.1:9101"}}, "runner": {"runner_module": "local_runner", "uid": "runner_type:local-shell-cmd", "enabled": true, "name": "local-shell-cmd", "runner_parameters": {"sudo": {"default": false, "type": "boolean", "description": "The command will be executed with sudo."}, "timeout": {"default": 60, "type": "integer", "description": "Action timeout in seconds. Action will get killed if it doesn't finish in timeout seconds."}, "cmd": {"type": "string", "description": "Arbitrary Linux command to be executed on the host."}, "kwarg_op": {"default": "--", "type": "string", "description": "Operator to use in front of keyword args i.e. \"--\" or \"-\"."}, "env": {"type": "object", "description": "Environment variables which will be available to the command(e.g. key1=val1,key2=val2)"}, "cwd": {"type": "string", "description": "Working directory where the command will be executed in"}}, "id": "58d0178550484300eb5228ba", "description": "A runner to execute local actions as a fixed user."}, "elapsed_seconds": 0.835787, "web_url": "https://7b0ec35f2fe8/#/history/58d0dddd504843005fcaa98e/general", "parent": "58d0dddb50484300b2a734ec", "result": {"failed": true, "stderr": "Traceback (most recent call last):\n  File \"/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2client/commands/resource.py\", line 162, in get_resource_by_pk\n    instance = self.manager.get_by_id(pk, **kwargs)\n  File \"/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2client/models/core.py\", line 39, in decorate\n    return func(*args, **kwargs)\n  File \"/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2client/models/core.py\", line 204, in get_by_id\n    self.handle_error(response)\n  File \"/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2client/models/core.py\", line 165, in handle_error\n    response.raise_for_status()\n  File \"/opt/stackstorm/st2/local/lib/python2.7/site-packages/requests/models.py\", line 862, in raise_for_status\n    raise HTTPError(http_error_msg, response=self)\nHTTPError: 401 Client Error: Unauthorized\nMESSAGE: Unauthorized - One of Token or API key required. for url: http://127.0.0.1:9101/triggertypes/core.st2.key_value_pair.create", "return_code": 1, "succeeded": false, "stdout": "ERROR: 401 Client Error: Unauthorized\nMESSAGE: Unauthorized - One of Token or API key required. for url: http://127.0.0.1:9101/triggertypes/core.st2.key_value_pair.create\n"}, "context": {"parent": {"user": "st2admin", "execution_id": "58d0dddb50484300b2a734ec"}, "chain": {"name": "check_if_key_create_trigger_is_registered", "parameters": null, "publish": null, "params": {"cmd": "st2 trigger get {{trigger_key_create}}", "env": {"ST2_AUTH_URL": "{{protocol}}://{{hostname}}:9100", "ST2_BASE_URL": "{{protocol}}://{{hostname}}", "ST2_AUTH_TOKEN": "{{token}}", "ST2_API_URL": "{{protocol}}://{{hostname}}:9101"}}, "notify": null, "on_failure": null, "ref": "core.local", "on_success": "check_if_key_update_trigger_is_registered"}, "user": "st2admin"}, "action": {"runner_type": "local-shell-cmd", "name": "local", "parameters": {"cmd": {"required": true, "type": "string", "description": "Arbitrary Linux command to be executed on the local host."}, "sudo": {"immutable": true}}, "tags": [], "enabled": true, "entry_point": "", "notify": {}, "uid": "action:core:local", "pack": "core", "ref": "core.local", "id": "58d0178550484300eb5228c1", "description": "Action that executes an arbitrary Linux command on the localhost."}, "liveaction": {"runner_info": {"hostname": "7b0ec35f2fe8", "pid": 85}, "parameters": {"cmd": "st2 trigger get core.st2.key_value_pair.create", "env": {"ST2_AUTH_URL": "http://127.0.0.1:9100", "ST2_BASE_URL": "http://127.0.0.1", "ST2_AUTH_TOKEN": "", "ST2_API_URL": "http://127.0.0.1:9101"}}, "action_is_workflow": false, "callback": {}, "action": "core.local", "id": "58d0dddd504843005fcaa98d"}, "id": "58d0dddd504843005fcaa98e", "end_timestamp": "2017-03-21T08:01:33.989328Z"}]
# -------- end 140054699285712 response ------------

id: 58d0dddb50484300b2a734ec
action.ref: tests.test_key_triggers
parameters: None
status: failed
error: Traceback (most recent call last):
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2client/commands/resource.py", line 162, in get_resource_by_pk
    instance = self.manager.get_by_id(pk, **kwargs)
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2client/models/core.py", line 39, in decorate
    return func(*args, **kwargs)
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2client/models/core.py", line 204, in get_by_id
    self.handle_error(response)
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2client/models/core.py", line 165, in handle_error
    response.raise_for_status()
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/requests/models.py", line 862, in raise_for_status
    raise HTTPError(http_error_msg, response=self)
HTTPError: 401 Client Error: Unauthorized
MESSAGE: Unauthorized - One of Token or API key required. for url: http://127.0.0.1:9101/triggertypes/core.st2.key_value_pair.create
traceback: None
failed_on: check_if_key_create_trigger_is_registered
start_timestamp: 2017-03-21T08:01:31.719729Z
end_timestamp: 2017-03-21T08:01:34.281867Z
+--------------------------+------------------------+----------------------------------------+------------+-------------------------------+
| id                       | status                 | task                                   | action     | start_timestamp               |
+--------------------------+------------------------+----------------------------------------+------------+-------------------------------+
| 58d0dddc504843005fcaa98c | succeeded (0s elapsed) | cleanup_environment                    | core.local | Tue, 21 Mar 2017 08:01:32 UTC |
| 58d0dddd504843005fcaa98e | failed (0s elapsed)    | check_if_key_create_trigger_is_registe | core.local | Tue, 21 Mar 2017 08:01:33 UTC |
|                          |                        | red                                    |            |                               |
+--------------------------+------------------------+----------------------------------------+------------+-------------------------------+

Note the last curl request contains an X-Auth-Token, but the response shows that ST2_AUTH_TOKEN is an empty string.

Reproduce the failure with st2-self-check or by running tests.test_key_triggers through the st2 CLI, using the shown cached-token configuration. Trace the action-chain child requests and compare them with the authenticated CLI requests; done means the child key and trigger checks receive authentication and the self-check completes successfully.