Triage: kpt live destroy core-landing-zone does not always remove project/folder.. resources even with a kubectl delete gcp --all - when iam resources are not ready - OAUTH SA failure is blocking the delete

Triage a way to managed a full lz delete and redeploy - with cycled project names (will need random ids in the future)
Issue here is a previously unmet iam role on the log sink is causing issues with deletion

https://console.cloud.google.com/kubernetes/object/logging.cnrm.cloud.google.com/logginglogsinks/northamerica-northeast1/krmapihost-kcc-oi3/logging/mgmt-project-cluster-disable-default-bucket?apiVersion=v1beta1&project=kcc-oi-629&orgonly=true&supportedpurview=project

see #446
```
status:
conditions:
- lastTransitionTime: "2023-10-21T21:13:45Z"
message: |
Delete call failed: error fetching live state: error reading underlying resource: summary: Error when reading or editing Project Logging Sink _Default: googleapi: Error 401: Request had invalid authentication credentials. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project.
Details:
[
{
"@type": "type.googleapis.com/google.rpc.ErrorInfo",
"domain": "googleapis.com",
"metadata": {
"email": "logging-sa@kcc-oi-629.iam.gserviceaccount.com",
"method": "google.logging.v2.ConfigServiceV2.GetSink",
"service": "logging.googleapis.com"
},
"reason": "ACCOUNT_STATE_INVALID"
}
]

More details:
Reason: authError, Message: Invalid Credentials
reason: DeleteFailed
status: "False"
type: Ready

```
```
michael@cloudshell:~/kcc-oi/github/pubsec-declarative-toolkit/solutions (kcc-oi-629)$ cd ../../../kpt
michael@cloudshell:~/kcc-oi/kpt (kcc-oi-629)$ kpt live destroy core-landing-zone
delete phase started
logginglogsink.logging.cnrm.cloud.google.com/mgmt-project-cluster-disable-default-bucket delete successful
delete phase finished
reconcile phase started
logginglogsink.logging.cnrm.cloud.google.com/mgmt-project-cluster-disable-default-bucket reconcile pending
^C
michael@cloudshell:~/kcc-oi/kpt (kcc-oi-629)$ kubectl get gcp
NAME AGE READY STATUS STATUS AGE
iamcustomrole.iam.cnrm.cloud.google.com/gke-firewall-admin 6h32m True UpToDate 6h32m
iamcustomrole.iam.cnrm.cloud.google.com/tier2-dnsrecord-admin 6h32m True UpToDate 6h32m
iamcustomrole.iam.cnrm.cloud.google.com/tier2-vpcpeering-admin 6h32m True UpToDate 6h32m
iamcustomrole.iam.cnrm.cloud.google.com/tier3-dnsrecord-admin 6h32m True UpToDate 6h32m
iamcustomrole.iam.cnrm.cloud.google.com/tier3-firewallrule-admin 6h32m True UpToDate 6h32m
iamcustomrole.iam.cnrm.cloud.google.com/tier3-subnetwork-admin 6h32m True UpToDate 6h32m
iamcustomrole.iam.cnrm.cloud.google.com/tier3-vpcsc-admin 6h32m True UpToDate 6h32m
iamcustomrole.iam.cnrm.cloud.google.com/tier4-secretmanager-admin 6h32m True UpToDate 6h32m

NAME AGE READY STATUS STATUS AGE
iampartialpolicy.iam.cnrm.cloud.google.com/gatekeeper-admin-sa-workload-identity-binding 6h32m True UpToDate 6h32m
iampartialpolicy.iam.cnrm.cloud.google.com/hierarchy-sa-workload-identity-binding 6h32m True UpToDate 6h32m
iampartialpolicy.iam.cnrm.cloud.google.com/logging-sa-workload-identity-binding 6h32m True UpToDate 6h32m
iampartialpolicy.iam.cnrm.cloud.google.com/networking-sa-workload-identity-binding 6h32m True UpToDate 6h32m
iampartialpolicy.iam.cnrm.cloud.google.com/policies-sa-workload-identity-binding 6h32m True UpToDate 6h31m
iampartialpolicy.iam.cnrm.cloud.google.com/projects-sa-workload-identity-binding 6h32m True UpToDate 6h32m

NAME AGE READY STATUS STATUS AGE
iampolicymember.iam.cnrm.cloud.google.com/config-control-sa-management-project-editor-permissions 6h32m True UpToDate 6h32m
iampolicymember.iam.cnrm.cloud.google.com/config-control-sa-management-project-serviceaccountadmin-permissions 6h32m True UpToDate 6h32m
iampolicymember.iam.cnrm.cloud.google.com/config-control-sa-orgroleadmin-permissions 6h32m True UpToDate 6h32m
iampolicymember.iam.cnrm.cloud.google.com/gatekeeper-admin-sa-metric-writer-permissions 6h32m True UpToDate 6h32m
iampolicymember.iam.cnrm.cloud.google.com/hierarchy-sa-folderadmin-permissions 6h32m True UpToDate 6h32m
iampolicymember.iam.cnrm.cloud.google.com/logging-sa-bigqueryadmin-permissions 6h32m True UpToDate 6h31m
iampolicymember.iam.cnrm.cloud.google.com/logging-sa-logadmin-permissions 6h32m True UpToDate 6h32m
iampolicymember.iam.cnrm.cloud.google.com/networking-sa-dns-permissions 6h32m True UpToDate 6h31m
iampolicymember.iam.cnrm.cloud.google.com/networking-sa-networkadmin-permissions 6h32m True UpToDate 6h31m
iampolicymember.iam.cnrm.cloud.google.com/networking-sa-security-permissions 6h32m True UpToDate 6h31m
iampolicymember.iam.cnrm.cloud.google.com/networking-sa-service-control-org-permissions 6h32m True UpToDate 6h32m
iampolicymember.iam.cnrm.cloud.google.com/networking-sa-servicedirectoryeditor-permissions 6h32m True UpToDate 6h31m
iampolicymember.iam.cnrm.cloud.google.com/networking-sa-xpnadmin-permissions 6h32m True UpToDate 6h32m
iampolicymember.iam.cnrm.cloud.google.com/policies-sa-orgpolicyadmin-permissions 6h32m True UpToDate 6h30m
iampolicymember.iam.cnrm.cloud.google.com/projects-sa-billinguser-permissions 6h32m True UpToDate 6h32m
iampolicymember.iam.cnrm.cloud.google.com/projects-sa-projectcreator-permissions 6h32m True UpToDate 6h31m
iampolicymember.iam.cnrm.cloud.google.com/projects-sa-projectdeleter-permissions 6h32m True UpToDate 6h31m
iampolicymember.iam.cnrm.cloud.google.com/projects-sa-projectiamadmin-permissions 6h32m True UpToDate 6h31m
iampolicymember.iam.cnrm.cloud.google.com/projects-sa-projectmover-permissions 6h32m True UpToDate 6h31m
iampolicymember.iam.cnrm.cloud.google.com/projects-sa-serviceusageadmin-permissions 6h32m True UpToDate 6h32m

NAME AGE READY STATUS STATUS AGE
iamserviceaccount.iam.cnrm.cloud.google.com/gatekeeper-admin-sa 6h32m True UpToDate 6h32m
iamserviceaccount.iam.cnrm.cloud.google.com/hierarchy-sa 6h32m True UpToDate 6h32m
iamserviceaccount.iam.cnrm.cloud.google.com/logging-sa 6h32m True UpToDate 6h32m
iamserviceaccount.iam.cnrm.cloud.google.com/networking-sa 6h32m True UpToDate 6h32m
iamserviceaccount.iam.cnrm.cloud.google.com/policies-sa 6h32m True UpToDate 6h31m
iamserviceaccount.iam.cnrm.cloud.google.com/projects-sa 6h32m True UpToDate 6h32m

NAME AGE READY STATUS STATUS AGE
service.serviceusage.cnrm.cloud.google.com/kcc-oi-629-accesscontextmanager 6h32m True UpToDate 6h32m
service.serviceusage.cnrm.cloud.google.com/kcc-oi-629-cloudbilling 6h32m True UpToDate 6h32m
service.serviceusage.cnrm.cloud.google.com/kcc-oi-629-cloudresourcemanager 6h32m True UpToDate 6h32m
service.serviceusage.cnrm.cloud.google.com/kcc-oi-629-serviceusage 6h32m True UpToDate 6h32m
michael@cloudshell:~/kcc-oi/kpt (kcc-oi-629)$ kubectl delete gcp --all
iamcustomrole.iam.cnrm.cloud.google.com "gke-firewall-admin" deleted
iamcustomrole.iam.cnrm.cloud.google.com "tier2-dnsrecord-admin" deleted
iamcustomrole.iam.cnrm.cloud.google.com "tier2-vpcpeering-admin" deleted
iamcustomrole.iam.cnrm.cloud.google.com "tier3-dnsrecord-admin" deleted
iamcustomrole.iam.cnrm.cloud.google.com "tier3-firewallrule-admin" deleted
iamcustomrole.iam.cnrm.cloud.google.com "tier3-subnetwork-admin" deleted
iamcustomrole.iam.cnrm.cloud.google.com "tier3-vpcsc-admin" deleted
iamcustomrole.iam.cnrm.cloud.google.com "tier4-secretmanager-admin" deleted
iampartialpolicy.iam.cnrm.cloud.google.com "gatekeeper-admin-sa-workload-identity-binding" deleted
iampartialpolicy.iam.cnrm.cloud.google.com "hierarchy-sa-workload-identity-binding" deleted
iampartialpolicy.iam.cnrm.cloud.google.com "logging-sa-workload-identity-binding" deleted
iampartialpolicy.iam.cnrm.cloud.google.com "networking-sa-workload-identity-binding" deleted
iampartialpolicy.iam.cnrm.cloud.google.com "policies-sa-workload-identity-binding" deleted
iampartialpolicy.iam.cnrm.cloud.google.com "projects-sa-workload-identity-binding" deleted
iampolicymember.iam.cnrm.cloud.google.com "config-control-sa-management-project-editor-permissions" deleted
iampolicymember.iam.cnrm.cloud.google.com "config-control-sa-management-project-serviceaccountadmin-permissions" deleted
iampolicymember.iam.cnrm.cloud.google.com "config-control-sa-orgroleadmin-permissions" deleted
iampolicymember.iam.cnrm.cloud.google.com "gatekeeper-admin-sa-metric-writer-permissions" deleted
iampolicymember.iam.cnrm.cloud.google.com "hierarchy-sa-folderadmin-permissions" deleted
iampolicymember.iam.cnrm.cloud.google.com "logging-sa-bigqueryadmin-permissions" deleted
iampolicymember.iam.cnrm.cloud.google.com "logging-sa-logadmin-permissions" deleted
iampolicymember.iam.cnrm.cloud.google.com "networking-sa-dns-permissions" deleted
iampolicymember.iam.cnrm.cloud.google.com "networking-sa-networkadmin-permissions" deleted
iampolicymember.iam.cnrm.cloud.google.com "networking-sa-security-permissions" deleted
iampolicymember.iam.cnrm.cloud.google.com "networking-sa-service-control-org-permissions" deleted
iampolicymember.iam.cnrm.cloud.google.com "networking-sa-servicedirectoryeditor-permissions" deleted
iampolicymember.iam.cnrm.cloud.google.com "networking-sa-xpnadmin-permissions" deleted
iampolicymember.iam.cnrm.cloud.google.com "policies-sa-orgpolicyadmin-permissions" deleted
iampolicymember.iam.cnrm.cloud.google.com "projects-sa-billinguser-permissions" deleted
iampolicymember.iam.cnrm.cloud.google.com "projects-sa-projectcreator-permissions" deleted
iampolicymember.iam.cnrm.cloud.google.com "projects-sa-projectdeleter-permissions" deleted
iampolicymember.iam.cnrm.cloud.google.com "projects-sa-projectiamadmin-permissions" deleted
iampolicymember.iam.cnrm.cloud.google.com "projects-sa-projectmover-permissions" deleted
iampolicymember.iam.cnrm.cloud.google.com "projects-sa-serviceusageadmin-permissions" deleted
iamserviceaccount.iam.cnrm.cloud.google.com "gatekeeper-admin-sa" deleted
iamserviceaccount.iam.cnrm.cloud.google.com "hierarchy-sa" deleted
iamserviceaccount.iam.cnrm.cloud.google.com "logging-sa" deleted
iamserviceaccount.iam.cnrm.cloud.google.com "networking-sa" deleted
iamserviceaccount.iam.cnrm.cloud.google.com "policies-sa" deleted
iamserviceaccount.iam.cnrm.cloud.google.com "projects-sa" deleted
service.serviceusage.cnrm.cloud.google.com "kcc-oi-629-accesscontextmanager" deleted
service.serviceusage.cnrm.cloud.google.com "kcc-oi-629-cloudbilling" deleted
service.serviceusage.cnrm.cloud.google.com "kcc-oi-629-cloudresourcemanager" deleted
service.serviceusage.cnrm.cloud.google.com "kcc-oi-629-serviceusage" deleted
michael@cloudshell:~/kcc-oi/kpt (kcc-oi-629)$ kpt live destroy core-landing-zone
delete phase started
logginglogsink.logging.cnrm.cloud.google.com/mgmt-project-cluster-disable-default-bucket delete successful
delete phase finished
reconcile phase started
logginglogsink.logging.cnrm.cloud.google.com/mgmt-project-cluster-disable-default-bucket reconcile pending
```
the reconcile usually hangs