+-------------------------------------------------------------------+
|                    VCF OPERATIONS (Mandatory)                      |
|         Fleet Management | Monitoring | Diagnostics                |
+-------------------------------------------------------------------+
|                     VCF AUTOMATION (Optional)                      |
|      Self-Service | Blueprints | Service Broker | Orchestrator     |
+-------------------------------------------------------------------+
|                       SDDC MANAGER                                 |
|       Lifecycle Management | Deployment | Orchestration            |
+-----------------+-----------------+-----------------+--------------+
|    vSphere      |      NSX        |     vSAN        |   vCenter    |
|   (Compute)     |  (Networking)   |   (Storage)     |   (Mgmt)     |
+-----------------+-----------------+-----------------+--------------+
|                     ESXi HYPERVISOR                                |
|                   Type 1 Bare-Metal                                |
+-------------------------------------------------------------------+

                   +-----------------------+
                   |   VCF Operations      |
                   |   192.168.1.77        |
                   +----------+------------+
                              |
                   +----------v------------+
                   |   Fleet Mgmt (Proxy)  |
                   |   192.168.1.78        |
                   +----------+------------+
                              |
                   +----------v------------+
                   |   SDDC Manager        |
                   |   192.168.1.241       |
                   +--+------+------+------+
                      |      |      |
           +----------+  +---+---+  +----------+
           |             |       |             |
    +------v------+ +----v----+ +------v------+
    | vCenter     | |  NSX    | | vSAN        |
    | .69         | |  .70/.71| | (4 hosts)   |
    +------+------+ +----+----+ +------+------+
           |              |            |
    +------v--------------v------------v------+
    |       ESXi Hosts (Transport Nodes)       |
    |  .74 (esxi01)  .75 (esxi02)             |
    |  .76 (esxi03)  .82 (esxi04)             |
    +-----------------------------------------+

VDS: vcenter-cl01-vds01
├── Port Group: vcenter-cl01-vds01-pg-vm-mgmt    (Management)
├── Port Group: vcenter-cl01-vds01-pg-vmotion     (vMotion)
└── Port Group: vcenter-cl01-vds01-pg-vsan         (vSAN)

# Forward Records (A)
192.168.1.69    vcenter.lab.local
192.168.1.70    nsx-vip.lab.local
192.168.1.71    nsx-node1.lab.local
192.168.1.74    esxi01.lab.local
192.168.1.75    esxi02.lab.local
192.168.1.76    esxi03.lab.local
192.168.1.82    esxi04.lab.local
192.168.1.77    vcf-ops.lab.local
192.168.1.78    fleet.lab.local
192.168.1.79    collector.lab.local
192.168.1.90    automation.lab.local
192.168.1.94    aria-lifecycle.lab.local
192.168.1.241   sddc-manager.lab.local

# SSH to NSX Manager as admin
set name-servers 192.168.1.230
set ntp-servers 192.168.1.230

# SSH to VCF Installer (192.168.1.240) as root
echo "vsan.esa.sddc.managed.disk.claim=true" >> /etc/vmware/vcf/domainmanager/application-prod.properties
systemctl restart domainmanager

sata0:0.virtualSSD = "1"
sata0:2.virtualSSD = "1"

# ===========================================
# NESTED VIRTUALIZATION SETTINGS
# ===========================================
vhv.enable = "TRUE"
vpmc.enable = "TRUE"
vvtd.enable = "TRUE"

# ===========================================
# PROMISCUOUS MODE FOR NESTED VM TRAFFIC
# ===========================================
ethernet0.noPromisc = "FALSE"
ethernet0.allowGuestConnectionControl = "TRUE"
ethernet1.noPromisc = "FALSE"
ethernet1.allowGuestConnectionControl = "TRUE"
ethernet2.noPromisc = "FALSE"
ethernet2.allowGuestConnectionControl = "TRUE"
ethernet3.noPromisc = "FALSE"
ethernet3.allowGuestConnectionControl = "TRUE"

# ===========================================
# MARK DISKS AS SSD FOR VSAN
# ===========================================
sata0:0.virtualSSD = "1"
sata0:2.virtualSSD = "1"

D:\VMs\esxi01.lab.local\esxi01.lab.local.vmx   (D: 2TB SSD)
E:\VMs\esxi02.lab.local\esxi02.lab.local.vmx   (E: 2TB SSD)
E:\VMs\esxi03.lab.local\esxi03.lab.local.vmx   (4TB HDD)
F:\VMs\esxi04.lab.local\esxi04.lab.local.vmx   (F: 4TB HDD)

# Run in PowerShell as Administrator
bcdedit /set hypervisorlaunchtype off
Disable-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V-All -NoRestart
Disable-WindowsOptionalFeature -Online -FeatureName VirtualMachinePlatform -NoRestart
Disable-WindowsOptionalFeature -Online -FeatureName Microsoft-Windows-Subsystem-Linux -NoRestart

bcdedit /enum | findstr hypervisor
# Should return nothing or "hypervisorlaunchtype Off"

Get-CimInstance -ClassName Win32_DeviceGuard -Namespace root\Microsoft\Windows\DeviceGuard
# VirtualizationBasedSecurityStatus should be 0

# ESXi hosts
192.168.1.74    esxi01.lab.local
192.168.1.75    esxi02.lab.local
192.168.1.76    esxi03.lab.local
192.168.1.82    esxi04.lab.local

# Core infrastructure
192.168.1.69    vcenter.lab.local
192.168.1.70    nsx-vip.lab.local
192.168.1.70    nsx-manager.lab.local
192.168.1.71    nsx-node1.lab.local
192.168.1.241   sddc-manager.lab.local

# VCF Operations ecosystem
192.168.1.77    vcf-ops.lab.local
192.168.1.78    fleet.lab.local
192.168.1.79    collector.lab.local
192.168.1.90    automation.lab.local
192.168.1.94    aria-lifecycle.lab.local

[ ] Physical host: Hyper-V disabled, Memory Integrity off, rebooted
[ ] VMware Workstation installed
[ ] 4 ESXi VMs created with correct specs (32 vCPU, 48GB RAM, 4x vmxnet3)
[ ] VMX files edited with nested virtualization + promiscuous mode + SSD marking
[ ] ESXi 9.0.1 installed on all 4 VMs from VMware ISO
[ ] DNS server running with all A and PTR records
[ ] NTP server accessible from all hosts
[ ] ESXi hosts have only vSwitch0 with vmk0 (clean state)
[ ] ESXi hosts not connected to any vCenter
[ ] SSH enabled on all ESXi hosts
[ ] Nested virtualization verified: cat /proc/cpuinfo | grep -E "vmx|svm"
[ ] SSD status verified: esxcli storage core device list | grep "Is SSD"
[ ] VCF Installer OVA downloaded from Broadcom Support Portal
[ ] Offline depot prepared (if not using online Broadcom depot)
[ ] Common password set on all ESXi hosts (used during VCF Installer wizard)

# ===========================================
# NESTED VIRTUALIZATION SETTINGS
# ===========================================

# Hardware virtualization passthrough
vhv.enable = "TRUE"

# Virtual Performance Counters
vpmc.enable = "TRUE"

# Virtual VT-d / IOMMU
vvtd.enable = "TRUE"

# ===========================================
# PROMISCUOUS MODE FOR NESTED VM TRAFFIC
# ===========================================

ethernet0.noPromisc = "FALSE"
ethernet0.allowGuestConnectionControl = "TRUE"
ethernet1.noPromisc = "FALSE"
ethernet1.allowGuestConnectionControl = "TRUE"
ethernet2.noPromisc = "FALSE"
ethernet2.allowGuestConnectionControl = "TRUE"
ethernet3.noPromisc = "FALSE"
ethernet3.allowGuestConnectionControl = "TRUE"

# ===========================================
# MARK DISKS AS SSD FOR VSAN
# ===========================================

sata0:0.virtualSSD = "1"
sata0:2.virtualSSD = "1"

# SSH to ESXi host
ssh root@192.168.1.74

# Verify nested virtualization is working
cat /proc/cpuinfo | grep -E "vmx|svm"
# Should output lines containing "vmx" or "svm"

# Verify disks detected as SSD
esxcli storage core device list | grep -E "Display Name|Is SSD"
# Each vSAN disk should show "Is SSD: true"

openssl req -x509 -newkey rsa:2048 `
  -keyout "C:\VCF-Depot\server.key" `
  -out "C:\VCF-Depot\server.crt" `
  -days 365 -nodes `
  -subj "/CN=192.168.1.52" `
  -addext "subjectAltName=IP:192.168.1.52"

#!/usr/bin/env python3
"""
HTTPS server for VCF Offline Depot
Serves files with TLS 1.2+ for SDDC Manager compatibility
"""

import http.server
import ssl
import os
import base64
import socketserver
from functools import partial

# Configuration
PORT = 8443
CERT_FILE = 'server.crt'
KEY_FILE = 'server.key'
USERNAME = 'admin'
PASSWORD = 'admin'


class AuthHandler(http.server.SimpleHTTPRequestHandler):
    protocol_version = "HTTP/1.1"

    def __init__(self, *args, directory=None, **kwargs):
        super().__init__(*args, directory=directory, **kwargs)

    def do_HEAD(self):
        if not self.authenticate():
            return
        super().do_HEAD()

    def do_GET(self):
        if not self.authenticate():
            return
        super().do_GET()

    def authenticate(self):
        auth_header = self.headers.get('Authorization')
        if auth_header is None:
            self.send_auth_request()
            return False

        try:
            auth_type, credentials = auth_header.split(' ', 1)
            if auth_type.lower() != 'basic':
                self.send_auth_request()
                return False

            decoded = base64.b64decode(credentials).decode('utf-8')
            username, password = decoded.split(':', 1)

            if username == USERNAME and password == PASSWORD:
                return True
        except Exception:
            pass

        self.send_auth_request()
        return False

    def send_auth_request(self):
        self.send_response(401)
        self.send_header('WWW-Authenticate', 'Basic realm="VCF Depot"')
        self.send_header('Content-type', 'text/html')
        self.send_header('Content-Length', '23')
        self.send_header('Connection', 'close')
        self.end_headers()
        self.wfile.write(b'Authentication required')

    def log_message(self, format, *args):
        print(f"{self.client_address[0]} - {format % args}")


class ThreadedHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
    daemon_threads = True


def run_server():
    os.chdir(os.path.dirname(os.path.abspath(__file__)))

    context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
    context.minimum_version = ssl.TLSVersion.TLSv1_2
    context.maximum_version = ssl.TLSVersion.TLSv1_3

    if hasattr(context, 'post_handshake_auth'):
        context.post_handshake_auth = False

    context.options |= ssl.OP_NO_TICKET
    context.options |= getattr(ssl, 'OP_NO_RENEGOTIATION', 0)
    context.load_cert_chain(CERT_FILE, KEY_FILE)

    try:
        context.set_ciphers('DEFAULT:!aNULL:!MD5:!DSS')
    except ssl.SSLError:
        pass

    handler = partial(AuthHandler, directory=os.getcwd())
    server = ThreadedHTTPServer(('0.0.0.0', PORT), handler)
    server.socket = context.wrap_socket(server.socket, server_side=True)

    print(f"VCF Offline Depot Server")
    print(f"========================")
    print(f"Serving: {os.getcwd()}")
    print(f"URL: https://192.168.1.52:{PORT}/")
    print(f"Credentials: {USERNAME} / {PASSWORD}")
    print(f"TLS: 1.2 - 1.3")
    print(f"Press Ctrl+C to stop")

    try:
        server.serve_forever()
    except KeyboardInterrupt:
        print("\nStopped.")
        server.shutdown()

if __name__ == '__main__':
    run_server()

# Extract metadata
Expand-Archive -Path "vcf-9.0.1.0-offline-depot-metadata.zip" -DestinationPath "C:\VCF-Depot\metadata-extract" -Force
Copy-Item "C:\VCF-Depot\metadata-extract\PROD\*" "C:\VCF-Depot\PROD\" -Recurse -Force

# Create component directories
New-Item -ItemType Directory -Path "C:\VCF-Depot\PROD\COMP\SDDC_MANAGER_VCF" -Force
New-Item -ItemType Directory -Path "C:\VCF-Depot\PROD\COMP\VCENTER" -Force
New-Item -ItemType Directory -Path "C:\VCF-Depot\PROD\COMP\NSX_T_MANAGER" -Force
New-Item -ItemType Directory -Path "C:\VCF-Depot\PROD\COMP\VRSLCM" -Force
New-Item -ItemType Directory -Path "C:\VCF-Depot\PROD\COMP\VROPS" -Force
New-Item -ItemType Directory -Path "C:\VCF-Depot\PROD\COMP\VCF_OPS_CLOUD_PROXY" -Force
New-Item -ItemType Directory -Path "C:\VCF-Depot\PROD\COMP\VRA" -Force
New-Item -ItemType Directory -Path "C:\VCF-Depot\PROD\COMP\VRO" -Force
New-Item -ItemType Directory -Path "C:\VCF-Depot\PROD\COMP\SDDC_MANAGER_VCF\lcm\productVersionCatalog" -Force

C:\VCF-Depot\
├── https_server.py
├── server.crt
├── server.key
└── PROD\
    ├── metadata\
    │   ├── manifest\v1\
    │   │   └── vcfManifest.json
    │   └── productVersionCatalog\v1\
    │       ├── productVersionCatalog.json
    │       └── productVersionCatalog.sig
    ├── vsan\hcl\
    │   ├── all.json
    │   └── lastupdatedtime.json
    └── COMP\
        ├── SDDC_MANAGER_VCF\
        │   ├── VCF-SDDC-Manager-Appliance-9.0.1.0.24962180.ova
        │   ├── Compatibility\
        │   │   └── VmwareCompatibilityData.json
        │   └── lcm\productVersionCatalog\
        │       └── productVersionCatalog.json
        ├── VCENTER\
        │   └── VMware-VCSA-all-9.0.1.0.24957454.iso
        ├── NSX_T_MANAGER\
        │   └── nsx-unified-appliance-9.0.1.0.24952114.ova
        ├── VRSLCM\
        │   └── VCF-OPS-Lifecycle-Manager-Appliance-9.0.1.0.24960371.ova
        ├── VROPS\
        │   └── Operations-Appliance-9.0.1.0.24960351.ova
        ├── VCF_OPS_CLOUD_PROXY\
        │   └── Operations-Cloud-Proxy-9.0.1.0.24960349.ova
        ├── VRA\
        │   └── vmsp-vcfa-combined-9.0.1.0.24965341.tar
        └── VRO\
            └── O11N_VA-9.0.1.0.24923009.ova

netsh advfirewall firewall add rule name="Allow 8443 Inbound" dir=in action=allow protocol=tcp localport=8443

cd C:\VCF-Depot
python https_server.py

curl -k -u admin:admin https://192.168.1.52:8443/PROD/metadata/productVersionCatalog/v1/productVersionCatalog.json

# Pull the depot server certificate
openssl s_client -connect 192.168.1.52:8443 </dev/null 2>/dev/null | openssl x509 > /tmp/depot.crt

# Find Java cacerts path
CACERTS=$(find /usr -name cacerts 2>/dev/null | head -1)
echo "Truststore: $CACERTS"

# Import certificate
keytool -import -trustcacerts -alias vcf-depot -file /tmp/depot.crt -keystore $CACERTS -storepass changeit -noprompt

# Restart services to pick up new certificate
systemctl restart commonsvcs domainmanager lcm operationsmanager

# SSH to VCF Installer as root
ssh root@192.168.1.240

# Add the vSAN ESA HCL bypass
echo "vsan.esa.sddc.managed.disk.claim=true" >> /etc/vmware/vcf/domainmanager/application-prod.properties

# Restart the domain manager service
systemctl restart domainmanager

# Verify the property was added
cat /etc/vmware/vcf/domainmanager/application-prod.properties | grep vsan

{
  "skipEsxThumbprintValidation": true,
  "managementPoolName": "mgmt-pool",
  "ceipEnabled": false,
  "fipsModeEnabled": true,
  "ntpServers": ["192.168.1.230"],
  "dnsSpec": {
    "nameserver": "192.168.1.230",
    "domain": "lab.local"
  },
  "sddcManagerSpec": {
    "hostname": "sddc-manager",
    "ipAddress": "192.168.1.241"
  },
  "networkSpecs": [
    { "networkType": "MANAGEMENT", "subnet": "192.168.1.0/24", "gateway": "192.168.1.1" },
    { "networkType": "VMOTION", "subnet": "192.168.11.0/24" },
    { "networkType": "VSAN", "subnet": "192.168.12.0/24" }
  ],
  "nsxtSpec": {
    "nsxtManagerSize": "small",
    "nsxtManagers": [
      { "hostname": "nsx-node1", "ip": "192.168.1.71" }
    ],
    "vip": "192.168.1.70",
    "vipFqdn": "nsx-vip.lab.local"
  },
  "vsanSpec": {
    "vsanName": "vcenter-cl01-ds-vsan01",
    "datastoreName": "vcenter-cl01-ds-vsan01",
    "esaEnabled": true
  },
  "hostSpecs": [
    { "hostname": "esxi01.lab.local", "ipAddress": "192.168.1.74" },
    { "hostname": "esxi02.lab.local", "ipAddress": "192.168.1.75" },
    { "hostname": "esxi03.lab.local", "ipAddress": "192.168.1.76" },
    { "hostname": "esxi04.lab.local", "ipAddress": "192.168.1.82" }
  ]
}

/usr/bin/ovftool --skipManifestCheck --powerOn --diskMode=thin --acceptAllEulas --allowExtraConfig --ipProtocol=IPv4 --ipAllocationPolicy=fixedPolicy --noSSLVerify --datastore=vcenter-cl01-ds-vsan01 --network=vcenter-cl01-vds01-pg-vm-mgmt --deploymentOption=xsmall --name=vcf-ops --prop:root_password='Success01!0909!!' --prop:ipv4_address.VMware_Aria_Operations=192.168.1.77 --prop:ipv4_type.VMware_Aria_Operations=Static --prop:domain.VMware_Aria_Operations=vcf-ops.lab.local --prop:ipv4_gateway.VMware_Aria_Operations=192.168.1.1 --prop:DNS.VMware_Aria_Operations=192.168.1.230 --prop:ipv4_netmask.VMware_Aria_Operations=255.255.255.0 --X:waitForIp --overwrite --X:logFile=/tmp/vcf-ops-manual.log --X:logLevel=verbose /nfs/vmware/vcf/nfs-mount/bundle/8a3336da-1b81-5144-b43e-d84eae7a8d8f/8a3336da-1b81-5144-b43e-d84eae7a8d8f/Operations-Appliance-9.0.2.0.25137838.ova "vi://administrator%40vsphere.local:Success01%210909%21%21@vcenter.lab.local/vcenter-dc01/host/vcenter-cl01"

/usr/bin/ovftool --skipManifestCheck --powerOn --diskMode=thin --acceptAllEulas --allowExtraConfig --ipProtocol=IPv4 --noSSLVerify --datastore=vcenter-cl01-ds-vsan01 --network=vcenter-cl01-vds01-pg-vm-mgmt --deploymentOption=small --name=nsx-manager --prop:nsx_role='NSX Manager' --prop:nsx_passwd_0='Success01!0909!!' --prop:nsx_cli_passwd_0='Success01!0909!!' --prop:nsx_cli_audit_passwd_0='Success01!0909!!' --prop:nsx_hostname=nsx-node1.lab.local --prop:nsx_ip_0=192.168.1.71 --prop:nsx_netmask_0=255.255.255.0 --prop:nsx_gateway_0=192.168.1.1 --prop:nsx_dns1_0=192.168.1.230 --prop:nsx_domain_0=lab.local --prop:nsx_ntp_0=192.168.1.230 --prop:nsx_isSSHEnabled=True --prop:nsx_allowSSHRootLogin=True --X:waitForIp --X:logFile=/tmp/nsx-manager.log --X:logLevel=verbose /nfs/vmware/vcf/nfs-mount/bundle/028849ee-d3e7-5748-9b90-47d503c6dd3e/028849ee-d3e7-5748-9b90-47d503c6dd3e/nsx-unified-appliance-9.0.1.0.24952114.ova "vi://administrator%40vsphere.local:Success01%210909%21%21@vcenter.lab.local/vcenter-dc01/host/vcenter-cl01"

/usr/bin/ovftool --skipManifestCheck --powerOn --diskMode=thin --acceptAllEulas --allowExtraConfig --ipProtocol=IPv4 --noSSLVerify --datastore=vcenter-cl01-ds-vsan01 --network=vcenter-cl01-vds01-pg-vm-mgmt --name=aria-lifecycle --prop:vami.hostname=automation.lab.local --prop:varoot-password='Success01!0909!!' --prop:admin-password='Success01!0909!!' --prop:va-ssh-enabled=True --prop:vami.ip0.VCF_OPS_Management_Appliance=192.168.1.90 --prop:vami.netmask0.VCF_OPS_Management_Appliance=255.255.255.0 --prop:vami.gateway.VCF_OPS_Management_Appliance=192.168.1.1 --prop:vami.DNS.VCF_OPS_Management_Appliance=192.168.1.230 --prop:vami.domain.VCF_OPS_Management_Appliance=lab.local --X:waitForIp --X:logFile=/tmp/aria-lifecycle.log --X:logLevel=verbose /nfs/vmware/vcf/nfs-mount/bundle/7301e3db-1ea7-5dd8-be67-c778becec936/7301e3db-1ea7-5dd8-be67-c778becec936/VCF-OPS-Lifecycle-Manager-Appliance-9.0.1.0.24960371.ova "vi://administrator%40vsphere.local:Success01%210909%21%21@vcenter.lab.local/vcenter-dc01/host/vcenter-cl01"

/usr/bin/ovftool /path/to/component.ova

# SSH to SDDC Manager as vcf, then su - to root
ssh vcf@192.168.1.241
su -

# Check all SDDC Manager services
systemctl status vcf-services

# Check individual critical services
systemctl status domainmanager
systemctl status lcm
systemctl status operationsmanager
systemctl status nginx
systemctl status postgresql

# Verify management domain via API
curl -k -X POST https://localhost/v1/tokens -H "Content-Type: application/json" -d '{"username":"admin@local","password":"Success01!0909!!"}'
# Use the returned accessToken for subsequent API calls

curl -k -X GET https://localhost/v1/domains -H "Authorization: Bearer <token>"
# Should show domain "mgmt" with status "ACTIVE"

curl -k -X GET https://localhost/v1/hosts -H "Authorization: Bearer <token>"
# Should show all 4 ESXi hosts with status "ACTIVE"

# On your workstation, download from Broadcom KB 344917
# File: vdt-2.2.7_02-05-2026.zip
# MD5: cc5780c93984fff13c91b8756d3b497d
# SHA256: 8801db4dfa3ed0ac19b8d33482d8dbff0634f0ac03f0d36926b438eab7cb43fc

# Upload to SDDC Manager (SCP works from external machine TO SDDC Manager)
scp vdt-2.2.7_02-05-2026.zip vcf@192.168.1.241:/home/vcf/

# SSH to SDDC Manager
ssh vcf@192.168.1.241

# Extract
unzip vdt-2.2.7_02-05-2026.zip

cd /home/vcf/vdt-2.2.7_02-05-2026
python vdt.py

/var/log/vmware/vcf/vdt/vdt-<timestamp>.txt
/var/log/vmware/vcf/vdt/vdt-<timestamp>.json

# Fix
chown root:vcf /nfs/vmware/vcf/nfs-mount/

# Verify
ls -la /nfs/vmware/vcf/
# Should show: drwxrwxr-x root vcf nfs-mount/

# Step 1: Create OpenSSL config on NSX Manager (SSH as root)
cat > /tmp/nsx-cert.conf << 'EOF'
[ req ]
default_bits = 2048
distinguished_name = req_distinguished_name
req_extensions = req_ext
x509_extensions = req_ext
prompt = no

[ req_distinguished_name ]
countryName = US
stateOrProvinceName = Lab
localityName = Lab
organizationName = lab.local
commonName = nsx-vip.lab.local

[ req_ext ]
basicConstraints = CA:FALSE
subjectAltName = @alt_names

[alt_names]
DNS.1 = nsx-vip.lab.local
DNS.2 = nsx-node1.lab.local
DNS.3 = nsx-manager.lab.local
IP.1 = 192.168.1.70
IP.2 = 192.168.1.71
EOF

# Step 2: Generate self-signed certificate
openssl req -x509 -nodes -days 825 -newkey rsa:2048 -keyout /tmp/nsx.key -out /tmp/nsx.crt -config /tmp/nsx-cert.conf -sha256

# Step 3: Create JSON payload (Python avoids PEM escaping issues)
python -c "
import json
cert = open('/tmp/nsx.crt').read()
key = open('/tmp/nsx.key').read()
print(json.dumps({'pem_encoded': cert, 'private_key': key}))
" > /tmp/nsx-import.json

# Step 4: Import certificate (single-line curl -- NSX shell has no backslash continuation)
curl -k -u admin:'Success01!0909!!' -X POST "https://192.168.1.71/api/v1/trust-management/certificates?action=import" -H "Content-Type: application/json" -d @/tmp/nsx-import.json
# Returns certificate ID, e.g.: 701d1416-5054-4038-8749-4ac495980ebd

# Step 5: Get node UUID
curl -k -u admin:'Success01!0909!!' https://192.168.1.71/api/v1/cluster
# Returns node UUID, e.g.: 95493642-ef4a-cb8e-ed7c-5bc20033f2c2

# Step 6: Apply to NSX Manager node
curl -k -u admin:'Success01!0909!!' -X POST "https://192.168.1.71/api/v1/trust-management/certificates/<cert-id>?action=apply_certificate&service_type=API&node_id=<node-uuid>"

# Step 7: Apply to cluster VIP
curl -k -u admin:'Success01!0909!!' -X POST "https://192.168.1.71/api/v1/trust-management/certificates/<cert-id>?action=apply_certificate&service_type=MGMT_CLUSTER"

# On SDDC Manager as root:

# Pull the active NSX certificate
openssl s_client -showcerts -connect 192.168.1.71:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM > /tmp/nsx-root.crt

# Import into VCF trust store
KEY=$(cat /etc/vmware/vcf/commonsvcs/trusted_certificates.key)
keytool -importcert -alias nsx-selfsigned -file /tmp/nsx-root.crt -keystore /etc/vmware/vcf/commonsvcs/trusted_certificates.store -storepass "$KEY" -noprompt

# Import into Java cacerts
keytool -importcert -alias nsx-selfsigned -file /tmp/nsx-root.crt -keystore /etc/alternatives/jre/lib/security/cacerts -storepass changeit -noprompt

# Restart SDDC Manager services
/opt/vmware/vcf/operationsmanager/scripts/cli/sddcmanager_restart_services.sh

3.1 VCF Operations First Login & Setup

VCF Operations (formerly VMware Aria Operations) is the mandatory central management console for the entire VCF 9.0 platform. The SDDC Manager UI is deprecated and will be removed in a future release. VCF Operations is now the primary interface for fleet management, lifecycle management, licensing, monitoring, certificate management, password management, and all Day 2 operations.

3.1.1 Environment Reference

Component	Address
VCF Operations	192.168.1.77 (vcf-ops.lab.local)
SDDC Manager	192.168.1.241 (sddc-manager.lab.local)
vCenter Server	192.168.1.69 (vcenter.lab.local)
Offline Depot Server	192.168.1.52:8443
ESXi Hosts	esxi01 (.74), esxi02 (.75), esxi03 (.76), esxi04 (.82)
NSX Manager	192.168.1.71 (nsx-node1.lab.local)
NSX VIP	192.168.1.70 (nsx-vip.lab.local)
Fleet Management (Cloud Proxy)	192.168.1.78 (fleet.lab.local)
DNS Server	192.168.1.230 (Windows AD DC for lab.local)
Mode	Air-gapped / Disconnected

3.1.2 Initial Access

1. Open a browser and navigate to https://192.168.1.77
2. Log in with the credentials configured during bringup:
   • Username: admin
   • Password: The password set during VCF Installer deployment
3. Upon first login, you land on the Fleet Management dashboard

3.1.3 Navigation Overview

The left navigation pane displays the main sections:

Section	Purpose
Fleet Management	Lifecycle management, depot configuration, component health
Infrastructure Operations	Monitoring, dashboards, alerts, diagnostics
Security & Compliance	Compliance benchmarks, drift detection
License Management	Registration and license file management
Administration	Integrations, accounts, access control, system settings

Note: If licensing has not been completed, some menu items may be grayed out. VCF Operations runs in evaluation mode for up to 90 days after deployment.

3.1.4 Initial Setup Wizard (Manual OVA Deployment Only)

If VCF Operations was deployed manually via OVA rather than through the VCF Installer, the initial setup wizard appears automatically on first access:

1. Click NEXT on the welcome page
2. Set Admin Password: Enter a new password for the admin user (minimum 8 characters, upper, lower, number, special character)
3. Select EXPRESS INSTALLATION to deploy a single-node configuration
4. Accept the EULA/license agreement
5. The wizard completes and brings you to the main VCF Operations interface

3.1.5 CEIP Opt-Out

VCF Operations ships with the Customer Experience Improvement Program (CEIP) enabled by default. For air-gapped labs this should be disabled:

1. Navigate to Administration > Management
2. Locate the CEIP or Customer Experience setting
3. Toggle to Disabled
4. Click Save

Tip: In a disconnected environment, CEIP data cannot be sent anyway, but disabling prevents unnecessary connection attempts that clutter logs.

---

3.2 License Registration (Air-Gapped)

VCF 9.0 uses a unified subscription-based license file model. The old 25-character license keys are replaced by license files. There are only two license types: VMware Cloud Foundation (cores) and VMware vSAN (TiBs). All other components (NSX, vCenter, VCF Automation, etc.) are automatically licensed when a primary license is assigned.

3.2.1 Download the Registration File

Navigation: VCF Operations > License Management > Registration

1. In the left navigation, click License Management
2. Click Registration
3. In the Download Registration File card, click Download
4. Save the .jws (JSON Web Signed) file to a local machine or USB drive

3.2.2 Upload Registration to VCF Business Services Console

This step is performed on a machine with internet access:

1. Transfer the .jws file to a computer with internet access via USB drive or secure transfer
2. Open a browser and navigate to https://vcf.broadcom.com
3. Log in with your Broadcom Support Portal credentials
4. Select the Site ID you want to register this VCF Operations instance against
5. Upload the registration file when prompted
6. Add licenses to your license server -- you must add licenses to each license server to complete registration
7. The Business Services Console generates a license file in exchange
8. Click Download to save the license file
9. Click Finish

3.2.3 Import the License File into VCF Operations

Navigation: VCF Operations > License Management > Registration

1. Return to VCF Operations at https://192.168.1.77
2. Navigate to License Management > Registration
3. Click Import License File
4. Click Browse and select the downloaded license file
5. Click Import
6. Upon completion, click Complete

3.2.4 Verification

• The license status should change from "Evaluation" to showing your valid license
• All previously grayed-out menu items become fully active
• Navigate to License Management and confirm license details show correct core counts and expiration

3.2.5 Ongoing License Usage Reporting (Every 180 Days)

Since the environment is air-gapped, you must manually report usage at least every 180 days:

1. Navigate to License Management > Registration
2. Click Generate Usage File and save it
3. Transfer the usage file to an internet-connected machine
4. Log in to https://vcf.broadcom.com
5. Navigate to License Management > VCF Operations Registrations
6. Find your VCF Operations instance, click the vertical ellipsis menu, select Upload Usage File
7. Upload the usage file, click Save and Next
8. The system generates an updated license file -- click Download
9. Click Finish
10. Transfer the new license file back and import it via License Management > Registration > Import License File

WARNING: If license usage data is not submitted within 180 days, licenses are treated as expired. Hosts are disconnected from vCenter and workload operations are blocked. In a lab environment, set a calendar reminder.

---

3.3 Fleet Management & Depot Configuration

3.3.1 Fleet Management Appliance Registration

The Fleet Management appliance handles lifecycle management functions formerly in SDDC Manager. If deployed via the VCF Installer, this may already be connected. If not:

Navigation: https://192.168.1.77/admin/ (the Admin UI, not the main UI)

1. Open a browser and navigate to https://192.168.1.77/admin/
2. Log in as admin with your VCF Operations admin password
3. Navigate to System Status > Fleet Management section
4. Click the Connect button
5. Node Address: Enter the FQDN of the VCF Operations Fleet Management appliance
6. Admin Password: Enter the Fleet Management appliance admin password
7. Click Test Connection to verify connectivity
8. Review the security certificate presented by the appliance
9. Accept the certificate and click Next
10. Enter the VCF Operations admin password when prompted
11. Click Finish
12. The Fleet Management status should show as Connected in the Admin UI

Lab Context: In the lab, Fleet Management was deployed at 192.168.1.78 via the VCF Operations Lifecycle import (not during bringup, which failed). The Cloud Proxy was deployed automatically during this process.

3.3.2 Configure the Offline Depot for VCF Management Components

In VCF 9.0, depot functionality has moved from SDDC Manager to VCF Operations. You must configure the depot before you can download binaries for additional components. Only one depot connection (online OR offline) can be ACTIVE at a time.

Navigation: VCF Operations > Fleet Management > Lifecycle > VCF Management > Depot Configuration

1. Navigate to Fleet Management > Lifecycle > VCF Management > Depot Configuration
2. Click Configure under the Offline Depot widget
3. Offline Depot Type: Keep as "Webserver"
4. Repository URL: Enter https://192.168.1.52:8443
5. Username: admin
6. Password: admin
7. Check "I accept the imported certificate" after reviewing the certificate details
8. Click OK

3.3.3 Verify Depot Connection

1. Navigate to Binary Management > Install Binaries tab
2. You should see available binaries listed for download (Operations for Logs, Operations for Networks, etc.)
3. Download status should show the binaries available for installation

3.3.4 Configure the Offline Depot for VCF Instance (SDDC Manager)

Navigation: VCF Operations > Fleet Management > Lifecycle > VCF Instances > (select your instance) > Depot Settings

1. Navigate to Fleet Management > Lifecycle > VCF Instances
2. Select your VCF Instance from the list
3. Click Depot Settings
4. Under Offline Depot, select Set Up
5. Enter the hostname of your depot server: 192.168.1.52:8443
6. Click Save

Note: Before configuring the SDDC Manager depot, you may need to trust the SSL certificate of your offline depot server. This was already done during the initial bringup (certificate imported into SDDC Manager's Java trust store).

3.3.5 Bundle Management & Update Scheduling

After depot configuration, binaries become available for download and deployment:

1. Navigate to Fleet Management > Lifecycle > VCF Management > Components
2. View available components and their current/available versions
3. Click Add next to a component to deploy it (e.g., operations-logs, operations-networks)
4. Updates to existing components are surfaced in the Updates Available section
5. Schedule updates during maintenance windows by selecting the component and clicking Schedule Update

Tip: Binary downloads from depot may intermittently fail. If a download disappears, retry it.

---

3.4 Data Source Connections (VCF Cloud Account)

This is the critical step that connects VCF Operations to your SDDC Manager, enabling automatic monitoring of all VCF domains including vCenter, NSX, and vSAN.

3.4.1 Add the VMware Cloud Foundation Account

Navigation: VCF Operations > Administration > Integrations > Accounts tab > Add

1. In the left navigation, click Administration
2. Click Integrations
3. Click the Accounts tab
4. Click Add
5. On the Account Types page, select VMware Cloud Foundation
6. Fill in the following fields:
   • Name: Lab VCF Instance (or any descriptive name)
   • Description: Management Domain - Lab Environment
   • Physical Data Center: Select existing or create new
7. Connection Details:
   • SDDC Manager FQDN: sddc-manager.lab.local (use FQDN rather than IP for VCF SSO to work properly)
8. Credentials:
   • Click the Add (+) icon to create new credentials
   • Credential Name: SDDC Manager Admin
   • Username: administrator@vsphere.local
   • Password: Enter the corresponding password
   • Click OK to save the credential
9. Collector:
   • Select which VCF Operations collector or collector group manages this account
   • Ensure the SDDC Manager FQDN is reachable from the selected collector
10. Click Validate Connection
11. A certificate dialog appears -- review the certificate and click OK to accept
12. Advanced Settings:
    • Enable Domain Monitoring on Creation: Toggle to True for automatic data collection on newly discovered domains
    • Configuration Limits: Optionally enter the name of a file containing VCF configuration max soft and hard limits
13. Management Options:
    • Select the option for monitoring plus license/plugin management
14. Click Add to create the account

3.4.2 Start Data Collection

1. On the Accounts tab, locate your new VMware Cloud Foundation account
2. Click the vertical ellipsis (three dots) menu next to the account
3. Select Start Collecting All

3.4.3 What Happens Automatically

After configuration, VCF Operations automatically:

• Discovers all VCF domains
• Creates vCenter adapter instances for each domain's vCenter
• Creates NSX adapter instances for each domain's NSX Manager
• Creates vSAN adapter instances for vSAN-enabled clusters
• Configures system-managed credentials for NSX 9.0 endpoints
• Begins collecting metrics, properties, and alerts from all discovered components

Note: Initial collection takes multiple cycles (standard cycle = 5 minutes). Allow 15-30 minutes for full data population.

3.4.4 Add Individual vCenter Account (If Not Auto-Discovered)

When you add a VCF account, vCenter accounts are normally auto-discovered. If you need to add one manually:

Navigation: VCF Operations > Administration > Integrations > Accounts tab > Add

1. Click Add on the Accounts tab
2. Select vCenter from the Account Types page
3. Display Name: vcenter.lab.local - 192.168.1.69
4. Description: Management Domain vCenter
5. vCenter Field: vcenter.lab.local or 192.168.1.69
6. Credentials: Click Add (+) -- enter administrator@vsphere.local and password
7. Collector: Select the VCF Operations collector
8. Click Validate Connection and accept the certificate
9. Optional Features:
   • Activate for Operational Actions: Check to enable remediation actions
   • Activate Log Collection: Check to enable log forwarding (requires VCF Operations for Logs)
   • Activate Network and Flow: Check to enable network monitoring
10. Click Add
11. On the Accounts tab, click the vertical ellipsis menu > Start Collecting

Important: vCenter accounts do NOT start monitoring automatically. You must manually initiate data collection.

3.4.5 Verify Data Collection

Navigation: VCF Operations > Administration > Integrations > Accounts

1. For each configured account (VCF, vCenter, NSX, vSAN), verify:
   • Collection Status: Green "Collecting" (not "Stopped" or "No data receiving")
   • Collection State: "Collecting Data"
2. Navigate to Infrastructure Operations > Inventory and verify:
   • vCenter instances (vcenter.lab.local)
   • ESXi hosts (esxi01, esxi02, esxi03, esxi04)
   • Clusters (management cluster)
   • Datastores (vSAN datastore)
   • Virtual Machines (all management VMs)
   • NSX objects (NSX Manager, transport nodes)
3. Navigate to Infrastructure Operations > VCF Health and verify all components show healthy status

Key Timing Notes:

Metric	Interval
Standard collection cycle	Every 5 minutes
Initial collection (full population)	15-30 minutes
Property-based diagnostic scans	Every 4 hours
Telegraf agent data collection	Every 4 minutes
Cloud proxy registration (first boot)	Up to 20 minutes

---

3.5 SSO / Identity & Access Management

VCF 9.0 introduces the VCF Identity Broker (VIDB), which provides federated SSO across all VCF components.

3.5.1 Configure VCF Single Sign-On for VCF Operations

Navigation: VCF Operations > Fleet Management > Identity & Access > VCF Management > Operations Appliance

1. Navigate to Fleet Management > Identity & Access > VCF Management
2. Select Operations Appliance
3. Click Configure
4. Select the Identity Broker instance from the dropdown
5. Accept the role assignment requirements
6. The system validates and displays the Identity Broker on the configuration list after processing

3.5.2 Verify Authentication Source

Navigation: VCF Operations > Administration > Control Panel > Authentication Sources

1. Navigate to Administration > Control Panel > Authentication Sources
2. Confirm that "VCF SSO" now appears in the list of available authentication sources

3.5.3 Import Directory Users and Groups

Navigation: VCF Operations > Administration > Control Panel > Access Control

1. Navigate to Administration > Control Panel > Access Control
2. Click the three-dot menu and select Import from Source (do NOT use the standard "Add" button -- that creates local groups only)
3. Select VCF SSO as the source
4. Search for your Active Directory groups (e.g., vcf-admins, vcf-readonly, Domain Admins)
5. Select the groups to import

3.5.4 Assign Permissions

1. Select the imported groups
2. Click the menu and choose Edit
3. Assign:
   • Role: The actions users can perform (e.g., Administrator, ReadOnly, ContentAdmin)
   • Scope: The objects those actions apply to (e.g., all objects, specific data centers)
4. Click Save
5. Test by logging out and logging back in using VCF SSO authentication

3.5.5 Add Active Directory Identity Source in vCenter

To add AD authentication to vCenter separately:

1. Log in to vCenter at https://192.168.1.69
2. Navigate to Administration > Single Sign-On > Configuration
3. Click Identity Sources > Add
4. Select Active Directory over LDAP (IWA is removed in vCenter 9.0)
5. Enter your AD domain details:
   • Domain name: lab.local
   • Base distinguished name for users: DC=lab,DC=local
   • Base distinguished name for groups: DC=lab,DC=local
   • Primary server URL: ldap://192.168.1.230:389
   • Bind user distinguished name: (your bind user DN)
   • Bind password: (bind user password)
6. Click Test Connection to verify
7. Click Add to save

Lab Context: The lab has AD/LDAP configured via the embedded identity broker with lab.local domain at 192.168.1.230. Attribute mappings: userName=sAMAccountName, firstName=givenName, lastName=sn, email=mail. Domain Admins group synced with nested groups enabled.

---

3.6 Certificate Management

VCF 9.0 introduces unified, non-disruptive TLS certificate management across all VCF components.

3.6.1 View All Certificates

Navigation: VCF Operations > Fleet Management > Certificates

1. Navigate to Fleet Management > Certificates
2. Select either VCF Management or VCF Instances tab
3. View the certificate inventory showing all TLS certificates across your environment
4. Certificates are displayed for: vCenter, ESX hosts, VCF Operations, VCF Automation, Fleet Management, SDDC Manager, NSX local manager
5. Review certificate expiration dates and status alerts

3.6.2 Configure a Certificate Authority -- Microsoft CA

Navigation: VCF Operations > Fleet Management > Certificates > Configure CA

1. Navigate to Fleet Management > Certificates
2. Select VCF Management or VCF Instances (and choose a specific instance)
3. Click Configure CA
4. Select Microsoft Certificate Authority
5. Fill in:
   • CA Server URL: Must begin with https:// and end with certsrv (e.g., https://ca.lab.local/certsrv)
   • User Name: Least-privileged service account (e.g., svc-vcf-ca)
   • Password: Service account password
   • Template Name: The issuing certificate template created in Microsoft CA
6. Click Save

Important: VCF management components only support Microsoft CA. VCF Instance components support both Microsoft CA and OpenSSL. You configure the CA separately for management components and instance components.

3.6.3 Configure a Certificate Authority -- OpenSSL

1. Click Configure CA
2. Select OpenSSL
3. Fill in:
   • Common Name: FQDN of SDDC Manager appliance
   • Country: Country of registration
   • Locality Name: City
   • Organization Name: Legal company name
   • Organization Unit Name: Department
   • State: Full state/province name (unabbreviated)
4. Click Save

3.6.4 Replace Default Certificates

After configuring a CA, replace default self-signed certificates with enterprise CA-signed certificates. Certificates eligible for non-disruptive auto-renewal include: ESX SSL, vCenter machine SSL, NSX LM/VIP, SDDC Manager SSL, and VCF Operations certificates.

3.6.5 Enable Automatic Renewal

On the Certificates page, enable auto-renewal for supported certificates. This prevents unexpected certificate expiration.

Lab Note: In a lab with no Microsoft CA, you can continue using self-signed certificates. The certificate management UI will show certificate expiration warnings, which is normal.

---

3.7 Password Management & Rotation

VCF 9.0 provides unified password management centralized in VCF Operations, replacing the password management previously found in SDDC Manager.

3.7.1 View Password Status

Navigation: VCF Operations > Fleet Management > Passwords

1. Navigate to Fleet Management > Passwords
2. Select either VCF Management or VCF Instances tab
3. Select your domain to view all managed account passwords
4. The dashboard shows:
   • Account names and types (root, admin, backup, consoleuser, support, admin@local, vmware-system-user)
   • Password status (valid, expiring soon, expired)
   • Last modified dates
   • Expiration dates

3.7.2 Managed Components and Accounts

VCF Management Components:

Component
Fleet Management
VCF Automation
VCF Identity Broker
VCF Operations
VCF Operations for Logs
VCF Operations for Networks

VCF Instance/Domain Components:

Component
ESX hosts (esxi01-04)
NSX Manager
vCenter Server
SDDC Manager

3.7.3 Password Functions Reference

Function	When to Use	What It Does
Update	You changed a password outside VCF	Updates VCF database to match the new password on the component
Rotate	Scheduled password change	Changes password on BOTH the component AND the VCF database
Remediate	A rotation failed mid-way	Re-syncs by accepting the current password on the component

3.7.4 Update a Password

1. Navigate to Fleet Management > Passwords
2. Select the component and account you want to update
3. Click Update Password
4. Enter the new desired password (this lets you specify the exact password, unlike rotation)
5. Confirm the new password
6. Click Update

3.7.5 Rotate Passwords

Password rotation generates a randomized password:

1. Navigate to Fleet Management > Passwords
2. Select accounts to rotate
3. Click Rotate
4. The system generates random passwords meeting complexity requirements
5. Set the rotation interval: 30 days, 60 days, or 90 days
6. You can also deactivate the schedule
7. Only a user with the ADMIN role can perform this task

Note: Auto-rotate is automatically enabled for vCenter Server. It may take up to 24 hours to configure the auto-rotate policy for a newly deployed vCenter.

3.7.6 Remediate Passwords

If a password gets out of sync between SDDC Manager and the actual component:

Prerequisites:

• No workflows running or scheduled
• Required permissions: Fleet Management > Passwords > Manage and Fleet Management > Passwords > View
• One account remediated at a time

Steps:

1. Navigate to Fleet Management > Passwords
2. Select either VCF Management or VCF Instances and choose your domain
3. Select the component showing a password issue
4. Click Remediate Password
5. Enter and confirm the manually-set password (the password currently on the component)
6. Click Remediate Password to complete

Tip: Password rotation options from VCF 5.x are not fully available in VCF Operations yet. Use the SDDC Manager API as a workaround for some rotation tasks if needed.

WARNING — Credential Rotation Cascade Failure: If a credential update or rotation fails mid-operation (commonly because NSX was temporarily unreachable during a boot storm or maintenance), the component resource can get stuck in ACTIVATING state with stale exclusive locks blocking all future password operations. Error messages: "Resources [host] are not available/ready" or "Unable to acquire resource level lock(s)". This requires a database-level fix on SDDC Manager — see Section 7.2.6 for the complete repair procedure.

---

3.8 Compliance Monitoring

3.8.1 Access Compliance

Navigation: VCF Operations > Security & Compliance > Compliance

1. Navigate to Security & Compliance > Compliance
2. Ensure your data sources (vCenter, VCF account) are configured and collecting before proceeding

3.8.2 Activate VMware SDDC Benchmarks

1. On the Compliance page, locate the VMware SDDC Benchmarks section
2. Click Activate for the benchmark you want to enable
3. Available score cards:
   • vSphere Security Configuration Guide
   • vSAN Security Configuration Guide
   • NSX Security Configuration Guide
4. Select an applicable policy when prompted
5. The system activates relevant alert definitions automatically

3.8.3 Activate Regulatory Compliance Benchmarks

Built-in standards (no additional download):

Standard	Notes
DISA Security Standards	Defense Information Systems Agency STIGs
FISMA Security Standards	Federal Information Security Management Act
HIPAA	Health Insurance Portability and Accountability Act

Standards requiring marketplace download (.PAK file):

Standard	Notes
PCI DSS Compliance Standards	Payment Card Industry Data Security Standard
CIS Security Standards	Center for Internet Security Benchmarks
NIST SP 800-171	Controlled Unclassified Information
NIST SP 800-53 R5	Security and Privacy Controls

For air-gapped environments, install marketplace packs manually:

Navigation: VCF Operations > Administration > Repository

1. Navigate to Administration > Repository
2. The Add Solution wizard opens
3. Page 1: Locate and upload the .PAK file
4. Page 2: Accept the EULA and install
5. Page 3: Review the installation
6. Click Add Account to configure the newly installed integration

3.8.4 Configure Drift Detection

Navigation: VCF Operations > Fleet Management > Configuration Drifts > Schedule Drift Detection

1. Navigate to Fleet Management > Configuration Drifts
2. Click Schedule Drift Detection
3. Step 1 - Configuration Details: Enter a name and description for the drift check (you can schedule drifts only for vCenter object types)
4. Step 2 - Define Scope: Select vCenter instances from the right panel and move them to the left Scope window (you can select a VCF folder as scope to automatically include all VCF instances in that folder)
5. Step 3 - Preview Scope: Click Preview Scope to validate which vCenter instances will be included
6. Step 4 - Filtering Criteria: Apply filters and add criteria specific to the vCenter object type
7. Step 5 - Schedule: Set the desired schedule interval and click Create
8. The system creates a new job visible in the automation central page

---

3.9 Alerts, Notifications & Dashboards

3.9.1 Configure Outbound Notification Plug-Ins

Navigation: VCF Operations > Infrastructure Operations > Configurations > Outbound Settings

1. Navigate to Infrastructure Operations > Configurations
2. Click the Outbound Settings tile
3. Click Add

3.9.2 Standard Email Plug-In

1. Select Standard Email Plugin from the Plug-In Type dropdown
2. Instance Name: Lab Email Notifications
3. Configure SMTP settings:
   • Use Secure Connection: Enable for SSL/TLS
   • Secure Connection Type: SSL or TLS
   • Requires Authentication: Check if your SMTP requires auth
   • SMTP Host: URL or IP of email server
   • SMTP Port: 25, 465, or 587
   • Sender Email Address: vcf-ops@lab.local
   • Sender Name: VCF Operations
   • Receiver Email Address: Default recipient
4. Click Save
5. Select the instance and click Activate

3.9.3 Other Available Plug-Ins

Plug-In	Use Case
Standard Email Plugin	SMTP email notifications
SNMP Trap Plugin	SNMP v1/v2c/v3 traps to network management systems
Webhook Notification Plugin	REST webhooks (supports Basic Auth, Bearer Token, OAuth, X.509, API Key)
Log File	Write alerts to log files
ServiceNow	ITSM integration
Slack	Chat-based alerting
Network Share	Write to network file shares

3.9.4 Create Notification Rules

Navigation: VCF Operations > Infrastructure Operations > Configurations > Notifications

1. Navigate to Infrastructure Operations > Configurations
2. Click the Notifications tile
3. Click Add on the toolbar

Step 1 - Basic Details:

• Name: Rule identifier (e.g., Critical Host Alerts)
• Notification Status: Toggle active/inactive
• Notification Type: Select "Alert"

Step 2 - Define Filtering Criteria:

• Object Scope: Object Type, specific Objects, Tags, Applications, Tiers (option to include child objects)
• Alert Scope: Alert Types/Subtypes, Alert Impact, or Alert Definition
• Criticality: Filter by severity (Critical, Immediate, Warning, Information)
• Notify On: Trigger conditions (new, updated, canceled)

Step 3 - Select Outbound Method:

• Choose from configured plug-in instances

Step 4 - Payload Template:

• Select default or custom template
• For Email: configure Recipients, Cc/Bcc, Notify Again interval, Max Notifications, Delay to Notify

Step 5 - Test:

• Click Initiate Process to test, select an alert definition and object, click Validate Configuration

Step 6 - Create:

• Click Create to finalize

3.9.5 Key Predefined Dashboards

Navigation: VCF Operations > Infrastructure Operations > Dashboards & Reports

Dashboard Category	What It Shows
Overview	Geo-map view of VCF instances, inventory sections, diagnostic findings, security risk highlights
Cluster Configuration	vSphere cluster configuration requiring attention
ESXi Configuration	ESXi host configurations needing review
Network Configuration	vSphere distributed switch configurations
VM Configuration	Virtual machine configurations
vSAN Configuration	vSAN configuration details
vSAN OSA Performance	Read/write latency, contention, utilization
vSAN ESA Performance	ESA-specific metrics
Security Operations	User auth, encryption status, CVE advisories, certificate health
Skyline Operational	Proactive monitoring and recommendation dashboard
Energy Efficiency	Virtualization efficiency, idle VM impact

3.9.6 Create a Custom Dashboard

1. From the left menu, click Dashboards & Reports
2. Click New Dashboard
3. Dashboard Name: Enter a name (using / in the name creates folder hierarchy, e.g., Lab/Overview)
4. The dashboard canvas opens for widget placement
5. Available widget types: Metric Chart, View, Health Chart, Sparkline, Mashup Chart, Rolling View
6. For each widget, click the pencil icon to configure data source, metrics, time range, and visual options
7. Widget Interactions: Set data from one widget as a filter for another
8. Share with user groups, mark as Favorite, or set as Dashboard Home (up to 5 dashboards on Product Home)

---

3.10 Backup Configuration

3.10.1 Fleet-Level Backups

Navigation: VCF Operations > Fleet Management > Lifecycle > Settings > SFTP Settings

1. Navigate to Fleet Management > Lifecycle > Settings
2. Click SFTP Settings
3. Configure the SFTP server details:
   • SFTP Host: IP or FQDN of your SFTP server
   • Port: Default 22
   • Username: SFTP account username
   • Password: SFTP account password
   • Path: Directory path for backup storage
4. Click Test Connection to verify
5. Click Save
6. Navigate to Backup Settings and configure the backup schedule:
   • Backup frequency: Daily, Weekly, or Custom
   • Retention: Number of backups to keep

3.10.2 Instance-Level Backups

Navigation: VCF Operations > Inventory > VCF Instance > Actions > Manage VCF Instance Settings

1. Navigate to Inventory > Select your VCF Instance
2. Click Actions > Manage VCF Instance Settings
3. Click Backup Settings
4. Configure instance-specific backup parameters
5. Click Save

---

3.11 VCF Operations for Logs

VCF Operations for Logs is not deployed automatically during initial bringup. It must be deployed as a Day 2 operation. Status: Deployed.

Setting	Value
FQDN	logs.lab.local
IP Address	192.168.1.242
VM Name	logs
Node Size	Small
Deployment Method	Fleet Management with custom cert

Known Issue — Self-Signed Certificate SAN Mismatch: The Fleet Management deployment wizard's "Generate self-signed certificate" option may produce a certificate whose SAN entries do not match the node FQDN/IP, causing a precheck error: "Certificate validation for component vrli:vrli-master — The hosts in the certificate doesn't match with the provided/product hosts." The workaround is to generate a custom certificate with OpenSSL and import it. See Section 3.11.1a.

3.11.1 Deploy via Fleet Management

Navigation: VCF Operations > Fleet Management > Lifecycle > VCF Management > Components

Prerequisites: Depot must be configured (see Section 3.3) and the operations-logs binary must be downloaded via Binary Management > INSTALL BINARIES tab. The OVA and PAK files must be in the offline depot under PROD\COMP\VRLI\.

1. Navigate to Fleet Management > Lifecycle > VCF Management
2. Under the Components section, click Add next to operations-logs
3. Select New Installation
4. Select deployment type: Simple for lab environments
5. Certificate Configuration:
   • Recommended: Import a custom certificate generated with proper SANs (see Section 3.11.1a)
   • Alternative: Generate self-signed certificate (may fail precheck — see warning above)
6. VM Location & OS Configuration:
   • Select vCenter (vcenter.lab.local), cluster (vcenter-cl01), VM network, and datastore (vcenter-cl01-ds-vsan01)
   • Click Edit Server Selection to choose DNS (192.168.1.230) and NTP servers
7. Component Configuration:
   • Click Add Password to set default password (15+ characters, must include special characters !@#$%^&*)
   • Node Size: Small (for lab)
   • FIPS Mode: Disable for lab
   • VM Compatibility: Update to latest hardware version
   • Time Sync: Select NTP servers
   • VM Name: logs
   • FQDN: logs.lab.local
   • IP Address: 192.168.1.242
8. Run Precheck validation
9. Click Deploy
10. Monitor deployment until completion

3.11.1a Certificate Workaround: Generate Custom Certificate

If the wizard's self-signed certificate fails precheck validation, generate a proper certificate with OpenSSL on SDDC Manager (SSH as vcf, then su - to root):

Step 1 — Verify DNS resolution:

nslookup logs.lab.local 192.168.1.230
nslookup 192.168.1.242 192.168.1.230
ping -c 2 logs.lab.local

Step 2 — Create OpenSSL config and generate certificate:

cat > /tmp/vrli-cert.cnf << 'EOF'
[req]
default_bits = 4096
prompt = no
default_md = sha256
distinguished_name = dn
req_extensions = v3_req
x509_extensions = v3_req

[dn]
C = US
ST = California
L = Lab
O = Lab
OU = VCF
CN = logs.lab.local

[v3_req]
basicConstraints = CA:FALSE
keyUsage = digitalSignature, keyEncipherment
extendedKeyUsage = serverAuth, clientAuth
subjectAltName = @alt_names

[alt_names]
DNS.1 = logs.lab.local
DNS.2 = logs
IP.1 = 192.168.1.242
EOF

openssl req -x509 -nodes -days 730 -newkey rsa:4096 \
  -keyout /tmp/vrli.key -out /tmp/vrli.crt \
  -config /tmp/vrli-cert.cnf

Step 3 — Verify SANs are correct:

openssl x509 -in /tmp/vrli.crt -noout -text | grep -A5 "Subject Alternative Name"
# Expected: DNS:logs.lab.local, DNS:logs, IP Address:192.168.1.242

Step 4 — Transfer cert to workstation:

Display the certificate and key, then copy-paste into local files (vrli.crt and vrli.key):

cat /tmp/vrli.crt
cat /tmp/vrli.key

Step 5 — Import in Fleet Management wizard:

1. In the deployment wizard's Certificate step, select Import
2. Upload vrli.crt (certificate) and vrli.key (private key) — must be PEM format
3. Continue to Component Configuration and complete the deployment as in Section 3.11.1
4. Run Precheck — should pass with the custom certificate

Step 6 — Verify deployment:

# Check appliance is reachable
curl -sk https://logs.lab.local:9543/api/v2/deployment/new -o /dev/null -w "%{http_code}"

# Check certificate on deployed appliance
openssl s_client -connect logs.lab.local:443 -servername logs.lab.local </dev/null 2>/dev/null | openssl x509 -noout -subject -issuer -dates

3.11.2 Integrate with VCF Operations

Navigation: VCF Operations > Administration > Control Panel > Log Management

1. Navigate to Administration > Control Panel > Log Management
2. Enter connection details for the VCF Operations for Logs appliance
3. Click Validate Connection
4. Authenticate using admin credentials

3.11.3 Enable Log Collection

1. Navigate to Administration > Integrations > Accounts
2. Find your VCF or vCenter account and click the ellipsis > Edit
3. Go to the Domains tab (for VCF account) or Log Operations section (for vCenter)
4. Click Activate Log Collection
5. Repeat for all workload domains
6. Click Save and verify the collector status shows healthy

3.11.4 Configure SDDC Manager Log Forwarding (Manual)

As of VCF 9.0, there is no automated way to configure the logs agent on SDDC Manager:

1. Download the deploy_vcf_ops_logs_agent.sh script
2. Upload to SDDC Manager appliance (use ssh vcf@192.168.1.241 "cat > /home/vcf/deploy_vcf_ops_logs_agent.sh" < deploy_vcf_ops_logs_agent.sh)
3. Ensure port 9543 from SDDC Manager to VCF Operations for Logs is open
4. SSH as root and run the script

Note: The log collection configuration for vCenter adapter instances is NOT included in configuration export/import operations. SCP does not work with SDDC Manager's restricted shell -- use the ssh cat > method for file transfers.

---

3.12 SDDC-to-VCF-Ops Task Migration Reference

The following tasks have moved from SDDC Manager to VCF Operations in VCF 9.0:

Task	VCF 9.0 Location in VCF Operations
DNS/NTP Configuration	Inventory > VCF Instance > Actions > Manage VCF Instance Settings > Network Settings
Workload Domain Creation	Inventory > VCF Instance > Add Workload Domain
Backup Configuration	Fleet Management > Lifecycle > Settings
Certificate Authority	Fleet Management > Certificates > Configure CA
Certificate Management	Fleet Management > Certificates
Password Management	Fleet Management > Passwords
Network Pools	vCenter: Global Inventory > Hosts > Network Pools
Host Commissioning	vCenter: Global Inventory > Unassigned Hosts
Cluster Creation	vCenter: New SDDC Cluster
Licensing	License Management (single file model)

Critical Note: While the SDDC Manager UI is still present in VCF 9.0, performing tasks there does not immediately sync to VCF Operations. Changes depend on scheduled synchronization intervals. Use VCF Operations as the primary interface for all Day 2 operations.

3.12.1 Known Issues (VCF Operations 9.0.1)

#	Issue	Impact
1	Relationships not updated after 2nd collection cycle in management packs built with the Management Pack Builder	Custom management packs may show stale data
2	Custom network adapters do not start after VCF Operations and VCF Operations for Networks are updated to VCF 9.0	Workaround required
3	VCF Operations for Networks stops collecting metrics when NSX is upgraded from 4.2.1 to 9.0	Re-configure after upgrade
4	Manually stopped adapter instances start collecting after a management pack upgrade	Monitor adapter states after upgrades
5	Binary downloads from depot may intermittently fail	Retry the download
6	Fleet Management appliance root password must be 15+ characters	Precheck will fail otherwise
7	Only one VCF Operations for Networks instance supported	Cannot add multiple
8	Log collection configuration for vCenter adapters not included in config export/import	Manually reconfigure after import
9	License expires if usage file not submitted within 180 days (disconnected mode)	Hosts disconnect, workloads blocked
10	Do not configure NTP during OVF deployment (KB 374792)	Configure it in the setup wizard instead
11	Password rotation options from VCF 5.x not fully available	Use SDDC Manager API as workaround
12	After workload domain redeployment, vCenter/vSAN adapter may enter Warning	Reconfigure adapter
13	Infrastructure Health Adapter "no data receiving" — stale SDDC Manager credential	Fix: Integrations → SDDC Mgr → ROTATE or set manually → VALIDATE → SAVE → reboot appliance
14	Adapter log paths changed in 9.x — /storage/log/vcops/log/adapters/<Name>/	Legacy /var/log/vmware/vcops/adapters/ does not exist
15	NSX adapter warnings when NSX is powered off	Expected — clears when NSX is back online
16	NSX adapter PKIX cert trust failure — self-signed cert not trusted	Import NSX cert into /usr/java/jre-vmware-17/lib/security/cacerts (password changeit), reboot
17	NSX System Managed Credential ROTATE fails	Uncheck System Managed, set manually (admin/password), VALIDATE, SAVE
18	Two separate NSX adapters exist — VCF uses VIP, NSX "Aria Admin" uses node FQDN	Both need credentials configured separately
19	Credential Update/Rotate/Remediate cascade failure — stuck tasks and locks	Full PostgreSQL repair required — see Section 7.2.6

3.12.2 Post-Configuration Verification Checklist

[ ] License Management -- license valid, not evaluation mode
[ ] Administration > Integrations > Accounts -- all adapters green "Collecting"
[ ] Fleet Management dashboard -- all components healthy, Connected
[ ] Depot configuration -- connected to offline depot, binaries available
[ ] Infrastructure Operations > VCF Instances -- shows VCF instance with all domains
[ ] All ESXi hosts (esxi01-04) visible in inventory
[ ] VCF Health -- certificates, NTP, DNS checks passing
[ ] Security & Compliance -- SDDC benchmarks activated
[ ] Fleet Management > Passwords -- all accounts valid
[ ] Fleet Management > Certificates -- all certificates visible with expiration dates

+-----------------------------------------------------------+
|                    NSX MANAGER CLUSTER                     |
|              (3-node for HA, 1-node for lab)               |
+-----------------------------------------------------------+
|                      TIER-0 GATEWAY                        |
|              (Provider Router - North-South)               |
|                    BGP/OSPF to Physical                    |
+-----------------------------------------------------------+
|                      TIER-1 GATEWAY                        |
|              (Tenant Router - Internal)                    |
|                   NAT, Load Balancing                      |
+-----------------------------------------------------------+
|                        SEGMENTS                            |
|              (Layer 2 - Overlay or VLAN)                   |
+-----------------------------------------------------------+

/usr/bin/ovftool --skipManifestCheck --powerOn --diskMode=thin --acceptAllEulas --allowExtraConfig --ipProtocol=IPv4 --noSSLVerify --datastore=vcenter-cl01-ds-vsan01 --network=vcenter-cl01-vds01-pg-vm-mgmt --deploymentOption=small --name=nsx-manager --prop:nsx_role='NSX Manager' --prop:nsx_passwd_0='Success01!0909!!' --prop:nsx_cli_passwd_0='Success01!0909!!' --prop:nsx_cli_audit_passwd_0='Success01!0909!!' --prop:nsx_hostname=nsx-node1.lab.local --prop:nsx_ip_0=192.168.1.71 --prop:nsx_netmask_0=255.255.255.0 --prop:nsx_gateway_0=192.168.1.1 --prop:nsx_dns1_0=192.168.1.230 --prop:nsx_domain_0=lab.local --prop:nsx_ntp_0=192.168.1.230 --prop:nsx_isSSHEnabled=True --prop:nsx_allowSSHRootLogin=True --X:waitForIp --X:logFile=/tmp/nsx-manager.log --X:logLevel=verbose /nfs/vmware/vcf/nfs-mount/bundle/028849ee-d3e7-5748-9b90-47d503c6dd3e/028849ee-d3e7-5748-9b90-47d503c6dd3e/nsx-unified-appliance-9.0.1.0.24952114.ova "vi://administrator%40vsphere.local:Success01%210909%21%21@vcenter.lab.local/vcenter-dc01/host/vcenter-cl01"

# SSH to NSX Manager
ssh admin@192.168.1.71

# Configure DNS
set name-servers 192.168.1.230

# Configure NTP
set ntp-servers 192.168.1.230

# Verify DNS
get name-servers

# Verify NTP
get ntp-servers

# SSH to NSX Manager as admin
ssh admin@192.168.1.71

# Check cluster status
get cluster status

# Check all service statuses
get cluster status verbose

# Verify via API
curl -k -u admin:'Success01!0909!!' https://192.168.1.71/api/v1/cluster/status

# SSH to ESXi host
ssh root@192.168.1.74

# Restart management network
/etc/init.d/hostd restart
/etc/init.d/vpxa restart

# SSH to ESXi host
ssh root@192.168.1.74

# Check NSX proxy status
/etc/init.d/nsx-proxy status

# Check NSX datapath (DFW)
/etc/init.d/nsx-datapath status

# Check NSX operations agent
/etc/init.d/nsx-opsagent status

# List VMkernel interfaces (confirm vmk50 hyperbus exists)
esxcli network ip interface list

# Check TEP connectivity to another host
vmkping 192.168.1.75

# View NSX logs
tail -50 /var/log/nsx-syslog.log

# Check NSX agent communication (port 1234)
esxcli network ip connection list | grep 1234

# Check NSX Manager memory from vCenter
# VM > Monitor > Performance > Memory

# Check service health via API
curl -k -u admin:'Success01!0909!!' https://192.168.1.71/api/v1/cluster/status

# SSH to ESXi host
ssh root@192.168.1.74

# Find TEP VMkernel
esxcfg-vmknic -l | grep -i tep

# For vmk0-as-TEP configuration, test management connectivity
vmkping 192.168.1.75

# Test with MTU 1600 (GENEVE overhead requires 1600+ bytes)
vmkping -d -s 1572 192.168.1.75

/etc/init.d/nsx-proxy status
/etc/init.d/nsx-datapath status
tail -50 /var/log/nsx-syslog.log

# Overall cluster status
get cluster status

# Detailed service list
get cluster status verbose

# Get manager node list
get managers

# Get all transport nodes
get transport-nodes

# Cluster status
curl -k -u admin:'Success01!0909!!' https://192.168.1.71/api/v1/cluster/status

# Transport node status
curl -k -u admin:'Success01!0909!!' https://192.168.1.71/api/v1/transport-nodes

# Transport node state
curl -k -u admin:'Success01!0909!!' https://192.168.1.71/api/v1/transport-nodes/state

# Compute managers
curl -k -u admin:'Success01!0909!!' https://192.168.1.71/api/v1/fabric/compute-managers

# List certificates
curl -k -u admin:'Success01!0909!!' https://192.168.1.71/api/v1/trust-management/certificates

# Node UUID (from cluster info)
curl -k -u admin:'Success01!0909!!' https://192.168.1.71/api/v1/cluster

cat > /tmp/nsx-cert.conf << 'EOF'
[ req ]
default_bits = 2048
distinguished_name = req_distinguished_name
req_extensions = req_ext
x509_extensions = req_ext
prompt = no

[ req_distinguished_name ]
countryName = US
stateOrProvinceName = Lab
localityName = Lab
organizationName = lab.local
commonName = nsx-vip.lab.local

[ req_ext ]
basicConstraints = CA:FALSE
subjectAltName = @alt_names

[alt_names]
DNS.1 = nsx-vip.lab.local
DNS.2 = nsx-node1.lab.local
DNS.3 = nsx-manager.lab.local
IP.1 = 192.168.1.70
IP.2 = 192.168.1.71
EOF

openssl req -x509 -nodes -days 825 -newkey rsa:2048 \
  -keyout /tmp/nsx.key -out /tmp/nsx.crt \
  -config /tmp/nsx-cert.conf -sha256

openssl x509 -in /tmp/nsx.crt -text -noout | grep -A4 "Subject Alternative Name"

python -c "
import json
cert = open('/tmp/nsx.crt').read()
key = open('/tmp/nsx.key').read()
print(json.dumps({'pem_encoded': cert, 'private_key': key}))
" > /tmp/nsx-import.json

curl -k -u admin:'Success01!0909!!' -X POST "https://192.168.1.71/api/v1/trust-management/certificates?action=import" -H "Content-Type: application/json" -d @/tmp/nsx-import.json

curl -k -u admin:'Success01!0909!!' https://192.168.1.71/api/v1/cluster

curl -k -u admin:'Success01!0909!!' -X POST "https://192.168.1.71/api/v1/trust-management/certificates/<CERT-ID>?action=apply_certificate&service_type=API&node_id=<NODE-UUID>"

curl -k -u admin:'Success01!0909!!' -X POST "https://192.168.1.71/api/v1/trust-management/certificates/<CERT-ID>?action=apply_certificate&service_type=MGMT_CLUSTER"

# Pull active NSX certificate
openssl s_client -showcerts -connect 192.168.1.71:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM > /tmp/nsx-root.crt

# Import into VCF trust store
KEY=$(cat /etc/vmware/vcf/commonsvcs/trusted_certificates.key)
keytool -importcert -alias nsx-selfsigned -file /tmp/nsx-root.crt \
  -keystore /etc/vmware/vcf/commonsvcs/trusted_certificates.store \
  -storepass "$KEY" -noprompt

# Import into Java cacerts
keytool -importcert -alias nsx-selfsigned -file /tmp/nsx-root.crt \
  -keystore /etc/alternatives/jre/lib/security/cacerts \
  -storepass changeit -noprompt

# Restart SDDC Manager services
/opt/vmware/vcf/operationsmanager/scripts/cli/sddcmanager_restart_services.sh

# Add to each ESXi VM's VMX file
sata0:0.virtualSSD = "1"
sata0:2.virtualSSD = "1"

sata0:3.virtualSSD = "1"

D:\VMs\esxi01.lab.local\esxi01.lab.local.vmx
E:\VMs\esxi02.lab.local\esxi02.lab.local.vmx
E:\VMs\esxi03.lab.local\esxi03.lab.local.vmx
F:\VMs\esxi04.lab.local\esxi04.lab.local.vmx

# Add the vSAN ESA HCL bypass property
echo "vsan.esa.sddc.managed.disk.claim=true" >> /etc/vmware/vcf/domainmanager/application-prod.properties

# Restart the domain manager service to apply
systemctl restart domainmanager

# Verify the property was written
cat /etc/vmware/vcf/domainmanager/application-prod.properties | grep vsan

# Check SSD status for all storage devices
esxcli storage core device list | grep -E "Display Name|Is SSD"

# Expected output for each disk:
#    Display Name: Local ATA Disk (t10.ATA...)
#    Is SSD: true

# List current SATP rules filtering for SSD
esxcli storage nmp satp rule list | grep enable_ssd

# Add a claim rule to mark a specific device as SSD
esxcli storage nmp satp rule add -s VMW_SATP_LOCAL \
  -d t10.ATA_____VMware_Virtual_SATA_Hard_Drive__________03000000000000000001 \
  -o enable_ssd

# Reclaim the device to apply the new rule
esxcli storage core claiming reclaim \
  -d t10.ATA_____VMware_Virtual_SATA_Hard_Drive__________03000000000000000001

# Verify the device is now marked as SSD
esxcli storage core device list -d t10.ATA_____VMware_Virtual_SATA_Hard_Drive__________03000000000000000001 | grep "Is SSD"

# On any ESXi host, list vSAN storage
esxcli vsan storage list

# Check vSAN cluster membership
esxcli vsan cluster get

# List datastores visible to the host
esxcli storage filesystem list | grep -i vsan

# Verify datastore is accessible in vCenter
# Navigate to: vcenter.lab.local > vcenter-dc01 > vcenter-cl01 > Datastores
# Datastore name: vcenter-cl01-ds-vsan01

# Comprehensive disk query with vSAN eligibility
vdq -iH

# Quick eligibility check
vdq -q

# List all vSAN storage devices and their state
esxcli vsan storage list

# List all storage devices with full details
esxcli storage core device list

# Filter for device name and SSD status
esxcli storage core device list | grep -E "^t10|^naa|Display Name|Is SSD|Size"

# Check partition tables on a specific disk
partedUtil getptbl /vmfs/devices/disks/t10.ATA_____VMware_Virtual_SATA_Hard_Drive__________03000000000000000001

{
    "Name": "t10.ATA_____VMware_Virtual_SATA_Hard_Drive__________03000000000000000001",
    "State": "Eligible for use by VSAN",
    "Reason": "None",
    "IsSSD": "1"
}

{
    "Name": "t10.ATA_____VMware_Virtual_SATA_Hard_Drive__________03000000000000000001",
    "State": "Ineligible for use by VSAN",
    "Reason": "Has partitions",
    "IsSSD": "1"
}

# Remove a specific disk from vSAN storage
esxcli vsan storage remove -d t10.ATA_____VMware_Virtual_SATA_Hard_Drive__________03000000000000000001

# Verify removal
esxcli vsan storage list
vdq -q

# Check existing partitions
partedUtil getptbl /vmfs/devices/disks/t10.ATA_____VMware_Virtual_SATA_Hard_Drive__________03000000000000000001

# Delete partition 1
partedUtil delete /vmfs/devices/disks/t10.ATA_____VMware_Virtual_SATA_Hard_Drive__________03000000000000000001 1

# Delete partition 2
partedUtil delete /vmfs/devices/disks/t10.ATA_____VMware_Virtual_SATA_Hard_Drive__________03000000000000000001 2

# Verify disk is now eligible
vdq -q

# List current disk groups
esxcli vsan storage list

# Remove an entire disk group by specifying the cache disk
esxcli vsan storage remove -d <cache-disk-device-name>

# List vSAN objects on a host
esxcli vsan debug object list

# Check for inaccessible objects
esxcli vsan debug object health summary get

ssh root@192.168.1.74

mkdir -p /vmfs/volumes/vcenter-cl01-ds-vsan01/sddc-manager/

# Disk 0 (32GB allocated, 2.6GB actual)
vmkfstools -i /vmfs/volumes/esxi01-local/sddc-manager/sddc-manager.vmdk /vmfs/volumes/vcenter-cl01-ds-vsan01/sddc-manager/sddc-manager.vmdk -d thin

# Disk 1 (16GB allocated, 2.6GB actual)
vmkfstools -i /vmfs/volumes/esxi01-local/sddc-manager/sddc-manager_1.vmdk /vmfs/volumes/vcenter-cl01-ds-vsan01/sddc-manager/sddc-manager_1.vmdk -d thin

# Disk 2 (240GB allocated, 3.0GB actual)
vmkfstools -i /vmfs/volumes/esxi01-local/sddc-manager/sddc-manager_2.vmdk /vmfs/volumes/vcenter-cl01-ds-vsan01/sddc-manager/sddc-manager_2.vmdk -d thin

# Disk 3 (512GB allocated, 99.5GB actual) — LARGEST DISK, takes longest
vmkfstools -i /vmfs/volumes/esxi01-local/sddc-manager/sddc-manager_3.vmdk /vmfs/volumes/vcenter-cl01-ds-vsan01/sddc-manager/sddc-manager_3.vmdk -d thin

# Disk 4 (26GB allocated, 30MB actual)
vmkfstools -i /vmfs/volumes/esxi01-local/sddc-manager/sddc-manager_4.vmdk /vmfs/volumes/vcenter-cl01-ds-vsan01/sddc-manager/sddc-manager_4.vmdk -d thin

# Disk 5 (88GB allocated, 64MB actual)
vmkfstools -i /vmfs/volumes/esxi01-local/sddc-manager/sddc-manager_5.vmdk /vmfs/volumes/vcenter-cl01-ds-vsan01/sddc-manager/sddc-manager_5.vmdk -d thin

# Copy VMX, NVRAM, and VMSD files
cp /vmfs/volumes/esxi01-local/sddc-manager/sddc-manager.vmx /vmfs/volumes/vcenter-cl01-ds-vsan01/sddc-manager/
cp /vmfs/volumes/esxi01-local/sddc-manager/sddc-manager.nvram /vmfs/volumes/vcenter-cl01-ds-vsan01/sddc-manager/
cp /vmfs/volumes/esxi01-local/sddc-manager/sddc-manager.vmsd /vmfs/volumes/vcenter-cl01-ds-vsan01/sddc-manager/

# Verify thin provisioned disks on vSAN
ls -la /vmfs/volumes/vcenter-cl01-ds-vsan01/sddc-manager/

# Check actual disk usage (thin should show much less)
du -sh /vmfs/volumes/vcenter-cl01-ds-vsan01/sddc-manager/

# Only after confirming the migrated VM works correctly
rm -rf /vmfs/volumes/esxi01-local/sddc-manager/

# Check vSAN cluster health summary
esxcli vsan health cluster list

# Run a specific health check
esxcli vsan health cluster get -t "Network health"

# Check vSAN cluster membership
esxcli vsan cluster get

# List all vSAN storage devices and their state
esxcli vsan storage list

# Check resync status
esxcli vsan debug resync summary get

# Check vSAN object health
esxcli vsan debug object health summary get

# Launch esxtop in disk (storage) mode
esxtop

# Press 'u' to switch to disk device view
# Press 'v' to switch to disk VM view

# Connect to RVC from vCenter shell
rvc administrator@vsphere.local@localhost

# Navigate to cluster
cd /vcenter.lab.local/vcenter-dc01/computers/vcenter-cl01

# Start vSAN Observer
vsan.observer . --run-webserver --force

# Check if disk is seen at all
esxcli storage core device list | grep -E "^t10|Is SSD"

# Check vSAN eligibility
vdq -q

# Check for stale partitions
partedUtil getptbl /vmfs/devices/disks/<device-name>

# Check vSAN cluster membership
esxcli vsan cluster get

# Check network connectivity between vSAN VMkernel ports
vmkping -I vmk2 192.168.12.120
vmkping -I vmk2 192.168.12.121
vmkping -I vmk2 192.168.12.122
vmkping -I vmk2 192.168.12.123

# Check VMkernel adapter status
esxcli network ip interface list

# Check object health summary
esxcli vsan debug object health summary get

# List objects with issues
esxcli vsan debug object list

# In vCenter: Cluster > Monitor > vSAN > Virtual Objects
# Filter for non-healthy objects

# Check resync summary
esxcli vsan debug resync summary get

# In vCenter: Cluster > Monitor > vSAN > Resyncing Objects
# Shows: Objects resyncing, bytes remaining, ETA

# vSAN trace files location
ls /var/log/vmkernel.log | head
ls /var/log/vobd.log | head

# Search for vSAN-related errors in vmkernel log
grep -i "vsan\|cmmds\|clom\|dom\|lsom" /var/log/vmkernel.log | tail -50

# vSAN specific logs
cat /var/log/vsanmgmt.log | tail -50
cat /var/log/vsantraced.log | tail -50

# Check vSAN daemon status
/etc/init.d/vsanmgmtd status
/etc/init.d/vsand status

ssh root@192.168.1.71

cat > /tmp/nsx-cert.conf << 'EOF'
[ req ]
default_bits = 2048
distinguished_name = req_distinguished_name
req_extensions = req_ext
x509_extensions = req_ext
prompt = no

[ req_distinguished_name ]
countryName = US
stateOrProvinceName = Lab
localityName = Lab
organizationName = lab.local
commonName = nsx-vip.lab.local

[ req_ext ]
basicConstraints = CA:FALSE
subjectAltName = @alt_names

[alt_names]
DNS.1 = nsx-vip.lab.local
DNS.2 = nsx-node1.lab.local
DNS.3 = nsx-manager.lab.local
IP.1 = 192.168.1.70
IP.2 = 192.168.1.71
EOF

openssl req -x509 -nodes -days 825 -newkey rsa:2048 \
  -keyout /tmp/nsx.key -out /tmp/nsx.crt \
  -config /tmp/nsx-cert.conf -sha256

openssl x509 -in /tmp/nsx.crt -text -noout | grep -A4 "Subject Alternative Name"

X509v3 Subject Alternative Name:
    DNS:nsx-vip.lab.local, DNS:nsx-node1.lab.local, DNS:nsx-manager.lab.local, IP Address:192.168.1.70, IP Address:192.168.1.71

# Check subject, issuer, validity period
openssl x509 -in /tmp/nsx.crt -text -noout | head -20

# Check key type and size
openssl x509 -in /tmp/nsx.crt -text -noout | grep "Public-Key"

python -c "
import json
cert = open('/tmp/nsx.crt').read()
key = open('/tmp/nsx.key').read()
print(json.dumps({'pem_encoded': cert, 'private_key': key}))
" > /tmp/nsx-import.json

python -c "import json; d=json.load(open('/tmp/nsx-import.json')); print('cert lines:', d['pem_encoded'].count('\n'), 'key lines:', d['private_key'].count('\n'))"

curl -k -u admin:'Success01!0909!!' -X POST "https://192.168.1.71/api/v1/trust-management/certificates?action=import" -H "Content-Type: application/json" -d @/tmp/nsx-import.json

{
  "results": [
    {
      "id": "701d1416-5054-4038-8749-4ac495980ebd",
      ...
    }
  ]
}

curl -k -u admin:'Success01!0909!!' https://192.168.1.71/api/v1/cluster

curl -k -u admin:'Success01!0909!!' -X POST "https://192.168.1.71/api/v1/trust-management/certificates/701d1416-5054-4038-8749-4ac495980ebd?action=apply_certificate&service_type=API&node_id=95493642-ef4a-cb8e-ed7c-5bc20033f2c2"

curl -k -u admin:'Success01!0909!!' -X POST "https://192.168.1.71/api/v1/trust-management/certificates/701d1416-5054-4038-8749-4ac495980ebd?action=apply_certificate&service_type=MGMT_CLUSTER"

# Verify node certificate (.71)
openssl s_client -connect 192.168.1.71:443 -showcerts </dev/null 2>/dev/null | openssl x509 -noout -text | grep -A2 "Subject Alternative Name"

# Verify VIP certificate (.70)
openssl s_client -connect 192.168.1.70:443 -showcerts </dev/null 2>/dev/null | openssl x509 -noout -text | grep -A2 "Subject Alternative Name"

X509v3 Subject Alternative Name:
    DNS:nsx-vip.lab.local, DNS:nsx-node1.lab.local, DNS:nsx-manager.lab.local, IP Address:192.168.1.70, IP Address:192.168.1.71

# Only the vcf user can SSH in (root and admin are rejected)
ssh vcf@192.168.1.241

# Switch to root
su -

openssl s_client -showcerts -connect 192.168.1.71:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM > /tmp/nsx-root.crt

openssl x509 -in /tmp/nsx-root.crt -noout -text | grep -A2 "Subject Alternative Name"
# Should show: DNS:nsx-vip.lab.local, DNS:nsx-node1.lab.local, DNS:nsx-manager.lab.local, IP Address:192.168.1.70, IP Address:192.168.1.71

# Read the trust store password
KEY=$(cat /etc/vmware/vcf/commonsvcs/trusted_certificates.key)

# Import the NSX certificate
keytool -importcert -alias nsx-selfsigned -file /tmp/nsx-root.crt \
  -keystore /etc/vmware/vcf/commonsvcs/trusted_certificates.store \
  -storepass "$KEY" -noprompt

keytool -importcert -alias nsx-selfsigned -file /tmp/nsx-root.crt \
  -keystore /etc/alternatives/jre/lib/security/cacerts \
  -storepass changeit -noprompt

/opt/vmware/vcf/operationsmanager/scripts/cli/sddcmanager_restart_services.sh

javax.net.ssl.SSLPeerUnverifiedException: Certificate for <esxi01.lab.local> doesn't match any of the subject alternative names: [localhost.localdomain]

# Check current hostname
esxcli system hostname get

# View current certificate SAN
openssl x509 -in /etc/vmware/ssl/rui.crt -text -noout | grep -A1 "Subject Alternative Name"

# View full certificate details (subject, issuer, validity)
openssl x509 -in /etc/vmware/ssl/rui.crt -text -noout

# Step 1: Ensure hostname is correct
esxcli system hostname set --fqdn=esxi01.lab.local

# Step 2: Verify hostname
esxcli system hostname get

# Step 3: Backup existing certificates
mv /etc/vmware/ssl/rui.crt /etc/vmware/ssl/rui.crt.bak
mv /etc/vmware/ssl/rui.key /etc/vmware/ssl/rui.key.bak

# Step 4: Generate new certificates
/sbin/generate-certificates

# Step 5: Restart all services to apply new certificate
services.sh restart

# Step 6: Verify new certificate has correct SAN
openssl x509 -in /etc/vmware/ssl/rui.crt -text -noout | grep -A1 "Subject Alternative Name"

esxcli system hostname set --fqdn=esxi02.lab.local
mv /etc/vmware/ssl/rui.crt /etc/vmware/ssl/rui.crt.bak
mv /etc/vmware/ssl/rui.key /etc/vmware/ssl/rui.key.bak
/sbin/generate-certificates
services.sh restart

esxcli system hostname set --fqdn=esxi03.lab.local
mv /etc/vmware/ssl/rui.crt /etc/vmware/ssl/rui.crt.bak
mv /etc/vmware/ssl/rui.key /etc/vmware/ssl/rui.key.bak
/sbin/generate-certificates
services.sh restart

esxcli system hostname set --fqdn=esxi04.lab.local
mv /etc/vmware/ssl/rui.crt /etc/vmware/ssl/rui.crt.bak
mv /etc/vmware/ssl/rui.key /etc/vmware/ssl/rui.key.bak
/sbin/generate-certificates
services.sh restart

# Get SHA-256 thumbprint for each host
echo | openssl s_client -connect 192.168.1.74:443 2>/dev/null | openssl x509 -noout -fingerprint -sha256
echo | openssl s_client -connect 192.168.1.75:443 2>/dev/null | openssl x509 -noout -fingerprint -sha256
echo | openssl s_client -connect 192.168.1.76:443 2>/dev/null | openssl x509 -noout -fingerprint -sha256
echo | openssl s_client -connect 192.168.1.82:443 2>/dev/null | openssl x509 -noout -fingerprint -sha256

keytool -list -keystore /etc/alternatives/jre/lib/security/cacerts -storepass changeit

keytool -list -v -keystore /etc/alternatives/jre/lib/security/cacerts -storepass changeit

keytool -list -alias nsx-selfsigned -keystore /etc/alternatives/jre/lib/security/cacerts -storepass changeit -v

# Import into Java cacerts
keytool -importcert -alias <alias-name> -file /tmp/cert.crt \
  -keystore /etc/alternatives/jre/lib/security/cacerts \
  -storepass changeit -noprompt

# Import into VCF trust store
KEY=$(cat /etc/vmware/vcf/commonsvcs/trusted_certificates.key)
keytool -importcert -alias <alias-name> -file /tmp/cert.crt \
  -keystore /etc/vmware/vcf/commonsvcs/trusted_certificates.store \
  -storepass "$KEY" -noprompt

# Delete from Java cacerts
keytool -delete -alias <alias-name> \
  -keystore /etc/alternatives/jre/lib/security/cacerts \
  -storepass changeit

# Delete from VCF trust store
KEY=$(cat /etc/vmware/vcf/commonsvcs/trusted_certificates.key)
keytool -delete -alias <alias-name> \
  -keystore /etc/vmware/vcf/commonsvcs/trusted_certificates.store \
  -storepass "$KEY"

keytool -exportcert -alias <alias-name> \
  -keystore /etc/alternatives/jre/lib/security/cacerts \
  -storepass changeit \
  -file /tmp/exported-cert.crt -rfc

keytool -list -alias nsx-selfsigned \
  -keystore /etc/alternatives/jre/lib/security/cacerts \
  -storepass changeit 2>&1 | head -1
# Returns "nsx-selfsigned, ..." if found, or error if not found

keytool -storepasswd \
  -keystore /etc/alternatives/jre/lib/security/cacerts \
  -storepass changeit \
  -new <new-password>

# Pull certificate from a remote server
openssl s_client -showcerts -connect 192.168.1.71:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM > /tmp/remote-cert.crt

# Verify it is the correct certificate
openssl x509 -in /tmp/remote-cert.crt -noout -subject -issuer -dates

# Import into both keystores
keytool -importcert -alias remote-server -file /tmp/remote-cert.crt \
  -keystore /etc/alternatives/jre/lib/security/cacerts \
  -storepass changeit -noprompt

KEY=$(cat /etc/vmware/vcf/commonsvcs/trusted_certificates.key)
keytool -importcert -alias remote-server -file /tmp/remote-cert.crt \
  -keystore /etc/vmware/vcf/commonsvcs/trusted_certificates.store \
  -storepass "$KEY" -noprompt