# HELP queue_time_wait_seconds histogram of queue wait times
# TYPE queue_time_wait_seconds histogram
queue_time_wait_seconds_bucket{le="300"} 22975
queue_time_wait_seconds_bucket{le="3600"} 24931
queue_time_wait_seconds_bucket{le="+Inf"} 25079
queue_time_wait_seconds_sum 9486530
queue_time_wait_seconds_count 25079

# HELP test_run_seconds histogram of test execution time
# TYPE test_run_seconds histogram
test_run_seconds_bucket{le="3600"} 23547
test_run_seconds_bucket{le="7200"} 25001
test_run_seconds_bucket{le="+Inf"} 25079
test_run_seconds_sum 46659290
test_run_seconds_count 25079

# HELP top_failures_pct Most unstable tests in the last 7 days (more than 10% failure rate)
# TYPE top_failures_pct gauge
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-consoles TestMachinesConsoles.testSerialConsole"} 30.88235294117647
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-nics TestMachinesNICs.testNICAdd"} 29.850746268656717
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-settings TestMachinesSettings.testAutostart"} 28.78787878787879
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testAddDiskSCSI"} 28.78787878787879
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-storage-pools TestMachinesStoragePools.testStoragePools"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-snapshots TestMachinesSnapshots.testSnapshots"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-snapshots TestMachinesSnapshots.testSnapshotRevert"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-snapshots TestMachinesSnapshots.testSnapshotCreate"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-settings TestMachinesSettings.testVCPU"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-settings TestMachinesSettings.testMultipleSettings"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-settings TestMachinesSettings.testDomainMemorySettings"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-settings TestMachinesSettings.testCPUModel"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-settings TestMachinesSettings.testBootOrder"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-nics TestMachinesNICs.testVmNICs"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-nics TestMachinesNICs.testNICPlugingAndUnpluging"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-nics TestMachinesNICs.testNICEdit"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-nics TestMachinesNICs.testNICDelete"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-networks TestMachinesNetworks.testNetworkSettings"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-networks TestMachinesNetworks.testNetworkAddStaticDCHPHosts"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-migrate TestMachinesMigration.testSharedStorageMigration"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-migrate TestMachinesMigration.testMoveTemporarilyMigration"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testRename"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testLibvirt"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testDelete"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testCloneSessionConnection"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testClone"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testBasicNonrootUserUnprivileged"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testBasicAdminUser"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testBasic"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-hostdevs TestMachinesHostDevs.testHostDevicesList"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-hostdevs TestMachinesHostDevs.testHostDevAddSingleDevice"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-hostdevs TestMachinesHostDevs.testHostDevAddSessionConnection"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-hostdevs TestMachinesHostDevs.testHostDevAddMultipleDevices"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-filesystems TestMachinesFilesystems.testBasicSystemConnection"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-filesystems TestMachinesFilesystems.testBasicSessionConnection"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testDisks"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testDiskEdit"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testDetachDisk"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testAddDiskNFS"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testAddDiskDirPool"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testAddDiskCustomPath"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testAddDiskAdditionalOptions"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-consoles TestMachinesConsoles.testSwitchConsoleFromSerialToGraphical"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-consoles TestMachinesConsoles.testInlineConsoleWithUrlRoot"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-consoles TestMachinesConsoles.testInlineConsole"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-consoles TestMachinesConsoles.testExternalConsole"} 27.692307692307693
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-migrate TestMachinesMigration.testFailMigrationVolumesNotCompatible"} 23.80952380952381
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-migrate TestMachinesMigration.testFailMigrationUriIncorrect"} 23.80952380952381
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-migrate TestMachinesMigration.testFailMigrationTCPPortClosed"} 23.80952380952381
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-migrate TestMachinesMigration.testFailMigrationStopLibvirtd"} 23.80952380952381

# HELP pr_retries histogram of how many retries it took to get PRs to green
# TYPE pr_retries histogram
pr_retries_bucket{le="0"} 3856
pr_retries_bucket{le="1"} 4623
pr_retries_bucket{le="2"} 4828
pr_retries_bucket{le="3"} 4891
pr_retries_bucket{le="4"} 4921
pr_retries_bucket{le="5"} 4934
pr_retries_bucket{le="6"} 4939
pr_retries_bucket{le="7"} 4944
pr_retries_bucket{le="8"} 4945
pr_retries_bucket{le="11"} 4947
pr_retries_bucket{le="+Inf"} 4947
pr_retries_sum 4947
pr_retries_count 4947

# HELP merged_failures How many PRs were merged with non-successful tests
# TYPE merged_failures counter
merged_failures 142

# HELP top_slowest_tests The slowest tests in the last 7 days
# TYPE top_slowest_tests gauge
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-system-info TestSystemInfo.testInconsistentCryptoPolicy"} 595
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-system-info TestSystemInfo.testCPUSecurityMitigationsEnable"} 504
top_slowest_tests{project="osbuild/cockpit-composer",test="test/verify/check-image TestImage.testOpenStack"} 499
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-system-info TestSystemInfo.testCryptoPolicies"} 476
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-system-services TestServices.testBasic"} 439
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-kdump TestKdumpNFS.testBasic"} 423
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-packagekit TestUpdates.testTracer"} 398
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-system-realms TestIPA.testClientCertAuthentication"} 389
top_slowest_tests{project="candlepin/subscription-manager",test="/integration-tests/submancockpit/test/check-subscriptions TestSubscriptions.testSubAndInAndFail"} 382
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-selinux TestSelinux.testTroubleshootAlerts"} 376
top_slowest_tests{project="candlepin/subscription-manager",test="integration-tests/check-subscriptions TestSubscriptions.testSubAndInAndFail"} 370
top_slowest_tests{project="candlepin/subscription-manager/subscription-manager-1.28",test="integration-tests/check-subscriptions TestSubscriptions.testSubAndInAndFail"} 369
top_slowest_tests{project="candlepin/subscription-manager/subscription-manager-1.28.21",test="integration-tests/check-subscriptions TestSubscriptions.testSubAndInAndFail"} 366
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-kdump TestKdump.testBasic"} 364
top_slowest_tests{project="candlepin/subscription-manager-cockpit",test="/test/check-subscriptions TestSubscriptions.testSubAndInAndFail"} 349
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-system-realms TestIPA.testQualifiedUsers"} 346
top_slowest_tests{project="cockpit-project/cockpit-ostree",test="/test/check-ostree OstreeRestartCase.testOstree"} 342
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-metrics TestHistoryMetrics.testPmProxySettings"} 334
top_slowest_tests{project="cockpit-project/cockpit-ostree",test="/test/check-ostree OstreeCase.testPageStatus"} 228
top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-shell-menu TestMenu.testBasic"} 228
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testBasic"} 225
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testAddDiskNFS"} 222
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testBasicLibvirtUserUnprivileged"} 222
top_slowest_tests{project="candlepin/subscription-manager",test="/integration-tests/submancockpit/test/check-subscriptions TestSubscriptions.testInsights"} 220
top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-system-realms TestIPA.testClientCertAuthentication"} 219
top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-system-realms TestIPA.testQualifiedUsers"} 215
top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-system-realms TestIPA.testUnqualifiedUsers"} 208
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testBasicAdminUser"} 202
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testAddDiskSCSI"} 197
top_slowest_tests{project="candlepin/subscription-manager",test="integration-tests/check-subscriptions TestSubscriptions.testInsights"} 195
top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-storage-luks TestStorageLuks.testLuks1Slots"} 193
top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-pages TestPages.testBasic"} 189
top_slowest_tests{project="candlepin/subscription-manager/subscription-manager-1.28",test="integration-tests/check-subscriptions TestSubscriptions.testInsights"} 177
top_slowest_tests{project="candlepin/subscription-manager/subscription-manager-1.28.29",test="integration-tests/check-subscriptions TestSubscriptions.testInsights"} 176
top_slowest_tests{project="candlepin/subscription-manager-cockpit",test="/test/check-subscriptions TestSubscriptions.testInsights"} 174
top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-system-realms TestAD.testUnqualifiedUsers"} 174
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-create TestMachinesCreate.testCreateUrlSource"} 173
top_slowest_tests{project="rhinstaller/anaconda",test="/ui/webui/test/check-progress TestInstallationProgress.testBasic"} 172
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-settings TestMachinesSettings.testAutostart"} 171
top_slowest_tests{project="candlepin/subscription-manager/subscription-manager-1.28.21",test="integration-tests/check-subscriptions TestSubscriptions.testInsights"} 170
top_slowest_tests{project="candlepin/subscription-manager/subscription-manager-1.28.29",test="integration-tests/check-subscriptions TestSubscriptions.testSubAndInAndFail"} 168
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-nics TestMachinesNICs.testNICDelete"} 168
top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-kdump TestKdumpNFS.testBasic"} 166
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testDetachDisk"} 160
top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-packagekit TestUpdates.testTracer"} 159
top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-system-info TestSystemInfo.testBasic"} 159
top_slowest_tests{project="candlepin/subscription-manager-cockpit",test="/test/check-subscriptions TestSubscriptionsPackages.testMissingPackages"} 157
top_slowest_tests{project="rhinstaller/anaconda",test="/ui/webui/test/check-basic TestQuit.testQuitInstallation"} 152
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-consoles TestMachinesConsoles.testInlineConsole"} 148
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-consoles TestMachinesConsoles.testInlineConsoleWithUrlRoot"} 148
top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-metrics TestHistoryMetrics.testPmProxySettings"} 148
top_slowest_tests{project="rhinstaller/anaconda",test="/ui/webui/test/check-storage TestStorageExtraDisks.testLocalDisksSyncNew"} 146
top_slowest_tests{project="cockpit-project/cockpit-podman",test="/test/check-application TestApplication.testRunImageSystem"} 143
top_slowest_tests{project="cockpit-project/cockpit-podman",test="/test/check-application TestApplication.testNotRunning"} 138
top_slowest_tests{project="cockpit-project/cockpit-ostree",test="/test/check-ostree OstreeRestartCase.testRebase"} 134
top_slowest_tests{project="candlepin/subscription-manager",test="/integration-tests/submancockpit/test/check-subscriptions TestSubscriptionsPackages.testMissingPackages"} 132
top_slowest_tests{project="candlepin/subscription-manager",test="/integration-tests/submancockpit/test/check-subscriptions TestSubscriptions.testRegisterWithKey"} 130
top_slowest_tests{project="candlepin/subscription-manager",test="/integration-tests/submancockpit/test/check-subscriptions TestSubscriptions.testRegisterWithoutAutoAttach"} 129
top_slowest_tests{project="candlepin/subscription-manager",test="/integration-tests/submancockpit/test/check-subscriptions TestSubscriptions.testRegister"} 128
top_slowest_tests{project="cockpit-project/cockpit-podman",test="/test/check-application TestApplication.testRunImageUser"} 124
top_slowest_tests{project="cockpit-project/cockpit-podman",test="/test/check-application TestApplication.testCheckpointRestoreCGroupsV1"} 122
top_slowest_tests{project="cockpit-project/cockpit-podman",test="/test/check-application TestApplication.testHealthcheckSystem"} 119
top_slowest_tests{project="cockpit-project/cockpit-podman",test="/test/check-application TestApplication.testBasicSystem"} 115
top_slowest_tests{project="candlepin/subscription-manager",test="integration-tests/check-subscriptions TestSubscriptionsPackages.testMissingPackages"} 113
top_slowest_tests{project="candlepin/subscription-manager",test="integration-tests/check-subscriptions TestSubscriptions.testRegisterWithKey"} 112
top_slowest_tests{project="osbuild/cockpit-composer",test="test/verify/check-package TestPackage.testRedoUndoCommit"} 111
top_slowest_tests{project="candlepin/subscription-manager",test="integration-tests/check-subscriptions TestSubscriptions.testRegisterWithoutAutoAttach"} 109
top_slowest_tests{project="candlepin/subscription-manager-cockpit",test="/test/check-subscriptions TestSubscriptions.testRegisterWithKey"} 106

# HELP s3_usage_bytes How much disk space an S3 store is using
# TYPE s3_usage_bytes gauge
s3_usage_bytes{store="https://cockpit-images.eu-central-1.linodeobjects.com/"} 101983358464
s3_usage_bytes{store="https://cockpit-images.us-east-1.linodeobjects.com/"} 101983358464