# HELP queue_time_wait_seconds histogram of queue wait times # TYPE queue_time_wait_seconds histogram queue_time_wait_seconds_bucket{le="300"} 22975 queue_time_wait_seconds_bucket{le="3600"} 24931 queue_time_wait_seconds_bucket{le="+Inf"} 25079 queue_time_wait_seconds_sum 9486530 queue_time_wait_seconds_count 25079 # HELP test_run_seconds histogram of test execution time # TYPE test_run_seconds histogram test_run_seconds_bucket{le="3600"} 23547 test_run_seconds_bucket{le="7200"} 25001 test_run_seconds_bucket{le="+Inf"} 25079 test_run_seconds_sum 46659290 test_run_seconds_count 25079 # HELP top_failures_pct Most unstable tests in the last 7 days (more than 10% failure rate) # TYPE top_failures_pct gauge top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-consoles TestMachinesConsoles.testSerialConsole"} 30.88235294117647 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-nics TestMachinesNICs.testNICAdd"} 29.850746268656717 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-settings TestMachinesSettings.testAutostart"} 28.78787878787879 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testAddDiskSCSI"} 28.78787878787879 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-storage-pools TestMachinesStoragePools.testStoragePools"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-snapshots TestMachinesSnapshots.testSnapshots"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-snapshots TestMachinesSnapshots.testSnapshotRevert"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-snapshots TestMachinesSnapshots.testSnapshotCreate"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-settings TestMachinesSettings.testVCPU"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-settings TestMachinesSettings.testMultipleSettings"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-settings TestMachinesSettings.testDomainMemorySettings"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-settings TestMachinesSettings.testCPUModel"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-settings TestMachinesSettings.testBootOrder"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-nics TestMachinesNICs.testVmNICs"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-nics TestMachinesNICs.testNICPlugingAndUnpluging"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-nics TestMachinesNICs.testNICEdit"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-nics TestMachinesNICs.testNICDelete"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-networks TestMachinesNetworks.testNetworkSettings"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-networks TestMachinesNetworks.testNetworkAddStaticDCHPHosts"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-migrate TestMachinesMigration.testSharedStorageMigration"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-migrate TestMachinesMigration.testMoveTemporarilyMigration"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testRename"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testLibvirt"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testDelete"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testCloneSessionConnection"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testClone"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testBasicNonrootUserUnprivileged"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testBasicAdminUser"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testBasic"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-hostdevs TestMachinesHostDevs.testHostDevicesList"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-hostdevs TestMachinesHostDevs.testHostDevAddSingleDevice"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-hostdevs TestMachinesHostDevs.testHostDevAddSessionConnection"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-hostdevs TestMachinesHostDevs.testHostDevAddMultipleDevices"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-filesystems TestMachinesFilesystems.testBasicSystemConnection"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-filesystems TestMachinesFilesystems.testBasicSessionConnection"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testDisks"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testDiskEdit"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testDetachDisk"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testAddDiskNFS"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testAddDiskDirPool"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testAddDiskCustomPath"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testAddDiskAdditionalOptions"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-consoles TestMachinesConsoles.testSwitchConsoleFromSerialToGraphical"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-consoles TestMachinesConsoles.testInlineConsoleWithUrlRoot"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-consoles TestMachinesConsoles.testInlineConsole"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-consoles TestMachinesConsoles.testExternalConsole"} 27.692307692307693 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-migrate TestMachinesMigration.testFailMigrationVolumesNotCompatible"} 23.80952380952381 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-migrate TestMachinesMigration.testFailMigrationUriIncorrect"} 23.80952380952381 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-migrate TestMachinesMigration.testFailMigrationTCPPortClosed"} 23.80952380952381 top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-migrate TestMachinesMigration.testFailMigrationStopLibvirtd"} 23.80952380952381 # HELP pr_retries histogram of how many retries it took to get PRs to green # TYPE pr_retries histogram pr_retries_bucket{le="0"} 3856 pr_retries_bucket{le="1"} 4623 pr_retries_bucket{le="2"} 4828 pr_retries_bucket{le="3"} 4891 pr_retries_bucket{le="4"} 4921 pr_retries_bucket{le="5"} 4934 pr_retries_bucket{le="6"} 4939 pr_retries_bucket{le="7"} 4944 pr_retries_bucket{le="8"} 4945 pr_retries_bucket{le="11"} 4947 pr_retries_bucket{le="+Inf"} 4947 pr_retries_sum 4947 pr_retries_count 4947 # HELP merged_failures How many PRs were merged with non-successful tests # TYPE merged_failures counter merged_failures 142 # HELP top_slowest_tests The slowest tests in the last 7 days # TYPE top_slowest_tests gauge top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-system-info TestSystemInfo.testInconsistentCryptoPolicy"} 595 top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-system-info TestSystemInfo.testCPUSecurityMitigationsEnable"} 504 top_slowest_tests{project="osbuild/cockpit-composer",test="test/verify/check-image TestImage.testOpenStack"} 499 top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-system-info TestSystemInfo.testCryptoPolicies"} 476 top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-system-services TestServices.testBasic"} 439 top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-kdump TestKdumpNFS.testBasic"} 423 top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-packagekit TestUpdates.testTracer"} 398 top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-system-realms TestIPA.testClientCertAuthentication"} 389 top_slowest_tests{project="candlepin/subscription-manager",test="/integration-tests/submancockpit/test/check-subscriptions TestSubscriptions.testSubAndInAndFail"} 382 top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-selinux TestSelinux.testTroubleshootAlerts"} 376 top_slowest_tests{project="candlepin/subscription-manager",test="integration-tests/check-subscriptions TestSubscriptions.testSubAndInAndFail"} 370 top_slowest_tests{project="candlepin/subscription-manager/subscription-manager-1.28",test="integration-tests/check-subscriptions TestSubscriptions.testSubAndInAndFail"} 369 top_slowest_tests{project="candlepin/subscription-manager/subscription-manager-1.28.21",test="integration-tests/check-subscriptions TestSubscriptions.testSubAndInAndFail"} 366 top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-kdump TestKdump.testBasic"} 364 top_slowest_tests{project="candlepin/subscription-manager-cockpit",test="/test/check-subscriptions TestSubscriptions.testSubAndInAndFail"} 349 top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-system-realms TestIPA.testQualifiedUsers"} 346 top_slowest_tests{project="cockpit-project/cockpit-ostree",test="/test/check-ostree OstreeRestartCase.testOstree"} 342 top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-metrics TestHistoryMetrics.testPmProxySettings"} 334 top_slowest_tests{project="cockpit-project/cockpit-ostree",test="/test/check-ostree OstreeCase.testPageStatus"} 228 top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-shell-menu TestMenu.testBasic"} 228 top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testBasic"} 225 top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testAddDiskNFS"} 222 top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testBasicLibvirtUserUnprivileged"} 222 top_slowest_tests{project="candlepin/subscription-manager",test="/integration-tests/submancockpit/test/check-subscriptions TestSubscriptions.testInsights"} 220 top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-system-realms TestIPA.testClientCertAuthentication"} 219 top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-system-realms TestIPA.testQualifiedUsers"} 215 top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-system-realms TestIPA.testUnqualifiedUsers"} 208 top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testBasicAdminUser"} 202 top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testAddDiskSCSI"} 197 top_slowest_tests{project="candlepin/subscription-manager",test="integration-tests/check-subscriptions TestSubscriptions.testInsights"} 195 top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-storage-luks TestStorageLuks.testLuks1Slots"} 193 top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-pages TestPages.testBasic"} 189 top_slowest_tests{project="candlepin/subscription-manager/subscription-manager-1.28",test="integration-tests/check-subscriptions TestSubscriptions.testInsights"} 177 top_slowest_tests{project="candlepin/subscription-manager/subscription-manager-1.28.29",test="integration-tests/check-subscriptions TestSubscriptions.testInsights"} 176 top_slowest_tests{project="candlepin/subscription-manager-cockpit",test="/test/check-subscriptions TestSubscriptions.testInsights"} 174 top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-system-realms TestAD.testUnqualifiedUsers"} 174 top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-create TestMachinesCreate.testCreateUrlSource"} 173 top_slowest_tests{project="rhinstaller/anaconda",test="/ui/webui/test/check-progress TestInstallationProgress.testBasic"} 172 top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-settings TestMachinesSettings.testAutostart"} 171 top_slowest_tests{project="candlepin/subscription-manager/subscription-manager-1.28.21",test="integration-tests/check-subscriptions TestSubscriptions.testInsights"} 170 top_slowest_tests{project="candlepin/subscription-manager/subscription-manager-1.28.29",test="integration-tests/check-subscriptions TestSubscriptions.testSubAndInAndFail"} 168 top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-nics TestMachinesNICs.testNICDelete"} 168 top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-kdump TestKdumpNFS.testBasic"} 166 top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testDetachDisk"} 160 top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-packagekit TestUpdates.testTracer"} 159 top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-system-info TestSystemInfo.testBasic"} 159 top_slowest_tests{project="candlepin/subscription-manager-cockpit",test="/test/check-subscriptions TestSubscriptionsPackages.testMissingPackages"} 157 top_slowest_tests{project="rhinstaller/anaconda",test="/ui/webui/test/check-basic TestQuit.testQuitInstallation"} 152 top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-consoles TestMachinesConsoles.testInlineConsole"} 148 top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-consoles TestMachinesConsoles.testInlineConsoleWithUrlRoot"} 148 top_slowest_tests{project="cockpit-project/cockpit/rhel-9.0",test="test/verify/check-metrics TestHistoryMetrics.testPmProxySettings"} 148 top_slowest_tests{project="rhinstaller/anaconda",test="/ui/webui/test/check-storage TestStorageExtraDisks.testLocalDisksSyncNew"} 146 top_slowest_tests{project="cockpit-project/cockpit-podman",test="/test/check-application TestApplication.testRunImageSystem"} 143 top_slowest_tests{project="cockpit-project/cockpit-podman",test="/test/check-application TestApplication.testNotRunning"} 138 top_slowest_tests{project="cockpit-project/cockpit-ostree",test="/test/check-ostree OstreeRestartCase.testRebase"} 134 top_slowest_tests{project="candlepin/subscription-manager",test="/integration-tests/submancockpit/test/check-subscriptions TestSubscriptionsPackages.testMissingPackages"} 132 top_slowest_tests{project="candlepin/subscription-manager",test="/integration-tests/submancockpit/test/check-subscriptions TestSubscriptions.testRegisterWithKey"} 130 top_slowest_tests{project="candlepin/subscription-manager",test="/integration-tests/submancockpit/test/check-subscriptions TestSubscriptions.testRegisterWithoutAutoAttach"} 129 top_slowest_tests{project="candlepin/subscription-manager",test="/integration-tests/submancockpit/test/check-subscriptions TestSubscriptions.testRegister"} 128 top_slowest_tests{project="cockpit-project/cockpit-podman",test="/test/check-application TestApplication.testRunImageUser"} 124 top_slowest_tests{project="cockpit-project/cockpit-podman",test="/test/check-application TestApplication.testCheckpointRestoreCGroupsV1"} 122 top_slowest_tests{project="cockpit-project/cockpit-podman",test="/test/check-application TestApplication.testHealthcheckSystem"} 119 top_slowest_tests{project="cockpit-project/cockpit-podman",test="/test/check-application TestApplication.testBasicSystem"} 115 top_slowest_tests{project="candlepin/subscription-manager",test="integration-tests/check-subscriptions TestSubscriptionsPackages.testMissingPackages"} 113 top_slowest_tests{project="candlepin/subscription-manager",test="integration-tests/check-subscriptions TestSubscriptions.testRegisterWithKey"} 112 top_slowest_tests{project="osbuild/cockpit-composer",test="test/verify/check-package TestPackage.testRedoUndoCommit"} 111 top_slowest_tests{project="candlepin/subscription-manager",test="integration-tests/check-subscriptions TestSubscriptions.testRegisterWithoutAutoAttach"} 109 top_slowest_tests{project="candlepin/subscription-manager-cockpit",test="/test/check-subscriptions TestSubscriptions.testRegisterWithKey"} 106 # HELP s3_usage_bytes How much disk space an S3 store is using # TYPE s3_usage_bytes gauge s3_usage_bytes{store="https://cockpit-images.eu-central-1.linodeobjects.com/"} 101983358464 s3_usage_bytes{store="https://cockpit-images.us-east-1.linodeobjects.com/"} 101983358464