From 0d9065e7d7550604b3c6108c3a1b55ef564d9a37 Mon Sep 17 00:00:00 2001 From: HusneShabbir Date: Tue, 16 Dec 2025 20:42:11 +0530 Subject: [PATCH 1/3] update scorecard e2e --- .../e2e/plugins/scorecard/scorecard.spec.ts | 118 ++++++---- .../scorecard/component-import-page.ts | 21 +- .../page-objects/scorecard/scorecard-page.ts | 21 +- .../utils/scorecard-response-utils.ts | 203 ------------------ e2e-tests/playwright/utils/scorecard-utils.ts | 49 ----- 5 files changed, 96 insertions(+), 316 deletions(-) delete mode 100644 e2e-tests/playwright/utils/scorecard-response-utils.ts delete mode 100644 e2e-tests/playwright/utils/scorecard-utils.ts diff --git a/e2e-tests/playwright/e2e/plugins/scorecard/scorecard.spec.ts b/e2e-tests/playwright/e2e/plugins/scorecard/scorecard.spec.ts index 4fe4f7ba5b..8f6bf305a5 100644 --- a/e2e-tests/playwright/e2e/plugins/scorecard/scorecard.spec.ts +++ b/e2e-tests/playwright/e2e/plugins/scorecard/scorecard.spec.ts @@ -16,16 +16,9 @@ import { test, expect } from "@playwright/test"; import { Common } from "../../../utils/common"; -import { mockScorecardResponse } from "../../../utils/scorecard-utils"; import { ComponentImportPage } from "../../../support/page-objects/scorecard/component-import-page"; import { Catalog } from "../../../support/pages/catalog"; import { ScorecardPage } from "../../../support/page-objects/scorecard/scorecard-page"; -import { - CUSTOM_SCORECARD_RESPONSE, - EMPTY_SCORECARD_RESPONSE, - UNAVAILABLE_METRIC_RESPONSE, - INVALID_THRESHOLD_RESPONSE, -} from "../../../utils/scorecard-response-utils"; test.describe.serial("Scorecard Plugin Tests", () => { let context; @@ -53,42 +46,33 @@ test.describe.serial("Scorecard Plugin Tests", () => { }); test("Import component and validate scorecard tabs for GitHub PRs and Jira tickets", async () => { - await mockScorecardResponse(page, CUSTOM_SCORECARD_RESPONSE); - - await catalog.go(); - await importPage.startComponentImport(); - await importPage.analyzeComponent( - "https://github.com/rhdh-pai-qe/backstage-catalog/blob/main/catalog-info.yaml", + await importPage.importAndOpenScorecard( + "https://github.com/rhdh-pai-qe/RHDH-scorecard-plugin-test/blob/main/all-scorecards.yaml", + catalog, + scorecardPage, ); - await importPage.viewImportedComponent(); - await scorecardPage.openTab(); - - await scorecardPage.verifyScorecardValues({ - "GitHub open PRs": "9", - "Jira open blocking tickets": "8", - }); for (const metric of scorecardPage.scorecardMetrics) { await scorecardPage.validateScorecardAriaFor(metric); } }); - test("Display empty state when scorecard API returns no metrics", async () => { - await mockScorecardResponse(page, EMPTY_SCORECARD_RESPONSE); - - await catalog.go(); - await catalog.goToByName("rhdh-app"); - await scorecardPage.openTab(); + test("Validate empty scorecard state", async () => { + await importPage.importAndOpenScorecard( + "https://github.com/rhdh-pai-qe/RHDH-scorecard-plugin-test/blob/main/no-scorecards.yaml", + catalog, + scorecardPage, + ); await scorecardPage.expectEmptyState(); }); test("Displays error state for unavailable data while rendering metrics", async () => { - await mockScorecardResponse(page, UNAVAILABLE_METRIC_RESPONSE); - - await catalog.go(); - await catalog.goToByName("rhdh-app"); - await scorecardPage.openTab(); + await importPage.importAndOpenScorecard( + "https://github.com/rhdh-pai-qe/RHDH-scorecard-plugin-test/blob/main/metrics-unavailable.yaml", + catalog, + scorecardPage, + ); const jiraMetric = scorecardPage.scorecardMetrics[1]; const githubMetric = scorecardPage.scorecardMetrics[0]; @@ -109,22 +93,19 @@ test.describe.serial("Scorecard Plugin Tests", () => { await expect(errorLocator).toBeVisible(); await errorLocator.hover(); - const errorTooltip = UNAVAILABLE_METRIC_RESPONSE.find( - (metric) => metric.id === "github.open-prs", - )?.error; - - expect(errorTooltip).toBeTruthy(); - await expect(page.getByText(errorTooltip!)).toBeVisible(); + const errorTooltip = + "GraphqlResponseError: Request failed due to following response errors: - Could not resolve to a Repository with the name 'dzemanov/react-app-t1'."; + await expect(page.getByText(errorTooltip)).toBeVisible(); await scorecardPage.validateScorecardAriaFor(jiraMetric); }); test("Display error state for invalid threshold config while rendering metrics", async () => { - await mockScorecardResponse(page, INVALID_THRESHOLD_RESPONSE); - - await catalog.go(); - await catalog.goToByName("rhdh-app"); - await scorecardPage.openTab(); + await importPage.importAndOpenScorecard( + "https://github.com/rhdh-pai-qe/RHDH-scorecard-plugin-test/blob/main/invalid-threshold.yaml", + catalog, + scorecardPage, + ); const githubMetric = scorecardPage.scorecardMetrics[0]; const jiraMetric = scorecardPage.scorecardMetrics[1]; @@ -145,12 +126,55 @@ test.describe.serial("Scorecard Plugin Tests", () => { await expect(errorLocator).toBeVisible(); await errorLocator.hover(); - const errorTooltip = INVALID_THRESHOLD_RESPONSE.find( - (metric) => metric.id === "github.open-prs", - )?.result?.thresholdResult?.error; + const errorTooltip = + "ThresholdConfigFormatError: Invalid threshold annotation 'scorecard.io/github.open_prs.thresholds.rules.error: >50d' in entity 'component:default/invalid-threshold': Cannot parse \"50d\" as number from expression: \">50d\""; + await expect(page.getByText(errorTooltip)).toBeVisible(); + + await scorecardPage.validateScorecardAriaFor(jiraMetric); + }); + + test("Validate only GitHub scorecard is displayed", async () => { + await importPage.importAndOpenScorecard( + "https://github.com/rhdh-pai-qe/RHDH-scorecard-plugin-test/blob/main/github-scorecard-only.yaml", + catalog, + scorecardPage, + ); + + const githubMetric = scorecardPage.scorecardMetrics[0]; + const jiraMetric = scorecardPage.scorecardMetrics[1]; + + const isGithubVisible = await scorecardPage.isScorecardVisible( + githubMetric.title, + ); + expect(isGithubVisible).toBe(true); + + const isJiraVisible = await scorecardPage.isScorecardVisible( + jiraMetric.title, + ); + expect(isJiraVisible).toBe(false); - expect(errorTooltip).toBeTruthy(); - await expect(page.getByText(errorTooltip!)).toBeVisible(); + await scorecardPage.validateScorecardAriaFor(githubMetric); + }); + + test("Validate only Jira scorecard is displayed", async () => { + await importPage.importAndOpenScorecard( + "https://github.com/rhdh-pai-qe/RHDH-scorecard-plugin-test/blob/main/jira-scorecard-only.yaml", + catalog, + scorecardPage, + ); + + const githubMetric = scorecardPage.scorecardMetrics[0]; + const jiraMetric = scorecardPage.scorecardMetrics[1]; + + const isGithubVisible = await scorecardPage.isScorecardVisible( + githubMetric.title, + ); + expect(isGithubVisible).toBe(false); + + const isJiraVisible = await scorecardPage.isScorecardVisible( + jiraMetric.title, + ); + expect(isJiraVisible).toBe(true); await scorecardPage.validateScorecardAriaFor(jiraMetric); }); diff --git a/e2e-tests/playwright/support/page-objects/scorecard/component-import-page.ts b/e2e-tests/playwright/support/page-objects/scorecard/component-import-page.ts index 6a9f7deee7..483967b2ae 100644 --- a/e2e-tests/playwright/support/page-objects/scorecard/component-import-page.ts +++ b/e2e-tests/playwright/support/page-objects/scorecard/component-import-page.ts @@ -15,6 +15,8 @@ */ import { Page } from "@playwright/test"; import { UIhelper } from "../../../utils/ui-helper"; +import { Catalog } from "../../../support/pages/catalog"; +import { ScorecardPage } from "./scorecard-page"; export class ComponentImportPage { readonly page: Page; @@ -34,12 +36,29 @@ export class ComponentImportPage { await this.uiHelper.fillTextInputByLabel("URL", url); await this.uiHelper.clickButton("Analyze"); await this.uiHelper.clickButton("Import"); - //wait for few seconds await this.page.waitForTimeout(5000); } async viewImportedComponent() { await this.uiHelper.clickButton("View Component"); + const entityNotFoundLocator = this.page.getByRole("button", { + name: "Warning: Entity not found", + }); + if (await entityNotFoundLocator.isVisible({ timeout: 10000 })) { + await this.page.reload(); + } await this.uiHelper.verifyText("Overview"); } + + async importAndOpenScorecard( + url: string, + catalog: Catalog, + scorecardPage: ScorecardPage, + ) { + await catalog.go(); + await this.startComponentImport(); + await this.analyzeComponent(url); + await this.viewImportedComponent(); + await scorecardPage.openTab(); + } } diff --git a/e2e-tests/playwright/support/page-objects/scorecard/scorecard-page.ts b/e2e-tests/playwright/support/page-objects/scorecard/scorecard-page.ts index f414ea21c3..7298e5ae38 100644 --- a/e2e-tests/playwright/support/page-objects/scorecard/scorecard-page.ts +++ b/e2e-tests/playwright/support/page-objects/scorecard/scorecard-page.ts @@ -15,7 +15,6 @@ */ import { Page, expect } from "@playwright/test"; -import { waitUntilApiCallSucceeds } from "../../../utils/scorecard-utils"; export class ScorecardPage { readonly page: Page; @@ -34,25 +33,15 @@ export class ScorecardPage { { title: "Jira open blocking tickets", description: - "Highlights the number of critical, blocking issues that are currently open in Jira.", + "Highlights the number of issues that are currently open in Jira.", }, ]; } async openTab() { - const scorecardTab = this.page.getByText("Scorecard"); + const scorecardTab = this.page.getByText("Scorecard", { exact: true }); await expect(scorecardTab).toBeVisible(); - await Promise.all([ - waitUntilApiCallSucceeds(this.page), - scorecardTab.click(), - ]); - } - - async verifyScorecardValues(expectedValues: { [key: string]: string }) { - for (const [metric, value] of Object.entries(expectedValues)) { - await expect(this.page.getByText(metric)).toBeVisible(); - await expect(this.page.getByText(value)).toBeVisible(); - } + await scorecardTab.click(); } async expectEmptyState() { @@ -79,9 +68,9 @@ export class ScorecardPage { - article: - text: ${title} - paragraph: ${description} - - paragraph: /Error/ - - paragraph: /Warning/ - paragraph: /Success/ + - paragraph: /Warning/ + - paragraph: /Error/ `); } diff --git a/e2e-tests/playwright/utils/scorecard-response-utils.ts b/e2e-tests/playwright/utils/scorecard-response-utils.ts deleted file mode 100644 index b423441a5e..0000000000 --- a/e2e-tests/playwright/utils/scorecard-response-utils.ts +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Copyright Red Hat, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -export const CUSTOM_SCORECARD_RESPONSE = [ - { - id: "github.open-prs", - status: "success", - metadata: { - title: "Github open PRs", - description: - "Current count of open Pull Requests for a given GitHub repository.", - type: "number", - history: true, - }, - result: { - value: 9, - timestamp: "2025-09-08T09:08:55.629Z", - thresholdResult: { - definition: { - rules: [ - { key: "error", expression: ">=200" }, - { key: "warning", expression: "10-200" }, - { key: "success", expression: "<10" }, - ], - }, - status: "success", - evaluation: "success", - }, - }, - }, - { - id: "jira.open-issues", - status: "success", - metadata: { - title: "Jira open blocking tickets", - description: - "Highlights the number of critical, blocking issues that are currently open in Jira.", - type: "number", - history: true, - }, - result: { - value: 8, - timestamp: "2025-09-08T09:08:55.629Z", - thresholdResult: { - definition: { - rules: [ - { key: "error", expression: ">=50" }, - { key: "warning", expression: "10-50" }, - { key: "success", expression: "<10" }, - ], - }, - status: "success", - evaluation: "success", - }, - }, - }, -]; - -export const EMPTY_SCORECARD_RESPONSE = []; - -export const UNAVAILABLE_METRIC_RESPONSE = [ - { - id: "jira.open-issues", - status: "success", - metadata: { - title: "Jira open blocking tickets", - description: - "Highlights the number of critical, blocking issues that are currently open in Jira.", - type: "number", - history: true, - }, - result: { - value: 54, - timestamp: "2025-09-12T15:28:56.898Z", - thresholdResult: { - definition: { - rules: [ - { - key: "error", - expression: ">50", - }, - { - key: "warning", - expression: "10-50", - }, - { - key: "success", - expression: "<10", - }, - ], - }, - status: "success", - evaluation: "error", - }, - }, - }, - { - id: "github.open-prs", - status: "error", - metadata: { - title: "Github open PRs", - description: - "Current count of open Pull Requests for a given GitHub repository.", - type: "number", - history: true, - }, - error: - "HttpError: API rate limit exceeded for 157.50.94.55. (But here's the good news: Authenticated requests get a higher rate limit. Check out the documentation for more details.) - https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting", - result: { - timestamp: "2025-09-23T12:17:02.496Z", - thresholdResult: { - definition: { - rules: [ - { - key: "success", - expression: "<10", - }, - { - key: "warning", - expression: "10-50", - }, - { - key: "error", - expression: ">50", - }, - ], - }, - status: "error", - error: "Unable to evaluate thresholds, metric value is missing", - }, - }, - }, -]; - -export const INVALID_THRESHOLD_RESPONSE = [ - { - id: "jira.open-issues", - status: "success", - metadata: { - title: "Jira open blocking tickets", - description: - "Highlights the number of critical, blocking issues that are currently open in Jira.", - type: "number", - history: true, - }, - result: { - value: 54, - timestamp: "2025-09-12T15:28:56.898Z", - thresholdResult: { - definition: { - rules: [ - { - key: "error", - expression: ">50", - }, - { - key: "warning", - expression: "10-50", - }, - { - key: "success", - expression: "<10", - }, - ], - }, - status: "success", - evaluation: "error", - }, - }, - }, - { - id: "github.open-prs", - status: "success", - metadata: { - title: "Github open PRs", - description: - "Current count of open Pull Requests for a given GitHub repository.", - type: "number", - history: true, - }, - result: { - value: 40, - timestamp: "2025-09-23T12:27:04.531Z", - thresholdResult: { - status: "error", - error: - "ThresholdConfigFormatError: Invalid threshold annotation 'scorecard.io/github.open-prs.thresholds.rules.warning: 10--15' in entity 'component:default/all-scorecards-service': Invalid threshold expression: \"10--15\".", - }, - }, - }, -]; diff --git a/e2e-tests/playwright/utils/scorecard-utils.ts b/e2e-tests/playwright/utils/scorecard-utils.ts deleted file mode 100644 index 939423ae3b..0000000000 --- a/e2e-tests/playwright/utils/scorecard-utils.ts +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Red Hat, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { Page, expect } from "@playwright/test"; - -export async function waitUntilApiCallSucceeds( - page: Page, - urlPart: string = "/api/scorecard/metrics/catalog/Component/default/rhdh-app", -): Promise { - const response = await page.waitForResponse( - async (res) => { - const urlMatches = res.url().includes(urlPart); - const isSuccess = res.status() === 200; - return urlMatches && isSuccess; - }, - { timeout: 60000 }, - ); - - expect(response.status()).toBe(200); -} - -const scorecardApiRoute = - "**/api/scorecard/metrics/catalog/Component/default/rhdh-app"; - -export async function mockScorecardResponse( - page: Page, - responseData: object, - status = 200, -) { - await page.route(scorecardApiRoute, async (route) => { - await route.fulfill({ - status, - contentType: "application/json", - body: JSON.stringify(responseData), - }); - }); -} From f6707165e38335968fbb65a31f798b7184e91b87 Mon Sep 17 00:00:00 2001 From: HusneShabbir Date: Tue, 16 Dec 2025 20:45:41 +0530 Subject: [PATCH 2/3] modify scorecard config --- .ibm/pipelines/env_variables.sh | 1 + .../value_files/values_showcase-rbac.yaml | 26 ++++++++++--------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/.ibm/pipelines/env_variables.sh b/.ibm/pipelines/env_variables.sh index 814d1b9514..7822bf8a4b 100755 --- a/.ibm/pipelines/env_variables.sh +++ b/.ibm/pipelines/env_variables.sh @@ -90,6 +90,7 @@ QE_USER8_ID=$(cat /tmp/secrets/QE_USER8_ID) QE_USER8_PASS=$(cat /tmp/secrets/QE_USER8_PASS) QE_USER9_ID=$(cat /tmp/secrets/QE_USER9_ID) QE_USER9_PASS=$(cat /tmp/secrets/QE_USER9_PASS) +JIRA_TOKEN=$(cat /tmp/secrets/jira_token) K8S_CLUSTER_TOKEN_TEMPORARY=$(cat /tmp/secrets/K8S_CLUSTER_TOKEN_TEMPORARY) diff --git a/.ibm/pipelines/value_files/values_showcase-rbac.yaml b/.ibm/pipelines/value_files/values_showcase-rbac.yaml index 0f77eb6fbf..c1fd8e5451 100644 --- a/.ibm/pipelines/value_files/values_showcase-rbac.yaml +++ b/.ibm/pipelines/value_files/values_showcase-rbac.yaml @@ -94,8 +94,8 @@ global: - package: ./dynamic-plugins/dist/backstage-community-plugin-analytics-provider-segment disabled: true #Enable Scorecard plugin. - - disabled: false - package: oci://ghcr.io/redhat-developer/rhdh-plugin-export-overlays/red-hat-developer-hub-backstage-plugin-scorecard:pr_1499__0.1.0!red-hat-developer-hub-backstage-plugin-scorecard + - package: oci://ghcr.io/redhat-developer/rhdh-plugin-export-overlays/red-hat-developer-hub-backstage-plugin-scorecard:bs_1.42.5__1.0.0!red-hat-developer-hub-backstage-plugin-scorecard + disabled: false pluginConfig: dynamicPlugins: frontend: @@ -103,7 +103,6 @@ global: entityTabs: - path: "/scorecard" title: Scorecard - titleKey: catalog.entityPage.scorecard.title mountPoint: entity.page.scorecard mountPoints: - mountPoint: entity.page.scorecard/cards @@ -111,22 +110,25 @@ global: config: layout: gridColumn: 1 / -1 - - disabled: false - package: oci://ghcr.io/redhat-developer/rhdh-plugin-export-overlays/red-hat-developer-hub-backstage-plugin-scorecard-backend:pr_1499__0.1.0!red-hat-developer-hub-backstage-plugin-scorecard-backend - - disabled: false - package: oci://ghcr.io/redhat-developer/rhdh-plugin-export-overlays/red-hat-developer-hub-backstage-plugin-scorecard-backend-module-github:pr_1499__0.1.0!red-hat-developer-hub-backstage-plugin-scorecard-backend-module-github + if: + allOf: + - isKind: component + - package: oci://ghcr.io/redhat-developer/rhdh-plugin-export-overlays/red-hat-developer-hub-backstage-plugin-scorecard-backend:bs_1.42.5__1.0.0!red-hat-developer-hub-backstage-plugin-scorecard-backend + disabled: false + - package: oci://ghcr.io/redhat-developer/rhdh-plugin-export-overlays/red-hat-developer-hub-backstage-plugin-scorecard-backend-module-github:bs_1.42.5__1.0.0!red-hat-developer-hub-backstage-plugin-scorecard-backend-module-github pluginConfig: integrations: github: - host: github.com token: "{gh-token}" - - disabled: false - package: oci://ghcr.io/redhat-developer/rhdh-plugin-export-overlays/red-hat-developer-hub-backstage-plugin-scorecard-backend-module-jira:pr_1499__0.1.0!red-hat-developer-hub-backstage-plugin-scorecard-backend-module-jira + disabled: false + - package: oci://ghcr.io/redhat-developer/rhdh-plugin-export-overlays/red-hat-developer-hub-backstage-plugin-scorecard-backend-module-jira:pr_1499__0.1.0!red-hat-developer-hub-backstage-plugin-scorecard-backend-module-jira pluginConfig: jira: - baseUrl: "{jira-base-url}" - token: "{jira-api-token}" - product: datacenter + baseUrl: https://redhat-team-f703ynt3.atlassian.net + token: ${JIRA_TOKEN} + product: cloud + disabled: false # Enable orchestrator plugins - Official release (Backstage 1.42.5) - package: "oci://ghcr.io/redhat-developer/rhdh-plugin-export-overlays/red-hat-developer-hub-backstage-plugin-orchestrator:bs_1.42.5__5.1.0!red-hat-developer-hub-backstage-plugin-orchestrator" disabled: false From 93284befc0a54e1286aacacbcfb79afed5d062ed Mon Sep 17 00:00:00 2001 From: HusneShabbir Date: Thu, 18 Dec 2025 17:35:40 +0530 Subject: [PATCH 3/3] add secrets --- .ibm/pipelines/auth/secrets-rhdh-secrets.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.ibm/pipelines/auth/secrets-rhdh-secrets.yaml b/.ibm/pipelines/auth/secrets-rhdh-secrets.yaml index 5370a758b7..364b4133d1 100644 --- a/.ibm/pipelines/auth/secrets-rhdh-secrets.yaml +++ b/.ibm/pipelines/auth/secrets-rhdh-secrets.yaml @@ -42,4 +42,5 @@ data: GITHUB_OAUTH_APP_ID: $GITHUB_OAUTH_APP_ID_ENCODED GITHUB_OAUTH_APP_SECRET: $GITHUB_OAUTH_APP_SECRET_ENCODED BACKEND_SECRET: $BACKEND_SECRET + JIRA_TOKEN: $JIRA_TOKEN type: Opaque