diff --git a/backend/plugins/gh-copilot/models/enterprise_metrics.go b/backend/plugins/gh-copilot/models/enterprise_metrics.go index 07663aa6dd5..e89a7c02429 100644 --- a/backend/plugins/gh-copilot/models/enterprise_metrics.go +++ b/backend/plugins/gh-copilot/models/enterprise_metrics.go @@ -44,6 +44,15 @@ type CopilotCodeMetrics struct { LocDeletedSum int `json:"locDeletedSum"` } +// CopilotCliMetrics contains CLI usage breakdown metrics. +type CopilotCliMetrics struct { + CliSessionCount int `json:"cliSessionCount" gorm:"comment:Number of CLI sessions"` + CliRequestCount int `json:"cliRequestCount" gorm:"comment:Number of CLI requests"` + CliPromptCount int `json:"cliPromptCount" gorm:"comment:Number of CLI prompts"` + CliOutputTokenSum int `json:"cliOutputTokenSum" gorm:"comment:Total output tokens from CLI"` + CliPromptTokenSum int `json:"cliPromptTokenSum" gorm:"comment:Total prompt tokens from CLI"` +} + // GhCopilotEnterpriseDailyMetrics captures daily enterprise-level aggregate Copilot metrics. type GhCopilotEnterpriseDailyMetrics struct { ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` @@ -57,12 +66,43 @@ type GhCopilotEnterpriseDailyMetrics struct { MonthlyActiveChatUsers int `json:"monthlyActiveChatUsers"` MonthlyActiveAgentUsers int `json:"monthlyActiveAgentUsers"` - PRTotalReviewed int `json:"prTotalReviewed" gorm:"comment:Total PRs reviewed"` - PRTotalCreated int `json:"prTotalCreated" gorm:"comment:Total PRs created"` - PRTotalCreatedByCopilot int `json:"prTotalCreatedByCopilot" gorm:"comment:PRs created by Copilot"` - PRTotalReviewedByCopilot int `json:"prTotalReviewedByCopilot" gorm:"comment:PRs reviewed by Copilot"` + // CLI active users + DailyActiveCliUsers int `json:"dailyActiveCliUsers" gorm:"comment:Daily active CLI users"` + + // Code review user counts + DailyActiveCopilotCodeReviewUsers int `json:"dailyActiveCopilotCodeReviewUsers"` + DailyPassiveCopilotCodeReviewUsers int `json:"dailyPassiveCopilotCodeReviewUsers"` + WeeklyActiveCopilotCodeReviewUsers int `json:"weeklyActiveCopilotCodeReviewUsers"` + WeeklyPassiveCopilotCodeReviewUsers int `json:"weeklyPassiveCopilotCodeReviewUsers"` + MonthlyActiveCopilotCodeReviewUsers int `json:"monthlyActiveCopilotCodeReviewUsers"` + MonthlyPassiveCopilotCodeReviewUsers int `json:"monthlyPassiveCopilotCodeReviewUsers"` + + // Chat panel mode breakdown + ChatPanelAgentMode int `json:"chatPanelAgentMode" gorm:"comment:Chat panel agent mode interactions"` + ChatPanelAskMode int `json:"chatPanelAskMode" gorm:"comment:Chat panel ask mode interactions"` + ChatPanelCustomMode int `json:"chatPanelCustomMode" gorm:"comment:Chat panel custom mode interactions"` + ChatPanelEditMode int `json:"chatPanelEditMode" gorm:"comment:Chat panel edit mode interactions"` + ChatPanelPlanMode int `json:"chatPanelPlanMode" gorm:"comment:Chat panel plan mode interactions"` + ChatPanelUnknownMode int `json:"chatPanelUnknownMode" gorm:"comment:Chat panel unknown mode interactions"` + + // Pull request metrics (expanded) + PRTotalReviewed int `json:"prTotalReviewed" gorm:"comment:Total PRs reviewed"` + PRTotalCreated int `json:"prTotalCreated" gorm:"comment:Total PRs created"` + PRTotalMerged int `json:"prTotalMerged" gorm:"comment:Total PRs merged"` + PRMedianMinutesToMerge float64 `json:"prMedianMinutesToMerge" gorm:"comment:Median minutes to merge PRs"` + PRTotalSuggestions int `json:"prTotalSuggestions" gorm:"comment:Total PR review suggestions"` + PRTotalAppliedSuggestions int `json:"prTotalAppliedSuggestions" gorm:"comment:Total applied PR suggestions"` + PRTotalCreatedByCopilot int `json:"prTotalCreatedByCopilot" gorm:"comment:PRs created by Copilot"` + PRTotalReviewedByCopilot int `json:"prTotalReviewedByCopilot" gorm:"comment:PRs reviewed by Copilot"` + PRTotalMergedCreatedByCopilot int `json:"prTotalMergedCreatedByCopilot" gorm:"comment:Merged PRs created by Copilot"` + PRTotalMergedReviewedByCopilot int `json:"prTotalMergedReviewedByCopilot" gorm:"comment:Merged PRs reviewed by Copilot"` + PRMedianMinToMergeCopilotAuthored float64 `json:"prMedianMinToMergeCopilotAuthored" gorm:"comment:Median min to merge Copilot-authored PRs"` + PRMedianMinToMergeCopilotReviewed float64 `json:"prMedianMinToMergeCopilotReviewed" gorm:"comment:Median min to merge Copilot-reviewed PRs"` + PRTotalCopilotSuggestions int `json:"prTotalCopilotSuggestions" gorm:"comment:Total Copilot review suggestions"` + PRTotalCopilotAppliedSuggestions int `json:"prTotalCopilotAppliedSuggestions" gorm:"comment:Total Copilot applied suggestions"` CopilotActivityMetrics `mapstructure:",squash"` + CopilotCliMetrics `mapstructure:",squash"` common.NoPKModel } diff --git a/backend/plugins/gh-copilot/models/migrationscripts/20260527_add_copilot_metrics_gaps.go b/backend/plugins/gh-copilot/models/migrationscripts/20260527_add_copilot_metrics_gaps.go new file mode 100644 index 00000000000..f5334e39c2a --- /dev/null +++ b/backend/plugins/gh-copilot/models/migrationscripts/20260527_add_copilot_metrics_gaps.go @@ -0,0 +1,153 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type addCopilotMetricsGaps struct{} + +// --- Enterprise daily metrics: new columns --- + +type enterpriseDailyMetrics20260527 struct { + // CLI + DailyActiveCliUsers int + + // Code review user counts + DailyActiveCopilotCodeReviewUsers int + DailyPassiveCopilotCodeReviewUsers int + WeeklyActiveCopilotCodeReviewUsers int + WeeklyPassiveCopilotCodeReviewUsers int + MonthlyActiveCopilotCodeReviewUsers int + MonthlyPassiveCopilotCodeReviewUsers int + + // Chat panel mode breakdown + ChatPanelAgentMode int + ChatPanelAskMode int + ChatPanelCustomMode int + ChatPanelEditMode int + ChatPanelPlanMode int + ChatPanelUnknownMode int + + // Expanded PR metrics + PRTotalMerged int + PRMedianMinutesToMerge float64 + PRTotalSuggestions int + PRTotalAppliedSuggestions int + PRTotalMergedCreatedByCopilot int + PRTotalMergedReviewedByCopilot int + PRMedianMinToMergeCopilotAuthored float64 + PRMedianMinToMergeCopilotReviewed float64 + PRTotalCopilotSuggestions int + PRTotalCopilotAppliedSuggestions int + + // CLI breakdown + CliSessionCount int + CliRequestCount int + CliPromptCount int + CliOutputTokenSum int + CliPromptTokenSum int +} + +func (enterpriseDailyMetrics20260527) TableName() string { + return "_tool_copilot_enterprise_daily_metrics" +} + +// --- User daily metrics: new columns --- + +type userDailyMetrics20260527 struct { + UsedCli bool + UsedCopilotCodeReviewActive bool + UsedCopilotCodeReviewPassive bool + + // CLI breakdown + CliSessionCount int + CliRequestCount int + CliPromptCount int + CliOutputTokenSum int + CliPromptTokenSum int +} + +func (userDailyMetrics20260527) TableName() string { + return "_tool_copilot_user_daily_metrics" +} + +// --- Seat: new columns --- + +type seat20260527 struct { + UserName string `gorm:"type:varchar(255)"` + UserEmail string `gorm:"type:varchar(255)"` + AssigningTeamId int64 + AssigningTeamName string `gorm:"type:varchar(255)"` + AssigningTeamSlug string `gorm:"type:varchar(255)"` +} + +func (seat20260527) TableName() string { + return "_tool_copilot_seats" +} + +// --- User-teams: new table --- + +type userTeam20260527 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + UserId int64 `gorm:"primaryKey"` + TeamId int64 `gorm:"primaryKey"` + + UserLogin string `gorm:"type:varchar(255);index"` + OrganizationId string `gorm:"type:varchar(100)"` + EnterpriseId string `gorm:"type:varchar(100)"` + TeamSlug string `gorm:"type:varchar(255)"` + + archived.NoPKModel +} + +func (userTeam20260527) TableName() string { + return "_tool_copilot_user_teams" +} + +func (script *addCopilotMetricsGaps) Up(basicRes context.BasicRes) errors.Error { + // Add new columns to existing tables + if err := migrationhelper.AutoMigrateTables(basicRes, + &enterpriseDailyMetrics20260527{}, + &userDailyMetrics20260527{}, + &seat20260527{}, + ); err != nil { + return err + } + + // Create new user-teams table + return migrationhelper.AutoMigrateTables(basicRes, + &userTeam20260527{}, + ) +} + +func (*addCopilotMetricsGaps) Version() uint64 { + return 20260527000000 +} + +func (*addCopilotMetricsGaps) Name() string { + return "Add Copilot metrics gaps: CLI, code review, chat modes, PR expansion, user-teams" +} diff --git a/backend/plugins/gh-copilot/models/migrationscripts/register.go b/backend/plugins/gh-copilot/models/migrationscripts/register.go index a9c1a770bfa..399735695e0 100644 --- a/backend/plugins/gh-copilot/models/migrationscripts/register.go +++ b/backend/plugins/gh-copilot/models/migrationscripts/register.go @@ -30,5 +30,6 @@ func All() []plugin.MigrationScript { new(migrateToUsageMetricsV2), new(addPRFieldsToEnterpriseMetrics), new(addOrganizationIdToUserMetrics), + new(addCopilotMetricsGaps), } } diff --git a/backend/plugins/gh-copilot/models/models.go b/backend/plugins/gh-copilot/models/models.go index f223c821827..5143ce5f8b7 100644 --- a/backend/plugins/gh-copilot/models/models.go +++ b/backend/plugins/gh-copilot/models/models.go @@ -45,5 +45,7 @@ func GetTablesInfo() []dal.Tabler { &GhCopilotUserMetricsByModelFeature{}, // Seat assignments &GhCopilotSeat{}, + // User-team mappings + &GhCopilotUserTeam{}, } } diff --git a/backend/plugins/gh-copilot/models/models_test.go b/backend/plugins/gh-copilot/models/models_test.go index 72ead8a65e9..bf754d0afc3 100644 --- a/backend/plugins/gh-copilot/models/models_test.go +++ b/backend/plugins/gh-copilot/models/models_test.go @@ -40,6 +40,7 @@ func TestGetTablesInfo(t *testing.T) { (&GhCopilotUserMetricsByLanguageModel{}).TableName(): false, (&GhCopilotUserMetricsByModelFeature{}).TableName(): false, (&GhCopilotSeat{}).TableName(): false, + (&GhCopilotUserTeam{}).TableName(): false, } if len(tables) != len(expected) { diff --git a/backend/plugins/gh-copilot/models/seat.go b/backend/plugins/gh-copilot/models/seat.go index 85ebf177ae4..d65c80f2e30 100644 --- a/backend/plugins/gh-copilot/models/seat.go +++ b/backend/plugins/gh-copilot/models/seat.go @@ -29,7 +29,12 @@ type GhCopilotSeat struct { Organization string `gorm:"primaryKey;type:varchar(255)"` UserLogin string `gorm:"primaryKey;type:varchar(255)"` UserId int64 `gorm:"index"` + UserName string `gorm:"type:varchar(255)" json:"userName"` + UserEmail string `gorm:"type:varchar(255)" json:"userEmail"` PlanType string `gorm:"type:varchar(32)"` + AssigningTeamId int64 `json:"assigningTeamId" gorm:"comment:Team that assigned the seat"` + AssigningTeamName string `json:"assigningTeamName" gorm:"type:varchar(255)"` + AssigningTeamSlug string `json:"assigningTeamSlug" gorm:"type:varchar(255)"` CreatedAt time.Time LastActivityAt *time.Time LastActivityEditor string diff --git a/backend/plugins/gh-copilot/models/user_metrics.go b/backend/plugins/gh-copilot/models/user_metrics.go index 1f17acad80a..ef68e8ffcc6 100644 --- a/backend/plugins/gh-copilot/models/user_metrics.go +++ b/backend/plugins/gh-copilot/models/user_metrics.go @@ -35,8 +35,12 @@ type GhCopilotUserDailyMetrics struct { UserLogin string `json:"userLogin" gorm:"type:varchar(255);index"` UsedAgent bool `json:"usedAgent"` UsedChat bool `json:"usedChat"` + UsedCli bool `json:"usedCli" gorm:"comment:Whether user used Copilot CLI"` + UsedCopilotCodeReviewActive bool `json:"usedCopilotCodeReviewActive" gorm:"comment:Whether user actively used code review"` + UsedCopilotCodeReviewPassive bool `json:"usedCopilotCodeReviewPassive" gorm:"comment:Whether user passively used code review"` CopilotActivityMetrics `mapstructure:",squash"` + CopilotCliMetrics `mapstructure:",squash"` common.NoPKModel } diff --git a/backend/plugins/gh-copilot/models/user_team.go b/backend/plugins/gh-copilot/models/user_team.go new file mode 100644 index 00000000000..582e48fdf42 --- /dev/null +++ b/backend/plugins/gh-copilot/models/user_team.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// GhCopilotUserTeam maps users to teams per day from the user-teams-1-day report. +// This enables team-level metrics aggregation by joining with per-user daily metrics. +type GhCopilotUserTeam struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Day time.Time `gorm:"primaryKey;type:date" json:"day"` + UserId int64 `gorm:"primaryKey" json:"userId"` + TeamId int64 `gorm:"primaryKey" json:"teamId"` + + UserLogin string `json:"userLogin" gorm:"type:varchar(255);index"` + OrganizationId string `json:"organizationId" gorm:"type:varchar(100)"` + EnterpriseId string `json:"enterpriseId" gorm:"type:varchar(100)"` + TeamSlug string `json:"teamSlug" gorm:"type:varchar(255)"` + + common.NoPKModel +} + +func (GhCopilotUserTeam) TableName() string { + return "_tool_copilot_user_teams" +} diff --git a/backend/plugins/gh-copilot/tasks/enterprise_metrics_extractor.go b/backend/plugins/gh-copilot/tasks/enterprise_metrics_extractor.go index 8686b8cc415..a5535a2155a 100644 --- a/backend/plugins/gh-copilot/tasks/enterprise_metrics_extractor.go +++ b/backend/plugins/gh-copilot/tasks/enterprise_metrics_extractor.go @@ -37,6 +37,24 @@ type enterpriseDayTotal struct { MonthlyActiveUsers int `json:"monthly_active_users"` MonthlyActiveChatUsers int `json:"monthly_active_chat_users"` MonthlyActiveAgentUsers int `json:"monthly_active_agent_users"` + DailyActiveCliUsers int `json:"daily_active_cli_users"` + + // Code review user counts + DailyActiveCopilotCodeReviewUsers int `json:"daily_active_copilot_code_review_users"` + DailyPassiveCopilotCodeReviewUsers int `json:"daily_passive_copilot_code_review_users"` + WeeklyActiveCopilotCodeReviewUsers int `json:"weekly_active_copilot_code_review_users"` + WeeklyPassiveCopilotCodeReviewUsers int `json:"weekly_passive_copilot_code_review_users"` + MonthlyActiveCopilotCodeReviewUsers int `json:"monthly_active_copilot_code_review_users"` + MonthlyPassiveCopilotCodeReviewUsers int `json:"monthly_passive_copilot_code_review_users"` + + // Chat panel mode breakdown + ChatPanelAgentMode int `json:"chat_panel_agent_mode"` + ChatPanelAskMode int `json:"chat_panel_ask_mode"` + ChatPanelCustomMode int `json:"chat_panel_custom_mode"` + ChatPanelEditMode int `json:"chat_panel_edit_mode"` + ChatPanelPlanMode int `json:"chat_panel_plan_mode"` + ChatPanelUnknownMode int `json:"chat_panel_unknown_mode"` + UserInitiatedInteractionCount int `json:"user_initiated_interaction_count"` CodeGenerationActivityCount int `json:"code_generation_activity_count"` CodeAcceptanceActivityCount int `json:"code_acceptance_activity_count"` @@ -49,6 +67,7 @@ type enterpriseDayTotal struct { TotalsByLanguageFeature []totalsByLangFeature `json:"totals_by_language_feature"` TotalsByLanguageModel []totalsByLangModel `json:"totals_by_language_model"` TotalsByModelFeature []totalsByModelFeature `json:"totals_by_model_feature"` + TotalsByCli *totalsByCli `json:"totals_by_cli"` PullRequests *pullRequestStats `json:"pull_requests"` } @@ -97,10 +116,32 @@ type totalsByLangModel struct { } type pullRequestStats struct { - TotalReviewed int `json:"total_reviewed"` - TotalCreated int `json:"total_created"` - TotalCreatedByCopilot int `json:"total_created_by_copilot"` - TotalReviewedByCopilot int `json:"total_reviewed_by_copilot"` + TotalReviewed int `json:"total_reviewed"` + TotalCreated int `json:"total_created"` + TotalMerged int `json:"total_merged"` + MedianMinutesToMerge float64 `json:"median_minutes_to_merge"` + TotalSuggestions int `json:"total_suggestions"` + TotalAppliedSuggestions int `json:"total_applied_suggestions"` + TotalCreatedByCopilot int `json:"total_created_by_copilot"` + TotalReviewedByCopilot int `json:"total_reviewed_by_copilot"` + TotalMergedCreatedByCopilot int `json:"total_merged_created_by_copilot"` + TotalMergedReviewedByCopilot int `json:"total_merged_reviewed_by_copilot"` + MedianMinToMergeCopilotAuthored float64 `json:"median_minutes_to_merge_copilot_authored"` + MedianMinToMergeCopilotReviewed float64 `json:"median_minutes_to_merge_copilot_reviewed"` + TotalCopilotSuggestions int `json:"total_copilot_suggestions"` + TotalCopilotAppliedSuggestions int `json:"total_copilot_applied_suggestions"` +} + +type totalsByCli struct { + SessionCount int `json:"session_count"` + RequestCount int `json:"request_count"` + PromptCount int `json:"prompt_count"` + TokenUsage *cliTokens `json:"token_usage"` +} + +type cliTokens struct { + OutputTokensSum int `json:"output_tokens_sum"` + PromptTokensSum int `json:"prompt_tokens_sum"` } type totalsByModelFeature struct { @@ -167,6 +208,22 @@ func ExtractEnterpriseMetrics(taskCtx plugin.SubTaskContext) errors.Error { MonthlyActiveUsers: dt.MonthlyActiveUsers, MonthlyActiveChatUsers: dt.MonthlyActiveChatUsers, MonthlyActiveAgentUsers: dt.MonthlyActiveAgentUsers, + DailyActiveCliUsers: dt.DailyActiveCliUsers, + + DailyActiveCopilotCodeReviewUsers: dt.DailyActiveCopilotCodeReviewUsers, + DailyPassiveCopilotCodeReviewUsers: dt.DailyPassiveCopilotCodeReviewUsers, + WeeklyActiveCopilotCodeReviewUsers: dt.WeeklyActiveCopilotCodeReviewUsers, + WeeklyPassiveCopilotCodeReviewUsers: dt.WeeklyPassiveCopilotCodeReviewUsers, + MonthlyActiveCopilotCodeReviewUsers: dt.MonthlyActiveCopilotCodeReviewUsers, + MonthlyPassiveCopilotCodeReviewUsers: dt.MonthlyPassiveCopilotCodeReviewUsers, + + ChatPanelAgentMode: dt.ChatPanelAgentMode, + ChatPanelAskMode: dt.ChatPanelAskMode, + ChatPanelCustomMode: dt.ChatPanelCustomMode, + ChatPanelEditMode: dt.ChatPanelEditMode, + ChatPanelPlanMode: dt.ChatPanelPlanMode, + ChatPanelUnknownMode: dt.ChatPanelUnknownMode, + CopilotActivityMetrics: models.CopilotActivityMetrics{ UserInitiatedInteractionCount: dt.UserInitiatedInteractionCount, CodeGenerationActivityCount: dt.CodeGenerationActivityCount, @@ -177,11 +234,32 @@ func ExtractEnterpriseMetrics(taskCtx plugin.SubTaskContext) errors.Error { LocDeletedSum: dt.LocDeletedSum, }, } + if dt.TotalsByCli != nil { + dailyMetrics.CopilotCliMetrics = models.CopilotCliMetrics{ + CliSessionCount: dt.TotalsByCli.SessionCount, + CliRequestCount: dt.TotalsByCli.RequestCount, + CliPromptCount: dt.TotalsByCli.PromptCount, + } + if dt.TotalsByCli.TokenUsage != nil { + dailyMetrics.CopilotCliMetrics.CliOutputTokenSum = dt.TotalsByCli.TokenUsage.OutputTokensSum + dailyMetrics.CopilotCliMetrics.CliPromptTokenSum = dt.TotalsByCli.TokenUsage.PromptTokensSum + } + } if dt.PullRequests != nil { dailyMetrics.PRTotalReviewed = dt.PullRequests.TotalReviewed dailyMetrics.PRTotalCreated = dt.PullRequests.TotalCreated + dailyMetrics.PRTotalMerged = dt.PullRequests.TotalMerged + dailyMetrics.PRMedianMinutesToMerge = dt.PullRequests.MedianMinutesToMerge + dailyMetrics.PRTotalSuggestions = dt.PullRequests.TotalSuggestions + dailyMetrics.PRTotalAppliedSuggestions = dt.PullRequests.TotalAppliedSuggestions dailyMetrics.PRTotalCreatedByCopilot = dt.PullRequests.TotalCreatedByCopilot dailyMetrics.PRTotalReviewedByCopilot = dt.PullRequests.TotalReviewedByCopilot + dailyMetrics.PRTotalMergedCreatedByCopilot = dt.PullRequests.TotalMergedCreatedByCopilot + dailyMetrics.PRTotalMergedReviewedByCopilot = dt.PullRequests.TotalMergedReviewedByCopilot + dailyMetrics.PRMedianMinToMergeCopilotAuthored = dt.PullRequests.MedianMinToMergeCopilotAuthored + dailyMetrics.PRMedianMinToMergeCopilotReviewed = dt.PullRequests.MedianMinToMergeCopilotReviewed + dailyMetrics.PRTotalCopilotSuggestions = dt.PullRequests.TotalCopilotSuggestions + dailyMetrics.PRTotalCopilotAppliedSuggestions = dt.PullRequests.TotalCopilotAppliedSuggestions } results = append(results, dailyMetrics) diff --git a/backend/plugins/gh-copilot/tasks/metrics_extractor.go b/backend/plugins/gh-copilot/tasks/metrics_extractor.go index 4d635c1723e..d89eababde6 100644 --- a/backend/plugins/gh-copilot/tasks/metrics_extractor.go +++ b/backend/plugins/gh-copilot/tasks/metrics_extractor.go @@ -38,12 +38,21 @@ type copilotSeatResponse struct { LastActivityAt *string `json:"last_activity_at"` LastActivityEditor string `json:"last_activity_editor"` Assignee copilotAssignee `json:"assignee"` + AssigningTeam *copilotTeam `json:"assigning_team"` } type copilotAssignee struct { Login string `json:"login"` Id int64 `json:"id"` Type string `json:"type"` + Name string `json:"name"` + Email string `json:"email"` +} + +type copilotTeam struct { + Id int64 `json:"id"` + Name string `json:"name"` + Slug string `json:"slug"` } // ExtractOrgMetrics parses org report data from the new report download API. @@ -100,6 +109,22 @@ func ExtractOrgMetrics(taskCtx plugin.SubTaskContext) errors.Error { MonthlyActiveUsers: dt.MonthlyActiveUsers, MonthlyActiveChatUsers: dt.MonthlyActiveChatUsers, MonthlyActiveAgentUsers: dt.MonthlyActiveAgentUsers, + DailyActiveCliUsers: dt.DailyActiveCliUsers, + + DailyActiveCopilotCodeReviewUsers: dt.DailyActiveCopilotCodeReviewUsers, + DailyPassiveCopilotCodeReviewUsers: dt.DailyPassiveCopilotCodeReviewUsers, + WeeklyActiveCopilotCodeReviewUsers: dt.WeeklyActiveCopilotCodeReviewUsers, + WeeklyPassiveCopilotCodeReviewUsers: dt.WeeklyPassiveCopilotCodeReviewUsers, + MonthlyActiveCopilotCodeReviewUsers: dt.MonthlyActiveCopilotCodeReviewUsers, + MonthlyPassiveCopilotCodeReviewUsers: dt.MonthlyPassiveCopilotCodeReviewUsers, + + ChatPanelAgentMode: dt.ChatPanelAgentMode, + ChatPanelAskMode: dt.ChatPanelAskMode, + ChatPanelCustomMode: dt.ChatPanelCustomMode, + ChatPanelEditMode: dt.ChatPanelEditMode, + ChatPanelPlanMode: dt.ChatPanelPlanMode, + ChatPanelUnknownMode: dt.ChatPanelUnknownMode, + CopilotActivityMetrics: models.CopilotActivityMetrics{ UserInitiatedInteractionCount: dt.UserInitiatedInteractionCount, CodeGenerationActivityCount: dt.CodeGenerationActivityCount, @@ -110,11 +135,32 @@ func ExtractOrgMetrics(taskCtx plugin.SubTaskContext) errors.Error { LocDeletedSum: dt.LocDeletedSum, }, } + if dt.TotalsByCli != nil { + dailyMetrics.CopilotCliMetrics = models.CopilotCliMetrics{ + CliSessionCount: dt.TotalsByCli.SessionCount, + CliRequestCount: dt.TotalsByCli.RequestCount, + CliPromptCount: dt.TotalsByCli.PromptCount, + } + if dt.TotalsByCli.TokenUsage != nil { + dailyMetrics.CopilotCliMetrics.CliOutputTokenSum = dt.TotalsByCli.TokenUsage.OutputTokensSum + dailyMetrics.CopilotCliMetrics.CliPromptTokenSum = dt.TotalsByCli.TokenUsage.PromptTokensSum + } + } if dt.PullRequests != nil { dailyMetrics.PRTotalReviewed = dt.PullRequests.TotalReviewed dailyMetrics.PRTotalCreated = dt.PullRequests.TotalCreated + dailyMetrics.PRTotalMerged = dt.PullRequests.TotalMerged + dailyMetrics.PRMedianMinutesToMerge = dt.PullRequests.MedianMinutesToMerge + dailyMetrics.PRTotalSuggestions = dt.PullRequests.TotalSuggestions + dailyMetrics.PRTotalAppliedSuggestions = dt.PullRequests.TotalAppliedSuggestions dailyMetrics.PRTotalCreatedByCopilot = dt.PullRequests.TotalCreatedByCopilot dailyMetrics.PRTotalReviewedByCopilot = dt.PullRequests.TotalReviewedByCopilot + dailyMetrics.PRTotalMergedCreatedByCopilot = dt.PullRequests.TotalMergedCreatedByCopilot + dailyMetrics.PRTotalMergedReviewedByCopilot = dt.PullRequests.TotalMergedReviewedByCopilot + dailyMetrics.PRMedianMinToMergeCopilotAuthored = dt.PullRequests.MedianMinToMergeCopilotAuthored + dailyMetrics.PRMedianMinToMergeCopilotReviewed = dt.PullRequests.MedianMinToMergeCopilotReviewed + dailyMetrics.PRTotalCopilotSuggestions = dt.PullRequests.TotalCopilotSuggestions + dailyMetrics.PRTotalCopilotAppliedSuggestions = dt.PullRequests.TotalCopilotAppliedSuggestions } results = append(results, dailyMetrics) diff --git a/backend/plugins/gh-copilot/tasks/register.go b/backend/plugins/gh-copilot/tasks/register.go index ee1dcc797fc..3c7e5b1eeb9 100644 --- a/backend/plugins/gh-copilot/tasks/register.go +++ b/backend/plugins/gh-copilot/tasks/register.go @@ -27,10 +27,12 @@ func GetSubTaskMetas() []plugin.SubTaskMeta { CollectCopilotSeatAssignmentsMeta, CollectEnterpriseMetricsMeta, CollectUserMetricsMeta, + CollectUserTeamsMeta, // Extractors ExtractSeatsMeta, ExtractOrgMetricsMeta, ExtractEnterpriseMetricsMeta, ExtractUserMetricsMeta, + ExtractUserTeamsMeta, } } diff --git a/backend/plugins/gh-copilot/tasks/seat_extractor.go b/backend/plugins/gh-copilot/tasks/seat_extractor.go index 48abc3c0ce1..1a1b6b13518 100644 --- a/backend/plugins/gh-copilot/tasks/seat_extractor.go +++ b/backend/plugins/gh-copilot/tasks/seat_extractor.go @@ -96,6 +96,8 @@ func ExtractSeats(taskCtx plugin.SubTaskContext) errors.Error { Organization: connection.Organization, UserLogin: seat.Assignee.Login, UserId: seat.Assignee.Id, + UserName: seat.Assignee.Name, + UserEmail: seat.Assignee.Email, PlanType: seat.PlanType, CreatedAt: createdAt, LastActivityAt: lastAct, @@ -104,6 +106,11 @@ func ExtractSeats(taskCtx plugin.SubTaskContext) errors.Error { PendingCancellationDate: pendingCancel, UpdatedAt: updatedAt, } + if seat.AssigningTeam != nil { + toolSeat.AssigningTeamId = seat.AssigningTeam.Id + toolSeat.AssigningTeamName = seat.AssigningTeam.Name + toolSeat.AssigningTeamSlug = seat.AssigningTeam.Slug + } return []interface{}{toolSeat}, nil }, diff --git a/backend/plugins/gh-copilot/tasks/subtasks.go b/backend/plugins/gh-copilot/tasks/subtasks.go index 24a2c95f1c5..61ed5799525 100644 --- a/backend/plugins/gh-copilot/tasks/subtasks.go +++ b/backend/plugins/gh-copilot/tasks/subtasks.go @@ -53,6 +53,14 @@ var CollectUserMetricsMeta = plugin.SubTaskMeta{ Description: "Collect GitHub Copilot enterprise user-level usage metrics reports", } +var CollectUserTeamsMeta = plugin.SubTaskMeta{ + Name: "collectUserTeams", + EntryPoint: CollectUserTeams, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Collect GitHub Copilot user-team mappings from user-teams-1-day report", +} + var ExtractOrgMetricsMeta = plugin.SubTaskMeta{ Name: "extractOrgMetrics", EntryPoint: ExtractOrgMetrics, @@ -88,3 +96,12 @@ var ExtractUserMetricsMeta = plugin.SubTaskMeta{ Description: "Extract Copilot user metrics into tool-layer tables", Dependencies: []*plugin.SubTaskMeta{&CollectUserMetricsMeta}, } + +var ExtractUserTeamsMeta = plugin.SubTaskMeta{ + Name: "extractUserTeams", + EntryPoint: ExtractUserTeams, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Extract Copilot user-team mappings into tool-layer table", + Dependencies: []*plugin.SubTaskMeta{&CollectUserTeamsMeta}, +} diff --git a/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go b/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go index 96f5570f758..91178ff5bf4 100644 --- a/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go +++ b/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go @@ -46,11 +46,15 @@ type userDailyReport struct { LocDeletedSum int `json:"loc_deleted_sum"` UsedAgent bool `json:"used_agent"` UsedChat bool `json:"used_chat"` + UsedCli bool `json:"used_cli"` + UsedCopilotCodeReviewActive bool `json:"used_copilot_code_review_active"` + UsedCopilotCodeReviewPassive bool `json:"used_copilot_code_review_passive"` TotalsByIde []userTotalsByIde `json:"totals_by_ide"` TotalsByFeature []totalsByFeature `json:"totals_by_feature"` TotalsByLanguageFeature []totalsByLangFeature `json:"totals_by_language_feature"` TotalsByLanguageModel []totalsByLangModel `json:"totals_by_language_model"` TotalsByModelFeature []totalsByModelFeature `json:"totals_by_model_feature"` + TotalsByCli *totalsByCli `json:"totals_by_cli"` } type userTotalsByIde struct { @@ -106,7 +110,7 @@ func ExtractUserMetrics(taskCtx plugin.SubTaskContext) errors.Error { var results []interface{} // Main user daily metrics - results = append(results, &models.GhCopilotUserDailyMetrics{ + userMetrics := &models.GhCopilotUserDailyMetrics{ ConnectionId: data.Options.ConnectionId, ScopeId: data.Options.ScopeId, Day: day, @@ -116,6 +120,9 @@ func ExtractUserMetrics(taskCtx plugin.SubTaskContext) errors.Error { UserLogin: u.UserLogin, UsedAgent: u.UsedAgent, UsedChat: u.UsedChat, + UsedCli: u.UsedCli, + UsedCopilotCodeReviewActive: u.UsedCopilotCodeReviewActive, + UsedCopilotCodeReviewPassive: u.UsedCopilotCodeReviewPassive, CopilotActivityMetrics: models.CopilotActivityMetrics{ UserInitiatedInteractionCount: u.UserInitiatedInteractionCount, CodeGenerationActivityCount: u.CodeGenerationActivityCount, @@ -125,7 +132,19 @@ func ExtractUserMetrics(taskCtx plugin.SubTaskContext) errors.Error { LocAddedSum: u.LocAddedSum, LocDeletedSum: u.LocDeletedSum, }, - }) + } + if u.TotalsByCli != nil { + userMetrics.CopilotCliMetrics = models.CopilotCliMetrics{ + CliSessionCount: u.TotalsByCli.SessionCount, + CliRequestCount: u.TotalsByCli.RequestCount, + CliPromptCount: u.TotalsByCli.PromptCount, + } + if u.TotalsByCli.TokenUsage != nil { + userMetrics.CopilotCliMetrics.CliOutputTokenSum = u.TotalsByCli.TokenUsage.OutputTokensSum + userMetrics.CopilotCliMetrics.CliPromptTokenSum = u.TotalsByCli.TokenUsage.PromptTokensSum + } + } + results = append(results, userMetrics) // User by IDE for _, ide := range u.TotalsByIde { diff --git a/backend/plugins/gh-copilot/tasks/user_teams_collector.go b/backend/plugins/gh-copilot/tasks/user_teams_collector.go new file mode 100644 index 00000000000..2ae0200d2ef --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/user_teams_collector.go @@ -0,0 +1,133 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const rawUserTeamsTable = "copilot_user_teams" + +// CollectUserTeams collects user-team mapping data from the user-teams-1-day report. +// This enables team-level metrics aggregation by joining with per-user daily metrics. +func CollectUserTeams(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*GhCopilotTaskData) + if !ok { + return errors.Default.New("task data is not GhCopilotTaskData") + } + connection := data.Connection + connection.Normalize() + + var urlTemplate string + + if connection.HasEnterprise() { + urlTemplate = fmt.Sprintf("enterprises/%s/copilot/metrics/reports/user-teams-1-day", connection.Enterprise) + } else if connection.Organization != "" { + urlTemplate = fmt.Sprintf("orgs/%s/copilot/metrics/reports/user-teams-1-day", connection.Organization) + } else { + return nil + } + + apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) + if err != nil { + return err + } + + rawArgs := helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawUserTeamsTable, + Options: copilotRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: connection.Endpoint, + }, + } + + collector, err := helper.NewStatefulApiCollector(rawArgs) + if err != nil { + return err + } + + now := time.Now().UTC() + start, until := computeReportDateRange(now, collector.GetSince()) + logger := taskCtx.GetLogger() + + dayIter := newDayIterator(start, until) + + err = collector.InitCollector(helper.ApiCollectorArgs{ + ApiClient: apiClient, + Input: dayIter, + UrlTemplate: urlTemplate, + Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { + input := reqData.Input.(*dayInput) + q := url.Values{} + q.Set("day", input.Day) + return q, nil + }, + Incremental: true, + Concurrency: 1, + AfterResponse: ignoreNoContent, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + body, readErr := io.ReadAll(res.Body) + res.Body.Close() + if readErr != nil { + return nil, errors.Default.Wrap(readErr, "failed to read report metadata") + } + if isEmptyReport(body) { + return nil, nil + } + + var meta reportMetadataResponse + if jsonErr := json.Unmarshal(body, &meta); jsonErr != nil { + return nil, errors.Default.Wrap(jsonErr, "failed to parse report metadata") + } + + var results []json.RawMessage + for _, link := range meta.DownloadLinks { + reportBody, dlErr := downloadReport(link, logger) + if dlErr != nil { + return nil, dlErr + } + if reportBody == nil { + continue + } + // User-teams reports are JSONL format + records, parseErr := parseJSONL(reportBody) + if parseErr != nil { + return nil, parseErr + } + results = append(results, records...) + } + return results, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/gh-copilot/tasks/user_teams_extractor.go b/backend/plugins/gh-copilot/tasks/user_teams_extractor.go new file mode 100644 index 00000000000..72a3de8abe9 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/user_teams_extractor.go @@ -0,0 +1,93 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" +) + +// userTeamRecord represents a single line from the user-teams-1-day JSONL report. +type userTeamRecord struct { + Day string `json:"day"` + UserId int64 `json:"user_id"` + UserLogin string `json:"user_login"` + OrganizationId string `json:"organization_id"` + EnterpriseId string `json:"enterprise_id"` + TeamId int64 `json:"team_id"` + Slug string `json:"slug"` +} + +// ExtractUserTeams parses user-team JSONL records into the GhCopilotUserTeam model. +func ExtractUserTeams(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*GhCopilotTaskData) + if !ok { + return errors.Default.New("task data is not GhCopilotTaskData") + } + connection := data.Connection + connection.Normalize() + + params := copilotRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: connection.Endpoint, + } + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawUserTeamsTable, + Options: params, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + var rec userTeamRecord + if err := errors.Convert(json.Unmarshal(row.Data, &rec)); err != nil { + return nil, err + } + + day, parseErr := time.Parse("2006-01-02", rec.Day) + if parseErr != nil { + return nil, errors.BadInput.Wrap(parseErr, "invalid day in user-teams report") + } + + return []interface{}{ + &models.GhCopilotUserTeam{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + UserId: rec.UserId, + TeamId: rec.TeamId, + UserLogin: rec.UserLogin, + OrganizationId: rec.OrganizationId, + EnterpriseId: rec.EnterpriseId, + TeamSlug: rec.Slug, + }, + }, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +}