-
Notifications
You must be signed in to change notification settings - Fork 0
Summary #13
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Summary #13
Changes from all commits
717f773
bd52f70
893a75f
cce4a5d
22a09ad
7644316
2ae5355
d941131
fa6f91a
3bc222f
8113cb0
80561d5
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -33,6 +33,8 @@ func sendRepositoryDispatch(token string, msg *ScenarioProgressMessage) error { | |||||
| "overall_status": msg.OverallStatus, | ||||||
| "failed_count": msg.FailedCount, | ||||||
| "total_scenarios": msg.TotalScenarios, | ||||||
| "missing_tests_in_pr": msg.MissingTestsInPR, | ||||||
| "should_run_tests": msg.ShouldRunTests, | ||||||
| }, | ||||||
| } | ||||||
|
|
||||||
|
|
@@ -64,8 +66,8 @@ func sendRepositoryDispatch(token string, msg *ScenarioProgressMessage) error { | |||||
| return fmt.Errorf("github API returned %d: %s", resp.StatusCode, string(respBody)) | ||||||
| } | ||||||
|
|
||||||
| log.Printf("repository_dispatch sent: repo=%s sha=%s run_id=%s overall_status=%s", | ||||||
| msg.Repository, msg.CommitSHA, msg.RunID, msg.OverallStatus) | ||||||
| log.Printf("repository_dispatch sent: repo=%s sha=%s run_id=%s overall_status=%s missing_tests_in_pr=%v should_run_tests=%v", | ||||||
| msg.Repository, msg.CommitSHA, msg.RunID, msg.OverallStatus, msg.MissingTestsInPR, msg.ShouldRunTests) | ||||||
|
||||||
| msg.Repository, msg.CommitSHA, msg.RunID, msg.OverallStatus, msg.MissingTestsInPR, msg.ShouldRunTests) | |
| msg.Repository, msg.CommitSHA, msg.RunID, msg.OverallStatus, msg.MissingTestsInPR, msg.ShouldRunTests) |
| Original file line number | Diff line number | Diff line change | ||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -80,17 +80,20 @@ type cmd struct { | |||||||||||
| } | ||||||||||||
|
|
||||||||||||
| type ScenarioProgressMessage struct { | ||||||||||||
| Status string `json:"status"` | ||||||||||||
| Scenario string `json:"scenario"` | ||||||||||||
| RunID string `json:"run_id"` | ||||||||||||
| Data string `json:"data"` | ||||||||||||
| TotalScenarios string `json:"total_scenarios"` | ||||||||||||
| Code string `json:"code"` | ||||||||||||
| OverallStatus string `json:"overall_status,omitempty"` | ||||||||||||
| FailedCount int64 `json:"failed_count,omitempty"` | ||||||||||||
| CommitSHA string `json:"commit_sha,omitempty"` | ||||||||||||
| Repository string `json:"repository,omitempty"` | ||||||||||||
| RunURL string `json:"run_url,omitempty"` | ||||||||||||
| Status string `json:"status"` | ||||||||||||
| Scenario string `json:"scenario"` | ||||||||||||
| RunID string `json:"run_id"` | ||||||||||||
| Data string `json:"data"` | ||||||||||||
| TotalScenarios string `json:"total_scenarios"` | ||||||||||||
| Code string `json:"code"` | ||||||||||||
| OverallStatus string `json:"overall_status,omitempty"` | ||||||||||||
| FailedCount int64 `json:"failed_count,omitempty"` | ||||||||||||
| FailedScenarios []string `json:"failed_scenarios,omitempty"` | ||||||||||||
| CommitSHA string `json:"commit_sha,omitempty"` | ||||||||||||
| Repository string `json:"repository,omitempty"` | ||||||||||||
| RunURL string `json:"run_url,omitempty"` | ||||||||||||
| MissingTestsInPR bool `json:"missing_tests_in_pr,omitempty"` | ||||||||||||
| ShouldRunTests bool `json:"should_run_tests,omitempty"` | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| func runE(cmd *cobra.Command, args []string) error { | ||||||||||||
|
|
@@ -277,6 +280,9 @@ func filterScenariosByAffectedServices(files []string, affectedServices []string | |||||||||||
|
|
||||||||||||
| func distributePubsub(app *appctx, runID string, tagFilters []string, metadata map[string]interface{}, forceAll bool) bool { | ||||||||||||
| id := runID | ||||||||||||
| if metadata == nil { | ||||||||||||
| metadata = make(map[string]interface{}) | ||||||||||||
| } | ||||||||||||
| final := combineFilesAndDir() | ||||||||||||
| if !forceAll { | ||||||||||||
| affectedServices := extractAffectedServices(metadata) | ||||||||||||
|
|
@@ -296,11 +302,6 @@ func distributePubsub(app *appctx, runID string, tagFilters []string, metadata m | |||||||||||
|
|
||||||||||||
| metadata["total_scenarios"] = fmt.Sprintf("%d", len(filtered)) | ||||||||||||
|
|
||||||||||||
| if len(filtered) > 0 { | ||||||||||||
| app.runSummary[id] = &runSummary{} | ||||||||||||
| app.runTracker[id] = len(filtered) | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| for _, f := range filtered { | ||||||||||||
| nc := cmd{ | ||||||||||||
| Code: "process", | ||||||||||||
|
|
@@ -333,6 +334,9 @@ func distributeSQS(app *appctx, runID string, tagFilters []string, metadata map[ | |||||||||||
| } | ||||||||||||
|
|
||||||||||||
| id := runID | ||||||||||||
| if metadata == nil { | ||||||||||||
| metadata = make(map[string]interface{}) | ||||||||||||
| } | ||||||||||||
| final := combineFilesAndDir() | ||||||||||||
| if !forceAll { | ||||||||||||
| affectedServices := extractAffectedServices(metadata) | ||||||||||||
|
|
@@ -351,11 +355,6 @@ func distributeSQS(app *appctx, runID string, tagFilters []string, metadata map[ | |||||||||||
| log.Printf("distributing %d/%d scenarios matching tags %v", len(filtered), len(final), tagFilters) | ||||||||||||
| metadata["total_scenarios"] = fmt.Sprintf("%d", len(filtered)) | ||||||||||||
|
|
||||||||||||
| if len(filtered) > 0 { | ||||||||||||
| app.runSummary[id] = &runSummary{} | ||||||||||||
| app.runTracker[id] = len(filtered) | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| for _, f := range filtered { | ||||||||||||
| nc := cmd{ | ||||||||||||
| Code: "process", | ||||||||||||
|
|
@@ -381,19 +380,11 @@ func distributeSQS(app *appctx, runID string, tagFilters []string, metadata map[ | |||||||||||
| return true | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| type runSummary struct { | ||||||||||||
| Success int | ||||||||||||
| Failed int | ||||||||||||
| FailedScenarios []string | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| type appctx struct { | ||||||||||||
| pub *lspubsub.PubsubPublisher // starter publisher topic | ||||||||||||
| rpub *lspubsub.PubsubPublisher // topic to publish reports | ||||||||||||
| mtx *sync.Mutex | ||||||||||||
| topicArn *string | ||||||||||||
| runSummary map[string]*runSummary | ||||||||||||
| runTracker map[string]int | ||||||||||||
| pub *lspubsub.PubsubPublisher // starter publisher topic | ||||||||||||
| rpub *lspubsub.PubsubPublisher // topic to publish reports | ||||||||||||
| mtx *sync.Mutex | ||||||||||||
| topicArn *string | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| // Our message processing callback. | ||||||||||||
|
|
@@ -498,58 +489,6 @@ func process(ctx any, data []byte) error { | |||||||||||
| Verbose: verbose, | ||||||||||||
| Metadata: c.Metadata, | ||||||||||||
| RunID: c.ID, | ||||||||||||
| OnScenarioDone: func(scenario, status string) { | ||||||||||||
| if c.ID == "" { | ||||||||||||
| return | ||||||||||||
| } | ||||||||||||
| rs, ok := app.runSummary[c.ID] | ||||||||||||
| if !ok { | ||||||||||||
| return | ||||||||||||
| } | ||||||||||||
| if status == "success" { | ||||||||||||
| rs.Success++ | ||||||||||||
| } else { | ||||||||||||
| rs.Failed++ | ||||||||||||
| rs.FailedScenarios = append(rs.FailedScenarios, filepath.Base(scenario)) | ||||||||||||
| } | ||||||||||||
| app.runTracker[c.ID]-- | ||||||||||||
| if app.runTracker[c.ID] <= 0 { | ||||||||||||
| if repslack != "" { | ||||||||||||
| var summaryText strings.Builder | ||||||||||||
| summaryTitle := "Test Run Complete" | ||||||||||||
| summaryColor := "good" | ||||||||||||
| total := rs.Success + rs.Failed | ||||||||||||
| if rs.Failed > 0 { | ||||||||||||
| summaryTitle = "Test Run Complete (With Failures)" | ||||||||||||
| summaryColor = "danger" | ||||||||||||
| } | ||||||||||||
| fmt.Fprintf(&summaryText, "*Run Summary*\nTotal: %d\nPassed: %d\nFailed: %d", total, rs.Success, rs.Failed) | ||||||||||||
| if len(rs.FailedScenarios) > 0 { | ||||||||||||
| summaryText.WriteString("\n\n*Failed scenarios:*") | ||||||||||||
| for _, name := range rs.FailedScenarios { | ||||||||||||
| fmt.Fprintf(&summaryText, "\n• %v", name) | ||||||||||||
| } | ||||||||||||
| } | ||||||||||||
| summaryPayload := SlackMessage{ | ||||||||||||
| Attachments: []SlackAttachment{ | ||||||||||||
| { | ||||||||||||
| Color: summaryColor, | ||||||||||||
| Title: summaryTitle, | ||||||||||||
| Text: summaryText.String(), | ||||||||||||
| Footer: fmt.Sprintf("oops • run: %v", c.ID), | ||||||||||||
| Timestamp: time.Now().Unix(), | ||||||||||||
| MrkdwnIn: []string{"text"}, | ||||||||||||
| }, | ||||||||||||
| }, | ||||||||||||
| } | ||||||||||||
| if err := summaryPayload.Notify(repslack); err != nil { | ||||||||||||
| log.Printf("Notify (slack summary) failed: %v", err) | ||||||||||||
| } | ||||||||||||
| } | ||||||||||||
| delete(app.runSummary, c.ID) | ||||||||||||
| delete(app.runTracker, c.ID) | ||||||||||||
| } | ||||||||||||
| }, | ||||||||||||
| }) | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
|
|
@@ -592,12 +531,25 @@ func handleScenarioCompletion(ctx any, data []byte) error { | |||||||||||
|
|
||||||||||||
| if msg.OverallStatus == "failure" || msg.FailedCount > 0 { | ||||||||||||
| color = "danger" | ||||||||||||
| title = "Tests Done." | ||||||||||||
| text = fmt.Sprintf("Test completed with %d/%s success scenarios. Please check the errors <%s|here>.", | ||||||||||||
| successCount, total, msg.RunURL) | ||||||||||||
| title = "Test Run Complete (With Failures)" | ||||||||||||
| var sb strings.Builder | ||||||||||||
| fmt.Fprintf(&sb, "*Run Summary*\nTotal: %s\nPassed: %d\nFailed: %d", total, successCount, msg.FailedCount) | ||||||||||||
| if len(msg.FailedScenarios) > 0 { | ||||||||||||
| sb.WriteString("\n\n*Failed scenarios:*") | ||||||||||||
| for _, name := range msg.FailedScenarios { | ||||||||||||
| fmt.Fprintf(&sb, "\n• %v", name) | ||||||||||||
| } | ||||||||||||
| } | ||||||||||||
| if msg.RunURL != "" { | ||||||||||||
| fmt.Fprintf(&sb, "\n\n<%s|View run>", msg.RunURL) | ||||||||||||
| } | ||||||||||||
| text = sb.String() | ||||||||||||
| } else { | ||||||||||||
| text = fmt.Sprintf("Test completed with %s/%s success scenarios.", | ||||||||||||
| total, total) | ||||||||||||
| title = "Test Run Complete" | ||||||||||||
| text = fmt.Sprintf("*Run Summary*\nTotal: %s\nPassed: %s\nFailed: 0", total, total) | ||||||||||||
| if msg.RunURL != "" { | ||||||||||||
| text += fmt.Sprintf("\n\n<%s|View run>", msg.RunURL) | ||||||||||||
| } | ||||||||||||
| } | ||||||||||||
jenn-if-err marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||||||||
|
|
||||||||||||
| payload := SlackMessage{ | ||||||||||||
|
|
@@ -606,8 +558,9 @@ func handleScenarioCompletion(ctx any, data []byte) error { | |||||||||||
| Color: color, | ||||||||||||
| Title: title, | ||||||||||||
| Text: text, | ||||||||||||
| Footer: "oops", | ||||||||||||
| Footer: fmt.Sprintf("oops • run: %v", msg.RunID), | ||||||||||||
| Timestamp: time.Now().Unix(), | ||||||||||||
| MrkdwnIn: []string{"text"}, | ||||||||||||
| }, | ||||||||||||
| }, | ||||||||||||
| } | ||||||||||||
|
|
@@ -640,9 +593,7 @@ func run(ctx context.Context, done chan error) { | |||||||||||
| } | ||||||||||||
|
|
||||||||||||
| app := &appctx{ | ||||||||||||
| mtx: &sync.Mutex{}, | ||||||||||||
| runSummary: make(map[string]*runSummary), | ||||||||||||
| runTracker: make(map[string]int), | ||||||||||||
| mtx: &sync.Mutex{}, | ||||||||||||
| } | ||||||||||||
| ctx0, cancelCtx0 := context.WithCancel(ctx) | ||||||||||||
| defer cancelCtx0() | ||||||||||||
|
|
@@ -721,7 +672,10 @@ func run(ctx context.Context, done chan error) { | |||||||||||
| } | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| if scenariopubsub != "" && githubtoken != "" && pubsub != "" { | ||||||||||||
|
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @Ynah537 can I remove this? Start_all doesn't have githubtoken, if this is here then the code couldn't proceed.
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It will still be safe since there is a githubtoken checker in the sendrepositorydispatch
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @jenn-if-err yes that would be fine |
||||||||||||
| if scenariopubsub != "" && pubsub != "" { | ||||||||||||
|
||||||||||||
| if scenariopubsub != "" && pubsub != "" { | |
| if scenariopubsub != "" && pubsub != "" { | |
| if githubtoken == "" { | |
| log.Printf("WARNING: githubtoken is empty; scenario progress listener will run, but any GitHub repository_dispatch operations may fail due to missing credentials") | |
| } |
Copilot
AI
Mar 12, 2026
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The progress listener now starts even when githubtoken is empty, and the previous warning about missing token was removed. If GitHub repository_dispatch is still attempted within the listener path, this change can lead to repeated failing calls (401) with reduced operator visibility. Consider restoring a warning when GitHub dispatch is configured but token is missing, or ensuring the dispatch call is explicitly skipped (with a clear log) when githubtoken == "".
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -399,11 +399,22 @@ func doScenario(in *doScenarioInput) error { | |
| attr["pubsub"] = pubsub | ||
| } | ||
| if in.Metadata != nil { | ||
| for _, key := range []string{"pr_number", "branch", "commit_sha", "actor", "trigger_type", "run_url", "repository", "workflow"} { | ||
| for _, key := range []string{ | ||
| "pr_number", "branch", "commit_sha", "actor", | ||
| "trigger_type", "run_url", "repository", "workflow", "total_scenarios", | ||
| } { | ||
| if v, ok := in.Metadata[key].(string); ok && v != "" { | ||
| attr[key] = v | ||
| } | ||
| } | ||
| if ta, ok := in.Metadata["test_analysis"].(map[string]interface{}); ok { | ||
| for _, key := range []string{"missing_tests_in_pr", "should_run_tests"} { | ||
| if v, ok := ta[key].(bool); ok { | ||
| attr[key] = fmt.Sprintf("%v", v) | ||
| } | ||
| } | ||
| } | ||
|
Comment on lines
+410
to
+416
|
||
|
|
||
| if b, err := json.Marshal(in.Metadata); err == nil { | ||
| attr["metadata"] = string(b) | ||
| } | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.