diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 4f899453b5f57..bdc122888808d 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -383,9 +383,9 @@ func prepareMigrationTasks() []*migration { newMigration(318, "Add anonymous_access_mode for repo_unit", v1_24.AddRepoUnitAnonymousAccessMode), newMigration(319, "Add ExclusiveOrder to Label table", v1_24.AddExclusiveOrderColumnToLabelTable), newMigration(320, "Migrate two_factor_policy to login_source table", v1_24.MigrateSkipTwoFactor), - // Gitea 1.24.0 ends at database version 321 newMigration(321, "Use LONGTEXT for some columns and fix review_state.updated_files column", v1_25.UseLongTextInSomeColumnsAndFixBugs), + newMigration(322, "Add webhook payload optimization columns", v1_25.AddWebhookPayloadOptimizationColumns), } return preparedMigrations } diff --git a/models/migrations/v1_25/v322.go b/models/migrations/v1_25/v322.go new file mode 100644 index 0000000000000..b1888e7c681fa --- /dev/null +++ b/models/migrations/v1_25/v322.go @@ -0,0 +1,23 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_25 + +import ( + "xorm.io/xorm" +) + +func AddWebhookPayloadOptimizationColumns(x *xorm.Engine) error { + type Webhook struct { + ExcludeFilesLimit int `xorm:"exclude_files_limit NOT NULL DEFAULT -1"` + ExcludeCommitsLimit int `xorm:"exclude_commits_limit NOT NULL DEFAULT -1"` + } + _, err := x.SyncWithOptions( + xorm.SyncOptions{ + IgnoreConstrains: true, + IgnoreIndices: true, + }, + new(Webhook), + ) + return err +} diff --git a/models/webhook/webhook.go b/models/webhook/webhook.go index 7d4b2e2237db0..e78a81b7dfd6a 100644 --- a/models/webhook/webhook.go +++ b/models/webhook/webhook.go @@ -139,6 +139,10 @@ type Webhook struct { // HeaderAuthorizationEncrypted should be accessed using HeaderAuthorization() and SetHeaderAuthorization() HeaderAuthorizationEncrypted string `xorm:"TEXT"` + // Payload size optimization options + ExcludeFilesLimit int `xorm:"exclude_files_limit"` // -1: do not trim, 0: trim all (none kept), >0: keep N file changes in commit payloads + ExcludeCommitsLimit int `xorm:"exclude_commits_limit"` // -1: do not trim, 0: trim all (none kept), >0: keep N commits in push payloads + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` } diff --git a/models/webhook/webhook_test.go b/models/webhook/webhook_test.go index edad8fc996c14..db1150772e5cd 100644 --- a/models/webhook/webhook_test.go +++ b/models/webhook/webhook_test.go @@ -330,3 +330,45 @@ func TestCleanupHookTaskTable_OlderThan_LeavesTaskEarlierThanAgeToDelete(t *test assert.NoError(t, CleanupHookTaskTable(t.Context(), OlderThan, 168*time.Hour, 0)) unittest.AssertExistsAndLoadBean(t, hookTask) } + +func TestWebhookPayloadOptimization(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + webhook := &Webhook{ + RepoID: 1, + URL: "http://example.com/webhook", + HTTPMethod: "POST", + ContentType: ContentTypeJSON, + Secret: "secret", + IsActive: true, + Type: webhook_module.GITEA, + ExcludeFilesLimit: 1, + ExcludeCommitsLimit: 0, + HookEvent: &webhook_module.HookEvent{ + PushOnly: true, + }, + } + + // Test creating webhook with payload optimization options + err := CreateWebhook(db.DefaultContext, webhook) + assert.NoError(t, err) + assert.NotZero(t, webhook.ID) + + // Test retrieving webhook and checking payload optimization options + retrievedWebhook, err := GetWebhookByID(db.DefaultContext, webhook.ID) + assert.NoError(t, err) + assert.Equal(t, 1, retrievedWebhook.ExcludeFilesLimit) + assert.Equal(t, 0, retrievedWebhook.ExcludeCommitsLimit) + + // Test updating webhook with different payload optimization options + retrievedWebhook.ExcludeFilesLimit = 0 + retrievedWebhook.ExcludeCommitsLimit = 2 + err = UpdateWebhook(db.DefaultContext, retrievedWebhook) + assert.NoError(t, err) + + // Verify the update + updatedWebhook, err := GetWebhookByID(db.DefaultContext, webhook.ID) + assert.NoError(t, err) + assert.Equal(t, 0, updatedWebhook.ExcludeFilesLimit) + assert.Equal(t, 2, updatedWebhook.ExcludeCommitsLimit) +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 65ba4e9cd3fc5..7fb88791be09b 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2424,6 +2424,11 @@ settings.event_package = Package settings.event_package_desc = Package created or deleted in a repository. settings.branch_filter = Branch filter settings.branch_filter_desc = Branch whitelist for push, branch creation and branch deletion events, specified as glob pattern. If empty or *, events for all branches are reported. See %[2]s documentation for syntax. Examples: master, {master,release*}. +settings.payload_optimization = Payload Size Optimization +settings.exclude_files_limit = Limit file changes +settings.exclude_files_limit_desc = -1: do not trim, 0: trim all (none kept), >0: keep N file changes +settings.exclude_commits_limit = Limit commits +settings.exclude_commits_limit_desc = -1: do not trim, 0: trim all (none kept), >0: keep N commits settings.authorization_header = Authorization Header settings.authorization_header_desc = Will be included as authorization header for requests when present. Examples: %s. settings.active = Active @@ -3282,7 +3287,7 @@ auths.tip.github = Register a new OAuth application on %s auths.tip.gitlab_new = Register a new application on %s auths.tip.google_plus = Obtain OAuth2 client credentials from the Google API console at %s auths.tip.openid_connect = Use the OpenID Connect Discovery URL "https://{server}/.well-known/openid-configuration" to specify the endpoints -auths.tip.twitter = Go to %s, create an application and ensure that the “Allow this application to be used to Sign in with Twitter” option is enabled +auths.tip.twitter = Go to %s, create an application and ensure that the "Allow this application to be used to Sign in with Twitter" option is enabled auths.tip.discord = Register a new application on %s auths.tip.gitea = Register a new OAuth2 application. Guide can be found at %s auths.tip.yandex = Create a new application at %s. Select following permissions from the "Yandex.Passport API" section: "Access to email address", "Access to user avatar" and "Access to username, first name and surname, gender" diff --git a/routers/web/repo/setting/webhook.go b/routers/web/repo/setting/webhook.go index f107449749364..7b6d69674d9c7 100644 --- a/routers/web/repo/setting/webhook.go +++ b/routers/web/repo/setting/webhook.go @@ -232,17 +232,19 @@ func createWebhook(ctx *context.Context, params webhookParams) { } w := &webhook.Webhook{ - RepoID: orCtx.RepoID, - URL: params.URL, - HTTPMethod: params.HTTPMethod, - ContentType: params.ContentType, - Secret: params.WebhookForm.Secret, - HookEvent: ParseHookEvent(params.WebhookForm), - IsActive: params.WebhookForm.Active, - Type: params.Type, - Meta: string(meta), - OwnerID: orCtx.OwnerID, - IsSystemWebhook: orCtx.IsSystemWebhook, + RepoID: orCtx.RepoID, + URL: params.URL, + HTTPMethod: params.HTTPMethod, + ContentType: params.ContentType, + Secret: params.WebhookForm.Secret, + HookEvent: ParseHookEvent(params.WebhookForm), + IsActive: params.WebhookForm.Active, + Type: params.Type, + Meta: string(meta), + OwnerID: orCtx.OwnerID, + IsSystemWebhook: orCtx.IsSystemWebhook, + ExcludeFilesLimit: params.WebhookForm.ExcludeFilesLimit, + ExcludeCommitsLimit: params.WebhookForm.ExcludeCommitsLimit, } err = w.SetHeaderAuthorization(params.WebhookForm.AuthorizationHeader) if err != nil { @@ -294,6 +296,8 @@ func editWebhook(ctx *context.Context, params webhookParams) { w.IsActive = params.WebhookForm.Active w.HTTPMethod = params.HTTPMethod w.Meta = string(meta) + w.ExcludeFilesLimit = params.WebhookForm.ExcludeFilesLimit + w.ExcludeCommitsLimit = params.WebhookForm.ExcludeCommitsLimit err = w.SetHeaderAuthorization(params.WebhookForm.AuthorizationHeader) if err != nil { diff --git a/services/forms/repo_form.go b/services/forms/repo_form.go index cb267f891ccb7..7d2cc58300ce3 100644 --- a/services/forms/repo_form.go +++ b/services/forms/repo_form.go @@ -239,6 +239,9 @@ type WebhookForm struct { BranchFilter string `binding:"GlobPattern"` AuthorizationHeader string Secret string + // Payload size optimization options + ExcludeFilesLimit int // -1: do not trim, 0: trim all (none kept), >0: keep N file changes in commit payloads + ExcludeCommitsLimit int // -1: do not trim, 0: trim all (none kept), >0: keep N commits in push payloads } // PushOnly if the hook will be triggered when push @@ -622,7 +625,7 @@ type UpdateAllowEditsForm struct { // | _// __ \| | _/ __ \\__ \ / ___// __ \ // | | \ ___/| |_\ ___/ / __ \_\___ \\ ___/ // |____|_ /\___ >____/\___ >____ /____ >\___ > -// \/ \/ \/ \/ \/ \/ +// \/ \/ \/ \/ \/ \/ // NewReleaseForm form for creating release type NewReleaseForm struct { diff --git a/services/webhook/notifier.go b/services/webhook/notifier.go index 672abd5c95d0e..16f9844be5808 100644 --- a/services/webhook/notifier.go +++ b/services/webhook/notifier.go @@ -7,6 +7,7 @@ import ( "context" actions_model "code.gitea.io/gitea/models/actions" + "code.gitea.io/gitea/models/db" git_model "code.gitea.io/gitea/models/git" issues_model "code.gitea.io/gitea/models/issues" "code.gitea.io/gitea/models/organization" @@ -15,10 +16,12 @@ import ( access_model "code.gitea.io/gitea/models/perm/access" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" + webhook_model "code.gitea.io/gitea/models/webhook" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/httplib" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" @@ -640,6 +643,95 @@ func (m *webhookNotifier) IssueChangeMilestone(ctx context.Context, doer *user_m } } +// applyWebhookPayloadOptimizations applies payload size optimizations based on webhook configurations +func (m *webhookNotifier) applyWebhookPayloadOptimizations(ctx context.Context, repo *repo_model.Repository, apiCommits []*api.PayloadCommit, apiHeadCommit *api.PayloadCommit) ([]*api.PayloadCommit, *api.PayloadCommit) { + // Get webhooks for this repository to check their configuration + webhooks, err := db.Find[webhook_model.Webhook](ctx, webhook_model.ListWebhookOptions{ + RepoID: repo.ID, + IsActive: optional.Some(true), + }) + if err != nil { + log.Error("Failed to get webhooks for repository %d: %v", repo.ID, err) + // Continue with default behavior if we can't get webhooks + return apiCommits, apiHeadCommit + } + + // Check if any webhook has payload optimization options enabled + hasFilesLimit := -1 + hasCommitsLimit := -1 + for _, webhook := range webhooks { + if webhook.HasEvent(webhook_module.HookEventPush) { + if webhook.ExcludeFilesLimit >= 0 && (hasFilesLimit == -1 || webhook.ExcludeFilesLimit < hasFilesLimit) { + hasFilesLimit = webhook.ExcludeFilesLimit + } + if webhook.ExcludeCommitsLimit >= 0 && (hasCommitsLimit == -1 || webhook.ExcludeCommitsLimit < hasCommitsLimit) { + hasCommitsLimit = webhook.ExcludeCommitsLimit + } + } + } + + // Apply payload optimizations based on webhook configurations + // -1 not trim, 0 trim all (none kept), >0 trim to N commits + if hasFilesLimit != -1 { + for _, commit := range apiCommits { + if commit.Added != nil { + if hasFilesLimit == 0 { + commit.Added = nil + } else if hasFilesLimit > 0 && len(commit.Added) > hasFilesLimit { + commit.Added = commit.Added[:hasFilesLimit] + } + } + if commit.Removed != nil { + if hasFilesLimit == 0 { + commit.Removed = nil + } else if hasFilesLimit > 0 && len(commit.Removed) > hasFilesLimit { + commit.Removed = commit.Removed[:hasFilesLimit] + } + } + if commit.Modified != nil { + if hasFilesLimit == 0 { + commit.Modified = nil + } else if hasFilesLimit > 0 && len(commit.Modified) > hasFilesLimit { + commit.Modified = commit.Modified[:hasFilesLimit] + } + } + } + if apiHeadCommit != nil { + if apiHeadCommit.Added != nil { + if hasFilesLimit == 0 { + apiHeadCommit.Added = nil + } else if hasFilesLimit > 0 && len(apiHeadCommit.Added) > hasFilesLimit { + apiHeadCommit.Added = apiHeadCommit.Added[:hasFilesLimit] + } + } + if apiHeadCommit.Removed != nil { + if hasFilesLimit == 0 { + apiHeadCommit.Removed = nil + } else if hasFilesLimit > 0 && len(apiHeadCommit.Removed) > hasFilesLimit { + apiHeadCommit.Removed = apiHeadCommit.Removed[:hasFilesLimit] + } + } + if apiHeadCommit.Modified != nil { + if hasFilesLimit == 0 { + apiHeadCommit.Modified = nil + } else if hasFilesLimit > 0 && len(apiHeadCommit.Modified) > hasFilesLimit { + apiHeadCommit.Modified = apiHeadCommit.Modified[:hasFilesLimit] + } + } + } + } + + if hasCommitsLimit != -1 { + if hasCommitsLimit == 0 { + apiCommits = nil + } else if hasCommitsLimit > 0 && len(apiCommits) > hasCommitsLimit { + apiCommits = apiCommits[:hasCommitsLimit] + } + } + + return apiCommits, apiHeadCommit +} + func (m *webhookNotifier) PushCommits(ctx context.Context, pusher *user_model.User, repo *repo_model.Repository, opts *repository.PushUpdateOptions, commits *repository.PushCommits) { apiPusher := convert.ToUser(ctx, pusher, nil) apiCommits, apiHeadCommit, err := commits.ToAPIPayloadCommits(ctx, repo) @@ -648,6 +740,9 @@ func (m *webhookNotifier) PushCommits(ctx context.Context, pusher *user_model.Us return } + // Apply payload optimizations + apiCommits, apiHeadCommit = m.applyWebhookPayloadOptimizations(ctx, repo, apiCommits, apiHeadCommit) + if err := PrepareWebhooks(ctx, EventSource{Repository: repo}, webhook_module.HookEventPush, &api.PushPayload{ Ref: opts.RefFullName.String(), Before: opts.OldCommitID, @@ -887,6 +982,9 @@ func (m *webhookNotifier) SyncPushCommits(ctx context.Context, pusher *user_mode return } + // Apply payload optimizations + apiCommits, apiHeadCommit = m.applyWebhookPayloadOptimizations(ctx, repo, apiCommits, apiHeadCommit) + if err := PrepareWebhooks(ctx, EventSource{Repository: repo}, webhook_module.HookEventPush, &api.PushPayload{ Ref: opts.RefFullName.String(), Before: opts.OldCommitID, diff --git a/services/webhook/webhook_test.go b/services/webhook/webhook_test.go index 5a805347e38a7..dd5497c62c7d4 100644 --- a/services/webhook/webhook_test.go +++ b/services/webhook/webhook_test.go @@ -91,3 +91,157 @@ func TestWebhookUserMail(t *testing.T) { assert.Equal(t, user.GetPlaceholderEmail(), convert.ToUser(db.DefaultContext, user, nil).Email) assert.Equal(t, user.Email, convert.ToUser(db.DefaultContext, user, user).Email) } + +func TestWebhookPayloadOptimization(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + var optimizedCommits []*api.PayloadCommit + var optimizedHeadCommit *api.PayloadCommit + + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + + // Clean up all webhooks for this repo to avoid interference + webhooks, err := db.Find[webhook_model.Webhook](db.DefaultContext, webhook_model.ListWebhookOptions{RepoID: repo.ID}) + assert.NoError(t, err) + for _, wh := range webhooks { + err = webhook_model.DeleteWebhookByID(db.DefaultContext, wh.ID) + assert.NoError(t, err) + } + + // Case: -1 (no trimming) + webhook := &webhook_model.Webhook{ + RepoID: repo.ID, + URL: "http://example.com/webhook", + HTTPMethod: "POST", + ContentType: webhook_model.ContentTypeJSON, + Secret: "secret", + IsActive: true, + Type: webhook_module.GITEA, + ExcludeFilesLimit: -1, + ExcludeCommitsLimit: -1, + HookEvent: &webhook_module.HookEvent{ + PushOnly: true, + }, + } + + err = webhook.UpdateEvent() + assert.NoError(t, err) + err = webhook_model.CreateWebhook(db.DefaultContext, webhook) + assert.NoError(t, err) + assert.NotZero(t, webhook.ID) + + apiCommits := []*api.PayloadCommit{ + { + ID: "abc123", + Message: "Test commit", + Added: []string{"file1.txt", "file2.txt"}, + Removed: []string{"oldfile.txt"}, + Modified: []string{"modified.txt"}, + }, + { + ID: "def456", + Message: "Another commit", + Added: []string{"file3.txt"}, + Removed: []string{}, + Modified: []string{"file1.txt"}, + }, + } + apiHeadCommit := &api.PayloadCommit{ + ID: "def456", + Message: "Another commit", + Added: []string{"file3.txt"}, + Removed: []string{}, + Modified: []string{"file1.txt"}, + } + optimizedCommits, optimizedHeadCommit = (&webhookNotifier{}).applyWebhookPayloadOptimizations(db.DefaultContext, repo, apiCommits, apiHeadCommit) + if assert.NotNil(t, optimizedCommits) && len(optimizedCommits) == 2 { + assert.Equal(t, []string{"file1.txt", "file2.txt"}, optimizedCommits[0].Added) + assert.Equal(t, []string{"oldfile.txt"}, optimizedCommits[0].Removed) + assert.Equal(t, []string{"modified.txt"}, optimizedCommits[0].Modified) + assert.Equal(t, []string{"file3.txt"}, optimizedCommits[1].Added) + assert.Equal(t, []string{}, optimizedCommits[1].Removed) + assert.Equal(t, []string{"file1.txt"}, optimizedCommits[1].Modified) + } + if assert.NotNil(t, optimizedHeadCommit) { + assert.Equal(t, []string{"file3.txt"}, optimizedHeadCommit.Added) + assert.Equal(t, []string{}, optimizedHeadCommit.Removed) + assert.Equal(t, []string{"file1.txt"}, optimizedHeadCommit.Modified) + } + + // Case: 0 (keep nothing) + webhook.ExcludeFilesLimit = 0 + webhook.ExcludeCommitsLimit = 0 + err = webhook_model.UpdateWebhook(db.DefaultContext, webhook) + assert.NoError(t, err) + apiCommits = []*api.PayloadCommit{ + { + ID: "abc123", + Message: "Test commit", + Added: []string{"file1.txt", "file2.txt"}, + Removed: []string{"oldfile.txt"}, + Modified: []string{"modified.txt"}, + }, + { + ID: "def456", + Message: "Another commit", + Added: []string{"file3.txt"}, + Removed: []string{}, + Modified: []string{"file1.txt"}, + }, + } + apiHeadCommit = &api.PayloadCommit{ + ID: "def456", + Message: "Another commit", + Added: []string{"file3.txt"}, + Removed: []string{}, + Modified: []string{"file1.txt"}, + } + optimizedCommits, optimizedHeadCommit = (&webhookNotifier{}).applyWebhookPayloadOptimizations(db.DefaultContext, repo, apiCommits, apiHeadCommit) + assert.Nil(t, optimizedCommits) + if assert.NotNil(t, optimizedHeadCommit) { + assert.Nil(t, optimizedHeadCommit.Added) + assert.Nil(t, optimizedHeadCommit.Removed) + assert.Nil(t, optimizedHeadCommit.Modified) + } + + // Case: 1 (keep only 1) + webhook.ExcludeFilesLimit = 1 + webhook.ExcludeCommitsLimit = 1 + err = webhook_model.UpdateWebhook(db.DefaultContext, webhook) + assert.NoError(t, err) + apiCommits = []*api.PayloadCommit{ + { + ID: "abc123", + Message: "Test commit", + Added: []string{"file1.txt", "file2.txt"}, + Removed: []string{"oldfile.txt"}, + Modified: []string{"modified.txt"}, + }, + { + ID: "def456", + Message: "Another commit", + Added: []string{"file3.txt"}, + Removed: []string{}, + Modified: []string{"file1.txt"}, + }, + } + apiHeadCommit = &api.PayloadCommit{ + ID: "def456", + Message: "Another commit", + Added: []string{"file3.txt"}, + Removed: []string{}, + Modified: []string{"file1.txt"}, + } + optimizedCommits, optimizedHeadCommit = (&webhookNotifier{}).applyWebhookPayloadOptimizations(db.DefaultContext, repo, apiCommits, apiHeadCommit) + if assert.NotNil(t, optimizedCommits) && len(optimizedCommits) == 1 { + assert.Equal(t, "abc123", optimizedCommits[0].ID) + assert.Equal(t, []string{"file1.txt"}, optimizedCommits[0].Added) + assert.Equal(t, []string{"oldfile.txt"}, optimizedCommits[0].Removed) + assert.Equal(t, []string{"modified.txt"}, optimizedCommits[0].Modified) + } + if assert.NotNil(t, optimizedHeadCommit) { + assert.Equal(t, []string{"file3.txt"}, optimizedHeadCommit.Added) + assert.Equal(t, []string{}, optimizedHeadCommit.Removed) + assert.Equal(t, []string{"file1.txt"}, optimizedHeadCommit.Modified) + } +} diff --git a/templates/repo/settings/webhook/settings.tmpl b/templates/repo/settings/webhook/settings.tmpl index a8ad1d6c9e5cf..0f423c7963e1a 100644 --- a/templates/repo/settings/webhook/settings.tmpl +++ b/templates/repo/settings/webhook/settings.tmpl @@ -47,6 +47,21 @@ {{ctx.Locale.Tr "repo.settings.branch_filter_desc" "https://pkg.go.dev/github.com/gobwas/glob#Compile" "github.com/gobwas/glob"}} + +
+

{{ctx.Locale.Tr "repo.settings.payload_optimization"}}

+
+ + + {{ctx.Locale.Tr "repo.settings.exclude_files_limit_desc"}} +
+
+ + + {{ctx.Locale.Tr "repo.settings.exclude_commits_limit_desc"}} +
+
+

{{ctx.Locale.Tr "repo.settings.event_desc"}}