Skip to content

Add webhook payload size optimization options #35129

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion models/migrations/migrations.go
Original file line number Diff line number Diff line change
Expand Up @@ -383,9 +383,9 @@ func prepareMigrationTasks() []*migration {
newMigration(318, "Add anonymous_access_mode for repo_unit", v1_24.AddRepoUnitAnonymousAccessMode),
newMigration(319, "Add ExclusiveOrder to Label table", v1_24.AddExclusiveOrderColumnToLabelTable),
newMigration(320, "Migrate two_factor_policy to login_source table", v1_24.MigrateSkipTwoFactor),

// Gitea 1.24.0 ends at database version 321
newMigration(321, "Use LONGTEXT for some columns and fix review_state.updated_files column", v1_25.UseLongTextInSomeColumnsAndFixBugs),
newMigration(322, "Add webhook payload optimization columns", v1_25.AddWebhookPayloadOptimizationColumns),
}
return preparedMigrations
}
Expand Down
23 changes: 23 additions & 0 deletions models/migrations/v1_25/v322.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
// Copyright 2025 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package v1_25

import (
"xorm.io/xorm"
)

func AddWebhookPayloadOptimizationColumns(x *xorm.Engine) error {
type Webhook struct {
ExcludeFilesLimit int `xorm:"exclude_files_limit NOT NULL DEFAULT -1"`
ExcludeCommitsLimit int `xorm:"exclude_commits_limit NOT NULL DEFAULT -1"`
}
_, err := x.SyncWithOptions(
xorm.SyncOptions{
IgnoreConstrains: true,
IgnoreIndices: true,
},
new(Webhook),
)
return err
}
4 changes: 4 additions & 0 deletions models/webhook/webhook.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,10 @@ type Webhook struct {
// HeaderAuthorizationEncrypted should be accessed using HeaderAuthorization() and SetHeaderAuthorization()
HeaderAuthorizationEncrypted string `xorm:"TEXT"`

// Payload size optimization options
ExcludeFilesLimit int `xorm:"exclude_files_limit"` // -1: do not trim, 0: trim all (none kept), >0: keep N file changes in commit payloads
ExcludeCommitsLimit int `xorm:"exclude_commits_limit"` // -1: do not trim, 0: trim all (none kept), >0: keep N commits in push payloads

CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
}
Expand Down
42 changes: 42 additions & 0 deletions models/webhook/webhook_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -330,3 +330,45 @@ func TestCleanupHookTaskTable_OlderThan_LeavesTaskEarlierThanAgeToDelete(t *test
assert.NoError(t, CleanupHookTaskTable(t.Context(), OlderThan, 168*time.Hour, 0))
unittest.AssertExistsAndLoadBean(t, hookTask)
}

func TestWebhookPayloadOptimization(t *testing.T) {
assert.NoError(t, unittest.PrepareTestDatabase())

webhook := &Webhook{
RepoID: 1,
URL: "http://example.com/webhook",
HTTPMethod: "POST",
ContentType: ContentTypeJSON,
Secret: "secret",
IsActive: true,
Type: webhook_module.GITEA,
ExcludeFilesLimit: 1,
ExcludeCommitsLimit: 0,
HookEvent: &webhook_module.HookEvent{
PushOnly: true,
},
}

// Test creating webhook with payload optimization options
err := CreateWebhook(db.DefaultContext, webhook)
assert.NoError(t, err)
assert.NotZero(t, webhook.ID)

// Test retrieving webhook and checking payload optimization options
retrievedWebhook, err := GetWebhookByID(db.DefaultContext, webhook.ID)
assert.NoError(t, err)
assert.Equal(t, 1, retrievedWebhook.ExcludeFilesLimit)
assert.Equal(t, 0, retrievedWebhook.ExcludeCommitsLimit)

// Test updating webhook with different payload optimization options
retrievedWebhook.ExcludeFilesLimit = 0
retrievedWebhook.ExcludeCommitsLimit = 2
err = UpdateWebhook(db.DefaultContext, retrievedWebhook)
assert.NoError(t, err)

// Verify the update
updatedWebhook, err := GetWebhookByID(db.DefaultContext, webhook.ID)
assert.NoError(t, err)
assert.Equal(t, 0, updatedWebhook.ExcludeFilesLimit)
assert.Equal(t, 2, updatedWebhook.ExcludeCommitsLimit)
}
7 changes: 6 additions & 1 deletion options/locale/locale_en-US.ini
Original file line number Diff line number Diff line change
Expand Up @@ -2424,6 +2424,11 @@ settings.event_package = Package
settings.event_package_desc = Package created or deleted in a repository.
settings.branch_filter = Branch filter
settings.branch_filter_desc = Branch whitelist for push, branch creation and branch deletion events, specified as glob pattern. If empty or <code>*</code>, events for all branches are reported. See <a href="%[1]s">%[2]s</a> documentation for syntax. Examples: <code>master</code>, <code>{master,release*}</code>.
settings.payload_optimization = Payload Size Optimization
settings.exclude_files_limit = Limit file changes
settings.exclude_files_limit_desc = -1: do not trim, 0: trim all (none kept), >0: keep N file changes
settings.exclude_commits_limit = Limit commits
settings.exclude_commits_limit_desc = -1: do not trim, 0: trim all (none kept), >0: keep N commits
settings.authorization_header = Authorization Header
settings.authorization_header_desc = Will be included as authorization header for requests when present. Examples: %s.
settings.active = Active
Expand Down Expand Up @@ -3282,7 +3287,7 @@ auths.tip.github = Register a new OAuth application on %s
auths.tip.gitlab_new = Register a new application on %s
auths.tip.google_plus = Obtain OAuth2 client credentials from the Google API console at %s
auths.tip.openid_connect = Use the OpenID Connect Discovery URL "https://{server}/.well-known/openid-configuration" to specify the endpoints
auths.tip.twitter = Go to %s, create an application and ensure that the Allow this application to be used to Sign in with Twitter option is enabled
auths.tip.twitter = Go to %s, create an application and ensure that the "Allow this application to be used to Sign in with Twitter" option is enabled
auths.tip.discord = Register a new application on %s
auths.tip.gitea = Register a new OAuth2 application. Guide can be found at %s
auths.tip.yandex = Create a new application at %s. Select following permissions from the "Yandex.Passport API" section: "Access to email address", "Access to user avatar" and "Access to username, first name and surname, gender"
Expand Down
26 changes: 15 additions & 11 deletions routers/web/repo/setting/webhook.go
Original file line number Diff line number Diff line change
Expand Up @@ -232,17 +232,19 @@ func createWebhook(ctx *context.Context, params webhookParams) {
}

w := &webhook.Webhook{
RepoID: orCtx.RepoID,
URL: params.URL,
HTTPMethod: params.HTTPMethod,
ContentType: params.ContentType,
Secret: params.WebhookForm.Secret,
HookEvent: ParseHookEvent(params.WebhookForm),
IsActive: params.WebhookForm.Active,
Type: params.Type,
Meta: string(meta),
OwnerID: orCtx.OwnerID,
IsSystemWebhook: orCtx.IsSystemWebhook,
RepoID: orCtx.RepoID,
URL: params.URL,
HTTPMethod: params.HTTPMethod,
ContentType: params.ContentType,
Secret: params.WebhookForm.Secret,
HookEvent: ParseHookEvent(params.WebhookForm),
IsActive: params.WebhookForm.Active,
Type: params.Type,
Meta: string(meta),
OwnerID: orCtx.OwnerID,
IsSystemWebhook: orCtx.IsSystemWebhook,
ExcludeFilesLimit: params.WebhookForm.ExcludeFilesLimit,
ExcludeCommitsLimit: params.WebhookForm.ExcludeCommitsLimit,
}
err = w.SetHeaderAuthorization(params.WebhookForm.AuthorizationHeader)
if err != nil {
Expand Down Expand Up @@ -294,6 +296,8 @@ func editWebhook(ctx *context.Context, params webhookParams) {
w.IsActive = params.WebhookForm.Active
w.HTTPMethod = params.HTTPMethod
w.Meta = string(meta)
w.ExcludeFilesLimit = params.WebhookForm.ExcludeFilesLimit
w.ExcludeCommitsLimit = params.WebhookForm.ExcludeCommitsLimit

err = w.SetHeaderAuthorization(params.WebhookForm.AuthorizationHeader)
if err != nil {
Expand Down
5 changes: 4 additions & 1 deletion services/forms/repo_form.go
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,9 @@ type WebhookForm struct {
BranchFilter string `binding:"GlobPattern"`
AuthorizationHeader string
Secret string
// Payload size optimization options
ExcludeFilesLimit int // -1: do not trim, 0: trim all (none kept), >0: keep N file changes in commit payloads
ExcludeCommitsLimit int // -1: do not trim, 0: trim all (none kept), >0: keep N commits in push payloads
}

// PushOnly if the hook will be triggered when push
Expand Down Expand Up @@ -622,7 +625,7 @@ type UpdateAllowEditsForm struct {
// | _// __ \| | _/ __ \\__ \ / ___// __ \
// | | \ ___/| |_\ ___/ / __ \_\___ \\ ___/
// |____|_ /\___ >____/\___ >____ /____ >\___ >
// \/ \/ \/ \/ \/ \/
// \/ \/ \/ \/ \/ \/

// NewReleaseForm form for creating release
type NewReleaseForm struct {
Expand Down
98 changes: 98 additions & 0 deletions services/webhook/notifier.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"context"

actions_model "code.gitea.io/gitea/models/actions"
"code.gitea.io/gitea/models/db"
git_model "code.gitea.io/gitea/models/git"
issues_model "code.gitea.io/gitea/models/issues"
"code.gitea.io/gitea/models/organization"
Expand All @@ -15,10 +16,12 @@ import (
access_model "code.gitea.io/gitea/models/perm/access"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
webhook_model "code.gitea.io/gitea/models/webhook"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/httplib"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/optional"
"code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
api "code.gitea.io/gitea/modules/structs"
Expand Down Expand Up @@ -640,6 +643,95 @@ func (m *webhookNotifier) IssueChangeMilestone(ctx context.Context, doer *user_m
}
}

// applyWebhookPayloadOptimizations applies payload size optimizations based on webhook configurations
func (m *webhookNotifier) applyWebhookPayloadOptimizations(ctx context.Context, repo *repo_model.Repository, apiCommits []*api.PayloadCommit, apiHeadCommit *api.PayloadCommit) ([]*api.PayloadCommit, *api.PayloadCommit) {
// Get webhooks for this repository to check their configuration
webhooks, err := db.Find[webhook_model.Webhook](ctx, webhook_model.ListWebhookOptions{
RepoID: repo.ID,
IsActive: optional.Some(true),
})
if err != nil {
log.Error("Failed to get webhooks for repository %d: %v", repo.ID, err)
// Continue with default behavior if we can't get webhooks
return apiCommits, apiHeadCommit
}

// Check if any webhook has payload optimization options enabled
hasFilesLimit := -1
hasCommitsLimit := -1
for _, webhook := range webhooks {
if webhook.HasEvent(webhook_module.HookEventPush) {
if webhook.ExcludeFilesLimit >= 0 && (hasFilesLimit == -1 || webhook.ExcludeFilesLimit < hasFilesLimit) {
hasFilesLimit = webhook.ExcludeFilesLimit
}
if webhook.ExcludeCommitsLimit >= 0 && (hasCommitsLimit == -1 || webhook.ExcludeCommitsLimit < hasCommitsLimit) {
hasCommitsLimit = webhook.ExcludeCommitsLimit
}
}
}

// Apply payload optimizations based on webhook configurations
// -1 not trim, 0 trim all (none kept), >0 trim to N commits
if hasFilesLimit != -1 {
for _, commit := range apiCommits {
if commit.Added != nil {
if hasFilesLimit == 0 {
commit.Added = nil
} else if hasFilesLimit > 0 && len(commit.Added) > hasFilesLimit {
commit.Added = commit.Added[:hasFilesLimit]
}
}
if commit.Removed != nil {
if hasFilesLimit == 0 {
commit.Removed = nil
} else if hasFilesLimit > 0 && len(commit.Removed) > hasFilesLimit {
commit.Removed = commit.Removed[:hasFilesLimit]
}
}
if commit.Modified != nil {
if hasFilesLimit == 0 {
commit.Modified = nil
} else if hasFilesLimit > 0 && len(commit.Modified) > hasFilesLimit {
commit.Modified = commit.Modified[:hasFilesLimit]
}
}
}
if apiHeadCommit != nil {
if apiHeadCommit.Added != nil {
if hasFilesLimit == 0 {
apiHeadCommit.Added = nil
} else if hasFilesLimit > 0 && len(apiHeadCommit.Added) > hasFilesLimit {
apiHeadCommit.Added = apiHeadCommit.Added[:hasFilesLimit]
}
}
if apiHeadCommit.Removed != nil {
if hasFilesLimit == 0 {
apiHeadCommit.Removed = nil
} else if hasFilesLimit > 0 && len(apiHeadCommit.Removed) > hasFilesLimit {
apiHeadCommit.Removed = apiHeadCommit.Removed[:hasFilesLimit]
}
}
if apiHeadCommit.Modified != nil {
if hasFilesLimit == 0 {
apiHeadCommit.Modified = nil
} else if hasFilesLimit > 0 && len(apiHeadCommit.Modified) > hasFilesLimit {
apiHeadCommit.Modified = apiHeadCommit.Modified[:hasFilesLimit]
}
}
}
}

if hasCommitsLimit != -1 {
if hasCommitsLimit == 0 {
apiCommits = nil
} else if hasCommitsLimit > 0 && len(apiCommits) > hasCommitsLimit {
apiCommits = apiCommits[:hasCommitsLimit]
}
}

return apiCommits, apiHeadCommit
}

func (m *webhookNotifier) PushCommits(ctx context.Context, pusher *user_model.User, repo *repo_model.Repository, opts *repository.PushUpdateOptions, commits *repository.PushCommits) {
apiPusher := convert.ToUser(ctx, pusher, nil)
apiCommits, apiHeadCommit, err := commits.ToAPIPayloadCommits(ctx, repo)
Expand All @@ -648,6 +740,9 @@ func (m *webhookNotifier) PushCommits(ctx context.Context, pusher *user_model.Us
return
}

// Apply payload optimizations
apiCommits, apiHeadCommit = m.applyWebhookPayloadOptimizations(ctx, repo, apiCommits, apiHeadCommit)

if err := PrepareWebhooks(ctx, EventSource{Repository: repo}, webhook_module.HookEventPush, &api.PushPayload{
Ref: opts.RefFullName.String(),
Before: opts.OldCommitID,
Expand Down Expand Up @@ -887,6 +982,9 @@ func (m *webhookNotifier) SyncPushCommits(ctx context.Context, pusher *user_mode
return
}

// Apply payload optimizations
apiCommits, apiHeadCommit = m.applyWebhookPayloadOptimizations(ctx, repo, apiCommits, apiHeadCommit)

if err := PrepareWebhooks(ctx, EventSource{Repository: repo}, webhook_module.HookEventPush, &api.PushPayload{
Ref: opts.RefFullName.String(),
Before: opts.OldCommitID,
Expand Down
Loading
Loading