From 000d87efe1d99de1447631a94138995067480465 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 16 Jul 2025 13:30:55 -0700 Subject: [PATCH 01/36] Fix deleting code comment bug --- models/issues/comment.go | 44 +++++++++++++++++++++++----- models/issues/comment_test.go | 15 ++++++++++ routers/api/v1/repo/issue_comment.go | 4 +-- routers/web/repo/issue_comment.go | 10 +++++-- services/issue/comments.go | 35 ++++++++++++++++++---- services/user/delete.go | 2 +- web_src/js/features/repo-issue.ts | 6 ++++ 7 files changed, 99 insertions(+), 17 deletions(-) diff --git a/models/issues/comment.go b/models/issues/comment.go index 4fdb0c1808fb4..5cd05f86d12cc 100644 --- a/models/issues/comment.go +++ b/models/issues/comment.go @@ -1122,21 +1122,21 @@ func UpdateComment(ctx context.Context, c *Comment, contentVersion int, doer *us } // DeleteComment deletes the comment -func DeleteComment(ctx context.Context, comment *Comment) error { +func DeleteComment(ctx context.Context, comment *Comment) (*Comment, error) { e := db.GetEngine(ctx) if _, err := e.ID(comment.ID).NoAutoCondition().Delete(comment); err != nil { - return err + return nil, err } if _, err := db.DeleteByBean(ctx, &ContentHistory{ CommentID: comment.ID, }); err != nil { - return err + return nil, err } if comment.Type.CountedAsConversation() { if err := UpdateIssueNumComments(ctx, comment.IssueID); err != nil { - return err + return nil, err } } if _, err := e.Table("action"). @@ -1144,14 +1144,44 @@ func DeleteComment(ctx context.Context, comment *Comment) error { Update(map[string]any{ "is_deleted": true, }); err != nil { - return err + return nil, err + } + + var deletedReviewComment *Comment + + // delete review & review comment if the code comment is the last comment of the review + if comment.Type == CommentTypeCode && comment.ReviewID > 0 { + res, err := db.GetEngine(ctx).ID(comment.ReviewID). + Where("NOT EXISTS (SELECT 1 FROM comment WHERE review_id = ? AND `type` = ?)", comment.ReviewID, CommentTypeCode). + Delete(new(Review)) + if err != nil { + return nil, err + } + if res > 0 { + var reviewComment Comment + has, err := db.GetEngine(ctx).Where("review_id = ?", comment.ReviewID). + And("type = ?", CommentTypeReview).Get(&reviewComment) + if err != nil { + return nil, err + } + if has && reviewComment.Content == "" { + if _, err := db.GetEngine(ctx).ID(reviewComment.ID).Delete(new(Comment)); err != nil { + return nil, err + } + deletedReviewComment = &reviewComment + } + comment.ReviewID = 0 // reset review ID to 0 for the notification + } } if err := comment.neuterCrossReferences(ctx); err != nil { - return err + return nil, err } - return DeleteReaction(ctx, &ReactionOptions{CommentID: comment.ID}) + if err := DeleteReaction(ctx, &ReactionOptions{CommentID: comment.ID}); err != nil { + return nil, err + } + return deletedReviewComment, nil } // UpdateCommentsMigrationsByType updates comments' migrations information via given git service type and original id and poster id diff --git a/models/issues/comment_test.go b/models/issues/comment_test.go index c08e3b970d3b2..c014cc8a61243 100644 --- a/models/issues/comment_test.go +++ b/models/issues/comment_test.go @@ -124,3 +124,18 @@ func Test_UpdateIssueNumComments(t *testing.T) { issue2 = unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 2}) assert.Equal(t, 1, issue2.NumComments) } + +func Test_DeleteCommentWithReview(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + comment := unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{ID: 7}) + assert.Equal(t, int64(10), comment.ReviewID) + review := unittest.AssertExistsAndLoadBean(t, &issues_model.Review{ID: comment.ReviewID}) + + // FIXME: the test fixtures needs a review type comment to be created + + // since this is the last comment of the review, it should be deleted when the comment is deleted + assert.NoError(t, issues_model.DeleteComment(db.DefaultContext, comment)) + + unittest.AssertNotExistsBean(t, &issues_model.Review{ID: review.ID}) +} diff --git a/routers/api/v1/repo/issue_comment.go b/routers/api/v1/repo/issue_comment.go index cc342a9313c71..ab5647fe7cbf7 100644 --- a/routers/api/v1/repo/issue_comment.go +++ b/routers/api/v1/repo/issue_comment.go @@ -721,12 +721,12 @@ func deleteIssueComment(ctx *context.APIContext) { if !ctx.IsSigned || (ctx.Doer.ID != comment.PosterID && !ctx.Repo.CanWriteIssuesOrPulls(comment.Issue.IsPull)) { ctx.Status(http.StatusForbidden) return - } else if comment.Type != issues_model.CommentTypeComment { + } else if comment.Type != issues_model.CommentTypeComment && comment.Type == issues_model.CommentTypeCode { ctx.Status(http.StatusNoContent) return } - if err = issue_service.DeleteComment(ctx, ctx.Doer, comment); err != nil { + if _, err = issue_service.DeleteComment(ctx, ctx.Doer, comment); err != nil { ctx.APIErrorInternal(err) return } diff --git a/routers/web/repo/issue_comment.go b/routers/web/repo/issue_comment.go index cb5b2d801952d..1ad6c588a7798 100644 --- a/routers/web/repo/issue_comment.go +++ b/routers/web/repo/issue_comment.go @@ -325,12 +325,18 @@ func DeleteComment(ctx *context.Context) { return } - if err = issue_service.DeleteComment(ctx, ctx.Doer, comment); err != nil { + deletedReviewComment, err := issue_service.DeleteComment(ctx, ctx.Doer, comment) + if err != nil { ctx.ServerError("DeleteComment", err) return } - ctx.Status(http.StatusOK) + res := map[string]any{} + if deletedReviewComment != nil { + res["deletedReviewCommentHashTag"] = deletedReviewComment.HashTag() + } + + ctx.JSON(http.StatusOK, res) } // ChangeCommentReaction create a reaction for comment diff --git a/services/issue/comments.go b/services/issue/comments.go index 10c81198d57e2..8ac405cb24318 100644 --- a/services/issue/comments.go +++ b/services/issue/comments.go @@ -15,6 +15,8 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" git_service "code.gitea.io/gitea/services/git" notify_service "code.gitea.io/gitea/services/notify" @@ -131,17 +133,40 @@ func UpdateComment(ctx context.Context, c *issues_model.Comment, contentVersion } // DeleteComment deletes the comment -func DeleteComment(ctx context.Context, doer *user_model.User, comment *issues_model.Comment) error { - err := db.WithTx(ctx, func(ctx context.Context) error { - return issues_model.DeleteComment(ctx, comment) +func DeleteComment(ctx context.Context, doer *user_model.User, comment *issues_model.Comment) (*issues_model.Comment, error) { + deletedReviewComment, err := db.WithTx2(ctx, func(ctx context.Context) (*issues_model.Comment, error) { + if err := comment.LoadAttachments(ctx); err != nil { + return nil, err + } + + deletedReviewComment, err := issues_model.DeleteComment(ctx, comment) + if err != nil { + return nil, err + } + + // delete comment attachments + if _, err := repo_model.DeleteAttachments(ctx, comment.Attachments, true); err != nil { + return nil, fmt.Errorf("delete attachments: %w", err) + } + + for _, attachment := range comment.Attachments { + if err := storage.Attachments.Delete(repo_model.AttachmentRelativePath(attachment.UUID)); err != nil { + // Even delete files failed, but the attachments has been removed from database, so we + // should not return error but only record the error on logs. + // users have to delete this attachments manually or we should have a + // synchronize between database attachment table and attachment storage + log.Error("delete attachment[uuid: %s] failed: %v", attachment.UUID, err) + } + } + return deletedReviewComment, nil }) if err != nil { - return err + return nil, err } notify_service.DeleteComment(ctx, doer, comment) - return nil + return deletedReviewComment, nil } // LoadCommentPushCommits Load push commits diff --git a/services/user/delete.go b/services/user/delete.go index 39c6ef052dca7..f05eb6464b43d 100644 --- a/services/user/delete.go +++ b/services/user/delete.go @@ -117,7 +117,7 @@ func deleteUser(ctx context.Context, u *user_model.User, purge bool) (err error) } for _, comment := range comments { - if err = issues_model.DeleteComment(ctx, comment); err != nil { + if _, err = issues_model.DeleteComment(ctx, comment); err != nil { return err } } diff --git a/web_src/js/features/repo-issue.ts b/web_src/js/features/repo-issue.ts index 49e8fc40a23a2..693ec4f68765e 100644 --- a/web_src/js/features/repo-issue.ts +++ b/web_src/js/features/repo-issue.ts @@ -150,6 +150,12 @@ export function initRepoIssueCommentDelete() { counter.textContent = String(num); } + const json: Record = await response.json(); + if (json.errorMessage) throw new Error(json.errorMessage); + + if (json.deletedReviewCommentHashTag) { + document.querySelector(`#${json.deletedReviewCommentHashTag}`)?.remove(); + } document.querySelector(`#${deleteButton.getAttribute('data-comment-id')}`)?.remove(); if (conversationHolder && !conversationHolder.querySelector('.comment')) { From 012b4e504f762b580614ebc237346765af6cd7db Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 16 Jul 2025 13:54:52 -0700 Subject: [PATCH 02/36] improve the test --- models/fixtures/attachment.yml | 13 +++++++++++ models/fixtures/comment.yml | 9 ++++++++ models/issues/comment_test.go | 15 ------------- services/issue/comments_test.go | 38 +++++++++++++++++++++++++++++++++ services/user/delete.go | 24 ++++++++++++++++++++- 5 files changed, 83 insertions(+), 16 deletions(-) create mode 100644 services/issue/comments_test.go diff --git a/models/fixtures/attachment.yml b/models/fixtures/attachment.yml index 7882d8bff2089..b86a15b28269f 100644 --- a/models/fixtures/attachment.yml +++ b/models/fixtures/attachment.yml @@ -153,3 +153,16 @@ download_count: 0 size: 0 created_unix: 946684800 + +- + id: 13 + uuid: a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a23 + repo_id: 1 + issue_id: 3 + release_id: 0 + uploader_id: 0 + comment_id: 7 + name: code_comment_uploaded_attachment.png + download_count: 0 + size: 0 + created_unix: 946684812 diff --git a/models/fixtures/comment.yml b/models/fixtures/comment.yml index 8fde386e226d4..7d472cdea409a 100644 --- a/models/fixtures/comment.yml +++ b/models/fixtures/comment.yml @@ -102,3 +102,12 @@ review_id: 22 assignee_id: 5 created_unix: 946684817 + +- + id: 12 + type: 22 # review + poster_id: 100 + issue_id: 3 + content: "" + review_id: 10 + created_unix: 946684812 diff --git a/models/issues/comment_test.go b/models/issues/comment_test.go index c014cc8a61243..c08e3b970d3b2 100644 --- a/models/issues/comment_test.go +++ b/models/issues/comment_test.go @@ -124,18 +124,3 @@ func Test_UpdateIssueNumComments(t *testing.T) { issue2 = unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 2}) assert.Equal(t, 1, issue2.NumComments) } - -func Test_DeleteCommentWithReview(t *testing.T) { - assert.NoError(t, unittest.PrepareTestDatabase()) - - comment := unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{ID: 7}) - assert.Equal(t, int64(10), comment.ReviewID) - review := unittest.AssertExistsAndLoadBean(t, &issues_model.Review{ID: comment.ReviewID}) - - // FIXME: the test fixtures needs a review type comment to be created - - // since this is the last comment of the review, it should be deleted when the comment is deleted - assert.NoError(t, issues_model.DeleteComment(db.DefaultContext, comment)) - - unittest.AssertNotExistsBean(t, &issues_model.Review{ID: review.ID}) -} diff --git a/services/issue/comments_test.go b/services/issue/comments_test.go new file mode 100644 index 0000000000000..2e548bc3cbe56 --- /dev/null +++ b/services/issue/comments_test.go @@ -0,0 +1,38 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package issue + +import ( + "testing" + + "code.gitea.io/gitea/models/db" + issues_model "code.gitea.io/gitea/models/issues" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + + "github.com/stretchr/testify/assert" +) + +func Test_DeleteCommentWithReview(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + comment := unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{ID: 7}) + assert.NoError(t, comment.LoadAttachments(t.Context())) + assert.Len(t, comment.Attachments, 1) + assert.Equal(t, int64(13), comment.Attachments[0].ID) + assert.Equal(t, int64(10), comment.ReviewID) + review := unittest.AssertExistsAndLoadBean(t, &issues_model.Review{ID: comment.ReviewID}) + user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) + + // since this is the last comment of the review, it should be deleted when the comment is deleted + deletedReviewComment, err := DeleteComment(db.DefaultContext, user1, comment) + assert.NoError(t, err) + assert.NotNil(t, deletedReviewComment) + + // the review should be deleted as well + unittest.AssertNotExistsBean(t, &issues_model.Review{ID: review.ID}) + // the attachment should be deleted as well + unittest.AssertNotExistsBean(t, &repo_model.Attachment{ID: comment.Attachments[0].ID}) +} diff --git a/services/user/delete.go b/services/user/delete.go index f05eb6464b43d..84848c50c3b6d 100644 --- a/services/user/delete.go +++ b/services/user/delete.go @@ -22,7 +22,9 @@ import ( pull_model "code.gitea.io/gitea/models/pull" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" "xorm.io/builder" ) @@ -105,7 +107,7 @@ func deleteUser(ctx context.Context, u *user_model.User, purge bool) (err error) if purge || (setting.Service.UserDeleteWithCommentsMaxTime != 0 && u.CreatedUnix.AsTime().Add(setting.Service.UserDeleteWithCommentsMaxTime).After(time.Now())) { - // Delete Comments + // Delete Comments with attachments const batchSize = 50 for { comments := make([]*issues_model.Comment, 0, batchSize) @@ -117,9 +119,29 @@ func deleteUser(ctx context.Context, u *user_model.User, purge bool) (err error) } for _, comment := range comments { + // Delete attachments of the comments + if err := comment.LoadAttachments(ctx); err != nil { + return err + } + if _, err = issues_model.DeleteComment(ctx, comment); err != nil { return err } + + // delete comment attachments + if _, err := repo_model.DeleteAttachments(ctx, comment.Attachments, true); err != nil { + return fmt.Errorf("delete attachments: %w", err) + } + + for _, attachment := range comment.Attachments { + if err := storage.Attachments.Delete(repo_model.AttachmentRelativePath(attachment.UUID)); err != nil { + // Even delete files failed, but the attachments has been removed from database, so we + // should not return error but only record the error on logs. + // users have to delete this attachments manually or we should have a + // synchronize between database attachment table and attachment storage + log.Error("delete attachment[uuid: %s] failed: %v", attachment.UUID, err) + } + } } } From 5cc338897cabcc55d595974eb355580df1bfd52f Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 16 Jul 2025 17:16:20 -0700 Subject: [PATCH 03/36] Remove AfterDelete function in comment --- cmd/admin_user_delete.go | 7 +- models/issues/comment.go | 12 -- models/repo/attachment.go | 58 ------- models/repo/attachment_test.go | 20 --- routers/api/v1/admin/user.go | 2 +- routers/api/v1/org/org.go | 2 +- routers/api/v1/repo/issue_attachment.go | 2 +- routers/api/v1/repo/issue_comment.go | 2 +- .../api/v1/repo/issue_comment_attachment.go | 2 +- routers/api/v1/repo/migrate.go | 2 +- routers/api/v1/repo/release_attachment.go | 2 +- routers/web/admin/users.go | 2 +- routers/web/org/setting.go | 2 +- routers/web/repo/attachment.go | 6 +- routers/web/repo/issue.go | 3 +- routers/web/repo/issue_comment.go | 2 +- routers/web/user/setting/account.go | 2 +- services/attachment/attachment.go | 41 +++++ services/attachment/attachment_test.go | 12 ++ services/cron/tasks_extended.go | 6 +- services/doctor/repository.go | 10 +- services/issue/comments.go | 36 ++-- services/issue/comments_test.go | 2 +- services/issue/issue.go | 158 ++++++++---------- services/issue/issue_test.go | 6 +- services/org/org.go | 4 +- services/org/org_test.go | 7 +- services/org/team_test.go | 8 +- services/release/release.go | 3 +- services/repository/check.go | 2 +- services/repository/create.go | 6 +- services/repository/create_test.go | 2 +- services/repository/delete.go | 15 +- services/repository/delete_test.go | 2 +- services/repository/fork.go | 2 +- services/repository/fork_test.go | 2 +- services/repository/repository.go | 2 +- services/repository/template.go | 2 +- services/user/delete.go | 15 +- services/user/user.go | 10 +- services/user/user_test.go | 22 ++- tests/integration/api_repo_test.go | 2 +- .../ephemeral_actions_runner_deletion_test.go | 7 +- tests/integration/git_push_test.go | 3 +- tests/integration/pull_compare_test.go | 4 +- tests/integration/repo_test.go | 3 +- 46 files changed, 240 insertions(+), 282 deletions(-) diff --git a/cmd/admin_user_delete.go b/cmd/admin_user_delete.go index f91041577c3e5..cbbf258f028fc 100644 --- a/cmd/admin_user_delete.go +++ b/cmd/admin_user_delete.go @@ -80,5 +80,10 @@ func runDeleteUser(ctx context.Context, c *cli.Command) error { return fmt.Errorf("the user %s does not match the provided id %d", user.Name, c.Int64("id")) } - return user_service.DeleteUser(ctx, user, c.Bool("purge")) + adminUser, err := user_model.GetAdminUser(ctx) + if err != nil { + return fmt.Errorf("failed to get admin user: %w", err) + } + + return user_service.DeleteUser(ctx, adminUser, user, c.Bool("purge")) } diff --git a/models/issues/comment.go b/models/issues/comment.go index 5cd05f86d12cc..603c6ad7ca28a 100644 --- a/models/issues/comment.go +++ b/models/issues/comment.go @@ -390,18 +390,6 @@ func (c *Comment) LoadPoster(ctx context.Context) (err error) { return err } -// AfterDelete is invoked from XORM after the object is deleted. -func (c *Comment) AfterDelete(ctx context.Context) { - if c.ID <= 0 { - return - } - - _, err := repo_model.DeleteAttachmentsByComment(ctx, c.ID, true) - if err != nil { - log.Info("Could not delete files for comment %d on issue #%d: %s", c.ID, c.IssueID, err) - } -} - // HTMLURL formats a URL-string to the issue-comment func (c *Comment) HTMLURL(ctx context.Context) string { err := c.LoadIssue(ctx) diff --git a/models/repo/attachment.go b/models/repo/attachment.go index 835bee540250d..cb70bcc52f14f 100644 --- a/models/repo/attachment.go +++ b/models/repo/attachment.go @@ -8,13 +8,10 @@ import ( "errors" "fmt" "net/url" - "os" "path" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" ) @@ -166,61 +163,6 @@ func GetAttachmentByReleaseIDFileName(ctx context.Context, releaseID int64, file return attach, nil } -// DeleteAttachment deletes the given attachment and optionally the associated file. -func DeleteAttachment(ctx context.Context, a *Attachment, remove bool) error { - _, err := DeleteAttachments(ctx, []*Attachment{a}, remove) - return err -} - -// DeleteAttachments deletes the given attachments and optionally the associated files. -func DeleteAttachments(ctx context.Context, attachments []*Attachment, remove bool) (int, error) { - if len(attachments) == 0 { - return 0, nil - } - - ids := make([]int64, 0, len(attachments)) - for _, a := range attachments { - ids = append(ids, a.ID) - } - - cnt, err := db.GetEngine(ctx).In("id", ids).NoAutoCondition().Delete(attachments[0]) - if err != nil { - return 0, err - } - - if remove { - for i, a := range attachments { - if err := storage.Attachments.Delete(a.RelativePath()); err != nil { - if !errors.Is(err, os.ErrNotExist) { - return i, err - } - log.Warn("Attachment file not found when deleting: %s", a.RelativePath()) - } - } - } - return int(cnt), nil -} - -// DeleteAttachmentsByIssue deletes all attachments associated with the given issue. -func DeleteAttachmentsByIssue(ctx context.Context, issueID int64, remove bool) (int, error) { - attachments, err := GetAttachmentsByIssueID(ctx, issueID) - if err != nil { - return 0, err - } - - return DeleteAttachments(ctx, attachments, remove) -} - -// DeleteAttachmentsByComment deletes all attachments associated with the given comment. -func DeleteAttachmentsByComment(ctx context.Context, commentID int64, remove bool) (int, error) { - attachments, err := GetAttachmentsByCommentID(ctx, commentID) - if err != nil { - return 0, err - } - - return DeleteAttachments(ctx, attachments, remove) -} - // UpdateAttachmentByUUID Updates attachment via uuid func UpdateAttachmentByUUID(ctx context.Context, attach *Attachment, cols ...string) error { if attach.UUID == "" { diff --git a/models/repo/attachment_test.go b/models/repo/attachment_test.go index c059ffd39a91e..91d44159f2269 100644 --- a/models/repo/attachment_test.go +++ b/models/repo/attachment_test.go @@ -42,26 +42,6 @@ func TestGetByCommentOrIssueID(t *testing.T) { assert.Len(t, attachments, 2) } -func TestDeleteAttachments(t *testing.T) { - assert.NoError(t, unittest.PrepareTestDatabase()) - - count, err := repo_model.DeleteAttachmentsByIssue(db.DefaultContext, 4, false) - assert.NoError(t, err) - assert.Equal(t, 2, count) - - count, err = repo_model.DeleteAttachmentsByComment(db.DefaultContext, 2, false) - assert.NoError(t, err) - assert.Equal(t, 2, count) - - err = repo_model.DeleteAttachment(db.DefaultContext, &repo_model.Attachment{ID: 8}, false) - assert.NoError(t, err) - - attachment, err := repo_model.GetAttachmentByUUID(db.DefaultContext, "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a18") - assert.Error(t, err) - assert.True(t, repo_model.IsErrAttachmentNotExist(err)) - assert.Nil(t, attachment) -} - func TestGetAttachmentByID(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) diff --git a/routers/api/v1/admin/user.go b/routers/api/v1/admin/user.go index 494bace585189..f99fc6062e471 100644 --- a/routers/api/v1/admin/user.go +++ b/routers/api/v1/admin/user.go @@ -300,7 +300,7 @@ func DeleteUser(ctx *context.APIContext) { return } - if err := user_service.DeleteUser(ctx, ctx.ContextUser, ctx.FormBool("purge")); err != nil { + if err := user_service.DeleteUser(ctx, ctx.Doer, ctx.ContextUser, ctx.FormBool("purge")); err != nil { if repo_model.IsErrUserOwnRepos(err) || org_model.IsErrUserHasOrgs(err) || packages_model.IsErrUserOwnPackages(err) || diff --git a/routers/api/v1/org/org.go b/routers/api/v1/org/org.go index cd676860658dc..f907858ea9015 100644 --- a/routers/api/v1/org/org.go +++ b/routers/api/v1/org/org.go @@ -421,7 +421,7 @@ func Delete(ctx *context.APIContext) { // "404": // "$ref": "#/responses/notFound" - if err := org.DeleteOrganization(ctx, ctx.Org.Organization, false); err != nil { + if err := org.DeleteOrganization(ctx, ctx.Doer, ctx.Org.Organization, false); err != nil { ctx.APIErrorInternal(err) return } diff --git a/routers/api/v1/repo/issue_attachment.go b/routers/api/v1/repo/issue_attachment.go index 3f751a295c37e..550abf4a7b373 100644 --- a/routers/api/v1/repo/issue_attachment.go +++ b/routers/api/v1/repo/issue_attachment.go @@ -318,7 +318,7 @@ func DeleteIssueAttachment(ctx *context.APIContext) { return } - if err := repo_model.DeleteAttachment(ctx, attachment, true); err != nil { + if err := attachment_service.DeleteAttachment(ctx, attachment); err != nil { ctx.APIErrorInternal(err) return } diff --git a/routers/api/v1/repo/issue_comment.go b/routers/api/v1/repo/issue_comment.go index ab5647fe7cbf7..a0d3d2861909b 100644 --- a/routers/api/v1/repo/issue_comment.go +++ b/routers/api/v1/repo/issue_comment.go @@ -726,7 +726,7 @@ func deleteIssueComment(ctx *context.APIContext) { return } - if _, err = issue_service.DeleteComment(ctx, ctx.Doer, comment); err != nil { + if _, err = issue_service.DeleteComment(ctx, ctx.Doer, comment, true); err != nil { ctx.APIErrorInternal(err) return } diff --git a/routers/api/v1/repo/issue_comment_attachment.go b/routers/api/v1/repo/issue_comment_attachment.go index 5f660c57504dd..704db1c7a3a83 100644 --- a/routers/api/v1/repo/issue_comment_attachment.go +++ b/routers/api/v1/repo/issue_comment_attachment.go @@ -330,7 +330,7 @@ func DeleteIssueCommentAttachment(ctx *context.APIContext) { return } - if err := repo_model.DeleteAttachment(ctx, attach, true); err != nil { + if err := attachment_service.DeleteAttachment(ctx, attach); err != nil { ctx.APIErrorInternal(err) return } diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go index c1e0b47d331a9..f2e0cad86cae6 100644 --- a/routers/api/v1/repo/migrate.go +++ b/routers/api/v1/repo/migrate.go @@ -203,7 +203,7 @@ func Migrate(ctx *context.APIContext) { } if repo != nil { - if errDelete := repo_service.DeleteRepositoryDirectly(ctx, repo.ID); errDelete != nil { + if errDelete := repo_service.DeleteRepositoryDirectly(ctx, ctx.Doer, repo.ID); errDelete != nil { log.Error("DeleteRepository: %v", errDelete) } } diff --git a/routers/api/v1/repo/release_attachment.go b/routers/api/v1/repo/release_attachment.go index defde81a1d2ae..ab47cd4fd35b1 100644 --- a/routers/api/v1/repo/release_attachment.go +++ b/routers/api/v1/repo/release_attachment.go @@ -394,7 +394,7 @@ func DeleteReleaseAttachment(ctx *context.APIContext) { } // FIXME Should prove the existence of the given repo, but results in unnecessary database requests - if err := repo_model.DeleteAttachment(ctx, attach, true); err != nil { + if err := attachment_service.DeleteAttachment(ctx, attach); err != nil { ctx.APIErrorInternal(err) return } diff --git a/routers/web/admin/users.go b/routers/web/admin/users.go index 27577cd35ba1c..85ed0c4e22093 100644 --- a/routers/web/admin/users.go +++ b/routers/web/admin/users.go @@ -496,7 +496,7 @@ func DeleteUser(ctx *context.Context) { return } - if err = user_service.DeleteUser(ctx, u, ctx.FormBool("purge")); err != nil { + if err = user_service.DeleteUser(ctx, ctx.Doer, u, ctx.FormBool("purge")); err != nil { switch { case repo_model.IsErrUserOwnRepos(err): ctx.Flash.Error(ctx.Tr("admin.users.still_own_repo")) diff --git a/routers/web/org/setting.go b/routers/web/org/setting.go index 2bc1e8bc43388..dc01e91d8a6b4 100644 --- a/routers/web/org/setting.go +++ b/routers/web/org/setting.go @@ -149,7 +149,7 @@ func SettingsDeleteOrgPost(ctx *context.Context) { return } - if err := org_service.DeleteOrganization(ctx, ctx.Org.Organization, false /* no purge */); err != nil { + if err := org_service.DeleteOrganization(ctx, ctx.Doer, ctx.Org.Organization, false /* no purge */); err != nil { if repo_model.IsErrUserOwnRepos(err) { ctx.JSONError(ctx.Tr("form.org_still_own_repo")) } else if packages_model.IsErrUserOwnPackages(err) { diff --git a/routers/web/repo/attachment.go b/routers/web/repo/attachment.go index f696669196100..9b7be58875cea 100644 --- a/routers/web/repo/attachment.go +++ b/routers/web/repo/attachment.go @@ -15,7 +15,7 @@ import ( "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/routers/common" - "code.gitea.io/gitea/services/attachment" + attachment_service "code.gitea.io/gitea/services/attachment" "code.gitea.io/gitea/services/context" "code.gitea.io/gitea/services/context/upload" repo_service "code.gitea.io/gitea/services/repository" @@ -45,7 +45,7 @@ func uploadAttachment(ctx *context.Context, repoID int64, allowedTypes string) { } defer file.Close() - attach, err := attachment.UploadAttachment(ctx, file, allowedTypes, header.Size, &repo_model.Attachment{ + attach, err := attachment_service.UploadAttachment(ctx, file, allowedTypes, header.Size, &repo_model.Attachment{ Name: header.Filename, UploaderID: ctx.Doer.ID, RepoID: repoID, @@ -77,7 +77,7 @@ func DeleteAttachment(ctx *context.Context) { ctx.HTTPError(http.StatusForbidden) return } - err = repo_model.DeleteAttachment(ctx, attach, true) + err = attachment_service.DeleteAttachment(ctx, attach) if err != nil { ctx.HTTPError(http.StatusInternalServerError, fmt.Sprintf("DeleteAttachment: %v", err)) return diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index 54b7e5df2a01c..84035b5983569 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -29,6 +29,7 @@ import ( "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers/common" + attachment_service "code.gitea.io/gitea/services/attachment" "code.gitea.io/gitea/services/context" "code.gitea.io/gitea/services/convert" "code.gitea.io/gitea/services/forms" @@ -605,7 +606,7 @@ func updateAttachments(ctx *context.Context, item any, files []string) error { if util.SliceContainsString(files, attachments[i].UUID) { continue } - if err := repo_model.DeleteAttachment(ctx, attachments[i], true); err != nil { + if err := attachment_service.DeleteAttachment(ctx, attachments[i]); err != nil { return err } } diff --git a/routers/web/repo/issue_comment.go b/routers/web/repo/issue_comment.go index 1ad6c588a7798..e2318afd03931 100644 --- a/routers/web/repo/issue_comment.go +++ b/routers/web/repo/issue_comment.go @@ -325,7 +325,7 @@ func DeleteComment(ctx *context.Context) { return } - deletedReviewComment, err := issue_service.DeleteComment(ctx, ctx.Doer, comment) + deletedReviewComment, err := issue_service.DeleteComment(ctx, ctx.Doer, comment, true) if err != nil { ctx.ServerError("DeleteComment", err) return diff --git a/routers/web/user/setting/account.go b/routers/web/user/setting/account.go index 6b17da50e5b4a..ee4cb65e12067 100644 --- a/routers/web/user/setting/account.go +++ b/routers/web/user/setting/account.go @@ -272,7 +272,7 @@ func DeleteAccount(ctx *context.Context) { return } - if err := user.DeleteUser(ctx, ctx.Doer, false); err != nil { + if err := user.DeleteUser(ctx, ctx.Doer, ctx.Doer, false); err != nil { switch { case repo_model.IsErrUserOwnRepos(err): ctx.Flash.Error(ctx.Tr("form.still_own_repo")) diff --git a/services/attachment/attachment.go b/services/attachment/attachment.go index ccb97c66c82b1..f55ec1ddb6e94 100644 --- a/services/attachment/attachment.go +++ b/services/attachment/attachment.go @@ -6,11 +6,14 @@ package attachment import ( "bytes" "context" + "errors" "fmt" "io" + "os" "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/context/upload" @@ -59,3 +62,41 @@ func UpdateAttachment(ctx context.Context, allowedTypes string, attach *repo_mod return repo_model.UpdateAttachment(ctx, attach) } + +// DeleteAttachment deletes the given attachment and optionally the associated file. +func DeleteAttachment(ctx context.Context, a *repo_model.Attachment) error { + _, err := DeleteAttachments(ctx, []*repo_model.Attachment{a}) + return err +} + +// DeleteAttachments deletes the given attachments and optionally the associated files. +func DeleteAttachments(ctx context.Context, attachments []*repo_model.Attachment) (int, error) { + if len(attachments) == 0 { + return 0, nil + } + + ids := make([]int64, 0, len(attachments)) + for _, a := range attachments { + ids = append(ids, a.ID) + } + + cnt, err := db.GetEngine(ctx).In("id", ids).NoAutoCondition().Delete(attachments[0]) + if err != nil { + return 0, err + } + + for _, a := range attachments { + if err := storage.Attachments.Delete(a.RelativePath()); err != nil { + if !errors.Is(err, os.ErrNotExist) { + // Even delete files failed, but the attachments has been removed from database, so we + // should not return error but only record the error on logs. + // users have to delete this attachments manually or we should have a + // synchronize between database attachment table and attachment storage + log.Error("delete attachment[uuid: %s] failed: %v", a.UUID, err) + } else { + log.Warn("Attachment file not found when deleting: %s", a.RelativePath()) + } + } + } + return int(cnt), nil +} diff --git a/services/attachment/attachment_test.go b/services/attachment/attachment_test.go index 65475836becab..77c6f42a3f473 100644 --- a/services/attachment/attachment_test.go +++ b/services/attachment/attachment_test.go @@ -44,3 +44,15 @@ func TestUploadAttachment(t *testing.T) { assert.Equal(t, user.ID, attachment.UploaderID) assert.Equal(t, int64(0), attachment.DownloadCount) } + +func TestDeleteAttachments(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + err := DeleteAttachment(db.DefaultContext, &repo_model.Attachment{ID: 8}) + assert.NoError(t, err) + + attachment, err := repo_model.GetAttachmentByUUID(db.DefaultContext, "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a18") + assert.Error(t, err) + assert.True(t, repo_model.IsErrAttachmentNotExist(err)) + assert.Nil(t, attachment) +} diff --git a/services/cron/tasks_extended.go b/services/cron/tasks_extended.go index 0018c5facc5d7..f3638f2279dc2 100644 --- a/services/cron/tasks_extended.go +++ b/services/cron/tasks_extended.go @@ -30,7 +30,11 @@ func registerDeleteInactiveUsers() { OlderThan: time.Minute * time.Duration(setting.Service.ActiveCodeLives), }, func(ctx context.Context, _ *user_model.User, config Config) error { olderThanConfig := config.(*OlderThanConfig) - return user_service.DeleteInactiveUsers(ctx, olderThanConfig.OlderThan) + adminUser, err := user_model.GetAdminUser(ctx) + if err != nil { + return err + } + return user_service.DeleteInactiveUsers(ctx, adminUser, olderThanConfig.OlderThan) }) } diff --git a/services/doctor/repository.go b/services/doctor/repository.go index 359c4a17e0d82..4409ed7957978 100644 --- a/services/doctor/repository.go +++ b/services/doctor/repository.go @@ -7,6 +7,7 @@ import ( "context" "code.gitea.io/gitea/models/db" + user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/storage" repo_service "code.gitea.io/gitea/services/repository" @@ -36,8 +37,11 @@ func deleteOrphanedRepos(ctx context.Context) (int64, error) { } batchSize := db.MaxBatchInsertSize("repository") - e := db.GetEngine(ctx) var deleted int64 + adminUser, err := user_model.GetAdminUser(ctx) + if err != nil { + return deleted, err + } for { select { @@ -45,7 +49,7 @@ func deleteOrphanedRepos(ctx context.Context) (int64, error) { return deleted, ctx.Err() default: var ids []int64 - if err := e.Table("`repository`"). + if err := db.GetEngine(ctx).Table("`repository`"). Join("LEFT", "`user`", "repository.owner_id=`user`.id"). Where(builder.IsNull{"`user`.id"}). Select("`repository`.id").Limit(batchSize).Find(&ids); err != nil { @@ -58,7 +62,7 @@ func deleteOrphanedRepos(ctx context.Context) (int64, error) { } for _, id := range ids { - if err := repo_service.DeleteRepositoryDirectly(ctx, id, true); err != nil { + if err := repo_service.DeleteRepositoryDirectly(ctx, adminUser, id, true); err != nil { return deleted, err } deleted++ diff --git a/services/issue/comments.go b/services/issue/comments.go index 8ac405cb24318..a638825b08751 100644 --- a/services/issue/comments.go +++ b/services/issue/comments.go @@ -15,9 +15,8 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/json" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" + attachment_service "code.gitea.io/gitea/services/attachment" git_service "code.gitea.io/gitea/services/git" notify_service "code.gitea.io/gitea/services/notify" ) @@ -132,11 +131,13 @@ func UpdateComment(ctx context.Context, c *issues_model.Comment, contentVersion return nil } -// DeleteComment deletes the comment -func DeleteComment(ctx context.Context, doer *user_model.User, comment *issues_model.Comment) (*issues_model.Comment, error) { - deletedReviewComment, err := db.WithTx2(ctx, func(ctx context.Context) (*issues_model.Comment, error) { - if err := comment.LoadAttachments(ctx); err != nil { - return nil, err +// deleteComment deletes the comment +func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAttachments bool) (*issues_model.Comment, error) { + return db.WithTx2(ctx, func(ctx context.Context) (*issues_model.Comment, error) { + if removeAttachments { + if err := comment.LoadAttachments(ctx); err != nil { + return nil, err + } } deletedReviewComment, err := issues_model.DeleteComment(ctx, comment) @@ -144,22 +145,19 @@ func DeleteComment(ctx context.Context, doer *user_model.User, comment *issues_m return nil, err } - // delete comment attachments - if _, err := repo_model.DeleteAttachments(ctx, comment.Attachments, true); err != nil { - return nil, fmt.Errorf("delete attachments: %w", err) - } - - for _, attachment := range comment.Attachments { - if err := storage.Attachments.Delete(repo_model.AttachmentRelativePath(attachment.UUID)); err != nil { - // Even delete files failed, but the attachments has been removed from database, so we - // should not return error but only record the error on logs. - // users have to delete this attachments manually or we should have a - // synchronize between database attachment table and attachment storage - log.Error("delete attachment[uuid: %s] failed: %v", attachment.UUID, err) + if removeAttachments { + // delete comment attachments + if _, err := attachment_service.DeleteAttachments(ctx, comment.Attachments); err != nil { + return nil, err } } + return deletedReviewComment, nil }) +} + +func DeleteComment(ctx context.Context, doer *user_model.User, comment *issues_model.Comment, removeAttachments bool) (*issues_model.Comment, error) { + deletedReviewComment, err := deleteComment(ctx, comment, removeAttachments) if err != nil { return nil, err } diff --git a/services/issue/comments_test.go b/services/issue/comments_test.go index 2e548bc3cbe56..bb71faa61326e 100644 --- a/services/issue/comments_test.go +++ b/services/issue/comments_test.go @@ -27,7 +27,7 @@ func Test_DeleteCommentWithReview(t *testing.T) { user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) // since this is the last comment of the review, it should be deleted when the comment is deleted - deletedReviewComment, err := DeleteComment(db.DefaultContext, user1, comment) + deletedReviewComment, err := DeleteComment(db.DefaultContext, user1, comment, true) assert.NoError(t, err) assert.NotNil(t, deletedReviewComment) diff --git a/services/issue/issue.go b/services/issue/issue.go index f03be3e18f6c7..073e3f491bf4b 100644 --- a/services/issue/issue.go +++ b/services/issue/issue.go @@ -190,13 +190,9 @@ func DeleteIssue(ctx context.Context, doer *user_model.User, gitRepo *git.Reposi } // delete entries in database - attachmentPaths, err := deleteIssue(ctx, issue) - if err != nil { + if err := deleteIssue(ctx, issue, true); err != nil { return err } - for _, attachmentPath := range attachmentPaths { - system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete issue attachment", attachmentPath) - } // delete pull request related git data if issue.IsPull && gitRepo != nil { @@ -260,107 +256,96 @@ func GetRefEndNamesAndURLs(issues []*issues_model.Issue, repoLink string) (map[i } // deleteIssue deletes the issue -func deleteIssue(ctx context.Context, issue *issues_model.Issue) ([]string, error) { - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return nil, err - } - defer committer.Close() - - if _, err := db.GetEngine(ctx).ID(issue.ID).NoAutoCondition().Delete(issue); err != nil { - return nil, err - } +func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachments bool) error { + return db.WithTx(ctx, func(ctx context.Context) error { + if _, err := db.GetEngine(ctx).ID(issue.ID).NoAutoCondition().Delete(issue); err != nil { + return err + } - // update the total issue numbers - if err := repo_model.UpdateRepoIssueNumbers(ctx, issue.RepoID, issue.IsPull, false); err != nil { - return nil, err - } - // if the issue is closed, update the closed issue numbers - if issue.IsClosed { - if err := repo_model.UpdateRepoIssueNumbers(ctx, issue.RepoID, issue.IsPull, true); err != nil { - return nil, err + // update the total issue numbers + if err := repo_model.UpdateRepoIssueNumbers(ctx, issue.RepoID, issue.IsPull, false); err != nil { + return err + } + // if the issue is closed, update the closed issue numbers + if issue.IsClosed { + if err := repo_model.UpdateRepoIssueNumbers(ctx, issue.RepoID, issue.IsPull, true); err != nil { + return err + } } - } - if err := issues_model.UpdateMilestoneCounters(ctx, issue.MilestoneID); err != nil { - return nil, fmt.Errorf("error updating counters for milestone id %d: %w", - issue.MilestoneID, err) - } + if err := issues_model.UpdateMilestoneCounters(ctx, issue.MilestoneID); err != nil { + return fmt.Errorf("error updating counters for milestone id %d: %w", + issue.MilestoneID, err) + } - if err := activities_model.DeleteIssueActions(ctx, issue.RepoID, issue.ID, issue.Index); err != nil { - return nil, err - } + if err := activities_model.DeleteIssueActions(ctx, issue.RepoID, issue.ID, issue.Index); err != nil { + return err + } - // find attachments related to this issue and remove them - if err := issue.LoadAttachments(ctx); err != nil { - return nil, err - } + if deleteAttachments { + // find attachments related to this issue and remove them + if err := issue.LoadAttachments(ctx); err != nil { + return err + } + } - var attachmentPaths []string - for i := range issue.Attachments { - attachmentPaths = append(attachmentPaths, issue.Attachments[i].RelativePath()) - } + // delete all database data still assigned to this issue + if err := db.DeleteBeans(ctx, + &issues_model.ContentHistory{IssueID: issue.ID}, + &issues_model.IssueLabel{IssueID: issue.ID}, + &issues_model.IssueDependency{IssueID: issue.ID}, + &issues_model.IssueAssignees{IssueID: issue.ID}, + &issues_model.IssueUser{IssueID: issue.ID}, + &activities_model.Notification{IssueID: issue.ID}, + &issues_model.Reaction{IssueID: issue.ID}, + &issues_model.IssueWatch{IssueID: issue.ID}, + &issues_model.Stopwatch{IssueID: issue.ID}, + &issues_model.TrackedTime{IssueID: issue.ID}, + &project_model.ProjectIssue{IssueID: issue.ID}, + &repo_model.Attachment{IssueID: issue.ID}, + &issues_model.PullRequest{IssueID: issue.ID}, + &issues_model.Comment{RefIssueID: issue.ID}, + &issues_model.IssueDependency{DependencyID: issue.ID}, + &issues_model.Comment{DependentIssueID: issue.ID}, + &issues_model.IssuePin{IssueID: issue.ID}, + ); err != nil { + return err + } - // delete all database data still assigned to this issue - if err := db.DeleteBeans(ctx, - &issues_model.ContentHistory{IssueID: issue.ID}, - &issues_model.Comment{IssueID: issue.ID}, - &issues_model.IssueLabel{IssueID: issue.ID}, - &issues_model.IssueDependency{IssueID: issue.ID}, - &issues_model.IssueAssignees{IssueID: issue.ID}, - &issues_model.IssueUser{IssueID: issue.ID}, - &activities_model.Notification{IssueID: issue.ID}, - &issues_model.Reaction{IssueID: issue.ID}, - &issues_model.IssueWatch{IssueID: issue.ID}, - &issues_model.Stopwatch{IssueID: issue.ID}, - &issues_model.TrackedTime{IssueID: issue.ID}, - &project_model.ProjectIssue{IssueID: issue.ID}, - &repo_model.Attachment{IssueID: issue.ID}, - &issues_model.PullRequest{IssueID: issue.ID}, - &issues_model.Comment{RefIssueID: issue.ID}, - &issues_model.IssueDependency{DependencyID: issue.ID}, - &issues_model.Comment{DependentIssueID: issue.ID}, - &issues_model.IssuePin{IssueID: issue.ID}, - ); err != nil { - return nil, err - } + for _, comment := range issue.Comments { + if _, err := deleteComment(ctx, comment, deleteAttachments); err != nil { + return fmt.Errorf("deleteComment [comment_id: %d]: %w", comment.ID, err) + } + } - if err := committer.Commit(); err != nil { - return nil, err - } - return attachmentPaths, nil + if deleteAttachments { + // Remove issue attachment files. + for i := range issue.Attachments { + system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete issue attachment", issue.Attachments[i].RelativePath()) + } + } + return nil + }) } // DeleteOrphanedIssues delete issues without a repo func DeleteOrphanedIssues(ctx context.Context) error { - var attachmentPaths []string - err := db.WithTx(ctx, func(ctx context.Context) error { + return db.WithTx(ctx, func(ctx context.Context) error { repoIDs, err := issues_model.GetOrphanedIssueRepoIDs(ctx) if err != nil { return err } for i := range repoIDs { - paths, err := DeleteIssuesByRepoID(ctx, repoIDs[i]) - if err != nil { + if err := DeleteIssuesByRepoID(ctx, repoIDs[i], true); err != nil { return err } - attachmentPaths = append(attachmentPaths, paths...) } return nil }) - if err != nil { - return err - } - - // Remove issue attachment files. - for i := range attachmentPaths { - system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete issue attachment", attachmentPaths[i]) - } - return nil } // DeleteIssuesByRepoID deletes issues by repositories id -func DeleteIssuesByRepoID(ctx context.Context, repoID int64) (attachmentPaths []string, err error) { +func DeleteIssuesByRepoID(ctx context.Context, repoID int64, deleteAttachments bool) error { for { issues := make([]*issues_model.Issue, 0, db.DefaultMaxInSize) if err := db.GetEngine(ctx). @@ -368,7 +353,7 @@ func DeleteIssuesByRepoID(ctx context.Context, repoID int64) (attachmentPaths [] OrderBy("id"). Limit(db.DefaultMaxInSize). Find(&issues); err != nil { - return nil, err + return err } if len(issues) == 0 { @@ -376,14 +361,11 @@ func DeleteIssuesByRepoID(ctx context.Context, repoID int64) (attachmentPaths [] } for _, issue := range issues { - issueAttachPaths, err := deleteIssue(ctx, issue) - if err != nil { - return nil, fmt.Errorf("deleteIssue [issue_id: %d]: %w", issue.ID, err) + if err := deleteIssue(ctx, issue, deleteAttachments); err != nil { + return fmt.Errorf("deleteIssue [issue_id: %d]: %w", issue.ID, err) } - - attachmentPaths = append(attachmentPaths, issueAttachPaths...) } } - return attachmentPaths, err + return nil } diff --git a/services/issue/issue_test.go b/services/issue/issue_test.go index bad0d65d1ed8f..a6bc1014ad580 100644 --- a/services/issue/issue_test.go +++ b/services/issue/issue_test.go @@ -44,7 +44,7 @@ func TestIssue_DeleteIssue(t *testing.T) { ID: issueIDs[2], } - _, err = deleteIssue(db.DefaultContext, issue) + err = deleteIssue(db.DefaultContext, issue, true) assert.NoError(t, err) issueIDs, err = issues_model.GetIssueIDsByRepoID(db.DefaultContext, 1) assert.NoError(t, err) @@ -55,7 +55,7 @@ func TestIssue_DeleteIssue(t *testing.T) { assert.NoError(t, err) issue, err = issues_model.GetIssueByID(db.DefaultContext, 4) assert.NoError(t, err) - _, err = deleteIssue(db.DefaultContext, issue) + err = deleteIssue(db.DefaultContext, issue, true) assert.NoError(t, err) assert.Len(t, attachments, 2) for i := range attachments { @@ -78,7 +78,7 @@ func TestIssue_DeleteIssue(t *testing.T) { assert.NoError(t, err) assert.False(t, left) - _, err = deleteIssue(db.DefaultContext, issue2) + err = deleteIssue(db.DefaultContext, issue2, true) assert.NoError(t, err) left, err = issues_model.IssueNoDependenciesLeft(db.DefaultContext, issue1) assert.NoError(t, err) diff --git a/services/org/org.go b/services/org/org.go index 3d30ae21a39d3..4e90d575c5735 100644 --- a/services/org/org.go +++ b/services/org/org.go @@ -47,7 +47,7 @@ func deleteOrganization(ctx context.Context, org *org_model.Organization) error } // DeleteOrganization completely and permanently deletes everything of organization. -func DeleteOrganization(ctx context.Context, org *org_model.Organization, purge bool) error { +func DeleteOrganization(ctx context.Context, doer *user_model.User, org *org_model.Organization, purge bool) error { ctx, committer, err := db.TxContext(ctx) if err != nil { return err @@ -55,7 +55,7 @@ func DeleteOrganization(ctx context.Context, org *org_model.Organization, purge defer committer.Close() if purge { - err := repo_service.DeleteOwnerRepositoriesDirectly(ctx, org.AsUser()) + err := repo_service.DeleteOwnerRepositoriesDirectly(ctx, doer, org.AsUser()) if err != nil { return err } diff --git a/services/org/org_test.go b/services/org/org_test.go index 791404c5c8919..146ad69281b63 100644 --- a/services/org/org_test.go +++ b/services/org/org_test.go @@ -22,17 +22,18 @@ func TestMain(m *testing.M) { func TestDeleteOrganization(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) org := unittest.AssertExistsAndLoadBean(t, &organization.Organization{ID: 6}) - assert.NoError(t, DeleteOrganization(db.DefaultContext, org, false)) + user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) + assert.NoError(t, DeleteOrganization(db.DefaultContext, user1, org, false)) unittest.AssertNotExistsBean(t, &organization.Organization{ID: 6}) unittest.AssertNotExistsBean(t, &organization.OrgUser{OrgID: 6}) unittest.AssertNotExistsBean(t, &organization.Team{OrgID: 6}) org = unittest.AssertExistsAndLoadBean(t, &organization.Organization{ID: 3}) - err := DeleteOrganization(db.DefaultContext, org, false) + err := DeleteOrganization(db.DefaultContext, user1, org, false) assert.Error(t, err) assert.True(t, repo_model.IsErrUserOwnRepos(err)) user := unittest.AssertExistsAndLoadBean(t, &organization.Organization{ID: 5}) - assert.Error(t, DeleteOrganization(db.DefaultContext, user, false)) + assert.Error(t, DeleteOrganization(db.DefaultContext, user1, user, false)) unittest.CheckConsistencyFor(t, &user_model.User{}, &organization.Team{}) } diff --git a/services/org/team_test.go b/services/org/team_test.go index c1a69d8ee73a0..5e9b6037d3983 100644 --- a/services/org/team_test.go +++ b/services/org/team_test.go @@ -280,8 +280,10 @@ func TestIncludesAllRepositoriesTeams(t *testing.T) { testTeamRepositories(team.ID, teamRepos[i]) } + user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) + // Remove repo and check teams repositories. - assert.NoError(t, repo_service.DeleteRepositoryDirectly(db.DefaultContext, repoIDs[0]), "DeleteRepository") + assert.NoError(t, repo_service.DeleteRepositoryDirectly(db.DefaultContext, user1, repoIDs[0]), "DeleteRepository") teamRepos[0] = repoIDs[1:] teamRepos[1] = repoIDs[1:] teamRepos[3] = repoIDs[1:3] @@ -293,8 +295,8 @@ func TestIncludesAllRepositoriesTeams(t *testing.T) { // Wipe created items. for i, rid := range repoIDs { if i > 0 { // first repo already deleted. - assert.NoError(t, repo_service.DeleteRepositoryDirectly(db.DefaultContext, rid), "DeleteRepository %d", i) + assert.NoError(t, repo_service.DeleteRepositoryDirectly(db.DefaultContext, user1, rid), "DeleteRepository %d", i) } } - assert.NoError(t, DeleteOrganization(db.DefaultContext, org, false), "DeleteOrganization") + assert.NoError(t, DeleteOrganization(db.DefaultContext, user1, org, false), "DeleteOrganization") } diff --git a/services/release/release.go b/services/release/release.go index 0b8a74252a08d..a232d52820609 100644 --- a/services/release/release.go +++ b/services/release/release.go @@ -22,6 +22,7 @@ import ( "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" + attachment_service "code.gitea.io/gitea/services/attachment" notify_service "code.gitea.io/gitea/services/notify" ) @@ -301,7 +302,7 @@ func UpdateRelease(ctx context.Context, doer *user_model.User, gitRepo *git.Repo deletedUUIDs.Add(attach.UUID) } - if _, err := repo_model.DeleteAttachments(ctx, attachments, true); err != nil { + if _, err := attachment_service.DeleteAttachments(ctx, attachments); err != nil { return fmt.Errorf("DeleteAttachments [uuids: %v]: %w", delAttachmentUUIDs, err) } } diff --git a/services/repository/check.go b/services/repository/check.go index ffcd5ac749b97..b475fbc487d29 100644 --- a/services/repository/check.go +++ b/services/repository/check.go @@ -162,7 +162,7 @@ func DeleteMissingRepositories(ctx context.Context, doer *user_model.User) error default: } log.Trace("Deleting %d/%d...", repo.OwnerID, repo.ID) - if err := DeleteRepositoryDirectly(ctx, repo.ID); err != nil { + if err := DeleteRepositoryDirectly(ctx, doer, repo.ID); err != nil { log.Error("Failed to DeleteRepository %-v: Error: %v", repo, err) if err2 := system_model.CreateRepositoryNotice("Failed to DeleteRepository %s [%d]: Error: %v", repo.FullName(), repo.ID, err); err2 != nil { log.Error("CreateRepositoryNotice: %v", err) diff --git a/services/repository/create.go b/services/repository/create.go index bed02e5d7e941..9758b3eb1c0d5 100644 --- a/services/repository/create.go +++ b/services/repository/create.go @@ -263,7 +263,7 @@ func CreateRepositoryDirectly(ctx context.Context, doer, owner *user_model.User, defer func() { if err != nil { // we can not use the ctx because it maybe canceled or timeout - cleanupRepository(repo.ID) + cleanupRepository(doer, repo.ID) } }() @@ -458,8 +458,8 @@ func createRepositoryInDB(ctx context.Context, doer, u *user_model.User, repo *r return nil } -func cleanupRepository(repoID int64) { - if errDelete := DeleteRepositoryDirectly(db.DefaultContext, repoID); errDelete != nil { +func cleanupRepository(doer *user_model.User, repoID int64) { + if errDelete := DeleteRepositoryDirectly(db.DefaultContext, doer, repoID); errDelete != nil { log.Error("cleanupRepository failed: %v", errDelete) // add system notice if err := system_model.CreateRepositoryNotice("DeleteRepositoryDirectly failed when cleanup repository: %v", errDelete); err != nil { diff --git a/services/repository/create_test.go b/services/repository/create_test.go index fe464c1441c5e..8e3fdf88a5578 100644 --- a/services/repository/create_test.go +++ b/services/repository/create_test.go @@ -35,7 +35,7 @@ func TestCreateRepositoryDirectly(t *testing.T) { unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{OwnerName: user2.Name, Name: createdRepo.Name}) - err = DeleteRepositoryDirectly(db.DefaultContext, createdRepo.ID) + err = DeleteRepositoryDirectly(db.DefaultContext, user2, createdRepo.ID) assert.NoError(t, err) // a failed creating because some mock data diff --git a/services/repository/delete.go b/services/repository/delete.go index c48d6e1d56e94..70ad9dd1802f3 100644 --- a/services/repository/delete.go +++ b/services/repository/delete.go @@ -49,7 +49,7 @@ func deleteDBRepository(ctx context.Context, repoID int64) error { // DeleteRepository deletes a repository for a user or organization. // make sure if you call this func to close open sessions (sqlite will otherwise get a deadlock) -func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams ...bool) error { +func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID int64, ignoreOrgTeams ...bool) error { ctx, committer, err := db.TxContext(ctx) if err != nil { return err @@ -193,8 +193,8 @@ func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams } // Delete Issues and related objects - var attachmentPaths []string - if attachmentPaths, err = issue_service.DeleteIssuesByRepoID(ctx, repoID); err != nil { + // attachments will be deleted later with repo_id + if err = issue_service.DeleteIssuesByRepoID(ctx, repoID, false); err != nil { return err } @@ -330,11 +330,6 @@ func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams system_model.RemoveStorageWithNotice(ctx, storage.LFS, "Delete orphaned LFS file", lfsObj) } - // Remove issue attachment files. - for _, attachment := range attachmentPaths { - system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete issue attachment", attachment) - } - // Remove release attachment files. for _, releaseAttachment := range releaseAttachments { system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete release attachment", releaseAttachment) @@ -373,7 +368,7 @@ func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams } // DeleteOwnerRepositoriesDirectly calls DeleteRepositoryDirectly for all repos of the given owner -func DeleteOwnerRepositoriesDirectly(ctx context.Context, owner *user_model.User) error { +func DeleteOwnerRepositoriesDirectly(ctx context.Context, doer, owner *user_model.User) error { for { repos, _, err := repo_model.GetUserRepositories(ctx, repo_model.SearchRepoOptions{ ListOptions: db.ListOptions{ @@ -391,7 +386,7 @@ func DeleteOwnerRepositoriesDirectly(ctx context.Context, owner *user_model.User break } for _, repo := range repos { - if err := DeleteRepositoryDirectly(ctx, repo.ID); err != nil { + if err := DeleteRepositoryDirectly(ctx, doer, repo.ID); err != nil { return fmt.Errorf("unable to delete repository %s for %s[%d]. Error: %w", repo.Name, owner.Name, owner.ID, err) } } diff --git a/services/repository/delete_test.go b/services/repository/delete_test.go index 869b8af11d57d..977c75e39e433 100644 --- a/services/repository/delete_test.go +++ b/services/repository/delete_test.go @@ -51,5 +51,5 @@ func TestDeleteOwnerRepositoriesDirectly(t *testing.T) { user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) - assert.NoError(t, repo_service.DeleteOwnerRepositoriesDirectly(db.DefaultContext, user)) + assert.NoError(t, repo_service.DeleteOwnerRepositoriesDirectly(db.DefaultContext, user, user)) } diff --git a/services/repository/fork.go b/services/repository/fork.go index 8bd3498b1715c..d0568e6072e46 100644 --- a/services/repository/fork.go +++ b/services/repository/fork.go @@ -124,7 +124,7 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork defer func() { if err != nil { // we can not use the ctx because it maybe canceled or timeout - cleanupRepository(repo.ID) + cleanupRepository(doer, repo.ID) } }() diff --git a/services/repository/fork_test.go b/services/repository/fork_test.go index 5375f790282ea..35c6effca5097 100644 --- a/services/repository/fork_test.go +++ b/services/repository/fork_test.go @@ -69,7 +69,7 @@ func TestForkRepositoryCleanup(t *testing.T) { assert.NoError(t, err) assert.True(t, exist) - err = DeleteRepositoryDirectly(db.DefaultContext, fork.ID) + err = DeleteRepositoryDirectly(db.DefaultContext, user2, fork.ID) assert.NoError(t, err) // a failed creating because some mock data diff --git a/services/repository/repository.go b/services/repository/repository.go index e574dc6c0181d..0cdce336d4939 100644 --- a/services/repository/repository.go +++ b/services/repository/repository.go @@ -69,7 +69,7 @@ func DeleteRepository(ctx context.Context, doer *user_model.User, repo *repo_mod notify_service.DeleteRepository(ctx, doer, repo) } - return DeleteRepositoryDirectly(ctx, repo.ID) + return DeleteRepositoryDirectly(ctx, doer, repo.ID) } // PushCreateRepo creates a repository when a new repository is pushed to an appropriate namespace diff --git a/services/repository/template.go b/services/repository/template.go index 6906a60083275..621bd95cb15c1 100644 --- a/services/repository/template.go +++ b/services/repository/template.go @@ -102,7 +102,7 @@ func GenerateRepository(ctx context.Context, doer, owner *user_model.User, templ defer func() { if err != nil { // we can not use the ctx because it maybe canceled or timeout - cleanupRepository(generateRepo.ID) + cleanupRepository(doer, generateRepo.ID) } }() diff --git a/services/user/delete.go b/services/user/delete.go index 84848c50c3b6d..c53e7806490cc 100644 --- a/services/user/delete.go +++ b/services/user/delete.go @@ -22,9 +22,8 @@ import ( pull_model "code.gitea.io/gitea/models/pull" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/storage" + attachment_service "code.gitea.io/gitea/services/attachment" "xorm.io/builder" ) @@ -129,19 +128,9 @@ func deleteUser(ctx context.Context, u *user_model.User, purge bool) (err error) } // delete comment attachments - if _, err := repo_model.DeleteAttachments(ctx, comment.Attachments, true); err != nil { + if _, err := attachment_service.DeleteAttachments(ctx, comment.Attachments); err != nil { return fmt.Errorf("delete attachments: %w", err) } - - for _, attachment := range comment.Attachments { - if err := storage.Attachments.Delete(repo_model.AttachmentRelativePath(attachment.UUID)); err != nil { - // Even delete files failed, but the attachments has been removed from database, so we - // should not return error but only record the error on logs. - // users have to delete this attachments manually or we should have a - // synchronize between database attachment table and attachment storage - log.Error("delete attachment[uuid: %s] failed: %v", attachment.UUID, err) - } - } } } diff --git a/services/user/user.go b/services/user/user.go index c7252430dea03..139ba8048be67 100644 --- a/services/user/user.go +++ b/services/user/user.go @@ -121,7 +121,7 @@ func RenameUser(ctx context.Context, u *user_model.User, newUserName string) err // DeleteUser completely and permanently deletes everything of a user, // but issues/comments/pulls will be kept and shown as someone has been deleted, // unless the user is younger than USER_DELETE_WITH_COMMENTS_MAX_DAYS. -func DeleteUser(ctx context.Context, u *user_model.User, purge bool) error { +func DeleteUser(ctx context.Context, doer, u *user_model.User, purge bool) error { if u.IsOrganization() { return fmt.Errorf("%s is an organization not a user", u.Name) } @@ -160,7 +160,7 @@ func DeleteUser(ctx context.Context, u *user_model.User, purge bool) error { // // An alternative option here would be write a DeleteAllRepositoriesForUserID function which would delete all of the repos // but such a function would likely get out of date - err := repo_service.DeleteOwnerRepositoriesDirectly(ctx, u) + err := repo_service.DeleteOwnerRepositoriesDirectly(ctx, doer, u) if err != nil { return err } @@ -190,7 +190,7 @@ func DeleteUser(ctx context.Context, u *user_model.User, purge bool) error { for _, org := range orgs { if err := org_service.RemoveOrgUser(ctx, org, u); err != nil { if organization.IsErrLastOrgOwner(err) { - err = org_service.DeleteOrganization(ctx, org, true) + err = org_service.DeleteOrganization(ctx, doer, org, true) if err != nil { return fmt.Errorf("unable to delete organization %d: %w", org.ID, err) } @@ -278,7 +278,7 @@ func DeleteUser(ctx context.Context, u *user_model.User, purge bool) error { } // DeleteInactiveUsers deletes all inactive users and their email addresses. -func DeleteInactiveUsers(ctx context.Context, olderThan time.Duration) error { +func DeleteInactiveUsers(ctx context.Context, doer *user_model.User, olderThan time.Duration) error { inactiveUsers, err := user_model.GetInactiveUsers(ctx, olderThan) if err != nil { return err @@ -286,7 +286,7 @@ func DeleteInactiveUsers(ctx context.Context, olderThan time.Duration) error { // FIXME: should only update authorized_keys file once after all deletions. for _, u := range inactiveUsers { - if err = DeleteUser(ctx, u, false); err != nil { + if err = DeleteUser(ctx, doer, u, false); err != nil { // Ignore inactive users that were ever active but then were set inactive by admin if repo_model.IsErrUserOwnRepos(err) || organization.IsErrUserHasOrgs(err) || packages_model.IsErrUserOwnPackages(err) { log.Warn("Inactive user %q has repositories, organizations or packages, skipping deletion: %v", u.Name, err) diff --git a/services/user/user_test.go b/services/user/user_test.go index 28a0df8628fdb..316552f6b5c07 100644 --- a/services/user/user_test.go +++ b/services/user/user_test.go @@ -27,6 +27,7 @@ func TestMain(m *testing.M) { } func TestDeleteUser(t *testing.T) { + user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) test := func(userID int64) { assert.NoError(t, unittest.PrepareTestDatabase()) user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: userID}) @@ -34,7 +35,7 @@ func TestDeleteUser(t *testing.T) { ownedRepos := make([]*repo_model.Repository, 0, 10) assert.NoError(t, db.GetEngine(db.DefaultContext).Find(&ownedRepos, &repo_model.Repository{OwnerID: userID})) if len(ownedRepos) > 0 { - err := DeleteUser(db.DefaultContext, user, false) + err := DeleteUser(db.DefaultContext, user1, user, false) assert.Error(t, err) assert.True(t, repo_model.IsErrUserOwnRepos(err)) return @@ -49,7 +50,7 @@ func TestDeleteUser(t *testing.T) { return } } - assert.NoError(t, DeleteUser(db.DefaultContext, user, false)) + assert.NoError(t, DeleteUser(db.DefaultContext, user1, user, false)) unittest.AssertNotExistsBean(t, &user_model.User{ID: userID}) unittest.CheckConsistencyFor(t, &user_model.User{}, &repo_model.Repository{}) } @@ -59,15 +60,16 @@ func TestDeleteUser(t *testing.T) { test(11) org := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 3}) - assert.Error(t, DeleteUser(db.DefaultContext, org, false)) + assert.Error(t, DeleteUser(db.DefaultContext, user1, org, false)) } func TestPurgeUser(t *testing.T) { + user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) test := func(userID int64) { assert.NoError(t, unittest.PrepareTestDatabase()) user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: userID}) - err := DeleteUser(db.DefaultContext, user, true) + err := DeleteUser(db.DefaultContext, user1, user, true) assert.NoError(t, err) unittest.AssertNotExistsBean(t, &user_model.User{ID: userID}) @@ -79,10 +81,11 @@ func TestPurgeUser(t *testing.T) { test(11) org := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 3}) - assert.Error(t, DeleteUser(db.DefaultContext, org, false)) + assert.Error(t, DeleteUser(db.DefaultContext, user1, org, false)) } func TestCreateUser(t *testing.T) { + user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) user := &user_model.User{ Name: "GiteaBot", Email: "GiteaBot@gitea.io", @@ -94,7 +97,7 @@ func TestCreateUser(t *testing.T) { assert.NoError(t, user_model.CreateUser(db.DefaultContext, user, &user_model.Meta{})) - assert.NoError(t, DeleteUser(db.DefaultContext, user, false)) + assert.NoError(t, DeleteUser(db.DefaultContext, user1, user, false)) } func TestRenameUser(t *testing.T) { @@ -172,6 +175,8 @@ func TestCreateUser_Issue5882(t *testing.T) { setting.Service.DefaultAllowCreateOrganization = true + user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) + for _, v := range tt { setting.Admin.DisableRegularOrgCreation = v.disableOrgCreation @@ -182,7 +187,7 @@ func TestCreateUser_Issue5882(t *testing.T) { assert.Equal(t, !u.AllowCreateOrganization, v.disableOrgCreation) - assert.NoError(t, DeleteUser(db.DefaultContext, v.user, false)) + assert.NoError(t, DeleteUser(db.DefaultContext, user1, v.user, false)) } } @@ -195,13 +200,14 @@ func TestDeleteInactiveUsers(t *testing.T) { err = db.Insert(db.DefaultContext, inactiveUserEmail) assert.NoError(t, err) } + user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) addUser("user-inactive-10", "user-inactive-10@test.com", timeutil.TimeStampNow().Add(-600), false) addUser("user-inactive-5", "user-inactive-5@test.com", timeutil.TimeStampNow().Add(-300), false) addUser("user-active-10", "user-active-10@test.com", timeutil.TimeStampNow().Add(-600), true) addUser("user-active-5", "user-active-5@test.com", timeutil.TimeStampNow().Add(-300), true) unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: "user-inactive-10"}) unittest.AssertExistsAndLoadBean(t, &user_model.EmailAddress{Email: "user-inactive-10@test.com"}) - assert.NoError(t, DeleteInactiveUsers(db.DefaultContext, 8*time.Minute)) + assert.NoError(t, DeleteInactiveUsers(db.DefaultContext, user1, 8*time.Minute)) unittest.AssertNotExistsBean(t, &user_model.User{Name: "user-inactive-10"}) unittest.AssertNotExistsBean(t, &user_model.EmailAddress{Email: "user-inactive-10@test.com"}) unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: "user-inactive-5"}) diff --git a/tests/integration/api_repo_test.go b/tests/integration/api_repo_test.go index a2c3a467c60d9..672c2a2c8bf9b 100644 --- a/tests/integration/api_repo_test.go +++ b/tests/integration/api_repo_test.go @@ -586,7 +586,7 @@ func TestAPIRepoTransfer(t *testing.T) { // cleanup repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: apiRepo.ID}) - _ = repo_service.DeleteRepositoryDirectly(db.DefaultContext, repo.ID) + _ = repo_service.DeleteRepositoryDirectly(db.DefaultContext, user, repo.ID) } func transfer(t *testing.T) *repo_model.Repository { diff --git a/tests/integration/ephemeral_actions_runner_deletion_test.go b/tests/integration/ephemeral_actions_runner_deletion_test.go index 40f8c643a8306..2da90b40d25c9 100644 --- a/tests/integration/ephemeral_actions_runner_deletion_test.go +++ b/tests/integration/ephemeral_actions_runner_deletion_test.go @@ -50,7 +50,9 @@ func testEphemeralActionsRunnerDeletionByRepository(t *testing.T) { task := unittest.AssertExistsAndLoadBean(t, &actions_model.ActionTask{ID: 52}) assert.Equal(t, actions_model.StatusRunning, task.Status) - err = repo_service.DeleteRepositoryDirectly(t.Context(), task.RepoID, true) + user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) + + err = repo_service.DeleteRepositoryDirectly(t.Context(), user1, task.RepoID, true) assert.NoError(t, err) _, err = actions_model.GetRunnerByID(t.Context(), 34350) @@ -68,8 +70,9 @@ func testEphemeralActionsRunnerDeletionByUser(t *testing.T) { assert.Equal(t, actions_model.StatusRunning, task.Status) user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) - err = user_service.DeleteUser(t.Context(), user, true) + err = user_service.DeleteUser(t.Context(), user1, user, true) assert.NoError(t, err) _, err = actions_model.GetRunnerByID(t.Context(), 34350) diff --git a/tests/integration/git_push_test.go b/tests/integration/git_push_test.go index d716847b5417f..c307cf6b606b6 100644 --- a/tests/integration/git_push_test.go +++ b/tests/integration/git_push_test.go @@ -191,7 +191,8 @@ func runTestGitPush(t *testing.T, u *url.URL, gitOperation func(t *testing.T, gi assert.Equal(t, commitID, branch.CommitID) } - require.NoError(t, repo_service.DeleteRepositoryDirectly(db.DefaultContext, repo.ID)) + user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) + require.NoError(t, repo_service.DeleteRepositoryDirectly(db.DefaultContext, user1, repo.ID)) } func TestPushPullRefs(t *testing.T) { diff --git a/tests/integration/pull_compare_test.go b/tests/integration/pull_compare_test.go index f95a2f1690929..039cdcfd5c3f7 100644 --- a/tests/integration/pull_compare_test.go +++ b/tests/integration/pull_compare_test.go @@ -13,6 +13,7 @@ import ( issues_model "code.gitea.io/gitea/models/issues" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/test" repo_service "code.gitea.io/gitea/services/repository" "code.gitea.io/gitea/tests" @@ -76,8 +77,9 @@ func TestPullCompare(t *testing.T) { repoForked := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{OwnerName: "user1", Name: "repo1"}) + user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) // delete the head repository and revisit the PR diff view - err := repo_service.DeleteRepositoryDirectly(db.DefaultContext, repoForked.ID) + err := repo_service.DeleteRepositoryDirectly(db.DefaultContext, user1, repoForked.ID) assert.NoError(t, err) req = NewRequest(t, "GET", prFilesURL) diff --git a/tests/integration/repo_test.go b/tests/integration/repo_test.go index adfe07519faed..1a735381d9a93 100644 --- a/tests/integration/repo_test.go +++ b/tests/integration/repo_test.go @@ -539,8 +539,9 @@ func TestGenerateRepository(t *testing.T) { assert.True(t, exist) unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{OwnerName: user2.Name, Name: generatedRepo.Name}) + user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) - err = repo_service.DeleteRepositoryDirectly(db.DefaultContext, generatedRepo.ID) + err = repo_service.DeleteRepositoryDirectly(db.DefaultContext, user1, generatedRepo.ID) assert.NoError(t, err) // a failed creating because some mock data From 2d423dc78c590ce83f5121a9bf3b80ab9d69123f Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 16 Jul 2025 17:19:21 -0700 Subject: [PATCH 04/36] Fix bug --- routers/api/v1/repo/issue_comment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/api/v1/repo/issue_comment.go b/routers/api/v1/repo/issue_comment.go index a0d3d2861909b..3807cb60d6bcc 100644 --- a/routers/api/v1/repo/issue_comment.go +++ b/routers/api/v1/repo/issue_comment.go @@ -721,7 +721,7 @@ func deleteIssueComment(ctx *context.APIContext) { if !ctx.IsSigned || (ctx.Doer.ID != comment.PosterID && !ctx.Repo.CanWriteIssuesOrPulls(comment.Issue.IsPull)) { ctx.Status(http.StatusForbidden) return - } else if comment.Type != issues_model.CommentTypeComment && comment.Type == issues_model.CommentTypeCode { + } else if comment.Type != issues_model.CommentTypeComment && comment.Type != issues_model.CommentTypeCode { ctx.Status(http.StatusNoContent) return } From 3abb7571a8b094b0263b878b6bf188c1a0a49c64 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 16 Jul 2025 19:33:20 -0700 Subject: [PATCH 05/36] improvements --- modules/util/cleanup.go | 17 ++++++ routers/api/v1/repo/issue_comment.go | 2 +- routers/web/repo/issue_comment.go | 2 +- services/issue/comments.go | 41 ++++++++++--- services/issue/comments_test.go | 2 +- services/issue/issue.go | 38 +++++++++--- services/issue/issue_test.go | 9 ++- services/release/release.go | 8 +-- services/repository/delete.go | 47 +++++++------- services/user/delete.go | 91 ++++++++++++++++++---------- services/user/user.go | 5 +- 11 files changed, 177 insertions(+), 85 deletions(-) create mode 100644 modules/util/cleanup.go diff --git a/modules/util/cleanup.go b/modules/util/cleanup.go new file mode 100644 index 0000000000000..687dac27b5ffd --- /dev/null +++ b/modules/util/cleanup.go @@ -0,0 +1,17 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package util + +type CleanUpFunc func() + +func NewCleanUpFunc() CleanUpFunc { + return func() {} +} + +func (f CleanUpFunc) Append(newF CleanUpFunc) CleanUpFunc { + return func() { + f() + newF() + } +} diff --git a/routers/api/v1/repo/issue_comment.go b/routers/api/v1/repo/issue_comment.go index 3807cb60d6bcc..feb9f1da64cfe 100644 --- a/routers/api/v1/repo/issue_comment.go +++ b/routers/api/v1/repo/issue_comment.go @@ -726,7 +726,7 @@ func deleteIssueComment(ctx *context.APIContext) { return } - if _, err = issue_service.DeleteComment(ctx, ctx.Doer, comment, true); err != nil { + if _, err = issue_service.DeleteComment(ctx, ctx.Doer, comment); err != nil { ctx.APIErrorInternal(err) return } diff --git a/routers/web/repo/issue_comment.go b/routers/web/repo/issue_comment.go index e2318afd03931..1ad6c588a7798 100644 --- a/routers/web/repo/issue_comment.go +++ b/routers/web/repo/issue_comment.go @@ -325,7 +325,7 @@ func DeleteComment(ctx *context.Context) { return } - deletedReviewComment, err := issue_service.DeleteComment(ctx, ctx.Doer, comment, true) + deletedReviewComment, err := issue_service.DeleteComment(ctx, ctx.Doer, comment) if err != nil { ctx.ServerError("DeleteComment", err) return diff --git a/services/issue/comments.go b/services/issue/comments.go index a638825b08751..4a6b987ea1642 100644 --- a/services/issue/comments.go +++ b/services/issue/comments.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "os" "code.gitea.io/gitea/models/db" issues_model "code.gitea.io/gitea/models/issues" @@ -15,8 +16,10 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" - attachment_service "code.gitea.io/gitea/services/attachment" + "code.gitea.io/gitea/modules/util" git_service "code.gitea.io/gitea/services/git" notify_service "code.gitea.io/gitea/services/notify" ) @@ -132,9 +135,11 @@ func UpdateComment(ctx context.Context, c *issues_model.Comment, contentVersion } // deleteComment deletes the comment -func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAttachments bool) (*issues_model.Comment, error) { - return db.WithTx2(ctx, func(ctx context.Context) (*issues_model.Comment, error) { +func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAttachments bool) (*issues_model.Comment, func(), error) { + storageCleanup := util.NewCleanUpFunc() + deletedReviewComment, err := db.WithTx2(ctx, func(ctx context.Context) (*issues_model.Comment, error) { if removeAttachments { + // load attachments before deleting the comment if err := comment.LoadAttachments(ctx); err != nil { return nil, err } @@ -147,20 +152,42 @@ func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAtt if removeAttachments { // delete comment attachments - if _, err := attachment_service.DeleteAttachments(ctx, comment.Attachments); err != nil { + _, err := db.GetEngine(ctx).Where("comment_id = ?", comment.ID).NoAutoCondition().Delete(&repo_model.Attachment{}) + if err != nil { return nil, err } - } + // the storage cleanup function to remove attachments could be called after all transactions are committed + storageCleanup = storageCleanup.Append(func() { + for _, a := range comment.Attachments { + if err := storage.Attachments.Delete(a.RelativePath()); err != nil { + if !errors.Is(err, os.ErrNotExist) { + // Even delete files failed, but the attachments has been removed from database, so we + // should not return error but only record the error on logs. + // users have to delete this attachments manually or we should have a + // synchronize between database attachment table and attachment storage + log.Error("delete attachment[uuid: %s] failed: %v", a.UUID, err) + } else { + log.Warn("Attachment file not found when deleting: %s", a.RelativePath()) + } + } + } + }) + } return deletedReviewComment, nil }) + if err != nil { + return nil, nil, err + } + return deletedReviewComment, storageCleanup, nil } -func DeleteComment(ctx context.Context, doer *user_model.User, comment *issues_model.Comment, removeAttachments bool) (*issues_model.Comment, error) { - deletedReviewComment, err := deleteComment(ctx, comment, removeAttachments) +func DeleteComment(ctx context.Context, doer *user_model.User, comment *issues_model.Comment) (*issues_model.Comment, error) { + deletedReviewComment, cleanup, err := deleteComment(ctx, comment, false) if err != nil { return nil, err } + cleanup() notify_service.DeleteComment(ctx, doer, comment) diff --git a/services/issue/comments_test.go b/services/issue/comments_test.go index bb71faa61326e..2e548bc3cbe56 100644 --- a/services/issue/comments_test.go +++ b/services/issue/comments_test.go @@ -27,7 +27,7 @@ func Test_DeleteCommentWithReview(t *testing.T) { user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) // since this is the last comment of the review, it should be deleted when the comment is deleted - deletedReviewComment, err := DeleteComment(db.DefaultContext, user1, comment, true) + deletedReviewComment, err := DeleteComment(db.DefaultContext, user1, comment) assert.NoError(t, err) assert.NotNil(t, deletedReviewComment) diff --git a/services/issue/issue.go b/services/issue/issue.go index 073e3f491bf4b..39e1c9a01cbf5 100644 --- a/services/issue/issue.go +++ b/services/issue/issue.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/modules/util" notify_service "code.gitea.io/gitea/services/notify" ) @@ -190,9 +191,11 @@ func DeleteIssue(ctx context.Context, doer *user_model.User, gitRepo *git.Reposi } // delete entries in database - if err := deleteIssue(ctx, issue, true); err != nil { + cleanup, err := deleteIssue(ctx, issue, true) + if err != nil { return err } + cleanup() // delete pull request related git data if issue.IsPull && gitRepo != nil { @@ -256,8 +259,9 @@ func GetRefEndNamesAndURLs(issues []*issues_model.Issue, repoLink string) (map[i } // deleteIssue deletes the issue -func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachments bool) error { - return db.WithTx(ctx, func(ctx context.Context) error { +func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachments bool) (util.CleanUpFunc, error) { + cleanup := util.NewCleanUpFunc() + if err := db.WithTx(ctx, func(ctx context.Context) error { if _, err := db.GetEngine(ctx).ID(issue.ID).NoAutoCondition().Delete(issue); err != nil { return err } @@ -302,7 +306,6 @@ func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachmen &issues_model.Stopwatch{IssueID: issue.ID}, &issues_model.TrackedTime{IssueID: issue.ID}, &project_model.ProjectIssue{IssueID: issue.ID}, - &repo_model.Attachment{IssueID: issue.ID}, &issues_model.PullRequest{IssueID: issue.ID}, &issues_model.Comment{RefIssueID: issue.ID}, &issues_model.IssueDependency{DependencyID: issue.ID}, @@ -313,19 +316,32 @@ func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachmen } for _, comment := range issue.Comments { - if _, err := deleteComment(ctx, comment, deleteAttachments); err != nil { + _, cleanupDeleteComment, err := deleteComment(ctx, comment, deleteAttachments) + if err != nil { return fmt.Errorf("deleteComment [comment_id: %d]: %w", comment.ID, err) } + cleanup = cleanup.Append(cleanupDeleteComment) } if deleteAttachments { - // Remove issue attachment files. - for i := range issue.Attachments { - system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete issue attachment", issue.Attachments[i].RelativePath()) + // delete issue attachments + _, err := db.GetEngine(ctx).Where("issue_id = ? AND comment_id = 0", issue.ID).NoAutoCondition().Delete(&issues_model.Issue{}) + if err != nil { + return err } + // the storage cleanup function to remove attachments could be called after all transactions are committed + cleanup = cleanup.Append(func() { + // Remove issue attachment files. + for i := range issue.Attachments { + system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete issue attachment", issue.Attachments[i].RelativePath()) + } + }) } return nil - }) + }); err != nil { + return nil, err + } + return cleanup, nil } // DeleteOrphanedIssues delete issues without a repo @@ -361,9 +377,11 @@ func DeleteIssuesByRepoID(ctx context.Context, repoID int64, deleteAttachments b } for _, issue := range issues { - if err := deleteIssue(ctx, issue, deleteAttachments); err != nil { + cleanup, err := deleteIssue(ctx, issue, deleteAttachments) + if err != nil { return fmt.Errorf("deleteIssue [issue_id: %d]: %w", issue.ID, err) } + cleanup() } } diff --git a/services/issue/issue_test.go b/services/issue/issue_test.go index a6bc1014ad580..f8cb85db3f32b 100644 --- a/services/issue/issue_test.go +++ b/services/issue/issue_test.go @@ -44,8 +44,9 @@ func TestIssue_DeleteIssue(t *testing.T) { ID: issueIDs[2], } - err = deleteIssue(db.DefaultContext, issue, true) + cleanup, err := deleteIssue(db.DefaultContext, issue, true) assert.NoError(t, err) + cleanup() issueIDs, err = issues_model.GetIssueIDsByRepoID(db.DefaultContext, 1) assert.NoError(t, err) assert.Len(t, issueIDs, 4) @@ -55,8 +56,9 @@ func TestIssue_DeleteIssue(t *testing.T) { assert.NoError(t, err) issue, err = issues_model.GetIssueByID(db.DefaultContext, 4) assert.NoError(t, err) - err = deleteIssue(db.DefaultContext, issue, true) + cleanup, err = deleteIssue(db.DefaultContext, issue, true) assert.NoError(t, err) + cleanup() assert.Len(t, attachments, 2) for i := range attachments { attachment, err := repo_model.GetAttachmentByUUID(db.DefaultContext, attachments[i].UUID) @@ -78,8 +80,9 @@ func TestIssue_DeleteIssue(t *testing.T) { assert.NoError(t, err) assert.False(t, left) - err = deleteIssue(db.DefaultContext, issue2, true) + cleanup, err = deleteIssue(db.DefaultContext, issue2, true) assert.NoError(t, err) + cleanup() left, err = issues_model.IssueNoDependenciesLeft(db.DefaultContext, issue1) assert.NoError(t, err) assert.True(t, left) diff --git a/services/release/release.go b/services/release/release.go index a232d52820609..42af8e85187b8 100644 --- a/services/release/release.go +++ b/services/release/release.go @@ -22,7 +22,6 @@ import ( "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" - attachment_service "code.gitea.io/gitea/services/attachment" notify_service "code.gitea.io/gitea/services/notify" ) @@ -302,8 +301,9 @@ func UpdateRelease(ctx context.Context, doer *user_model.User, gitRepo *git.Repo deletedUUIDs.Add(attach.UUID) } - if _, err := attachment_service.DeleteAttachments(ctx, attachments); err != nil { - return fmt.Errorf("DeleteAttachments [uuids: %v]: %w", delAttachmentUUIDs, err) + _, err = db.GetEngine(ctx).In("uuid", deletedUUIDs.Values()).NoAutoCondition().Delete(&repo_model.Attachment{}) + if err != nil { + return err } } @@ -339,7 +339,7 @@ func UpdateRelease(ctx context.Context, doer *user_model.User, gitRepo *git.Repo return err } - for _, uuid := range delAttachmentUUIDs { + for _, uuid := range deletedUUIDs.Values() { if err := storage.Attachments.Delete(repo_model.AttachmentRelativePath(uuid)); err != nil { // Even delete files failed, but the attachments has been removed from database, so we // should not return error but only record the error on logs. diff --git a/services/repository/delete.go b/services/repository/delete.go index 70ad9dd1802f3..3b000b8219678 100644 --- a/services/repository/delete.go +++ b/services/repository/delete.go @@ -115,15 +115,22 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID } } - attachments := make([]*repo_model.Attachment, 0, 20) - if err = sess.Join("INNER", "`release`", "`release`.id = `attachment`.release_id"). + // some attachments have release_id but repo_id = 0 + releaseAttachments := make([]*repo_model.Attachment, 0, 20) + if err = db.GetEngine(ctx).Join("INNER", "`release`", "`release`.id = `attachment`.release_id"). Where("`release`.repo_id = ?", repoID). - Find(&attachments); err != nil { + Find(&releaseAttachments); err != nil { return err } - releaseAttachments := make([]string, 0, len(attachments)) - for i := 0; i < len(attachments); i++ { - releaseAttachments = append(releaseAttachments, attachments[i].RelativePath()) + // Delete attachments with release_id but repo_id = 0 + if len(releaseAttachments) > 0 { + ids := make([]int64, 0, len(releaseAttachments)) + for _, attach := range releaseAttachments { + ids = append(ids, attach.ID) + } + if _, err := db.GetEngine(ctx).In("id", ids).Delete(&repo_model.Attachment{}); err != nil { + return fmt.Errorf("delete release attachments failed: %w", err) + } } if _, err := db.Exec(ctx, "UPDATE `user` SET num_stars=num_stars-1 WHERE id IN (SELECT `uid` FROM `star` WHERE repo_id = ?)", repo.ID); err != nil { @@ -267,21 +274,14 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID } } - // Get all attachments with both issue_id and release_id are zero - var newAttachments []*repo_model.Attachment + // Get all attachments with repo_id = repo.ID. some release attachments have repo_id = 0 should be deleted before + var repoAttachments []*repo_model.Attachment if err := sess.Where(builder.Eq{ - "repo_id": repo.ID, - "issue_id": 0, - "release_id": 0, - }).Find(&newAttachments); err != nil { + "repo_id": repo.ID, + }).Find(&repoAttachments); err != nil { return err } - newAttachmentPaths := make([]string, 0, len(newAttachments)) - for _, attach := range newAttachments { - newAttachmentPaths = append(newAttachmentPaths, attach.RelativePath()) - } - if _, err := sess.Where("repo_id=?", repo.ID).Delete(new(repo_model.Attachment)); err != nil { return err } @@ -330,14 +330,13 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID system_model.RemoveStorageWithNotice(ctx, storage.LFS, "Delete orphaned LFS file", lfsObj) } - // Remove release attachment files. - for _, releaseAttachment := range releaseAttachments { - system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete release attachment", releaseAttachment) + // Remove release attachments + for _, attachment := range releaseAttachments { + system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete release attachment", attachment.RelativePath()) } - - // Remove attachment with no issue_id and release_id. - for _, newAttachment := range newAttachmentPaths { - system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete issue attachment", newAttachment) + // Remove attachment with repo_id = repo.ID. + for _, attachment := range repoAttachments { + system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete repo attachment", attachment.RelativePath()) } if len(repo.Avatar) > 0 { diff --git a/services/user/delete.go b/services/user/delete.go index c53e7806490cc..89941243fc7b9 100644 --- a/services/user/delete.go +++ b/services/user/delete.go @@ -5,7 +5,9 @@ package user import ( "context" + "errors" "fmt" + "os" "time" _ "image/jpeg" // Needed for jpeg support @@ -22,25 +24,27 @@ import ( pull_model "code.gitea.io/gitea/models/pull" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - attachment_service "code.gitea.io/gitea/services/attachment" + "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/modules/util" "xorm.io/builder" ) // deleteUser deletes models associated to an user. -func deleteUser(ctx context.Context, u *user_model.User, purge bool) (err error) { - e := db.GetEngine(ctx) +func deleteUser(ctx context.Context, u *user_model.User, purge bool) (cleanup util.CleanUpFunc, err error) { + cleanup = util.NewCleanUpFunc() // ***** START: Watch ***** watchedRepoIDs, err := db.FindIDs(ctx, "watch", "watch.repo_id", builder.Eq{"watch.user_id": u.ID}. And(builder.Neq{"watch.mode": repo_model.WatchModeDont})) if err != nil { - return fmt.Errorf("get all watches: %w", err) + return nil, fmt.Errorf("get all watches: %w", err) } if err = db.DecrByIDs(ctx, watchedRepoIDs, "num_watches", new(repo_model.Repository)); err != nil { - return fmt.Errorf("decrease repository num_watches: %w", err) + return nil, fmt.Errorf("decrease repository num_watches: %w", err) } // ***** END: Watch ***** @@ -48,9 +52,9 @@ func deleteUser(ctx context.Context, u *user_model.User, purge bool) (err error) starredRepoIDs, err := db.FindIDs(ctx, "star", "star.repo_id", builder.Eq{"star.uid": u.ID}) if err != nil { - return fmt.Errorf("get all stars: %w", err) + return nil, fmt.Errorf("get all stars: %w", err) } else if err = db.DecrByIDs(ctx, starredRepoIDs, "num_stars", new(repo_model.Repository)); err != nil { - return fmt.Errorf("decrease repository num_stars: %w", err) + return nil, fmt.Errorf("decrease repository num_stars: %w", err) } // ***** END: Star ***** @@ -58,17 +62,17 @@ func deleteUser(ctx context.Context, u *user_model.User, purge bool) (err error) followeeIDs, err := db.FindIDs(ctx, "follow", "follow.follow_id", builder.Eq{"follow.user_id": u.ID}) if err != nil { - return fmt.Errorf("get all followees: %w", err) + return nil, fmt.Errorf("get all followees: %w", err) } else if err = db.DecrByIDs(ctx, followeeIDs, "num_followers", new(user_model.User)); err != nil { - return fmt.Errorf("decrease user num_followers: %w", err) + return nil, fmt.Errorf("decrease user num_followers: %w", err) } followerIDs, err := db.FindIDs(ctx, "follow", "follow.user_id", builder.Eq{"follow.follow_id": u.ID}) if err != nil { - return fmt.Errorf("get all followers: %w", err) + return nil, fmt.Errorf("get all followers: %w", err) } else if err = db.DecrByIDs(ctx, followerIDs, "num_following", new(user_model.User)); err != nil { - return fmt.Errorf("decrease user num_following: %w", err) + return nil, fmt.Errorf("decrease user num_following: %w", err) } // ***** END: Follow ***** @@ -97,11 +101,11 @@ func deleteUser(ctx context.Context, u *user_model.User, purge bool) (err error) &user_model.Blocking{BlockeeID: u.ID}, &actions_model.ActionRunnerToken{OwnerID: u.ID}, ); err != nil { - return fmt.Errorf("deleteBeans: %w", err) + return nil, fmt.Errorf("deleteBeans: %w", err) } if err := auth_model.DeleteOAuth2RelictsByUserID(ctx, u.ID); err != nil { - return err + return nil, fmt.Errorf("deleteOAuth2RelictsByUserID: %w", err) } if purge || (setting.Service.UserDeleteWithCommentsMaxTime != 0 && @@ -110,8 +114,8 @@ func deleteUser(ctx context.Context, u *user_model.User, purge bool) (err error) const batchSize = 50 for { comments := make([]*issues_model.Comment, 0, batchSize) - if err = e.Where("type=? AND poster_id=?", issues_model.CommentTypeComment, u.ID).Limit(batchSize, 0).Find(&comments); err != nil { - return err + if err = db.GetEngine(ctx).Where("type=? AND poster_id=?", issues_model.CommentTypeComment, u.ID).Limit(batchSize, 0).Find(&comments); err != nil { + return nil, err } if len(comments) == 0 { break @@ -120,23 +124,44 @@ func deleteUser(ctx context.Context, u *user_model.User, purge bool) (err error) for _, comment := range comments { // Delete attachments of the comments if err := comment.LoadAttachments(ctx); err != nil { - return err + return nil, err } if _, err = issues_model.DeleteComment(ctx, comment); err != nil { - return err + return nil, err } - // delete comment attachments - if _, err := attachment_service.DeleteAttachments(ctx, comment.Attachments); err != nil { - return fmt.Errorf("delete attachments: %w", err) + ids := make([]int64, 0, len(comment.Attachments)) + for _, a := range comment.Attachments { + ids = append(ids, a.ID) } + + _, err := db.GetEngine(ctx).In("id", ids).NoAutoCondition().Delete(&repo_model.Attachment{}) + if err != nil { + return nil, err + } + + cleanup = cleanup.Append(func() { + for _, a := range comment.Attachments { + if err := storage.Attachments.Delete(a.RelativePath()); err != nil { + if !errors.Is(err, os.ErrNotExist) { + // Even delete files failed, but the attachments has been removed from database, so we + // should not return error but only record the error on logs. + // users have to delete this attachments manually or we should have a + // synchronize between database attachment table and attachment storage + log.Error("delete attachment[uuid: %s] failed: %v", a.UUID, err) + } else { + log.Warn("Attachment file not found when deleting: %s", a.RelativePath()) + } + } + } + }) } } // Delete Reactions if err = issues_model.DeleteReaction(ctx, &issues_model.ReactionOptions{DoerID: u.ID}); err != nil { - return err + return nil, err } } @@ -150,15 +175,15 @@ func deleteUser(ctx context.Context, u *user_model.User, purge bool) (err error) // though that query will be quite complex and tricky to maintain (compare `getRepoAssignees()`). // Also, as we didn't update branch protections when removing entries from `access` table, // it's safer to iterate all protected branches. - if err = e.Limit(batchSize, start).Find(&protections); err != nil { - return fmt.Errorf("findProtectedBranches: %w", err) + if err = db.GetEngine(ctx).Limit(batchSize, start).Find(&protections); err != nil { + return nil, fmt.Errorf("findProtectedBranches: %w", err) } if len(protections) == 0 { break } for _, p := range protections { if err := git_model.RemoveUserIDFromProtectedBranch(ctx, p, u.ID); err != nil { - return err + return nil, err } } } @@ -167,7 +192,7 @@ func deleteUser(ctx context.Context, u *user_model.User, purge bool) (err error) // ***** START: PublicKey ***** if _, err = db.DeleteByBean(ctx, &asymkey_model.PublicKey{OwnerID: u.ID}); err != nil { - return fmt.Errorf("deletePublicKeys: %w", err) + return nil, fmt.Errorf("deletePublicKeys: %w", err) } // ***** END: PublicKey ***** @@ -176,37 +201,37 @@ func deleteUser(ctx context.Context, u *user_model.User, purge bool) (err error) OwnerID: u.ID, }) if err != nil { - return fmt.Errorf("ListGPGKeys: %w", err) + return nil, fmt.Errorf("ListGPGKeys: %w", err) } // Delete GPGKeyImport(s). for _, key := range keys { if _, err = db.DeleteByBean(ctx, &asymkey_model.GPGKeyImport{KeyID: key.KeyID}); err != nil { - return fmt.Errorf("deleteGPGKeyImports: %w", err) + return nil, fmt.Errorf("deleteGPGKeyImports: %w", err) } } if _, err = db.DeleteByBean(ctx, &asymkey_model.GPGKey{OwnerID: u.ID}); err != nil { - return fmt.Errorf("deleteGPGKeys: %w", err) + return nil, fmt.Errorf("deleteGPGKeys: %w", err) } // ***** END: GPGPublicKey ***** // Clear assignee. if _, err = db.DeleteByBean(ctx, &issues_model.IssueAssignees{AssigneeID: u.ID}); err != nil { - return fmt.Errorf("clear assignee: %w", err) + return nil, fmt.Errorf("clear assignee: %w", err) } // ***** START: ExternalLoginUser ***** if err = user_model.RemoveAllAccountLinks(ctx, u); err != nil { - return fmt.Errorf("ExternalLoginUser: %w", err) + return nil, fmt.Errorf("ExternalLoginUser: %w", err) } // ***** END: ExternalLoginUser ***** if err := auth_model.DeleteAuthTokensByUserID(ctx, u.ID); err != nil { - return fmt.Errorf("DeleteAuthTokensByUserID: %w", err) + return nil, fmt.Errorf("DeleteAuthTokensByUserID: %w", err) } if _, err = db.DeleteByID[user_model.User](ctx, u.ID); err != nil { - return fmt.Errorf("delete: %w", err) + return nil, fmt.Errorf("delete: %w", err) } - return nil + return cleanup, nil } diff --git a/services/user/user.go b/services/user/user.go index 139ba8048be67..627633d3232d6 100644 --- a/services/user/user.go +++ b/services/user/user.go @@ -243,7 +243,8 @@ func DeleteUser(ctx context.Context, doer, u *user_model.User, purge bool) error return packages_model.ErrUserOwnPackages{UID: u.ID} } - if err := deleteUser(ctx, u, purge); err != nil { + cleanup, err := deleteUser(ctx, u, purge) + if err != nil { return fmt.Errorf("DeleteUser: %w", err) } @@ -252,6 +253,8 @@ func DeleteUser(ctx context.Context, doer, u *user_model.User, purge bool) error } _ = committer.Close() + cleanup() + if err = asymkey_service.RewriteAllPublicKeys(ctx); err != nil { return err } From 8f97a2d0b166e96b03b27ef92f3beaffd6c72d89 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 16 Jul 2025 19:45:30 -0700 Subject: [PATCH 06/36] improvements --- models/repo/attachment.go | 13 +++++++++++++ services/attachment/attachment.go | 11 +---------- services/issue/comments.go | 3 +-- services/user/delete.go | 8 +------- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/models/repo/attachment.go b/models/repo/attachment.go index cb70bcc52f14f..d0690a1319aaa 100644 --- a/models/repo/attachment.go +++ b/models/repo/attachment.go @@ -185,6 +185,19 @@ func UpdateAttachment(ctx context.Context, atta *Attachment) error { return err } +func DeleteAttachments(ctx context.Context, attachments []*Attachment) (int64, error) { + if len(attachments) == 0 { + return 0, nil + } + + ids := make([]int64, 0, len(attachments)) + for _, a := range attachments { + ids = append(ids, a.ID) + } + + return db.GetEngine(ctx).In("id", ids).NoAutoCondition().Delete(attachments[0]) +} + // DeleteAttachmentsByRelease deletes all attachments associated with the given release. func DeleteAttachmentsByRelease(ctx context.Context, releaseID int64) error { _, err := db.GetEngine(ctx).Where("release_id = ?", releaseID).Delete(&Attachment{}) diff --git a/services/attachment/attachment.go b/services/attachment/attachment.go index f55ec1ddb6e94..d430819357c91 100644 --- a/services/attachment/attachment.go +++ b/services/attachment/attachment.go @@ -71,16 +71,7 @@ func DeleteAttachment(ctx context.Context, a *repo_model.Attachment) error { // DeleteAttachments deletes the given attachments and optionally the associated files. func DeleteAttachments(ctx context.Context, attachments []*repo_model.Attachment) (int, error) { - if len(attachments) == 0 { - return 0, nil - } - - ids := make([]int64, 0, len(attachments)) - for _, a := range attachments { - ids = append(ids, a.ID) - } - - cnt, err := db.GetEngine(ctx).In("id", ids).NoAutoCondition().Delete(attachments[0]) + cnt, err := repo_model.DeleteAttachments(ctx, attachments) if err != nil { return 0, err } diff --git a/services/issue/comments.go b/services/issue/comments.go index 4a6b987ea1642..c415c0f91a71a 100644 --- a/services/issue/comments.go +++ b/services/issue/comments.go @@ -152,8 +152,7 @@ func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAtt if removeAttachments { // delete comment attachments - _, err := db.GetEngine(ctx).Where("comment_id = ?", comment.ID).NoAutoCondition().Delete(&repo_model.Attachment{}) - if err != nil { + if _, err := repo_model.DeleteAttachments(ctx, comment.Attachments); err != nil { return nil, err } diff --git a/services/user/delete.go b/services/user/delete.go index 89941243fc7b9..63874f392c389 100644 --- a/services/user/delete.go +++ b/services/user/delete.go @@ -131,13 +131,7 @@ func deleteUser(ctx context.Context, u *user_model.User, purge bool) (cleanup ut return nil, err } - ids := make([]int64, 0, len(comment.Attachments)) - for _, a := range comment.Attachments { - ids = append(ids, a.ID) - } - - _, err := db.GetEngine(ctx).In("id", ids).NoAutoCondition().Delete(&repo_model.Attachment{}) - if err != nil { + if _, err := repo_model.DeleteAttachments(ctx, comment.Attachments); err != nil { return nil, err } From 17424891fa0268ab34a4cfa51ba6a1a6dedb9ef8 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 16 Jul 2025 19:54:36 -0700 Subject: [PATCH 07/36] improvements --- services/issue/issue.go | 26 +++++++++++++++++--------- services/repository/delete.go | 5 ++++- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/services/issue/issue.go b/services/issue/issue.go index 39e1c9a01cbf5..446c7cdbf6331 100644 --- a/services/issue/issue.go +++ b/services/issue/issue.go @@ -346,22 +346,30 @@ func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachmen // DeleteOrphanedIssues delete issues without a repo func DeleteOrphanedIssues(ctx context.Context) error { - return db.WithTx(ctx, func(ctx context.Context) error { + cleanup := util.NewCleanUpFunc() + if err := db.WithTx(ctx, func(ctx context.Context) error { repoIDs, err := issues_model.GetOrphanedIssueRepoIDs(ctx) if err != nil { return err } for i := range repoIDs { - if err := DeleteIssuesByRepoID(ctx, repoIDs[i], true); err != nil { + deleteIssuesCleanup, err := DeleteIssuesByRepoID(ctx, repoIDs[i], true) + if err != nil { return err } + cleanup = cleanup.Append(deleteIssuesCleanup) } return nil - }) + }); err != nil { + return err + } + cleanup() + return nil } // DeleteIssuesByRepoID deletes issues by repositories id -func DeleteIssuesByRepoID(ctx context.Context, repoID int64, deleteAttachments bool) error { +func DeleteIssuesByRepoID(ctx context.Context, repoID int64, deleteAttachments bool) (util.CleanUpFunc, error) { + cleanup := util.NewCleanUpFunc() for { issues := make([]*issues_model.Issue, 0, db.DefaultMaxInSize) if err := db.GetEngine(ctx). @@ -369,7 +377,7 @@ func DeleteIssuesByRepoID(ctx context.Context, repoID int64, deleteAttachments b OrderBy("id"). Limit(db.DefaultMaxInSize). Find(&issues); err != nil { - return err + return nil, err } if len(issues) == 0 { @@ -377,13 +385,13 @@ func DeleteIssuesByRepoID(ctx context.Context, repoID int64, deleteAttachments b } for _, issue := range issues { - cleanup, err := deleteIssue(ctx, issue, deleteAttachments) + deleteIssueCleanUp, err := deleteIssue(ctx, issue, deleteAttachments) if err != nil { - return fmt.Errorf("deleteIssue [issue_id: %d]: %w", issue.ID, err) + return nil, fmt.Errorf("deleteIssue [issue_id: %d]: %w", issue.ID, err) } - cleanup() + cleanup = cleanup.Append(deleteIssueCleanUp) } } - return nil + return cleanup, nil } diff --git a/services/repository/delete.go b/services/repository/delete.go index 3b000b8219678..57057707f34ac 100644 --- a/services/repository/delete.go +++ b/services/repository/delete.go @@ -201,7 +201,8 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID // Delete Issues and related objects // attachments will be deleted later with repo_id - if err = issue_service.DeleteIssuesByRepoID(ctx, repoID, false); err != nil { + cleanup, err := issue_service.DeleteIssuesByRepoID(ctx, repoID, false) + if err != nil { return err } @@ -297,6 +298,8 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID committer.Close() + cleanup() + if needRewriteKeysFile { if err := asymkey_service.RewriteAllPublicKeys(ctx); err != nil { log.Error("RewriteAllPublicKeys failed: %v", err) From 63173a6b3107c23eda4cd02bdbf8a99f8df40136 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 17 Jul 2025 00:04:27 -0700 Subject: [PATCH 08/36] fix test --- services/attachment/attachment_test.go | 3 ++- services/issue/comments.go | 2 +- services/issue/issue.go | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/services/attachment/attachment_test.go b/services/attachment/attachment_test.go index 77c6f42a3f473..9562b8f731845 100644 --- a/services/attachment/attachment_test.go +++ b/services/attachment/attachment_test.go @@ -47,8 +47,9 @@ func TestUploadAttachment(t *testing.T) { func TestDeleteAttachments(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) + attachment8 := unittest.AssertExistsAndLoadBean(t, &repo_model.Attachment{ID: 8}) - err := DeleteAttachment(db.DefaultContext, &repo_model.Attachment{ID: 8}) + err := DeleteAttachment(db.DefaultContext, attachment8) assert.NoError(t, err) attachment, err := repo_model.GetAttachmentByUUID(db.DefaultContext, "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a18") diff --git a/services/issue/comments.go b/services/issue/comments.go index c415c0f91a71a..273af900ffc80 100644 --- a/services/issue/comments.go +++ b/services/issue/comments.go @@ -182,7 +182,7 @@ func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAtt } func DeleteComment(ctx context.Context, doer *user_model.User, comment *issues_model.Comment) (*issues_model.Comment, error) { - deletedReviewComment, cleanup, err := deleteComment(ctx, comment, false) + deletedReviewComment, cleanup, err := deleteComment(ctx, comment, true) if err != nil { return nil, err } diff --git a/services/issue/issue.go b/services/issue/issue.go index 446c7cdbf6331..80ac9c1d6ae99 100644 --- a/services/issue/issue.go +++ b/services/issue/issue.go @@ -325,7 +325,7 @@ func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachmen if deleteAttachments { // delete issue attachments - _, err := db.GetEngine(ctx).Where("issue_id = ? AND comment_id = 0", issue.ID).NoAutoCondition().Delete(&issues_model.Issue{}) + _, err := db.GetEngine(ctx).Where("issue_id = ? AND comment_id = 0", issue.ID).NoAutoCondition().Delete(&repo_model.Attachment{}) if err != nil { return err } From 6b66c30fc34532e8b3bc292899600163d6c8f642 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 17 Jul 2025 17:42:15 -0700 Subject: [PATCH 09/36] revert unnecessary change --- cmd/admin_user_delete.go | 7 +----- models/issues/pull_list_test.go | 5 ++--- routers/api/v1/admin/user.go | 2 +- routers/api/v1/org/org.go | 2 +- routers/api/v1/repo/migrate.go | 2 +- routers/web/admin/users.go | 2 +- routers/web/org/setting.go | 2 +- routers/web/user/setting/account.go | 2 +- services/cron/tasks_extended.go | 8 ++----- services/doctor/repository.go | 7 +----- services/org/org.go | 4 ++-- services/org/org_test.go | 7 +++--- services/org/team_test.go | 8 +++---- services/repository/check.go | 5 ++--- services/repository/create.go | 6 ++--- services/repository/create_test.go | 2 +- services/repository/delete.go | 6 ++--- services/repository/delete_test.go | 2 +- services/repository/fork.go | 2 +- services/repository/fork_test.go | 2 +- services/repository/repository.go | 2 +- services/repository/template.go | 2 +- services/user/user.go | 10 ++++----- services/user/user_test.go | 22 +++++++------------ tests/integration/api_repo_test.go | 2 +- .../ephemeral_actions_runner_deletion_test.go | 7 ++---- tests/integration/git_push_test.go | 3 +-- tests/integration/pull_compare_test.go | 4 +--- tests/integration/repo_test.go | 3 +-- 29 files changed, 53 insertions(+), 85 deletions(-) diff --git a/cmd/admin_user_delete.go b/cmd/admin_user_delete.go index cbbf258f028fc..f91041577c3e5 100644 --- a/cmd/admin_user_delete.go +++ b/cmd/admin_user_delete.go @@ -80,10 +80,5 @@ func runDeleteUser(ctx context.Context, c *cli.Command) error { return fmt.Errorf("the user %s does not match the provided id %d", user.Name, c.Int64("id")) } - adminUser, err := user_model.GetAdminUser(ctx) - if err != nil { - return fmt.Errorf("failed to get admin user: %w", err) - } - - return user_service.DeleteUser(ctx, adminUser, user, c.Bool("purge")) + return user_service.DeleteUser(ctx, user, c.Bool("purge")) } diff --git a/models/issues/pull_list_test.go b/models/issues/pull_list_test.go index eb2de006d60a4..feb59df216045 100644 --- a/models/issues/pull_list_test.go +++ b/models/issues/pull_list_test.go @@ -39,9 +39,8 @@ func TestPullRequestList_LoadReviewCommentsCounts(t *testing.T) { reviewComments, err := prs.LoadReviewCommentsCounts(db.DefaultContext) assert.NoError(t, err) assert.Len(t, reviewComments, 2) - for _, pr := range prs { - assert.Equal(t, 1, reviewComments[pr.IssueID]) - } + assert.Equal(t, 1, reviewComments[prs[0].IssueID]) + assert.Equal(t, 2, reviewComments[prs[1].IssueID]) } func TestPullRequestList_LoadReviews(t *testing.T) { diff --git a/routers/api/v1/admin/user.go b/routers/api/v1/admin/user.go index f99fc6062e471..494bace585189 100644 --- a/routers/api/v1/admin/user.go +++ b/routers/api/v1/admin/user.go @@ -300,7 +300,7 @@ func DeleteUser(ctx *context.APIContext) { return } - if err := user_service.DeleteUser(ctx, ctx.Doer, ctx.ContextUser, ctx.FormBool("purge")); err != nil { + if err := user_service.DeleteUser(ctx, ctx.ContextUser, ctx.FormBool("purge")); err != nil { if repo_model.IsErrUserOwnRepos(err) || org_model.IsErrUserHasOrgs(err) || packages_model.IsErrUserOwnPackages(err) || diff --git a/routers/api/v1/org/org.go b/routers/api/v1/org/org.go index f907858ea9015..cd676860658dc 100644 --- a/routers/api/v1/org/org.go +++ b/routers/api/v1/org/org.go @@ -421,7 +421,7 @@ func Delete(ctx *context.APIContext) { // "404": // "$ref": "#/responses/notFound" - if err := org.DeleteOrganization(ctx, ctx.Doer, ctx.Org.Organization, false); err != nil { + if err := org.DeleteOrganization(ctx, ctx.Org.Organization, false); err != nil { ctx.APIErrorInternal(err) return } diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go index f2e0cad86cae6..c1e0b47d331a9 100644 --- a/routers/api/v1/repo/migrate.go +++ b/routers/api/v1/repo/migrate.go @@ -203,7 +203,7 @@ func Migrate(ctx *context.APIContext) { } if repo != nil { - if errDelete := repo_service.DeleteRepositoryDirectly(ctx, ctx.Doer, repo.ID); errDelete != nil { + if errDelete := repo_service.DeleteRepositoryDirectly(ctx, repo.ID); errDelete != nil { log.Error("DeleteRepository: %v", errDelete) } } diff --git a/routers/web/admin/users.go b/routers/web/admin/users.go index 85ed0c4e22093..27577cd35ba1c 100644 --- a/routers/web/admin/users.go +++ b/routers/web/admin/users.go @@ -496,7 +496,7 @@ func DeleteUser(ctx *context.Context) { return } - if err = user_service.DeleteUser(ctx, ctx.Doer, u, ctx.FormBool("purge")); err != nil { + if err = user_service.DeleteUser(ctx, u, ctx.FormBool("purge")); err != nil { switch { case repo_model.IsErrUserOwnRepos(err): ctx.Flash.Error(ctx.Tr("admin.users.still_own_repo")) diff --git a/routers/web/org/setting.go b/routers/web/org/setting.go index dc01e91d8a6b4..2bc1e8bc43388 100644 --- a/routers/web/org/setting.go +++ b/routers/web/org/setting.go @@ -149,7 +149,7 @@ func SettingsDeleteOrgPost(ctx *context.Context) { return } - if err := org_service.DeleteOrganization(ctx, ctx.Doer, ctx.Org.Organization, false /* no purge */); err != nil { + if err := org_service.DeleteOrganization(ctx, ctx.Org.Organization, false /* no purge */); err != nil { if repo_model.IsErrUserOwnRepos(err) { ctx.JSONError(ctx.Tr("form.org_still_own_repo")) } else if packages_model.IsErrUserOwnPackages(err) { diff --git a/routers/web/user/setting/account.go b/routers/web/user/setting/account.go index ee4cb65e12067..6b17da50e5b4a 100644 --- a/routers/web/user/setting/account.go +++ b/routers/web/user/setting/account.go @@ -272,7 +272,7 @@ func DeleteAccount(ctx *context.Context) { return } - if err := user.DeleteUser(ctx, ctx.Doer, ctx.Doer, false); err != nil { + if err := user.DeleteUser(ctx, ctx.Doer, false); err != nil { switch { case repo_model.IsErrUserOwnRepos(err): ctx.Flash.Error(ctx.Tr("form.still_own_repo")) diff --git a/services/cron/tasks_extended.go b/services/cron/tasks_extended.go index f3638f2279dc2..2a0746c24571d 100644 --- a/services/cron/tasks_extended.go +++ b/services/cron/tasks_extended.go @@ -30,11 +30,7 @@ func registerDeleteInactiveUsers() { OlderThan: time.Minute * time.Duration(setting.Service.ActiveCodeLives), }, func(ctx context.Context, _ *user_model.User, config Config) error { olderThanConfig := config.(*OlderThanConfig) - adminUser, err := user_model.GetAdminUser(ctx) - if err != nil { - return err - } - return user_service.DeleteInactiveUsers(ctx, adminUser, olderThanConfig.OlderThan) + return user_service.DeleteInactiveUsers(ctx, olderThanConfig.OlderThan) }) } @@ -115,7 +111,7 @@ func registerDeleteMissingRepositories() { RunAtStart: false, Schedule: "@every 72h", }, func(ctx context.Context, user *user_model.User, _ Config) error { - return repo_service.DeleteMissingRepositories(ctx, user) + return repo_service.DeleteMissingRepositories(ctx) }) } diff --git a/services/doctor/repository.go b/services/doctor/repository.go index 4409ed7957978..4a8d00b5716e5 100644 --- a/services/doctor/repository.go +++ b/services/doctor/repository.go @@ -7,7 +7,6 @@ import ( "context" "code.gitea.io/gitea/models/db" - user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/storage" repo_service "code.gitea.io/gitea/services/repository" @@ -38,10 +37,6 @@ func deleteOrphanedRepos(ctx context.Context) (int64, error) { batchSize := db.MaxBatchInsertSize("repository") var deleted int64 - adminUser, err := user_model.GetAdminUser(ctx) - if err != nil { - return deleted, err - } for { select { @@ -62,7 +57,7 @@ func deleteOrphanedRepos(ctx context.Context) (int64, error) { } for _, id := range ids { - if err := repo_service.DeleteRepositoryDirectly(ctx, adminUser, id, true); err != nil { + if err := repo_service.DeleteRepositoryDirectly(ctx, id, true); err != nil { return deleted, err } deleted++ diff --git a/services/org/org.go b/services/org/org.go index 4e90d575c5735..3d30ae21a39d3 100644 --- a/services/org/org.go +++ b/services/org/org.go @@ -47,7 +47,7 @@ func deleteOrganization(ctx context.Context, org *org_model.Organization) error } // DeleteOrganization completely and permanently deletes everything of organization. -func DeleteOrganization(ctx context.Context, doer *user_model.User, org *org_model.Organization, purge bool) error { +func DeleteOrganization(ctx context.Context, org *org_model.Organization, purge bool) error { ctx, committer, err := db.TxContext(ctx) if err != nil { return err @@ -55,7 +55,7 @@ func DeleteOrganization(ctx context.Context, doer *user_model.User, org *org_mod defer committer.Close() if purge { - err := repo_service.DeleteOwnerRepositoriesDirectly(ctx, doer, org.AsUser()) + err := repo_service.DeleteOwnerRepositoriesDirectly(ctx, org.AsUser()) if err != nil { return err } diff --git a/services/org/org_test.go b/services/org/org_test.go index 146ad69281b63..791404c5c8919 100644 --- a/services/org/org_test.go +++ b/services/org/org_test.go @@ -22,18 +22,17 @@ func TestMain(m *testing.M) { func TestDeleteOrganization(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) org := unittest.AssertExistsAndLoadBean(t, &organization.Organization{ID: 6}) - user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) - assert.NoError(t, DeleteOrganization(db.DefaultContext, user1, org, false)) + assert.NoError(t, DeleteOrganization(db.DefaultContext, org, false)) unittest.AssertNotExistsBean(t, &organization.Organization{ID: 6}) unittest.AssertNotExistsBean(t, &organization.OrgUser{OrgID: 6}) unittest.AssertNotExistsBean(t, &organization.Team{OrgID: 6}) org = unittest.AssertExistsAndLoadBean(t, &organization.Organization{ID: 3}) - err := DeleteOrganization(db.DefaultContext, user1, org, false) + err := DeleteOrganization(db.DefaultContext, org, false) assert.Error(t, err) assert.True(t, repo_model.IsErrUserOwnRepos(err)) user := unittest.AssertExistsAndLoadBean(t, &organization.Organization{ID: 5}) - assert.Error(t, DeleteOrganization(db.DefaultContext, user1, user, false)) + assert.Error(t, DeleteOrganization(db.DefaultContext, user, false)) unittest.CheckConsistencyFor(t, &user_model.User{}, &organization.Team{}) } diff --git a/services/org/team_test.go b/services/org/team_test.go index 5e9b6037d3983..c1a69d8ee73a0 100644 --- a/services/org/team_test.go +++ b/services/org/team_test.go @@ -280,10 +280,8 @@ func TestIncludesAllRepositoriesTeams(t *testing.T) { testTeamRepositories(team.ID, teamRepos[i]) } - user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) - // Remove repo and check teams repositories. - assert.NoError(t, repo_service.DeleteRepositoryDirectly(db.DefaultContext, user1, repoIDs[0]), "DeleteRepository") + assert.NoError(t, repo_service.DeleteRepositoryDirectly(db.DefaultContext, repoIDs[0]), "DeleteRepository") teamRepos[0] = repoIDs[1:] teamRepos[1] = repoIDs[1:] teamRepos[3] = repoIDs[1:3] @@ -295,8 +293,8 @@ func TestIncludesAllRepositoriesTeams(t *testing.T) { // Wipe created items. for i, rid := range repoIDs { if i > 0 { // first repo already deleted. - assert.NoError(t, repo_service.DeleteRepositoryDirectly(db.DefaultContext, user1, rid), "DeleteRepository %d", i) + assert.NoError(t, repo_service.DeleteRepositoryDirectly(db.DefaultContext, rid), "DeleteRepository %d", i) } } - assert.NoError(t, DeleteOrganization(db.DefaultContext, user1, org, false), "DeleteOrganization") + assert.NoError(t, DeleteOrganization(db.DefaultContext, org, false), "DeleteOrganization") } diff --git a/services/repository/check.go b/services/repository/check.go index b475fbc487d29..e0daecbc4685c 100644 --- a/services/repository/check.go +++ b/services/repository/check.go @@ -12,7 +12,6 @@ import ( "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" system_model "code.gitea.io/gitea/models/system" - user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" @@ -145,7 +144,7 @@ func gatherMissingRepoRecords(ctx context.Context) (repo_model.RepositoryList, e } // DeleteMissingRepositories deletes all repository records that lost Git files. -func DeleteMissingRepositories(ctx context.Context, doer *user_model.User) error { +func DeleteMissingRepositories(ctx context.Context) error { repos, err := gatherMissingRepoRecords(ctx) if err != nil { return err @@ -162,7 +161,7 @@ func DeleteMissingRepositories(ctx context.Context, doer *user_model.User) error default: } log.Trace("Deleting %d/%d...", repo.OwnerID, repo.ID) - if err := DeleteRepositoryDirectly(ctx, doer, repo.ID); err != nil { + if err := DeleteRepositoryDirectly(ctx, repo.ID); err != nil { log.Error("Failed to DeleteRepository %-v: Error: %v", repo, err) if err2 := system_model.CreateRepositoryNotice("Failed to DeleteRepository %s [%d]: Error: %v", repo.FullName(), repo.ID, err); err2 != nil { log.Error("CreateRepositoryNotice: %v", err) diff --git a/services/repository/create.go b/services/repository/create.go index 9758b3eb1c0d5..bed02e5d7e941 100644 --- a/services/repository/create.go +++ b/services/repository/create.go @@ -263,7 +263,7 @@ func CreateRepositoryDirectly(ctx context.Context, doer, owner *user_model.User, defer func() { if err != nil { // we can not use the ctx because it maybe canceled or timeout - cleanupRepository(doer, repo.ID) + cleanupRepository(repo.ID) } }() @@ -458,8 +458,8 @@ func createRepositoryInDB(ctx context.Context, doer, u *user_model.User, repo *r return nil } -func cleanupRepository(doer *user_model.User, repoID int64) { - if errDelete := DeleteRepositoryDirectly(db.DefaultContext, doer, repoID); errDelete != nil { +func cleanupRepository(repoID int64) { + if errDelete := DeleteRepositoryDirectly(db.DefaultContext, repoID); errDelete != nil { log.Error("cleanupRepository failed: %v", errDelete) // add system notice if err := system_model.CreateRepositoryNotice("DeleteRepositoryDirectly failed when cleanup repository: %v", errDelete); err != nil { diff --git a/services/repository/create_test.go b/services/repository/create_test.go index 8e3fdf88a5578..fe464c1441c5e 100644 --- a/services/repository/create_test.go +++ b/services/repository/create_test.go @@ -35,7 +35,7 @@ func TestCreateRepositoryDirectly(t *testing.T) { unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{OwnerName: user2.Name, Name: createdRepo.Name}) - err = DeleteRepositoryDirectly(db.DefaultContext, user2, createdRepo.ID) + err = DeleteRepositoryDirectly(db.DefaultContext, createdRepo.ID) assert.NoError(t, err) // a failed creating because some mock data diff --git a/services/repository/delete.go b/services/repository/delete.go index 57057707f34ac..0527546d1dc74 100644 --- a/services/repository/delete.go +++ b/services/repository/delete.go @@ -49,7 +49,7 @@ func deleteDBRepository(ctx context.Context, repoID int64) error { // DeleteRepository deletes a repository for a user or organization. // make sure if you call this func to close open sessions (sqlite will otherwise get a deadlock) -func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID int64, ignoreOrgTeams ...bool) error { +func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams ...bool) error { ctx, committer, err := db.TxContext(ctx) if err != nil { return err @@ -370,7 +370,7 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID } // DeleteOwnerRepositoriesDirectly calls DeleteRepositoryDirectly for all repos of the given owner -func DeleteOwnerRepositoriesDirectly(ctx context.Context, doer, owner *user_model.User) error { +func DeleteOwnerRepositoriesDirectly(ctx context.Context, owner *user_model.User) error { for { repos, _, err := repo_model.GetUserRepositories(ctx, repo_model.SearchRepoOptions{ ListOptions: db.ListOptions{ @@ -388,7 +388,7 @@ func DeleteOwnerRepositoriesDirectly(ctx context.Context, doer, owner *user_mode break } for _, repo := range repos { - if err := DeleteRepositoryDirectly(ctx, doer, repo.ID); err != nil { + if err := DeleteRepositoryDirectly(ctx, repo.ID); err != nil { return fmt.Errorf("unable to delete repository %s for %s[%d]. Error: %w", repo.Name, owner.Name, owner.ID, err) } } diff --git a/services/repository/delete_test.go b/services/repository/delete_test.go index 977c75e39e433..869b8af11d57d 100644 --- a/services/repository/delete_test.go +++ b/services/repository/delete_test.go @@ -51,5 +51,5 @@ func TestDeleteOwnerRepositoriesDirectly(t *testing.T) { user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) - assert.NoError(t, repo_service.DeleteOwnerRepositoriesDirectly(db.DefaultContext, user, user)) + assert.NoError(t, repo_service.DeleteOwnerRepositoriesDirectly(db.DefaultContext, user)) } diff --git a/services/repository/fork.go b/services/repository/fork.go index d0568e6072e46..8bd3498b1715c 100644 --- a/services/repository/fork.go +++ b/services/repository/fork.go @@ -124,7 +124,7 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork defer func() { if err != nil { // we can not use the ctx because it maybe canceled or timeout - cleanupRepository(doer, repo.ID) + cleanupRepository(repo.ID) } }() diff --git a/services/repository/fork_test.go b/services/repository/fork_test.go index 35c6effca5097..5375f790282ea 100644 --- a/services/repository/fork_test.go +++ b/services/repository/fork_test.go @@ -69,7 +69,7 @@ func TestForkRepositoryCleanup(t *testing.T) { assert.NoError(t, err) assert.True(t, exist) - err = DeleteRepositoryDirectly(db.DefaultContext, user2, fork.ID) + err = DeleteRepositoryDirectly(db.DefaultContext, fork.ID) assert.NoError(t, err) // a failed creating because some mock data diff --git a/services/repository/repository.go b/services/repository/repository.go index 0cdce336d4939..e574dc6c0181d 100644 --- a/services/repository/repository.go +++ b/services/repository/repository.go @@ -69,7 +69,7 @@ func DeleteRepository(ctx context.Context, doer *user_model.User, repo *repo_mod notify_service.DeleteRepository(ctx, doer, repo) } - return DeleteRepositoryDirectly(ctx, doer, repo.ID) + return DeleteRepositoryDirectly(ctx, repo.ID) } // PushCreateRepo creates a repository when a new repository is pushed to an appropriate namespace diff --git a/services/repository/template.go b/services/repository/template.go index 621bd95cb15c1..6906a60083275 100644 --- a/services/repository/template.go +++ b/services/repository/template.go @@ -102,7 +102,7 @@ func GenerateRepository(ctx context.Context, doer, owner *user_model.User, templ defer func() { if err != nil { // we can not use the ctx because it maybe canceled or timeout - cleanupRepository(doer, generateRepo.ID) + cleanupRepository(generateRepo.ID) } }() diff --git a/services/user/user.go b/services/user/user.go index 627633d3232d6..d92763d2d7a5f 100644 --- a/services/user/user.go +++ b/services/user/user.go @@ -121,7 +121,7 @@ func RenameUser(ctx context.Context, u *user_model.User, newUserName string) err // DeleteUser completely and permanently deletes everything of a user, // but issues/comments/pulls will be kept and shown as someone has been deleted, // unless the user is younger than USER_DELETE_WITH_COMMENTS_MAX_DAYS. -func DeleteUser(ctx context.Context, doer, u *user_model.User, purge bool) error { +func DeleteUser(ctx context.Context, u *user_model.User, purge bool) error { if u.IsOrganization() { return fmt.Errorf("%s is an organization not a user", u.Name) } @@ -160,7 +160,7 @@ func DeleteUser(ctx context.Context, doer, u *user_model.User, purge bool) error // // An alternative option here would be write a DeleteAllRepositoriesForUserID function which would delete all of the repos // but such a function would likely get out of date - err := repo_service.DeleteOwnerRepositoriesDirectly(ctx, doer, u) + err := repo_service.DeleteOwnerRepositoriesDirectly(ctx, u) if err != nil { return err } @@ -190,7 +190,7 @@ func DeleteUser(ctx context.Context, doer, u *user_model.User, purge bool) error for _, org := range orgs { if err := org_service.RemoveOrgUser(ctx, org, u); err != nil { if organization.IsErrLastOrgOwner(err) { - err = org_service.DeleteOrganization(ctx, doer, org, true) + err = org_service.DeleteOrganization(ctx, org, true) if err != nil { return fmt.Errorf("unable to delete organization %d: %w", org.ID, err) } @@ -281,7 +281,7 @@ func DeleteUser(ctx context.Context, doer, u *user_model.User, purge bool) error } // DeleteInactiveUsers deletes all inactive users and their email addresses. -func DeleteInactiveUsers(ctx context.Context, doer *user_model.User, olderThan time.Duration) error { +func DeleteInactiveUsers(ctx context.Context, olderThan time.Duration) error { inactiveUsers, err := user_model.GetInactiveUsers(ctx, olderThan) if err != nil { return err @@ -289,7 +289,7 @@ func DeleteInactiveUsers(ctx context.Context, doer *user_model.User, olderThan t // FIXME: should only update authorized_keys file once after all deletions. for _, u := range inactiveUsers { - if err = DeleteUser(ctx, doer, u, false); err != nil { + if err = DeleteUser(ctx, u, false); err != nil { // Ignore inactive users that were ever active but then were set inactive by admin if repo_model.IsErrUserOwnRepos(err) || organization.IsErrUserHasOrgs(err) || packages_model.IsErrUserOwnPackages(err) { log.Warn("Inactive user %q has repositories, organizations or packages, skipping deletion: %v", u.Name, err) diff --git a/services/user/user_test.go b/services/user/user_test.go index 316552f6b5c07..28a0df8628fdb 100644 --- a/services/user/user_test.go +++ b/services/user/user_test.go @@ -27,7 +27,6 @@ func TestMain(m *testing.M) { } func TestDeleteUser(t *testing.T) { - user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) test := func(userID int64) { assert.NoError(t, unittest.PrepareTestDatabase()) user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: userID}) @@ -35,7 +34,7 @@ func TestDeleteUser(t *testing.T) { ownedRepos := make([]*repo_model.Repository, 0, 10) assert.NoError(t, db.GetEngine(db.DefaultContext).Find(&ownedRepos, &repo_model.Repository{OwnerID: userID})) if len(ownedRepos) > 0 { - err := DeleteUser(db.DefaultContext, user1, user, false) + err := DeleteUser(db.DefaultContext, user, false) assert.Error(t, err) assert.True(t, repo_model.IsErrUserOwnRepos(err)) return @@ -50,7 +49,7 @@ func TestDeleteUser(t *testing.T) { return } } - assert.NoError(t, DeleteUser(db.DefaultContext, user1, user, false)) + assert.NoError(t, DeleteUser(db.DefaultContext, user, false)) unittest.AssertNotExistsBean(t, &user_model.User{ID: userID}) unittest.CheckConsistencyFor(t, &user_model.User{}, &repo_model.Repository{}) } @@ -60,16 +59,15 @@ func TestDeleteUser(t *testing.T) { test(11) org := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 3}) - assert.Error(t, DeleteUser(db.DefaultContext, user1, org, false)) + assert.Error(t, DeleteUser(db.DefaultContext, org, false)) } func TestPurgeUser(t *testing.T) { - user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) test := func(userID int64) { assert.NoError(t, unittest.PrepareTestDatabase()) user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: userID}) - err := DeleteUser(db.DefaultContext, user1, user, true) + err := DeleteUser(db.DefaultContext, user, true) assert.NoError(t, err) unittest.AssertNotExistsBean(t, &user_model.User{ID: userID}) @@ -81,11 +79,10 @@ func TestPurgeUser(t *testing.T) { test(11) org := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 3}) - assert.Error(t, DeleteUser(db.DefaultContext, user1, org, false)) + assert.Error(t, DeleteUser(db.DefaultContext, org, false)) } func TestCreateUser(t *testing.T) { - user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) user := &user_model.User{ Name: "GiteaBot", Email: "GiteaBot@gitea.io", @@ -97,7 +94,7 @@ func TestCreateUser(t *testing.T) { assert.NoError(t, user_model.CreateUser(db.DefaultContext, user, &user_model.Meta{})) - assert.NoError(t, DeleteUser(db.DefaultContext, user1, user, false)) + assert.NoError(t, DeleteUser(db.DefaultContext, user, false)) } func TestRenameUser(t *testing.T) { @@ -175,8 +172,6 @@ func TestCreateUser_Issue5882(t *testing.T) { setting.Service.DefaultAllowCreateOrganization = true - user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) - for _, v := range tt { setting.Admin.DisableRegularOrgCreation = v.disableOrgCreation @@ -187,7 +182,7 @@ func TestCreateUser_Issue5882(t *testing.T) { assert.Equal(t, !u.AllowCreateOrganization, v.disableOrgCreation) - assert.NoError(t, DeleteUser(db.DefaultContext, user1, v.user, false)) + assert.NoError(t, DeleteUser(db.DefaultContext, v.user, false)) } } @@ -200,14 +195,13 @@ func TestDeleteInactiveUsers(t *testing.T) { err = db.Insert(db.DefaultContext, inactiveUserEmail) assert.NoError(t, err) } - user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) addUser("user-inactive-10", "user-inactive-10@test.com", timeutil.TimeStampNow().Add(-600), false) addUser("user-inactive-5", "user-inactive-5@test.com", timeutil.TimeStampNow().Add(-300), false) addUser("user-active-10", "user-active-10@test.com", timeutil.TimeStampNow().Add(-600), true) addUser("user-active-5", "user-active-5@test.com", timeutil.TimeStampNow().Add(-300), true) unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: "user-inactive-10"}) unittest.AssertExistsAndLoadBean(t, &user_model.EmailAddress{Email: "user-inactive-10@test.com"}) - assert.NoError(t, DeleteInactiveUsers(db.DefaultContext, user1, 8*time.Minute)) + assert.NoError(t, DeleteInactiveUsers(db.DefaultContext, 8*time.Minute)) unittest.AssertNotExistsBean(t, &user_model.User{Name: "user-inactive-10"}) unittest.AssertNotExistsBean(t, &user_model.EmailAddress{Email: "user-inactive-10@test.com"}) unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: "user-inactive-5"}) diff --git a/tests/integration/api_repo_test.go b/tests/integration/api_repo_test.go index 672c2a2c8bf9b..a2c3a467c60d9 100644 --- a/tests/integration/api_repo_test.go +++ b/tests/integration/api_repo_test.go @@ -586,7 +586,7 @@ func TestAPIRepoTransfer(t *testing.T) { // cleanup repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: apiRepo.ID}) - _ = repo_service.DeleteRepositoryDirectly(db.DefaultContext, user, repo.ID) + _ = repo_service.DeleteRepositoryDirectly(db.DefaultContext, repo.ID) } func transfer(t *testing.T) *repo_model.Repository { diff --git a/tests/integration/ephemeral_actions_runner_deletion_test.go b/tests/integration/ephemeral_actions_runner_deletion_test.go index 2da90b40d25c9..40f8c643a8306 100644 --- a/tests/integration/ephemeral_actions_runner_deletion_test.go +++ b/tests/integration/ephemeral_actions_runner_deletion_test.go @@ -50,9 +50,7 @@ func testEphemeralActionsRunnerDeletionByRepository(t *testing.T) { task := unittest.AssertExistsAndLoadBean(t, &actions_model.ActionTask{ID: 52}) assert.Equal(t, actions_model.StatusRunning, task.Status) - user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) - - err = repo_service.DeleteRepositoryDirectly(t.Context(), user1, task.RepoID, true) + err = repo_service.DeleteRepositoryDirectly(t.Context(), task.RepoID, true) assert.NoError(t, err) _, err = actions_model.GetRunnerByID(t.Context(), 34350) @@ -70,9 +68,8 @@ func testEphemeralActionsRunnerDeletionByUser(t *testing.T) { assert.Equal(t, actions_model.StatusRunning, task.Status) user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) - user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) - err = user_service.DeleteUser(t.Context(), user1, user, true) + err = user_service.DeleteUser(t.Context(), user, true) assert.NoError(t, err) _, err = actions_model.GetRunnerByID(t.Context(), 34350) diff --git a/tests/integration/git_push_test.go b/tests/integration/git_push_test.go index c307cf6b606b6..d716847b5417f 100644 --- a/tests/integration/git_push_test.go +++ b/tests/integration/git_push_test.go @@ -191,8 +191,7 @@ func runTestGitPush(t *testing.T, u *url.URL, gitOperation func(t *testing.T, gi assert.Equal(t, commitID, branch.CommitID) } - user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) - require.NoError(t, repo_service.DeleteRepositoryDirectly(db.DefaultContext, user1, repo.ID)) + require.NoError(t, repo_service.DeleteRepositoryDirectly(db.DefaultContext, repo.ID)) } func TestPushPullRefs(t *testing.T) { diff --git a/tests/integration/pull_compare_test.go b/tests/integration/pull_compare_test.go index 039cdcfd5c3f7..f95a2f1690929 100644 --- a/tests/integration/pull_compare_test.go +++ b/tests/integration/pull_compare_test.go @@ -13,7 +13,6 @@ import ( issues_model "code.gitea.io/gitea/models/issues" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" - user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/test" repo_service "code.gitea.io/gitea/services/repository" "code.gitea.io/gitea/tests" @@ -77,9 +76,8 @@ func TestPullCompare(t *testing.T) { repoForked := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{OwnerName: "user1", Name: "repo1"}) - user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) // delete the head repository and revisit the PR diff view - err := repo_service.DeleteRepositoryDirectly(db.DefaultContext, user1, repoForked.ID) + err := repo_service.DeleteRepositoryDirectly(db.DefaultContext, repoForked.ID) assert.NoError(t, err) req = NewRequest(t, "GET", prFilesURL) diff --git a/tests/integration/repo_test.go b/tests/integration/repo_test.go index 1a735381d9a93..adfe07519faed 100644 --- a/tests/integration/repo_test.go +++ b/tests/integration/repo_test.go @@ -539,9 +539,8 @@ func TestGenerateRepository(t *testing.T) { assert.True(t, exist) unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{OwnerName: user2.Name, Name: generatedRepo.Name}) - user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) - err = repo_service.DeleteRepositoryDirectly(db.DefaultContext, user1, generatedRepo.ID) + err = repo_service.DeleteRepositoryDirectly(db.DefaultContext, generatedRepo.ID) assert.NoError(t, err) // a failed creating because some mock data From cdb6147f38b085fce3750847b665f751666477df Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 17 Jul 2025 17:45:10 -0700 Subject: [PATCH 10/36] revert unnecessary change --- services/cron/tasks_extended.go | 2 +- services/repository/check.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/services/cron/tasks_extended.go b/services/cron/tasks_extended.go index 2a0746c24571d..0018c5facc5d7 100644 --- a/services/cron/tasks_extended.go +++ b/services/cron/tasks_extended.go @@ -111,7 +111,7 @@ func registerDeleteMissingRepositories() { RunAtStart: false, Schedule: "@every 72h", }, func(ctx context.Context, user *user_model.User, _ Config) error { - return repo_service.DeleteMissingRepositories(ctx) + return repo_service.DeleteMissingRepositories(ctx, user) }) } diff --git a/services/repository/check.go b/services/repository/check.go index e0daecbc4685c..ffcd5ac749b97 100644 --- a/services/repository/check.go +++ b/services/repository/check.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" system_model "code.gitea.io/gitea/models/system" + user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" @@ -144,7 +145,7 @@ func gatherMissingRepoRecords(ctx context.Context) (repo_model.RepositoryList, e } // DeleteMissingRepositories deletes all repository records that lost Git files. -func DeleteMissingRepositories(ctx context.Context) error { +func DeleteMissingRepositories(ctx context.Context, doer *user_model.User) error { repos, err := gatherMissingRepoRecords(ctx) if err != nil { return err From 85e6668c25609451fcb8d50470b07edfad33efca Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 17 Jul 2025 21:05:38 -0700 Subject: [PATCH 11/36] Rename the function for post database transaction --- modules/util/cleanup.go | 17 ---------------- modules/util/post_tx_action.go | 18 +++++++++++++++++ services/issue/comments.go | 2 +- services/issue/issue.go | 36 +++++++++++++++++----------------- services/issue/issue_test.go | 12 ++++++------ services/repository/delete.go | 4 ++-- services/user/delete.go | 8 ++++---- services/user/user.go | 4 ++-- 8 files changed, 51 insertions(+), 50 deletions(-) delete mode 100644 modules/util/cleanup.go create mode 100644 modules/util/post_tx_action.go diff --git a/modules/util/cleanup.go b/modules/util/cleanup.go deleted file mode 100644 index 687dac27b5ffd..0000000000000 --- a/modules/util/cleanup.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2025 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package util - -type CleanUpFunc func() - -func NewCleanUpFunc() CleanUpFunc { - return func() {} -} - -func (f CleanUpFunc) Append(newF CleanUpFunc) CleanUpFunc { - return func() { - f() - newF() - } -} diff --git a/modules/util/post_tx_action.go b/modules/util/post_tx_action.go new file mode 100644 index 0000000000000..b5b0e636173d7 --- /dev/null +++ b/modules/util/post_tx_action.go @@ -0,0 +1,18 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package util + +// PostTxAction is a function that is executed after a database transaction +type PostTxAction func() + +func NewPostTxAction() PostTxAction { + return func() {} +} + +func (f PostTxAction) Append(appendF PostTxAction) PostTxAction { + return func() { + f() + appendF() + } +} diff --git a/services/issue/comments.go b/services/issue/comments.go index 273af900ffc80..e92330ceba70a 100644 --- a/services/issue/comments.go +++ b/services/issue/comments.go @@ -136,7 +136,7 @@ func UpdateComment(ctx context.Context, c *issues_model.Comment, contentVersion // deleteComment deletes the comment func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAttachments bool) (*issues_model.Comment, func(), error) { - storageCleanup := util.NewCleanUpFunc() + storageCleanup := util.NewPostTxAction() deletedReviewComment, err := db.WithTx2(ctx, func(ctx context.Context) (*issues_model.Comment, error) { if removeAttachments { // load attachments before deleting the comment diff --git a/services/issue/issue.go b/services/issue/issue.go index 80ac9c1d6ae99..04b3e3f179386 100644 --- a/services/issue/issue.go +++ b/services/issue/issue.go @@ -191,16 +191,16 @@ func DeleteIssue(ctx context.Context, doer *user_model.User, gitRepo *git.Reposi } // delete entries in database - cleanup, err := deleteIssue(ctx, issue, true) + postTxActions, err := deleteIssue(ctx, issue, true) if err != nil { return err } - cleanup() + postTxActions() // delete pull request related git data if issue.IsPull && gitRepo != nil { if err := gitRepo.RemoveReference(issue.PullRequest.GetGitHeadRefName()); err != nil { - return err + log.Error("DeleteIssue: RemoveReference %s: %v", issue.PullRequest.GetGitHeadRefName(), err) } } @@ -259,8 +259,8 @@ func GetRefEndNamesAndURLs(issues []*issues_model.Issue, repoLink string) (map[i } // deleteIssue deletes the issue -func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachments bool) (util.CleanUpFunc, error) { - cleanup := util.NewCleanUpFunc() +func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachments bool) (util.PostTxAction, error) { + postTxActions := util.NewPostTxAction() if err := db.WithTx(ctx, func(ctx context.Context) error { if _, err := db.GetEngine(ctx).ID(issue.ID).NoAutoCondition().Delete(issue); err != nil { return err @@ -316,11 +316,11 @@ func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachmen } for _, comment := range issue.Comments { - _, cleanupDeleteComment, err := deleteComment(ctx, comment, deleteAttachments) + _, postTxActionsDeleteComment, err := deleteComment(ctx, comment, deleteAttachments) if err != nil { return fmt.Errorf("deleteComment [comment_id: %d]: %w", comment.ID, err) } - cleanup = cleanup.Append(cleanupDeleteComment) + postTxActions = postTxActions.Append(postTxActionsDeleteComment) } if deleteAttachments { @@ -330,7 +330,7 @@ func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachmen return err } // the storage cleanup function to remove attachments could be called after all transactions are committed - cleanup = cleanup.Append(func() { + postTxActions = postTxActions.Append(func() { // Remove issue attachment files. for i := range issue.Attachments { system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete issue attachment", issue.Attachments[i].RelativePath()) @@ -341,35 +341,35 @@ func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachmen }); err != nil { return nil, err } - return cleanup, nil + return postTxActions, nil } // DeleteOrphanedIssues delete issues without a repo func DeleteOrphanedIssues(ctx context.Context) error { - cleanup := util.NewCleanUpFunc() + postTxActions := util.NewPostTxAction() if err := db.WithTx(ctx, func(ctx context.Context) error { repoIDs, err := issues_model.GetOrphanedIssueRepoIDs(ctx) if err != nil { return err } for i := range repoIDs { - deleteIssuesCleanup, err := DeleteIssuesByRepoID(ctx, repoIDs[i], true) + postTxActionsDeleteIssues, err := DeleteIssuesByRepoID(ctx, repoIDs[i], true) if err != nil { return err } - cleanup = cleanup.Append(deleteIssuesCleanup) + postTxActions = postTxActions.Append(postTxActionsDeleteIssues) } return nil }); err != nil { return err } - cleanup() + postTxActions() return nil } // DeleteIssuesByRepoID deletes issues by repositories id -func DeleteIssuesByRepoID(ctx context.Context, repoID int64, deleteAttachments bool) (util.CleanUpFunc, error) { - cleanup := util.NewCleanUpFunc() +func DeleteIssuesByRepoID(ctx context.Context, repoID int64, deleteAttachments bool) (util.PostTxAction, error) { + postTxActions := util.NewPostTxAction() for { issues := make([]*issues_model.Issue, 0, db.DefaultMaxInSize) if err := db.GetEngine(ctx). @@ -385,13 +385,13 @@ func DeleteIssuesByRepoID(ctx context.Context, repoID int64, deleteAttachments b } for _, issue := range issues { - deleteIssueCleanUp, err := deleteIssue(ctx, issue, deleteAttachments) + postTxActionsDeleteIssue, err := deleteIssue(ctx, issue, deleteAttachments) if err != nil { return nil, fmt.Errorf("deleteIssue [issue_id: %d]: %w", issue.ID, err) } - cleanup = cleanup.Append(deleteIssueCleanUp) + postTxActions = postTxActions.Append(postTxActionsDeleteIssue) } } - return cleanup, nil + return postTxActions, nil } diff --git a/services/issue/issue_test.go b/services/issue/issue_test.go index f8cb85db3f32b..27a10ce9de4d9 100644 --- a/services/issue/issue_test.go +++ b/services/issue/issue_test.go @@ -44,9 +44,9 @@ func TestIssue_DeleteIssue(t *testing.T) { ID: issueIDs[2], } - cleanup, err := deleteIssue(db.DefaultContext, issue, true) + postTxActions, err := deleteIssue(db.DefaultContext, issue, true) assert.NoError(t, err) - cleanup() + postTxActions() issueIDs, err = issues_model.GetIssueIDsByRepoID(db.DefaultContext, 1) assert.NoError(t, err) assert.Len(t, issueIDs, 4) @@ -56,9 +56,9 @@ func TestIssue_DeleteIssue(t *testing.T) { assert.NoError(t, err) issue, err = issues_model.GetIssueByID(db.DefaultContext, 4) assert.NoError(t, err) - cleanup, err = deleteIssue(db.DefaultContext, issue, true) + postTxActions, err = deleteIssue(db.DefaultContext, issue, true) assert.NoError(t, err) - cleanup() + postTxActions() assert.Len(t, attachments, 2) for i := range attachments { attachment, err := repo_model.GetAttachmentByUUID(db.DefaultContext, attachments[i].UUID) @@ -80,9 +80,9 @@ func TestIssue_DeleteIssue(t *testing.T) { assert.NoError(t, err) assert.False(t, left) - cleanup, err = deleteIssue(db.DefaultContext, issue2, true) + postTxActions, err = deleteIssue(db.DefaultContext, issue2, true) assert.NoError(t, err) - cleanup() + postTxActions() left, err = issues_model.IssueNoDependenciesLeft(db.DefaultContext, issue1) assert.NoError(t, err) assert.True(t, left) diff --git a/services/repository/delete.go b/services/repository/delete.go index 0527546d1dc74..f6bb58222dc6e 100644 --- a/services/repository/delete.go +++ b/services/repository/delete.go @@ -201,7 +201,7 @@ func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams // Delete Issues and related objects // attachments will be deleted later with repo_id - cleanup, err := issue_service.DeleteIssuesByRepoID(ctx, repoID, false) + postTxActions, err := issue_service.DeleteIssuesByRepoID(ctx, repoID, false) if err != nil { return err } @@ -298,7 +298,7 @@ func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams committer.Close() - cleanup() + postTxActions() if needRewriteKeysFile { if err := asymkey_service.RewriteAllPublicKeys(ctx); err != nil { diff --git a/services/user/delete.go b/services/user/delete.go index 63874f392c389..b88d1fa54b792 100644 --- a/services/user/delete.go +++ b/services/user/delete.go @@ -33,8 +33,8 @@ import ( ) // deleteUser deletes models associated to an user. -func deleteUser(ctx context.Context, u *user_model.User, purge bool) (cleanup util.CleanUpFunc, err error) { - cleanup = util.NewCleanUpFunc() +func deleteUser(ctx context.Context, u *user_model.User, purge bool) (postTxActions util.PostTxAction, err error) { + postTxActions = util.NewPostTxAction() // ***** START: Watch ***** watchedRepoIDs, err := db.FindIDs(ctx, "watch", "watch.repo_id", @@ -135,7 +135,7 @@ func deleteUser(ctx context.Context, u *user_model.User, purge bool) (cleanup ut return nil, err } - cleanup = cleanup.Append(func() { + postTxActions = postTxActions.Append(func() { for _, a := range comment.Attachments { if err := storage.Attachments.Delete(a.RelativePath()); err != nil { if !errors.Is(err, os.ErrNotExist) { @@ -227,5 +227,5 @@ func deleteUser(ctx context.Context, u *user_model.User, purge bool) (cleanup ut return nil, fmt.Errorf("delete: %w", err) } - return cleanup, nil + return postTxActions, nil } diff --git a/services/user/user.go b/services/user/user.go index d92763d2d7a5f..26d7b11cdccd3 100644 --- a/services/user/user.go +++ b/services/user/user.go @@ -243,7 +243,7 @@ func DeleteUser(ctx context.Context, u *user_model.User, purge bool) error { return packages_model.ErrUserOwnPackages{UID: u.ID} } - cleanup, err := deleteUser(ctx, u, purge) + postTxActions, err := deleteUser(ctx, u, purge) if err != nil { return fmt.Errorf("DeleteUser: %w", err) } @@ -253,7 +253,7 @@ func DeleteUser(ctx context.Context, u *user_model.User, purge bool) error { } _ = committer.Close() - cleanup() + postTxActions() if err = asymkey_service.RewriteAllPublicKeys(ctx); err != nil { return err From f868da0afa617fa92f03f0680bb632dea366df01 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 17 Jul 2025 21:08:46 -0700 Subject: [PATCH 12/36] Rename --- services/issue/comments.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/services/issue/comments.go b/services/issue/comments.go index e92330ceba70a..e5d67c2afd7aa 100644 --- a/services/issue/comments.go +++ b/services/issue/comments.go @@ -135,8 +135,8 @@ func UpdateComment(ctx context.Context, c *issues_model.Comment, contentVersion } // deleteComment deletes the comment -func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAttachments bool) (*issues_model.Comment, func(), error) { - storageCleanup := util.NewPostTxAction() +func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAttachments bool) (*issues_model.Comment, util.PostTxAction, error) { + postTxActions := util.NewPostTxAction() deletedReviewComment, err := db.WithTx2(ctx, func(ctx context.Context) (*issues_model.Comment, error) { if removeAttachments { // load attachments before deleting the comment @@ -157,7 +157,7 @@ func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAtt } // the storage cleanup function to remove attachments could be called after all transactions are committed - storageCleanup = storageCleanup.Append(func() { + postTxActions = postTxActions.Append(func() { for _, a := range comment.Attachments { if err := storage.Attachments.Delete(a.RelativePath()); err != nil { if !errors.Is(err, os.ErrNotExist) { @@ -178,15 +178,15 @@ func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAtt if err != nil { return nil, nil, err } - return deletedReviewComment, storageCleanup, nil + return deletedReviewComment, postTxActions, nil } func DeleteComment(ctx context.Context, doer *user_model.User, comment *issues_model.Comment) (*issues_model.Comment, error) { - deletedReviewComment, cleanup, err := deleteComment(ctx, comment, true) + deletedReviewComment, postTxActions, err := deleteComment(ctx, comment, true) if err != nil { return nil, err } - cleanup() + postTxActions() notify_service.DeleteComment(ctx, doer, comment) From 97556a88de11aacef708a9754950f0e15d8888fb Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 18 Jul 2025 17:43:55 -0700 Subject: [PATCH 13/36] Improve attachments deletions --- models/db/file_status.go | 12 ++ models/issues/comment.go | 3 + models/issues/issue_update.go | 3 + models/migrations/v1_25/v321.go | 41 ++++++ models/repo/attachment.go | 95 ++++++++++---- modules/util/post_tx_action.go | 18 --- routers/api/v1/repo/issue_attachment.go | 5 + .../api/v1/repo/issue_comment_attachment.go | 5 + routers/api/v1/repo/release_attachment.go | 1 - routers/init.go | 2 + routers/web/repo/attachment.go | 6 +- services/attachment/attachment.go | 95 ++++++++++++-- services/cron/tasks_extended.go | 12 ++ services/issue/comments.go | 38 ++---- services/issue/issue.go | 51 ++++---- services/issue/issue_test.go | 13 +- services/release/release.go | 118 +++++++++--------- services/repository/delete.go | 24 ++-- services/user/delete.go | 30 +---- services/user/user.go | 5 +- 20 files changed, 355 insertions(+), 222 deletions(-) create mode 100644 models/db/file_status.go create mode 100644 models/migrations/v1_25/v321.go delete mode 100644 modules/util/post_tx_action.go diff --git a/models/db/file_status.go b/models/db/file_status.go new file mode 100644 index 0000000000000..6def378a0e5ee --- /dev/null +++ b/models/db/file_status.go @@ -0,0 +1,12 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package db + +// FileStatus represents the status of a file in the disk. +type FileStatus int + +const ( + FileStatusNormal FileStatus = iota // FileStatusNormal indicates the file is normal and exists on disk. + FileStatusToBeDeleted // FileStatusToBeDeleted indicates the file is marked for deletion but still exists on disk. +) diff --git a/models/issues/comment.go b/models/issues/comment.go index 603c6ad7ca28a..e73ac4cf09555 100644 --- a/models/issues/comment.go +++ b/models/issues/comment.go @@ -599,6 +599,9 @@ func UpdateCommentAttachments(ctx context.Context, c *Comment, uuids []string) e return fmt.Errorf("getAttachmentsByUUIDs [uuids: %v]: %w", uuids, err) } for i := range attachments { + if attachments[i].IssueID != 0 || attachments[i].CommentID != 0 { + return util.NewPermissionDeniedErrorf("update comment attachments permission denied") + } attachments[i].IssueID = c.IssueID attachments[i].CommentID = c.ID if err := repo_model.UpdateAttachment(ctx, attachments[i]); err != nil { diff --git a/models/issues/issue_update.go b/models/issues/issue_update.go index 9b99787e3bccf..bb49470dd56f4 100644 --- a/models/issues/issue_update.go +++ b/models/issues/issue_update.go @@ -305,6 +305,9 @@ func UpdateIssueAttachments(ctx context.Context, issueID int64, uuids []string) return fmt.Errorf("getAttachmentsByUUIDs [uuids: %v]: %w", uuids, err) } for i := range attachments { + if attachments[i].IssueID != 0 { + return util.NewPermissionDeniedErrorf("update issue attachments permission denied") + } attachments[i].IssueID = issueID if err := repo_model.UpdateAttachment(ctx, attachments[i]); err != nil { return fmt.Errorf("update attachment [id: %d]: %w", attachments[i].ID, err) diff --git a/models/migrations/v1_25/v321.go b/models/migrations/v1_25/v321.go new file mode 100644 index 0000000000000..4dae05114a984 --- /dev/null +++ b/models/migrations/v1_25/v321.go @@ -0,0 +1,41 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_24 + +import ( + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/xorm" +) + +func AddFileStatusToAttachment(x *xorm.Engine) error { + type Attachment struct { + ID int64 `xorm:"pk autoincr"` + UUID string `xorm:"uuid UNIQUE"` + RepoID int64 `xorm:"INDEX"` // this should not be zero + IssueID int64 `xorm:"INDEX"` // maybe zero when creating + ReleaseID int64 `xorm:"INDEX"` // maybe zero when creating + UploaderID int64 `xorm:"INDEX DEFAULT 0"` // Notice: will be zero before this column added + CommentID int64 `xorm:"INDEX"` + Name string + DownloadCount int64 `xorm:"DEFAULT 0"` + Status db.FileStatus `xorm:"INDEX DEFAULT 0"` + DeleteFailedCount int `xorm:"DEFAULT 0"` // Number of times the deletion failed, used to prevent infinite loop + LastDeleteFailedTime timeutil.TimeStamp // Last time the deletion failed, used to prevent infinite loop + Size int64 `xorm:"DEFAULT 0"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` + CustomDownloadURL string `xorm:"-"` + } + + if err := x.Sync(new(Attachment)); err != nil { + return err + } + + if _, err := x.Exec("UPDATE `attachment` SET status = ? WHERE status IS NULL", db.FileStatusNormal); err != nil { + return err + } + + return nil +} diff --git a/models/repo/attachment.go b/models/repo/attachment.go index d0690a1319aaa..3d3e931fe3561 100644 --- a/models/repo/attachment.go +++ b/models/repo/attachment.go @@ -18,18 +18,22 @@ import ( // Attachment represent a attachment of issue/comment/release. type Attachment struct { - ID int64 `xorm:"pk autoincr"` - UUID string `xorm:"uuid UNIQUE"` - RepoID int64 `xorm:"INDEX"` // this should not be zero - IssueID int64 `xorm:"INDEX"` // maybe zero when creating - ReleaseID int64 `xorm:"INDEX"` // maybe zero when creating - UploaderID int64 `xorm:"INDEX DEFAULT 0"` // Notice: will be zero before this column added - CommentID int64 `xorm:"INDEX"` - Name string - DownloadCount int64 `xorm:"DEFAULT 0"` - Size int64 `xorm:"DEFAULT 0"` - CreatedUnix timeutil.TimeStamp `xorm:"created"` - CustomDownloadURL string `xorm:"-"` + ID int64 `xorm:"pk autoincr"` + UUID string `xorm:"uuid UNIQUE"` + RepoID int64 `xorm:"INDEX"` // this should not be zero + IssueID int64 `xorm:"INDEX"` // maybe zero when creating + ReleaseID int64 `xorm:"INDEX"` // maybe zero when creating + UploaderID int64 `xorm:"INDEX DEFAULT 0"` // Notice: will be zero before this column added + CommentID int64 `xorm:"INDEX"` + Name string + DownloadCount int64 `xorm:"DEFAULT 0"` + Status db.FileStatus `xorm:"INDEX DEFAULT 0"` + DeleteFailedCount int `xorm:"DEFAULT 0"` // Number of times the deletion failed, used to prevent infinite loop + LastDeleteFailedReason string `xorm:"TEXT"` // Last reason the deletion failed, used to prevent infinite loop + LastDeleteFailedTime timeutil.TimeStamp // Last time the deletion failed, used to prevent infinite loop + Size int64 `xorm:"DEFAULT 0"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` + CustomDownloadURL string `xorm:"-"` } func init() { @@ -88,7 +92,9 @@ func (err ErrAttachmentNotExist) Unwrap() error { // GetAttachmentByID returns attachment by given id func GetAttachmentByID(ctx context.Context, id int64) (*Attachment, error) { attach := &Attachment{} - if has, err := db.GetEngine(ctx).ID(id).Get(attach); err != nil { + if has, err := db.GetEngine(ctx).ID(id). + And("status = ?", db.FileStatusNormal). + Get(attach); err != nil { return nil, err } else if !has { return nil, ErrAttachmentNotExist{ID: id, UUID: ""} @@ -99,7 +105,9 @@ func GetAttachmentByID(ctx context.Context, id int64) (*Attachment, error) { // GetAttachmentByUUID returns attachment by given UUID. func GetAttachmentByUUID(ctx context.Context, uuid string) (*Attachment, error) { attach := &Attachment{} - has, err := db.GetEngine(ctx).Where("uuid=?", uuid).Get(attach) + has, err := db.GetEngine(ctx).Where("uuid=?", uuid). + And("status = ?", db.FileStatusNormal). + Get(attach) if err != nil { return nil, err } else if !has { @@ -116,18 +124,24 @@ func GetAttachmentsByUUIDs(ctx context.Context, uuids []string) ([]*Attachment, // Silently drop invalid uuids. attachments := make([]*Attachment, 0, len(uuids)) - return attachments, db.GetEngine(ctx).In("uuid", uuids).Find(&attachments) + return attachments, db.GetEngine(ctx).In("uuid", uuids). + And("status = ?", db.FileStatusNormal). + Find(&attachments) } // ExistAttachmentsByUUID returns true if attachment exists with the given UUID func ExistAttachmentsByUUID(ctx context.Context, uuid string) (bool, error) { - return db.GetEngine(ctx).Where("`uuid`=?", uuid).Exist(new(Attachment)) + return db.GetEngine(ctx).Where("`uuid`=?", uuid). + And("status = ?", db.FileStatusNormal). + Exist(new(Attachment)) } // GetAttachmentsByIssueID returns all attachments of an issue. func GetAttachmentsByIssueID(ctx context.Context, issueID int64) ([]*Attachment, error) { attachments := make([]*Attachment, 0, 10) - return attachments, db.GetEngine(ctx).Where("issue_id = ? AND comment_id = 0", issueID).Find(&attachments) + return attachments, db.GetEngine(ctx).Where("issue_id = ? AND comment_id = 0", issueID). + And("status = ?", db.FileStatusNormal). + Find(&attachments) } // GetAttachmentsByIssueIDImagesLatest returns the latest image attachments of an issue. @@ -142,19 +156,23 @@ func GetAttachmentsByIssueIDImagesLatest(ctx context.Context, issueID int64) ([] OR name like '%.jxl' OR name like '%.png' OR name like '%.svg' - OR name like '%.webp')`, issueID).Desc("comment_id").Limit(5).Find(&attachments) + OR name like '%.webp')`, issueID). + And("status = ?", db.FileStatusNormal). + Desc("comment_id").Limit(5).Find(&attachments) } // GetAttachmentsByCommentID returns all attachments if comment by given ID. func GetAttachmentsByCommentID(ctx context.Context, commentID int64) ([]*Attachment, error) { attachments := make([]*Attachment, 0, 10) - return attachments, db.GetEngine(ctx).Where("comment_id=?", commentID).Find(&attachments) + return attachments, db.GetEngine(ctx).Where("comment_id=?", commentID). + And("status = ?", db.FileStatusNormal). + Find(&attachments) } // GetAttachmentByReleaseIDFileName returns attachment by given releaseId and fileName. func GetAttachmentByReleaseIDFileName(ctx context.Context, releaseID int64, fileName string) (*Attachment, error) { attach := &Attachment{ReleaseID: releaseID, Name: fileName} - has, err := db.GetEngine(ctx).Get(attach) + has, err := db.GetEngine(ctx).Where("status = ?", db.FileStatusNormal).Get(attach) if err != nil { return nil, err } else if !has { @@ -185,7 +203,8 @@ func UpdateAttachment(ctx context.Context, atta *Attachment) error { return err } -func DeleteAttachments(ctx context.Context, attachments []*Attachment) (int64, error) { +// MarkAttachmentsDeleted marks the given attachments as deleted +func MarkAttachmentsDeleted(ctx context.Context, attachments []*Attachment) (int64, error) { if len(attachments) == 0 { return 0, nil } @@ -195,15 +214,41 @@ func DeleteAttachments(ctx context.Context, attachments []*Attachment) (int64, e ids = append(ids, a.ID) } - return db.GetEngine(ctx).In("id", ids).NoAutoCondition().Delete(attachments[0]) + return db.GetEngine(ctx).Table("attachment").In("id", ids).Update(map[string]any{ + "status": db.FileStatusToBeDeleted, + }) } -// DeleteAttachmentsByRelease deletes all attachments associated with the given release. -func DeleteAttachmentsByRelease(ctx context.Context, releaseID int64) error { - _, err := db.GetEngine(ctx).Where("release_id = ?", releaseID).Delete(&Attachment{}) +// MarkAttachmentsDeletedByRelease marks all attachments associated with the given release as deleted. +func MarkAttachmentsDeletedByRelease(ctx context.Context, releaseID int64) error { + _, err := db.GetEngine(ctx).Table("attachment").Where("release_id = ?", releaseID).Update(map[string]any{ + "status": db.FileStatusToBeDeleted, + }) return err } +// DeleteAttachmentByID deletes the attachment which has been marked as deleted by given id +func DeleteAttachmentByID(ctx context.Context, id int64) error { + cnt, err := db.GetEngine(ctx).ID(id).Where("status = ?", db.FileStatusToBeDeleted).Delete(new(Attachment)) + if err != nil { + return fmt.Errorf("delete attachment by id: %w", err) + } + if cnt != 1 { + return fmt.Errorf("the attachment with id %d was not found or is not marked for deletion", id) + } + return nil +} + +func UpdateAttachmentFailure(ctx context.Context, attachment *Attachment, err error) error { + attachment.DeleteFailedCount++ + _, updateErr := db.GetEngine(ctx).Table("attachment").ID(attachment.ID).Update(map[string]any{ + "delete_failed_count": attachment.DeleteFailedCount, + "last_delete_failed_reason": err.Error(), + "last_delete_failed_time": timeutil.TimeStampNow(), + }) + return updateErr +} + // CountOrphanedAttachments returns the number of bad attachments func CountOrphanedAttachments(ctx context.Context) (int64, error) { return db.GetEngine(ctx).Where("(issue_id > 0 and issue_id not in (select id from issue)) or (release_id > 0 and release_id not in (select id from `release`))"). diff --git a/modules/util/post_tx_action.go b/modules/util/post_tx_action.go deleted file mode 100644 index b5b0e636173d7..0000000000000 --- a/modules/util/post_tx_action.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2025 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package util - -// PostTxAction is a function that is executed after a database transaction -type PostTxAction func() - -func NewPostTxAction() PostTxAction { - return func() {} -} - -func (f PostTxAction) Append(appendF PostTxAction) PostTxAction { - return func() { - f() - appendF() - } -} diff --git a/routers/api/v1/repo/issue_attachment.go b/routers/api/v1/repo/issue_attachment.go index 550abf4a7b373..aab6844304609 100644 --- a/routers/api/v1/repo/issue_attachment.go +++ b/routers/api/v1/repo/issue_attachment.go @@ -6,6 +6,7 @@ package repo import ( "net/http" + "code.gitea.io/gitea/models/db" issues_model "code.gitea.io/gitea/models/issues" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/log" @@ -360,6 +361,10 @@ func getIssueAttachmentSafeRead(ctx *context.APIContext, issue *issues_model.Iss if !attachmentBelongsToRepoOrIssue(ctx, attachment, issue) { return nil } + if attachment.Status != db.FileStatusNormal { + ctx.APIErrorNotFound() + return nil + } return attachment } diff --git a/routers/api/v1/repo/issue_comment_attachment.go b/routers/api/v1/repo/issue_comment_attachment.go index 704db1c7a3a83..5385275c127c7 100644 --- a/routers/api/v1/repo/issue_comment_attachment.go +++ b/routers/api/v1/repo/issue_comment_attachment.go @@ -7,6 +7,7 @@ import ( "errors" "net/http" + "code.gitea.io/gitea/models/db" issues_model "code.gitea.io/gitea/models/issues" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" @@ -391,6 +392,10 @@ func getIssueCommentAttachmentSafeRead(ctx *context.APIContext, comment *issues_ if !attachmentBelongsToRepoOrComment(ctx, attachment, comment) { return nil } + if attachment.Status != db.FileStatusNormal { + ctx.APIErrorNotFound() + return nil + } return attachment } diff --git a/routers/api/v1/repo/release_attachment.go b/routers/api/v1/repo/release_attachment.go index ab47cd4fd35b1..fa25c0cdfddaf 100644 --- a/routers/api/v1/repo/release_attachment.go +++ b/routers/api/v1/repo/release_attachment.go @@ -393,7 +393,6 @@ func DeleteReleaseAttachment(ctx *context.APIContext) { return } // FIXME Should prove the existence of the given repo, but results in unnecessary database requests - if err := attachment_service.DeleteAttachment(ctx, attach); err != nil { ctx.APIErrorInternal(err) return diff --git a/routers/init.go b/routers/init.go index 744feee2f0db0..b8bcd937bf90d 100644 --- a/routers/init.go +++ b/routers/init.go @@ -36,6 +36,7 @@ import ( web_routers "code.gitea.io/gitea/routers/web" actions_service "code.gitea.io/gitea/services/actions" asymkey_service "code.gitea.io/gitea/services/asymkey" + attachment_service "code.gitea.io/gitea/services/attachment" "code.gitea.io/gitea/services/auth" "code.gitea.io/gitea/services/auth/source/oauth2" "code.gitea.io/gitea/services/automerge" @@ -174,6 +175,7 @@ func InitWebInstalled(ctx context.Context) { mustInitCtx(ctx, actions_service.Init) mustInit(repo_service.InitLicenseClassifier) + mustInit(attachment_service.Init) // Finally start up the cron cron.NewContext(ctx) diff --git a/routers/web/repo/attachment.go b/routers/web/repo/attachment.go index 9b7be58875cea..ff8d098c1f2a6 100644 --- a/routers/web/repo/attachment.go +++ b/routers/web/repo/attachment.go @@ -70,7 +70,11 @@ func DeleteAttachment(ctx *context.Context) { file := ctx.FormString("file") attach, err := repo_model.GetAttachmentByUUID(ctx, file) if err != nil { - ctx.HTTPError(http.StatusBadRequest, err.Error()) + if repo_model.IsErrAttachmentNotExist(err) { + ctx.HTTPError(http.StatusNotFound) + } else { + ctx.ServerError("GetAttachmentByUUID", err) + } return } if !ctx.IsSigned || (ctx.Doer.ID != attach.UploaderID) { diff --git a/services/attachment/attachment.go b/services/attachment/attachment.go index d430819357c91..a41c8d87292be 100644 --- a/services/attachment/attachment.go +++ b/services/attachment/attachment.go @@ -13,7 +13,9 @@ import ( "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/queue" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/context/upload" @@ -71,23 +73,96 @@ func DeleteAttachment(ctx context.Context, a *repo_model.Attachment) error { // DeleteAttachments deletes the given attachments and optionally the associated files. func DeleteAttachments(ctx context.Context, attachments []*repo_model.Attachment) (int, error) { - cnt, err := repo_model.DeleteAttachments(ctx, attachments) + cnt, err := repo_model.MarkAttachmentsDeleted(ctx, attachments) if err != nil { return 0, err } + CleanAttachments(ctx, attachments) + + return int(cnt), nil +} + +var cleanQueue *queue.WorkerPoolQueue[int64] + +func Init() error { + cleanQueue = queue.CreateSimpleQueue(graceful.GetManager().ShutdownContext(), "attachments-clean", handler) + if cleanQueue == nil { + return errors.New("Unable to create attachments-clean queue") + } + return nil +} + +// CleanAttachments adds the attachments to the clean queue for deletion. +func CleanAttachments(ctx context.Context, attachments []*repo_model.Attachment) { for _, a := range attachments { - if err := storage.Attachments.Delete(a.RelativePath()); err != nil { + if err := cleanQueue.Push(a.ID); err != nil { + log.Error("Failed to push attachment ID %d to clean queue: %v", a.ID, err) + continue + } + } +} + +func handler(attachmentIDs ...int64) []int64 { + return cleanAttachments(graceful.GetManager().ShutdownContext(), attachmentIDs) +} + +func cleanAttachments(ctx context.Context, attachmentIDs []int64) []int64 { + var failed []int64 + for _, attachmentID := range attachmentIDs { + attachment, exist, err := db.GetByID[repo_model.Attachment](ctx, attachmentID) + if err != nil { + log.Error("Failed to get attachment by ID %d: %v", attachmentID, err) + continue + } + if !exist { + continue + } + if attachment.Status != db.FileStatusToBeDeleted { + log.Trace("Attachment %s is not marked for deletion, skipping", attachment.RelativePath()) + continue + } + + if err := storage.Attachments.Delete(attachment.RelativePath()); err != nil { if !errors.Is(err, os.ErrNotExist) { - // Even delete files failed, but the attachments has been removed from database, so we - // should not return error but only record the error on logs. - // users have to delete this attachments manually or we should have a - // synchronize between database attachment table and attachment storage - log.Error("delete attachment[uuid: %s] failed: %v", a.UUID, err) - } else { - log.Warn("Attachment file not found when deleting: %s", a.RelativePath()) + log.Error("delete attachment[uuid: %s] failed: %v", attachment.UUID, err) + failed = append(failed, attachment.ID) + if err := repo_model.UpdateAttachmentFailure(ctx, attachment, err); err != nil { + log.Error("Failed to update attachment failure for ID %d: %v", attachment.ID, err) + } + continue } } + if err := repo_model.DeleteAttachmentByID(ctx, attachment.ID); err != nil { + log.Error("Failed to delete attachment by ID %d(will be tried later): %v", attachment.ID, err) + failed = append(failed, attachment.ID) + } else { + log.Trace("Attachment %s deleted from database", attachment.RelativePath()) + } } - return int(cnt), nil + return failed +} + +// ScanTobeDeletedAttachments scans for attachments that are marked as to be deleted and send to +// clean queue +func ScanTobeDeletedAttachments(ctx context.Context) error { + attachments := make([]*repo_model.Attachment, 0, 10) + lastID := int64(0) + for { + if err := db.GetEngine(ctx). + Where("id > ? AND status = ?", lastID, db.FileStatusToBeDeleted). + Limit(100). + Find(&attachments); err != nil { + return fmt.Errorf("scan to-be-deleted attachments: %w", err) + } + + if len(attachments) == 0 { + log.Trace("No more attachments to be deleted") + break + } + CleanAttachments(ctx, attachments) + lastID = attachments[len(attachments)-1].ID + } + + return nil } diff --git a/services/cron/tasks_extended.go b/services/cron/tasks_extended.go index 0018c5facc5d7..7c41ff399f471 100644 --- a/services/cron/tasks_extended.go +++ b/services/cron/tasks_extended.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/updatechecker" asymkey_service "code.gitea.io/gitea/services/asymkey" + attachment_service "code.gitea.io/gitea/services/attachment" repo_service "code.gitea.io/gitea/services/repository" archiver_service "code.gitea.io/gitea/services/repository/archiver" user_service "code.gitea.io/gitea/services/user" @@ -223,6 +224,16 @@ func registerRebuildIssueIndexer() { }) } +func registerCleanAttachments() { + RegisterTaskFatal("clean_attachments", &BaseConfig{ + Enabled: false, + RunAtStart: false, + Schedule: "@every 24h", + }, func(ctx context.Context, _ *user_model.User, _ Config) error { + return attachment_service.ScanTobeDeletedAttachments(ctx) + }) +} + func initExtendedTasks() { registerDeleteInactiveUsers() registerDeleteRepositoryArchives() @@ -238,4 +249,5 @@ func initExtendedTasks() { registerDeleteOldSystemNotices() registerGCLFS() registerRebuildIssueIndexer() + registerCleanAttachments() } diff --git a/services/issue/comments.go b/services/issue/comments.go index e5d67c2afd7aa..a2c5e28d2342b 100644 --- a/services/issue/comments.go +++ b/services/issue/comments.go @@ -7,7 +7,6 @@ import ( "context" "errors" "fmt" - "os" "code.gitea.io/gitea/models/db" issues_model "code.gitea.io/gitea/models/issues" @@ -16,10 +15,8 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/json" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/services/attachment" git_service "code.gitea.io/gitea/services/git" notify_service "code.gitea.io/gitea/services/notify" ) @@ -135,8 +132,7 @@ func UpdateComment(ctx context.Context, c *issues_model.Comment, contentVersion } // deleteComment deletes the comment -func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAttachments bool) (*issues_model.Comment, util.PostTxAction, error) { - postTxActions := util.NewPostTxAction() +func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAttachments bool) (*issues_model.Comment, error) { deletedReviewComment, err := db.WithTx2(ctx, func(ctx context.Context) (*issues_model.Comment, error) { if removeAttachments { // load attachments before deleting the comment @@ -151,42 +147,26 @@ func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAtt } if removeAttachments { - // delete comment attachments - if _, err := repo_model.DeleteAttachments(ctx, comment.Attachments); err != nil { + // mark comment attachments as deleted + if _, err := repo_model.MarkAttachmentsDeleted(ctx, comment.Attachments); err != nil { return nil, err } - - // the storage cleanup function to remove attachments could be called after all transactions are committed - postTxActions = postTxActions.Append(func() { - for _, a := range comment.Attachments { - if err := storage.Attachments.Delete(a.RelativePath()); err != nil { - if !errors.Is(err, os.ErrNotExist) { - // Even delete files failed, but the attachments has been removed from database, so we - // should not return error but only record the error on logs. - // users have to delete this attachments manually or we should have a - // synchronize between database attachment table and attachment storage - log.Error("delete attachment[uuid: %s] failed: %v", a.UUID, err) - } else { - log.Warn("Attachment file not found when deleting: %s", a.RelativePath()) - } - } - } - }) } return deletedReviewComment, nil }) if err != nil { - return nil, nil, err + return nil, err } - return deletedReviewComment, postTxActions, nil + return deletedReviewComment, nil } func DeleteComment(ctx context.Context, doer *user_model.User, comment *issues_model.Comment) (*issues_model.Comment, error) { - deletedReviewComment, postTxActions, err := deleteComment(ctx, comment, true) + deletedReviewComment, err := deleteComment(ctx, comment, true) if err != nil { return nil, err } - postTxActions() + + attachment.CleanAttachments(ctx, comment.Attachments) notify_service.DeleteComment(ctx, doer, comment) diff --git a/services/issue/issue.go b/services/issue/issue.go index 04b3e3f179386..54f5fd9e1a4d5 100644 --- a/services/issue/issue.go +++ b/services/issue/issue.go @@ -13,13 +13,11 @@ import ( access_model "code.gitea.io/gitea/models/perm/access" project_model "code.gitea.io/gitea/models/project" repo_model "code.gitea.io/gitea/models/repo" - system_model "code.gitea.io/gitea/models/system" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/storage" - "code.gitea.io/gitea/modules/util" + attachment_service "code.gitea.io/gitea/services/attachment" notify_service "code.gitea.io/gitea/services/notify" ) @@ -191,11 +189,12 @@ func DeleteIssue(ctx context.Context, doer *user_model.User, gitRepo *git.Reposi } // delete entries in database - postTxActions, err := deleteIssue(ctx, issue, true) + toBeCleanedAttachments, err := deleteIssue(ctx, issue, true) if err != nil { return err } - postTxActions() + + attachment_service.CleanAttachments(ctx, toBeCleanedAttachments) // delete pull request related git data if issue.IsPull && gitRepo != nil { @@ -259,8 +258,8 @@ func GetRefEndNamesAndURLs(issues []*issues_model.Issue, repoLink string) (map[i } // deleteIssue deletes the issue -func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachments bool) (util.PostTxAction, error) { - postTxActions := util.NewPostTxAction() +func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachments bool) ([]*repo_model.Attachment, error) { + toBeCleanedAttachments := make([]*repo_model.Attachment, 0) if err := db.WithTx(ctx, func(ctx context.Context) error { if _, err := db.GetEngine(ctx).ID(issue.ID).NoAutoCondition().Delete(issue); err != nil { return err @@ -316,60 +315,56 @@ func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachmen } for _, comment := range issue.Comments { - _, postTxActionsDeleteComment, err := deleteComment(ctx, comment, deleteAttachments) + _, err := deleteComment(ctx, comment, deleteAttachments) if err != nil { return fmt.Errorf("deleteComment [comment_id: %d]: %w", comment.ID, err) } - postTxActions = postTxActions.Append(postTxActionsDeleteComment) + toBeCleanedAttachments = append(toBeCleanedAttachments, comment.Attachments...) } if deleteAttachments { // delete issue attachments - _, err := db.GetEngine(ctx).Where("issue_id = ? AND comment_id = 0", issue.ID).NoAutoCondition().Delete(&repo_model.Attachment{}) - if err != nil { + if err := issue.LoadAttachments(ctx); err != nil { + return err + } + if _, err := repo_model.MarkAttachmentsDeleted(ctx, issue.Attachments); err != nil { return err } - // the storage cleanup function to remove attachments could be called after all transactions are committed - postTxActions = postTxActions.Append(func() { - // Remove issue attachment files. - for i := range issue.Attachments { - system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete issue attachment", issue.Attachments[i].RelativePath()) - } - }) + toBeCleanedAttachments = append(toBeCleanedAttachments, issue.Attachments...) } return nil }); err != nil { return nil, err } - return postTxActions, nil + return toBeCleanedAttachments, nil } // DeleteOrphanedIssues delete issues without a repo func DeleteOrphanedIssues(ctx context.Context) error { - postTxActions := util.NewPostTxAction() + toBeCleanedAttachments := make([]*repo_model.Attachment, 0) if err := db.WithTx(ctx, func(ctx context.Context) error { repoIDs, err := issues_model.GetOrphanedIssueRepoIDs(ctx) if err != nil { return err } for i := range repoIDs { - postTxActionsDeleteIssues, err := DeleteIssuesByRepoID(ctx, repoIDs[i], true) + toBeCleanedIssueAttachments, err := DeleteIssuesByRepoID(ctx, repoIDs[i], true) if err != nil { return err } - postTxActions = postTxActions.Append(postTxActionsDeleteIssues) + toBeCleanedAttachments = append(toBeCleanedAttachments, toBeCleanedIssueAttachments...) } return nil }); err != nil { return err } - postTxActions() + attachment_service.CleanAttachments(ctx, toBeCleanedAttachments) return nil } // DeleteIssuesByRepoID deletes issues by repositories id -func DeleteIssuesByRepoID(ctx context.Context, repoID int64, deleteAttachments bool) (util.PostTxAction, error) { - postTxActions := util.NewPostTxAction() +func DeleteIssuesByRepoID(ctx context.Context, repoID int64, deleteAttachments bool) ([]*repo_model.Attachment, error) { + toBeCleanedAttachments := make([]*repo_model.Attachment, 0) for { issues := make([]*issues_model.Issue, 0, db.DefaultMaxInSize) if err := db.GetEngine(ctx). @@ -385,13 +380,13 @@ func DeleteIssuesByRepoID(ctx context.Context, repoID int64, deleteAttachments b } for _, issue := range issues { - postTxActionsDeleteIssue, err := deleteIssue(ctx, issue, deleteAttachments) + toBeCleanedIssueAttachments, err := deleteIssue(ctx, issue, deleteAttachments) if err != nil { return nil, fmt.Errorf("deleteIssue [issue_id: %d]: %w", issue.ID, err) } - postTxActions = postTxActions.Append(postTxActionsDeleteIssue) + toBeCleanedAttachments = append(toBeCleanedAttachments, toBeCleanedIssueAttachments...) } } - return postTxActions, nil + return toBeCleanedAttachments, nil } diff --git a/services/issue/issue_test.go b/services/issue/issue_test.go index 27a10ce9de4d9..ec15c0a5e01b0 100644 --- a/services/issue/issue_test.go +++ b/services/issue/issue_test.go @@ -11,6 +11,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" + attachment_service "code.gitea.io/gitea/services/attachment" "github.com/stretchr/testify/assert" ) @@ -44,9 +45,9 @@ func TestIssue_DeleteIssue(t *testing.T) { ID: issueIDs[2], } - postTxActions, err := deleteIssue(db.DefaultContext, issue, true) + toBeCleanedAttachments, err := deleteIssue(db.DefaultContext, issue, true) assert.NoError(t, err) - postTxActions() + attachment_service.CleanAttachments(db.DefaultContext, toBeCleanedAttachments) issueIDs, err = issues_model.GetIssueIDsByRepoID(db.DefaultContext, 1) assert.NoError(t, err) assert.Len(t, issueIDs, 4) @@ -56,9 +57,9 @@ func TestIssue_DeleteIssue(t *testing.T) { assert.NoError(t, err) issue, err = issues_model.GetIssueByID(db.DefaultContext, 4) assert.NoError(t, err) - postTxActions, err = deleteIssue(db.DefaultContext, issue, true) + toBeCleanedAttachments, err = deleteIssue(db.DefaultContext, issue, true) assert.NoError(t, err) - postTxActions() + attachment_service.CleanAttachments(db.DefaultContext, toBeCleanedAttachments) assert.Len(t, attachments, 2) for i := range attachments { attachment, err := repo_model.GetAttachmentByUUID(db.DefaultContext, attachments[i].UUID) @@ -80,9 +81,9 @@ func TestIssue_DeleteIssue(t *testing.T) { assert.NoError(t, err) assert.False(t, left) - postTxActions, err = deleteIssue(db.DefaultContext, issue2, true) + toBeCleanedAttachments, err = deleteIssue(db.DefaultContext, issue2, true) assert.NoError(t, err) - postTxActions() + attachment_service.CleanAttachments(db.DefaultContext, toBeCleanedAttachments) left, err = issues_model.IssueNoDependenciesLeft(db.DefaultContext, issue1) assert.NoError(t, err) assert.True(t, left) diff --git a/services/release/release.go b/services/release/release.go index 42af8e85187b8..ae840836d2545 100644 --- a/services/release/release.go +++ b/services/release/release.go @@ -19,9 +19,9 @@ import ( "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/repository" - "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" + attachment_service "code.gitea.io/gitea/services/attachment" notify_service "code.gitea.io/gitea/services/notify" ) @@ -288,6 +288,7 @@ func UpdateRelease(ctx context.Context, doer *user_model.User, gitRepo *git.Repo } deletedUUIDs := make(container.Set[string]) + deletedAttachments := make([]*repo_model.Attachment, 0, len(delAttachmentUUIDs)) if len(delAttachmentUUIDs) > 0 { // Check attachments attachments, err := repo_model.GetAttachmentsByUUIDs(ctx, delAttachmentUUIDs) @@ -299,12 +300,13 @@ func UpdateRelease(ctx context.Context, doer *user_model.User, gitRepo *git.Repo return util.NewPermissionDeniedErrorf("delete attachment of release permission denied") } deletedUUIDs.Add(attach.UUID) + deletedAttachments = append(deletedAttachments, attach) } - _, err = db.GetEngine(ctx).In("uuid", deletedUUIDs.Values()).NoAutoCondition().Delete(&repo_model.Attachment{}) - if err != nil { - return err + if _, err := repo_model.MarkAttachmentsDeleted(ctx, deletedAttachments); err != nil { + return fmt.Errorf("DeleteAttachments [uuids: %v]: %w", deletedUUIDs.Values(), err) } + // files will be deleted after database transaction is committed successfully } if len(editAttachments) > 0 { @@ -339,15 +341,7 @@ func UpdateRelease(ctx context.Context, doer *user_model.User, gitRepo *git.Repo return err } - for _, uuid := range deletedUUIDs.Values() { - if err := storage.Attachments.Delete(repo_model.AttachmentRelativePath(uuid)); err != nil { - // Even delete files failed, but the attachments has been removed from database, so we - // should not return error but only record the error on logs. - // users have to delete this attachments manually or we should have a - // synchronize between database attachment table and attachment storage - log.Error("delete attachment[uuid: %s] failed: %v", uuid, err) - } - } + attachment_service.CleanAttachments(ctx, deletedAttachments) if !rel.IsDraft { if !isTagCreated && !isConvertedFromTag { @@ -361,65 +355,65 @@ func UpdateRelease(ctx context.Context, doer *user_model.User, gitRepo *git.Repo // DeleteReleaseByID deletes a release and corresponding Git tag by given ID. func DeleteReleaseByID(ctx context.Context, repo *repo_model.Repository, rel *repo_model.Release, doer *user_model.User, delTag bool) error { - if delTag { - protectedTags, err := git_model.GetProtectedTags(ctx, rel.RepoID) - if err != nil { - return fmt.Errorf("GetProtectedTags: %w", err) - } - isAllowed, err := git_model.IsUserAllowedToControlTag(ctx, protectedTags, rel.TagName, rel.PublisherID) - if err != nil { - return err - } - if !isAllowed { - return ErrProtectedTagName{ - TagName: rel.TagName, + if err := db.WithTx(ctx, func(ctx context.Context) error { + if delTag { + protectedTags, err := git_model.GetProtectedTags(ctx, rel.RepoID) + if err != nil { + return fmt.Errorf("GetProtectedTags: %w", err) + } + isAllowed, err := git_model.IsUserAllowedToControlTag(ctx, protectedTags, rel.TagName, rel.PublisherID) + if err != nil { + return err + } + if !isAllowed { + return ErrProtectedTagName{ + TagName: rel.TagName, + } } - } - if stdout, _, err := git.NewCommand("tag", "-d").AddDashesAndList(rel.TagName). - RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}); err != nil && !strings.Contains(err.Error(), "not found") { - log.Error("DeleteReleaseByID (git tag -d): %d in %v Failed:\nStdout: %s\nError: %v", rel.ID, repo, stdout, err) - return fmt.Errorf("git tag -d: %w", err) - } + if stdout, _, err := git.NewCommand("tag", "-d").AddDashesAndList(rel.TagName). + RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}); err != nil && !strings.Contains(err.Error(), "not found") { + log.Error("DeleteReleaseByID (git tag -d): %d in %v Failed:\nStdout: %s\nError: %v", rel.ID, repo, stdout, err) + return fmt.Errorf("git tag -d: %w", err) + } - refName := git.RefNameFromTag(rel.TagName) - objectFormat := git.ObjectFormatFromName(repo.ObjectFormatName) - notify_service.PushCommits( - ctx, doer, repo, - &repository.PushUpdateOptions{ - RefFullName: refName, - OldCommitID: rel.Sha1, - NewCommitID: objectFormat.EmptyObjectID().String(), - }, repository.NewPushCommits()) - notify_service.DeleteRef(ctx, doer, repo, refName) - - if _, err := db.DeleteByID[repo_model.Release](ctx, rel.ID); err != nil { - return fmt.Errorf("DeleteReleaseByID: %w", err) - } - } else { - rel.IsTag = true + refName := git.RefNameFromTag(rel.TagName) + objectFormat := git.ObjectFormatFromName(repo.ObjectFormatName) + notify_service.PushCommits( + ctx, doer, repo, + &repository.PushUpdateOptions{ + RefFullName: refName, + OldCommitID: rel.Sha1, + NewCommitID: objectFormat.EmptyObjectID().String(), + }, repository.NewPushCommits()) + notify_service.DeleteRef(ctx, doer, repo, refName) + + if _, err := db.DeleteByID[repo_model.Release](ctx, rel.ID); err != nil { + return fmt.Errorf("DeleteReleaseByID: %w", err) + } + } else { + rel.IsTag = true - if err := repo_model.UpdateRelease(ctx, rel); err != nil { - return fmt.Errorf("Update: %w", err) + if err := repo_model.UpdateRelease(ctx, rel); err != nil { + return fmt.Errorf("Update: %w", err) + } } - } - - rel.Repo = repo - if err := rel.LoadAttributes(ctx); err != nil { - return fmt.Errorf("LoadAttributes: %w", err) - } - if err := repo_model.DeleteAttachmentsByRelease(ctx, rel.ID); err != nil { - return fmt.Errorf("DeleteAttachments: %w", err) - } + rel.Repo = repo + if err := rel.LoadAttributes(ctx); err != nil { + return fmt.Errorf("LoadAttributes: %w", err) + } - for i := range rel.Attachments { - attachment := rel.Attachments[i] - if err := storage.Attachments.Delete(attachment.RelativePath()); err != nil { - log.Error("Delete attachment %s of release %s failed: %v", attachment.UUID, rel.ID, err) + if err := repo_model.MarkAttachmentsDeletedByRelease(ctx, rel.ID); err != nil { + return fmt.Errorf("DeleteAttachments: %w", err) } + return nil + }); err != nil { + return err } + attachment_service.CleanAttachments(ctx, rel.Attachments) + if !rel.IsDraft { notify_service.DeleteRelease(ctx, doer, rel) } diff --git a/services/repository/delete.go b/services/repository/delete.go index f6bb58222dc6e..84f7fecbc7364 100644 --- a/services/repository/delete.go +++ b/services/repository/delete.go @@ -29,6 +29,7 @@ import ( "code.gitea.io/gitea/modules/storage" actions_service "code.gitea.io/gitea/services/actions" asymkey_service "code.gitea.io/gitea/services/asymkey" + attachment_service "code.gitea.io/gitea/services/attachment" issue_service "code.gitea.io/gitea/services/issue" "xorm.io/builder" @@ -122,15 +123,9 @@ func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams Find(&releaseAttachments); err != nil { return err } - // Delete attachments with release_id but repo_id = 0 - if len(releaseAttachments) > 0 { - ids := make([]int64, 0, len(releaseAttachments)) - for _, attach := range releaseAttachments { - ids = append(ids, attach.ID) - } - if _, err := db.GetEngine(ctx).In("id", ids).Delete(&repo_model.Attachment{}); err != nil { - return fmt.Errorf("delete release attachments failed: %w", err) - } + + if _, err := repo_model.MarkAttachmentsDeleted(ctx, releaseAttachments); err != nil { + return fmt.Errorf("delete release attachments: %w", err) } if _, err := db.Exec(ctx, "UPDATE `user` SET num_stars=num_stars-1 WHERE id IN (SELECT `uid` FROM `star` WHERE repo_id = ?)", repo.ID); err != nil { @@ -200,9 +195,8 @@ func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams } // Delete Issues and related objects - // attachments will be deleted later with repo_id - postTxActions, err := issue_service.DeleteIssuesByRepoID(ctx, repoID, false) - if err != nil { + // attachments will be deleted later with repo_id, so we don't need to delete them here + if _, err := issue_service.DeleteIssuesByRepoID(ctx, repoID, false); err != nil { return err } @@ -282,8 +276,7 @@ func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams }).Find(&repoAttachments); err != nil { return err } - - if _, err := sess.Where("repo_id=?", repo.ID).Delete(new(repo_model.Attachment)); err != nil { + if _, err := repo_model.MarkAttachmentsDeleted(ctx, repoAttachments); err != nil { return err } @@ -298,7 +291,8 @@ func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams committer.Close() - postTxActions() + attachment_service.CleanAttachments(ctx, releaseAttachments) + attachment_service.CleanAttachments(ctx, repoAttachments) if needRewriteKeysFile { if err := asymkey_service.RewriteAllPublicKeys(ctx); err != nil { diff --git a/services/user/delete.go b/services/user/delete.go index b88d1fa54b792..af287a074a354 100644 --- a/services/user/delete.go +++ b/services/user/delete.go @@ -5,9 +5,7 @@ package user import ( "context" - "errors" "fmt" - "os" "time" _ "image/jpeg" // Needed for jpeg support @@ -24,17 +22,14 @@ import ( pull_model "code.gitea.io/gitea/models/pull" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/storage" - "code.gitea.io/gitea/modules/util" "xorm.io/builder" ) // deleteUser deletes models associated to an user. -func deleteUser(ctx context.Context, u *user_model.User, purge bool) (postTxActions util.PostTxAction, err error) { - postTxActions = util.NewPostTxAction() +func deleteUser(ctx context.Context, u *user_model.User, purge bool) (toBeCleanedAttachments []*repo_model.Attachment, err error) { + toBeCleanedAttachments = make([]*repo_model.Attachment, 0) // ***** START: Watch ***** watchedRepoIDs, err := db.FindIDs(ctx, "watch", "watch.repo_id", @@ -131,25 +126,10 @@ func deleteUser(ctx context.Context, u *user_model.User, purge bool) (postTxActi return nil, err } - if _, err := repo_model.DeleteAttachments(ctx, comment.Attachments); err != nil { + if _, err := repo_model.MarkAttachmentsDeleted(ctx, comment.Attachments); err != nil { return nil, err } - - postTxActions = postTxActions.Append(func() { - for _, a := range comment.Attachments { - if err := storage.Attachments.Delete(a.RelativePath()); err != nil { - if !errors.Is(err, os.ErrNotExist) { - // Even delete files failed, but the attachments has been removed from database, so we - // should not return error but only record the error on logs. - // users have to delete this attachments manually or we should have a - // synchronize between database attachment table and attachment storage - log.Error("delete attachment[uuid: %s] failed: %v", a.UUID, err) - } else { - log.Warn("Attachment file not found when deleting: %s", a.RelativePath()) - } - } - } - }) + toBeCleanedAttachments = append(toBeCleanedAttachments, comment.Attachments...) } } @@ -227,5 +207,5 @@ func deleteUser(ctx context.Context, u *user_model.User, purge bool) (postTxActi return nil, fmt.Errorf("delete: %w", err) } - return postTxActions, nil + return toBeCleanedAttachments, nil } diff --git a/services/user/user.go b/services/user/user.go index 26d7b11cdccd3..7f43d3aa6d5bf 100644 --- a/services/user/user.go +++ b/services/user/user.go @@ -24,6 +24,7 @@ import ( "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/agit" asymkey_service "code.gitea.io/gitea/services/asymkey" + attachment_service "code.gitea.io/gitea/services/attachment" org_service "code.gitea.io/gitea/services/org" "code.gitea.io/gitea/services/packages" container_service "code.gitea.io/gitea/services/packages/container" @@ -243,7 +244,7 @@ func DeleteUser(ctx context.Context, u *user_model.User, purge bool) error { return packages_model.ErrUserOwnPackages{UID: u.ID} } - postTxActions, err := deleteUser(ctx, u, purge) + toBeCleanedAttachments, err := deleteUser(ctx, u, purge) if err != nil { return fmt.Errorf("DeleteUser: %w", err) } @@ -253,7 +254,7 @@ func DeleteUser(ctx context.Context, u *user_model.User, purge bool) error { } _ = committer.Close() - postTxActions() + attachment_service.CleanAttachments(ctx, toBeCleanedAttachments) if err = asymkey_service.RewriteAllPublicKeys(ctx); err != nil { return err From b89b27721874a5739b52bbcef4d769794453cbe4 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 18 Jul 2025 17:56:51 -0700 Subject: [PATCH 14/36] improvements --- models/migrations/v1_25/main_test.go | 14 ++++++++++ models/migrations/v1_25/v321.go | 2 +- models/migrations/v1_25/v321_test.go | 36 +++++++++++++++++++++++++ models/repo/attachment.go | 6 ++--- routers/api/v1/repo/issue_attachment.go | 5 ---- services/attachment/attachment.go | 8 ++++-- 6 files changed, 60 insertions(+), 11 deletions(-) create mode 100644 models/migrations/v1_25/main_test.go create mode 100644 models/migrations/v1_25/v321_test.go diff --git a/models/migrations/v1_25/main_test.go b/models/migrations/v1_25/main_test.go new file mode 100644 index 0000000000000..8ac213c908f78 --- /dev/null +++ b/models/migrations/v1_25/main_test.go @@ -0,0 +1,14 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_25 + +import ( + "testing" + + "code.gitea.io/gitea/models/migrations/base" +) + +func TestMain(m *testing.M) { + base.MainTest(m) +} diff --git a/models/migrations/v1_25/v321.go b/models/migrations/v1_25/v321.go index 4dae05114a984..3d8e8f7e3fe6a 100644 --- a/models/migrations/v1_25/v321.go +++ b/models/migrations/v1_25/v321.go @@ -1,7 +1,7 @@ // Copyright 2024 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package v1_24 +package v1_25 import ( "code.gitea.io/gitea/models/db" diff --git a/models/migrations/v1_25/v321_test.go b/models/migrations/v1_25/v321_test.go new file mode 100644 index 0000000000000..0febbe861fab8 --- /dev/null +++ b/models/migrations/v1_25/v321_test.go @@ -0,0 +1,36 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_25 + +import ( + "testing" + + "code.gitea.io/gitea/models/migrations/base" + "code.gitea.io/gitea/modules/timeutil" + + "github.com/stretchr/testify/assert" +) + +func Test_AddFileStatusToAttachment(t *testing.T) { + type Attachment struct { + ID int64 `xorm:"pk autoincr"` + UUID string `xorm:"uuid UNIQUE"` + RepoID int64 `xorm:"INDEX"` // this should not be zero + IssueID int64 `xorm:"INDEX"` // maybe zero when creating + ReleaseID int64 `xorm:"INDEX"` // maybe zero when creating + UploaderID int64 `xorm:"INDEX DEFAULT 0"` // Notice: will be zero before this column added + CommentID int64 `xorm:"INDEX"` + Name string + DownloadCount int64 `xorm:"DEFAULT 0"` + Size int64 `xorm:"DEFAULT 0"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` + CustomDownloadURL string `xorm:"-"` + } + + // Prepare and load the testing database + x, deferable := base.PrepareTestEnv(t, 0, new(Attachment)) + defer deferable() + + assert.NoError(t, AddFileStatusToAttachment(x)) +} diff --git a/models/repo/attachment.go b/models/repo/attachment.go index 3d3e931fe3561..af891384e0933 100644 --- a/models/repo/attachment.go +++ b/models/repo/attachment.go @@ -227,8 +227,8 @@ func MarkAttachmentsDeletedByRelease(ctx context.Context, releaseID int64) error return err } -// DeleteAttachmentByID deletes the attachment which has been marked as deleted by given id -func DeleteAttachmentByID(ctx context.Context, id int64) error { +// DeleteMarkedAttachmentByID deletes the attachment which has been marked as deleted by given id +func DeleteMarkedAttachmentByID(ctx context.Context, id int64) error { cnt, err := db.GetEngine(ctx).ID(id).Where("status = ?", db.FileStatusToBeDeleted).Delete(new(Attachment)) if err != nil { return fmt.Errorf("delete attachment by id: %w", err) @@ -239,7 +239,7 @@ func DeleteAttachmentByID(ctx context.Context, id int64) error { return nil } -func UpdateAttachmentFailure(ctx context.Context, attachment *Attachment, err error) error { +func UpdateMarkedAttachmentFailure(ctx context.Context, attachment *Attachment, err error) error { attachment.DeleteFailedCount++ _, updateErr := db.GetEngine(ctx).Table("attachment").ID(attachment.ID).Update(map[string]any{ "delete_failed_count": attachment.DeleteFailedCount, diff --git a/routers/api/v1/repo/issue_attachment.go b/routers/api/v1/repo/issue_attachment.go index aab6844304609..550abf4a7b373 100644 --- a/routers/api/v1/repo/issue_attachment.go +++ b/routers/api/v1/repo/issue_attachment.go @@ -6,7 +6,6 @@ package repo import ( "net/http" - "code.gitea.io/gitea/models/db" issues_model "code.gitea.io/gitea/models/issues" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/log" @@ -361,10 +360,6 @@ func getIssueAttachmentSafeRead(ctx *context.APIContext, issue *issues_model.Iss if !attachmentBelongsToRepoOrIssue(ctx, attachment, issue) { return nil } - if attachment.Status != db.FileStatusNormal { - ctx.APIErrorNotFound() - return nil - } return attachment } diff --git a/services/attachment/attachment.go b/services/attachment/attachment.go index a41c8d87292be..18e3d38444800 100644 --- a/services/attachment/attachment.go +++ b/services/attachment/attachment.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/system" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/queue" @@ -127,13 +128,16 @@ func cleanAttachments(ctx context.Context, attachmentIDs []int64) []int64 { if !errors.Is(err, os.ErrNotExist) { log.Error("delete attachment[uuid: %s] failed: %v", attachment.UUID, err) failed = append(failed, attachment.ID) - if err := repo_model.UpdateAttachmentFailure(ctx, attachment, err); err != nil { + if attachment.DeleteFailedCount%3 == 0 { + _ = system.CreateNotice(ctx, system.NoticeRepository, fmt.Sprintf("Failed to delete attachment %s (%d times): %v", attachment.RelativePath(), attachment.DeleteFailedCount+1, err)) + } + if err := repo_model.UpdateMarkedAttachmentFailure(ctx, attachment, err); err != nil { log.Error("Failed to update attachment failure for ID %d: %v", attachment.ID, err) } continue } } - if err := repo_model.DeleteAttachmentByID(ctx, attachment.ID); err != nil { + if err := repo_model.DeleteMarkedAttachmentByID(ctx, attachment.ID); err != nil { log.Error("Failed to delete attachment by ID %d(will be tried later): %v", attachment.ID, err) failed = append(failed, attachment.ID) } else { From 488e6581aff7656cd2ea25a03172793f1fa827af Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 18 Jul 2025 18:09:03 -0700 Subject: [PATCH 15/36] improvements --- models/migrations/v1_25/v321.go | 2 +- services/attachment/attachment.go | 8 +++---- services/issue/comments.go | 8 ++----- services/issue/issue.go | 35 ++++++++++++++----------------- services/issue/issue_test.go | 6 +++--- services/release/release.go | 4 ++-- services/repository/delete.go | 4 ++-- services/user/user.go | 2 +- 8 files changed, 31 insertions(+), 38 deletions(-) diff --git a/models/migrations/v1_25/v321.go b/models/migrations/v1_25/v321.go index 3d8e8f7e3fe6a..174e6c40c914a 100644 --- a/models/migrations/v1_25/v321.go +++ b/models/migrations/v1_25/v321.go @@ -1,4 +1,4 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. +// Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package v1_25 diff --git a/services/attachment/attachment.go b/services/attachment/attachment.go index 18e3d38444800..8a214ea2f53e5 100644 --- a/services/attachment/attachment.go +++ b/services/attachment/attachment.go @@ -79,7 +79,7 @@ func DeleteAttachments(ctx context.Context, attachments []*repo_model.Attachment return 0, err } - CleanAttachments(ctx, attachments) + AddAttachmentsToCleanQueue(ctx, attachments) return int(cnt), nil } @@ -94,8 +94,8 @@ func Init() error { return nil } -// CleanAttachments adds the attachments to the clean queue for deletion. -func CleanAttachments(ctx context.Context, attachments []*repo_model.Attachment) { +// AddAttachmentsToCleanQueue adds the attachments to the clean queue for deletion. +func AddAttachmentsToCleanQueue(ctx context.Context, attachments []*repo_model.Attachment) { for _, a := range attachments { if err := cleanQueue.Push(a.ID); err != nil { log.Error("Failed to push attachment ID %d to clean queue: %v", a.ID, err) @@ -164,7 +164,7 @@ func ScanTobeDeletedAttachments(ctx context.Context) error { log.Trace("No more attachments to be deleted") break } - CleanAttachments(ctx, attachments) + AddAttachmentsToCleanQueue(ctx, attachments) lastID = attachments[len(attachments)-1].ID } diff --git a/services/issue/comments.go b/services/issue/comments.go index a2c5e28d2342b..3b6a7af2cd672 100644 --- a/services/issue/comments.go +++ b/services/issue/comments.go @@ -133,7 +133,7 @@ func UpdateComment(ctx context.Context, c *issues_model.Comment, contentVersion // deleteComment deletes the comment func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAttachments bool) (*issues_model.Comment, error) { - deletedReviewComment, err := db.WithTx2(ctx, func(ctx context.Context) (*issues_model.Comment, error) { + return db.WithTx2(ctx, func(ctx context.Context) (*issues_model.Comment, error) { if removeAttachments { // load attachments before deleting the comment if err := comment.LoadAttachments(ctx); err != nil { @@ -154,10 +154,6 @@ func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAtt } return deletedReviewComment, nil }) - if err != nil { - return nil, err - } - return deletedReviewComment, nil } func DeleteComment(ctx context.Context, doer *user_model.User, comment *issues_model.Comment) (*issues_model.Comment, error) { @@ -166,7 +162,7 @@ func DeleteComment(ctx context.Context, doer *user_model.User, comment *issues_m return nil, err } - attachment.CleanAttachments(ctx, comment.Attachments) + attachment.AddAttachmentsToCleanQueue(ctx, comment.Attachments) notify_service.DeleteComment(ctx, doer, comment) diff --git a/services/issue/issue.go b/services/issue/issue.go index 54f5fd9e1a4d5..fe369fe1e68ed 100644 --- a/services/issue/issue.go +++ b/services/issue/issue.go @@ -194,7 +194,7 @@ func DeleteIssue(ctx context.Context, doer *user_model.User, gitRepo *git.Reposi return err } - attachment_service.CleanAttachments(ctx, toBeCleanedAttachments) + attachment_service.AddAttachmentsToCleanQueue(ctx, toBeCleanedAttachments) // delete pull request related git data if issue.IsPull && gitRepo != nil { @@ -259,36 +259,36 @@ func GetRefEndNamesAndURLs(issues []*issues_model.Issue, repoLink string) (map[i // deleteIssue deletes the issue func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachments bool) ([]*repo_model.Attachment, error) { - toBeCleanedAttachments := make([]*repo_model.Attachment, 0) - if err := db.WithTx(ctx, func(ctx context.Context) error { + return db.WithTx2(ctx, func(ctx context.Context) ([]*repo_model.Attachment, error) { + toBeCleanedAttachments := make([]*repo_model.Attachment, 0) if _, err := db.GetEngine(ctx).ID(issue.ID).NoAutoCondition().Delete(issue); err != nil { - return err + return nil, err } // update the total issue numbers if err := repo_model.UpdateRepoIssueNumbers(ctx, issue.RepoID, issue.IsPull, false); err != nil { - return err + return nil, err } // if the issue is closed, update the closed issue numbers if issue.IsClosed { if err := repo_model.UpdateRepoIssueNumbers(ctx, issue.RepoID, issue.IsPull, true); err != nil { - return err + return nil, err } } if err := issues_model.UpdateMilestoneCounters(ctx, issue.MilestoneID); err != nil { - return fmt.Errorf("error updating counters for milestone id %d: %w", + return nil, fmt.Errorf("error updating counters for milestone id %d: %w", issue.MilestoneID, err) } if err := activities_model.DeleteIssueActions(ctx, issue.RepoID, issue.ID, issue.Index); err != nil { - return err + return nil, err } if deleteAttachments { // find attachments related to this issue and remove them if err := issue.LoadAttachments(ctx); err != nil { - return err + return nil, err } } @@ -311,13 +311,13 @@ func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachmen &issues_model.Comment{DependentIssueID: issue.ID}, &issues_model.IssuePin{IssueID: issue.ID}, ); err != nil { - return err + return nil, err } for _, comment := range issue.Comments { _, err := deleteComment(ctx, comment, deleteAttachments) if err != nil { - return fmt.Errorf("deleteComment [comment_id: %d]: %w", comment.ID, err) + return nil, fmt.Errorf("deleteComment [comment_id: %d]: %w", comment.ID, err) } toBeCleanedAttachments = append(toBeCleanedAttachments, comment.Attachments...) } @@ -325,18 +325,15 @@ func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachmen if deleteAttachments { // delete issue attachments if err := issue.LoadAttachments(ctx); err != nil { - return err + return nil, err } if _, err := repo_model.MarkAttachmentsDeleted(ctx, issue.Attachments); err != nil { - return err + return nil, err } toBeCleanedAttachments = append(toBeCleanedAttachments, issue.Attachments...) } - return nil - }); err != nil { - return nil, err - } - return toBeCleanedAttachments, nil + return toBeCleanedAttachments, nil + }) } // DeleteOrphanedIssues delete issues without a repo @@ -358,7 +355,7 @@ func DeleteOrphanedIssues(ctx context.Context) error { }); err != nil { return err } - attachment_service.CleanAttachments(ctx, toBeCleanedAttachments) + attachment_service.AddAttachmentsToCleanQueue(ctx, toBeCleanedAttachments) return nil } diff --git a/services/issue/issue_test.go b/services/issue/issue_test.go index ec15c0a5e01b0..5bf8426d7b982 100644 --- a/services/issue/issue_test.go +++ b/services/issue/issue_test.go @@ -47,7 +47,7 @@ func TestIssue_DeleteIssue(t *testing.T) { toBeCleanedAttachments, err := deleteIssue(db.DefaultContext, issue, true) assert.NoError(t, err) - attachment_service.CleanAttachments(db.DefaultContext, toBeCleanedAttachments) + attachment_service.AddAttachmentsToCleanQueue(db.DefaultContext, toBeCleanedAttachments) issueIDs, err = issues_model.GetIssueIDsByRepoID(db.DefaultContext, 1) assert.NoError(t, err) assert.Len(t, issueIDs, 4) @@ -59,7 +59,7 @@ func TestIssue_DeleteIssue(t *testing.T) { assert.NoError(t, err) toBeCleanedAttachments, err = deleteIssue(db.DefaultContext, issue, true) assert.NoError(t, err) - attachment_service.CleanAttachments(db.DefaultContext, toBeCleanedAttachments) + attachment_service.AddAttachmentsToCleanQueue(db.DefaultContext, toBeCleanedAttachments) assert.Len(t, attachments, 2) for i := range attachments { attachment, err := repo_model.GetAttachmentByUUID(db.DefaultContext, attachments[i].UUID) @@ -83,7 +83,7 @@ func TestIssue_DeleteIssue(t *testing.T) { toBeCleanedAttachments, err = deleteIssue(db.DefaultContext, issue2, true) assert.NoError(t, err) - attachment_service.CleanAttachments(db.DefaultContext, toBeCleanedAttachments) + attachment_service.AddAttachmentsToCleanQueue(db.DefaultContext, toBeCleanedAttachments) left, err = issues_model.IssueNoDependenciesLeft(db.DefaultContext, issue1) assert.NoError(t, err) assert.True(t, left) diff --git a/services/release/release.go b/services/release/release.go index ae840836d2545..417b16ce51e9d 100644 --- a/services/release/release.go +++ b/services/release/release.go @@ -341,7 +341,7 @@ func UpdateRelease(ctx context.Context, doer *user_model.User, gitRepo *git.Repo return err } - attachment_service.CleanAttachments(ctx, deletedAttachments) + attachment_service.AddAttachmentsToCleanQueue(ctx, deletedAttachments) if !rel.IsDraft { if !isTagCreated && !isConvertedFromTag { @@ -412,7 +412,7 @@ func DeleteReleaseByID(ctx context.Context, repo *repo_model.Repository, rel *re return err } - attachment_service.CleanAttachments(ctx, rel.Attachments) + attachment_service.AddAttachmentsToCleanQueue(ctx, rel.Attachments) if !rel.IsDraft { notify_service.DeleteRelease(ctx, doer, rel) diff --git a/services/repository/delete.go b/services/repository/delete.go index 84f7fecbc7364..e0669ec9ca503 100644 --- a/services/repository/delete.go +++ b/services/repository/delete.go @@ -291,8 +291,8 @@ func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams committer.Close() - attachment_service.CleanAttachments(ctx, releaseAttachments) - attachment_service.CleanAttachments(ctx, repoAttachments) + attachment_service.AddAttachmentsToCleanQueue(ctx, releaseAttachments) + attachment_service.AddAttachmentsToCleanQueue(ctx, repoAttachments) if needRewriteKeysFile { if err := asymkey_service.RewriteAllPublicKeys(ctx); err != nil { diff --git a/services/user/user.go b/services/user/user.go index 7f43d3aa6d5bf..ad17732df1595 100644 --- a/services/user/user.go +++ b/services/user/user.go @@ -254,7 +254,7 @@ func DeleteUser(ctx context.Context, u *user_model.User, purge bool) error { } _ = committer.Close() - attachment_service.CleanAttachments(ctx, toBeCleanedAttachments) + attachment_service.AddAttachmentsToCleanQueue(ctx, toBeCleanedAttachments) if err = asymkey_service.RewriteAllPublicKeys(ctx); err != nil { return err From e00c7d32af10150287df1e7ad1c14fb19abe68a4 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 18 Jul 2025 18:12:42 -0700 Subject: [PATCH 16/36] improvements --- services/repository/delete.go | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/services/repository/delete.go b/services/repository/delete.go index e0669ec9ca503..9b3d1c07e855c 100644 --- a/services/repository/delete.go +++ b/services/repository/delete.go @@ -291,9 +291,6 @@ func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams committer.Close() - attachment_service.AddAttachmentsToCleanQueue(ctx, releaseAttachments) - attachment_service.AddAttachmentsToCleanQueue(ctx, repoAttachments) - if needRewriteKeysFile { if err := asymkey_service.RewriteAllPublicKeys(ctx); err != nil { log.Error("RewriteAllPublicKeys failed: %v", err) @@ -327,14 +324,8 @@ func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams system_model.RemoveStorageWithNotice(ctx, storage.LFS, "Delete orphaned LFS file", lfsObj) } - // Remove release attachments - for _, attachment := range releaseAttachments { - system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete release attachment", attachment.RelativePath()) - } - // Remove attachment with repo_id = repo.ID. - for _, attachment := range repoAttachments { - system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete repo attachment", attachment.RelativePath()) - } + attachment_service.AddAttachmentsToCleanQueue(ctx, releaseAttachments) + attachment_service.AddAttachmentsToCleanQueue(ctx, repoAttachments) if len(repo.Avatar) > 0 { if err := storage.RepoAvatars.Delete(repo.CustomAvatarRelativePath()); err != nil { From 27c83deb6d40fcba37f74c6ee241c4f08dfa1abb Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 18 Jul 2025 18:16:14 -0700 Subject: [PATCH 17/36] improvements --- routers/api/v1/repo/issue_comment_attachment.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/routers/api/v1/repo/issue_comment_attachment.go b/routers/api/v1/repo/issue_comment_attachment.go index 5385275c127c7..704db1c7a3a83 100644 --- a/routers/api/v1/repo/issue_comment_attachment.go +++ b/routers/api/v1/repo/issue_comment_attachment.go @@ -7,7 +7,6 @@ import ( "errors" "net/http" - "code.gitea.io/gitea/models/db" issues_model "code.gitea.io/gitea/models/issues" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" @@ -392,10 +391,6 @@ func getIssueCommentAttachmentSafeRead(ctx *context.APIContext, comment *issues_ if !attachmentBelongsToRepoOrComment(ctx, attachment, comment) { return nil } - if attachment.Status != db.FileStatusNormal { - ctx.APIErrorNotFound() - return nil - } return attachment } From 0379d4b97169156562b12ce6b5170285233b7d0e Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 18 Jul 2025 19:07:21 -0700 Subject: [PATCH 18/36] Fix test --- options/locale/locale_en-US.ini | 1 + services/attachment/attachment_test.go | 8 +++++++- services/issue/main_test.go | 9 ++++++++- services/release/release_test.go | 8 +++++++- services/repository/main_test.go | 9 ++++++++- services/user/user_test.go | 8 +++++++- 6 files changed, 38 insertions(+), 5 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index c1a3d37037fe3..ab925669a56a9 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3065,6 +3065,7 @@ dashboard.sync_branch.started = Branches Sync started dashboard.sync_tag.started = Tags Sync started dashboard.rebuild_issue_indexer = Rebuild issue indexer dashboard.sync_repo_licenses = Sync repo licenses +dashboard.clean_attachments = Clean up deleted attachments users.user_manage_panel = User Account Management users.new_account = Create User Account diff --git a/services/attachment/attachment_test.go b/services/attachment/attachment_test.go index 9562b8f731845..f4e178c3623ec 100644 --- a/services/attachment/attachment_test.go +++ b/services/attachment/attachment_test.go @@ -12,6 +12,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/setting" _ "code.gitea.io/gitea/models/actions" @@ -19,7 +20,12 @@ import ( ) func TestMain(m *testing.M) { - unittest.MainTest(m) + unittest.MainTest(m, &unittest.TestOptions{ + SetUp: func() error { + setting.LoadQueueSettings() + return Init() + }, + }) } func TestUploadAttachment(t *testing.T) { diff --git a/services/issue/main_test.go b/services/issue/main_test.go index 819c5d98c3f6c..544cf2aad7eb4 100644 --- a/services/issue/main_test.go +++ b/services/issue/main_test.go @@ -7,11 +7,18 @@ import ( "testing" "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/services/attachment" _ "code.gitea.io/gitea/models" _ "code.gitea.io/gitea/models/actions" ) func TestMain(m *testing.M) { - unittest.MainTest(m) + unittest.MainTest(m, &unittest.TestOptions{ + SetUp: func() error { + setting.LoadQueueSettings() + return attachment.Init() + }, + }) } diff --git a/services/release/release_test.go b/services/release/release_test.go index 36a9f667d697e..50da93446a35e 100644 --- a/services/release/release_test.go +++ b/services/release/release_test.go @@ -14,6 +14,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/services/attachment" _ "code.gitea.io/gitea/models/actions" @@ -22,7 +23,12 @@ import ( ) func TestMain(m *testing.M) { - unittest.MainTest(m) + unittest.MainTest(m, &unittest.TestOptions{ + SetUp: func() error { + setting.LoadQueueSettings() + return attachment.Init() + }, + }) } func TestRelease_Create(t *testing.T) { diff --git a/services/repository/main_test.go b/services/repository/main_test.go index 7ad1540aee48a..01d04cc10d4f8 100644 --- a/services/repository/main_test.go +++ b/services/repository/main_test.go @@ -7,8 +7,15 @@ import ( "testing" "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/services/attachment" ) func TestMain(m *testing.M) { - unittest.MainTest(m) + unittest.MainTest(m, &unittest.TestOptions{ + SetUp: func() error { + setting.LoadQueueSettings() + return attachment.Init() + }, + }) } diff --git a/services/user/user_test.go b/services/user/user_test.go index 28a0df8628fdb..f687095135e18 100644 --- a/services/user/user_test.go +++ b/services/user/user_test.go @@ -17,13 +17,19 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/services/attachment" org_service "code.gitea.io/gitea/services/org" "github.com/stretchr/testify/assert" ) func TestMain(m *testing.M) { - unittest.MainTest(m) + unittest.MainTest(m, &unittest.TestOptions{ + SetUp: func() error { + setting.LoadQueueSettings() + return attachment.Init() + }, + }) } func TestDeleteUser(t *testing.T) { From 804022bdb54116144102b9052c18682bd4397840 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 18 Jul 2025 21:13:10 -0700 Subject: [PATCH 19/36] Fix test --- models/repo/release.go | 1 + models/user/main_test.go | 9 ++++++++- services/issue/comments_test.go | 4 +++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/models/repo/release.go b/models/repo/release.go index 59f4caf5aa9e0..b3aae97560c16 100644 --- a/models/repo/release.go +++ b/models/repo/release.go @@ -378,6 +378,7 @@ func GetReleaseAttachments(ctx context.Context, rels ...*Release) (err error) { err = db.GetEngine(ctx). Asc("release_id", "name"). In("release_id", sortedRels.ID). + And("status = ?", db.FileStatusNormal). Find(&attachments) if err != nil { return err diff --git a/models/user/main_test.go b/models/user/main_test.go index a626d323a71a5..2ca502bbeaabd 100644 --- a/models/user/main_test.go +++ b/models/user/main_test.go @@ -7,6 +7,8 @@ import ( "testing" "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/services/attachment" _ "code.gitea.io/gitea/models" _ "code.gitea.io/gitea/models/actions" @@ -15,5 +17,10 @@ import ( ) func TestMain(m *testing.M) { - unittest.MainTest(m) + unittest.MainTest(m, &unittest.TestOptions{ + SetUp: func() error { + setting.LoadQueueSettings() + return attachment.Init() + }, + }) } diff --git a/services/issue/comments_test.go b/services/issue/comments_test.go index 2e548bc3cbe56..4088bd1020d25 100644 --- a/services/issue/comments_test.go +++ b/services/issue/comments_test.go @@ -34,5 +34,7 @@ func Test_DeleteCommentWithReview(t *testing.T) { // the review should be deleted as well unittest.AssertNotExistsBean(t, &issues_model.Review{ID: review.ID}) // the attachment should be deleted as well - unittest.AssertNotExistsBean(t, &repo_model.Attachment{ID: comment.Attachments[0].ID}) + newAttachment, err := repo_model.GetAttachmentByID(t.Context(), comment.Attachments[0].ID) + assert.Error(t, err) + assert.Nil(t, newAttachment) } From c89d939b54e36f2c826028cba9b27603338d5ae8 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 19 Jul 2025 11:00:56 -0700 Subject: [PATCH 20/36] Fix test --- models/issues/comment.go | 5 +- services/org/org.go | 49 ++--- services/repository/delete.go | 346 +++++++++++++++++----------------- services/user/user.go | 65 +++---- 4 files changed, 227 insertions(+), 238 deletions(-) diff --git a/models/issues/comment.go b/models/issues/comment.go index e73ac4cf09555..1ffe1b650ef43 100644 --- a/models/issues/comment.go +++ b/models/issues/comment.go @@ -1114,8 +1114,7 @@ func UpdateComment(ctx context.Context, c *Comment, contentVersion int, doer *us // DeleteComment deletes the comment func DeleteComment(ctx context.Context, comment *Comment) (*Comment, error) { - e := db.GetEngine(ctx) - if _, err := e.ID(comment.ID).NoAutoCondition().Delete(comment); err != nil { + if _, err := db.GetEngine(ctx).ID(comment.ID).NoAutoCondition().Delete(comment); err != nil { return nil, err } @@ -1130,7 +1129,7 @@ func DeleteComment(ctx context.Context, comment *Comment) (*Comment, error) { return nil, err } } - if _, err := e.Table("action"). + if _, err := db.GetEngine(ctx).Table("action"). Where("comment_id = ?", comment.ID). Update(map[string]any{ "is_deleted": true, diff --git a/services/org/org.go b/services/org/org.go index 3d30ae21a39d3..13221e2895816 100644 --- a/services/org/org.go +++ b/services/org/org.go @@ -48,12 +48,11 @@ func deleteOrganization(ctx context.Context, org *org_model.Organization) error // DeleteOrganization completely and permanently deletes everything of organization. func DeleteOrganization(ctx context.Context, org *org_model.Organization, purge bool) error { - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err - } - defer committer.Close() - + // The repositories deletion of the organization cannot be under a transaction, + // because it cannot be rolled back because the content in disk will be deleted + // in the DeleteOwnerRepositoriesDirectly function. + // Even not all repositories deleted successfully, we still delete the organization again. + // TODO: We should mark all the repositories as deleted and delete them in a background job. if purge { err := repo_service.DeleteOwnerRepositoriesDirectly(ctx, org.AsUser()) if err != nil { @@ -61,26 +60,28 @@ func DeleteOrganization(ctx context.Context, org *org_model.Organization, purge } } - // Check ownership of repository. - count, err := repo_model.CountRepositories(ctx, repo_model.CountRepositoryOptions{OwnerID: org.ID}) - if err != nil { - return fmt.Errorf("GetRepositoryCount: %w", err) - } else if count > 0 { - return repo_model.ErrUserOwnRepos{UID: org.ID} - } - - // Check ownership of packages. - if ownsPackages, err := packages_model.HasOwnerPackages(ctx, org.ID); err != nil { - return fmt.Errorf("HasOwnerPackages: %w", err) - } else if ownsPackages { - return packages_model.ErrUserOwnPackages{UID: org.ID} - } + err := db.WithTx(ctx, func(ctx context.Context) error { + // Check ownership of repository. + count, err := repo_model.CountRepositories(ctx, repo_model.CountRepositoryOptions{OwnerID: org.ID}) + if err != nil { + return fmt.Errorf("GetRepositoryCount: %w", err) + } else if count > 0 { + return repo_model.ErrUserOwnRepos{UID: org.ID} + } - if err := deleteOrganization(ctx, org); err != nil { - return fmt.Errorf("DeleteOrganization: %w", err) - } + // Check ownership of packages. + if ownsPackages, err := packages_model.HasOwnerPackages(ctx, org.ID); err != nil { + return fmt.Errorf("HasOwnerPackages: %w", err) + } else if ownsPackages { + return packages_model.ErrUserOwnPackages{UID: org.ID} + } - if err := committer.Commit(); err != nil { + if err := deleteOrganization(ctx, org); err != nil { + return fmt.Errorf("DeleteOrganization: %w", err) + } + return nil + }) + if err != nil { return err } diff --git a/services/repository/delete.go b/services/repository/delete.go index 9b3d1c07e855c..5647843e4a752 100644 --- a/services/repository/delete.go +++ b/services/repository/delete.go @@ -51,15 +51,8 @@ func deleteDBRepository(ctx context.Context, repoID int64) error { // DeleteRepository deletes a repository for a user or organization. // make sure if you call this func to close open sessions (sqlite will otherwise get a deadlock) func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams ...bool) error { - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err - } - defer committer.Close() - sess := db.GetEngine(ctx) - repo := &repo_model.Repository{} - has, err := sess.ID(repoID).Get(repo) + has, err := db.GetEngine(ctx).ID(repoID).Get(repo) if err != nil { return err } else if !has { @@ -82,215 +75,216 @@ func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams return fmt.Errorf("list actions artifacts of repo %v: %w", repoID, err) } - // In case owner is a organization, we have to change repo specific teams - // if ignoreOrgTeams is not true - var org *user_model.User - if len(ignoreOrgTeams) == 0 || !ignoreOrgTeams[0] { - if org, err = user_model.GetUserByID(ctx, repo.OwnerID); err != nil { - return err + var needRewriteKeysFile bool + releaseAttachments := make([]*repo_model.Attachment, 0, 20) + var repoAttachments []*repo_model.Attachment + var archivePaths []string + var lfsPaths []string + + err = db.WithTx(ctx, func(ctx context.Context) error { + // In case owner is a organization, we have to change repo specific teams + // if ignoreOrgTeams is not true + var org *user_model.User + if len(ignoreOrgTeams) == 0 || !ignoreOrgTeams[0] { + if org, err = user_model.GetUserByID(ctx, repo.OwnerID); err != nil { + return err + } } - } - - // Delete Deploy Keys - deleted, err := asymkey_service.DeleteRepoDeployKeys(ctx, repoID) - if err != nil { - return err - } - needRewriteKeysFile := deleted > 0 - if err := deleteDBRepository(ctx, repoID); err != nil { - return err - } - - if org != nil && org.IsOrganization() { - teams, err := organization.FindOrgTeams(ctx, org.ID) + // Delete Deploy Keys + deleted, err := asymkey_service.DeleteRepoDeployKeys(ctx, repoID) if err != nil { return err } - for _, t := range teams { - if !organization.HasTeamRepo(ctx, t.OrgID, t.ID, repoID) { - continue - } else if err = removeRepositoryFromTeam(ctx, t, repo, false); err != nil { + needRewriteKeysFile = deleted > 0 + + if err := deleteDBRepository(ctx, repoID); err != nil { + return err + } + + if org != nil && org.IsOrganization() { + teams, err := organization.FindOrgTeams(ctx, org.ID) + if err != nil { return err } + for _, t := range teams { + if !organization.HasTeamRepo(ctx, t.OrgID, t.ID, repoID) { + continue + } else if err = removeRepositoryFromTeam(ctx, t, repo, false); err != nil { + return err + } + } } - } - - // some attachments have release_id but repo_id = 0 - releaseAttachments := make([]*repo_model.Attachment, 0, 20) - if err = db.GetEngine(ctx).Join("INNER", "`release`", "`release`.id = `attachment`.release_id"). - Where("`release`.repo_id = ?", repoID). - Find(&releaseAttachments); err != nil { - return err - } - if _, err := repo_model.MarkAttachmentsDeleted(ctx, releaseAttachments); err != nil { - return fmt.Errorf("delete release attachments: %w", err) - } + // some attachments have release_id but repo_id = 0 + if err = db.GetEngine(ctx).Join("INNER", "`release`", "`release`.id = `attachment`.release_id"). + Where("`release`.repo_id = ?", repoID). + Find(&releaseAttachments); err != nil { + return err + } - if _, err := db.Exec(ctx, "UPDATE `user` SET num_stars=num_stars-1 WHERE id IN (SELECT `uid` FROM `star` WHERE repo_id = ?)", repo.ID); err != nil { - return err - } + if _, err := repo_model.MarkAttachmentsDeleted(ctx, releaseAttachments); err != nil { + return fmt.Errorf("delete release attachments: %w", err) + } - if _, err := db.GetEngine(ctx).In("hook_id", builder.Select("id").From("webhook").Where(builder.Eq{"webhook.repo_id": repo.ID})). - Delete(&webhook.HookTask{}); err != nil { - return err - } + if _, err := db.Exec(ctx, "UPDATE `user` SET num_stars=num_stars-1 WHERE id IN (SELECT `uid` FROM `star` WHERE repo_id = ?)", repo.ID); err != nil { + return err + } - // CleanupEphemeralRunnersByPickedTaskOfRepo deletes ephemeral global/org/user that have started any task of this repo - // The cannot pick a second task hardening for ephemeral runners expect that task objects remain available until runner deletion - // This method will delete affected ephemeral global/org/user runners - // &actions_model.ActionRunner{RepoID: repoID} does only handle ephemeral repository runners - if err := actions_service.CleanupEphemeralRunnersByPickedTaskOfRepo(ctx, repoID); err != nil { - return fmt.Errorf("cleanupEphemeralRunners: %w", err) - } + if _, err := db.GetEngine(ctx).In("hook_id", builder.Select("id").From("webhook").Where(builder.Eq{"webhook.repo_id": repo.ID})). + Delete(&webhook.HookTask{}); err != nil { + return err + } - if err := db.DeleteBeans(ctx, - &access_model.Access{RepoID: repo.ID}, - &activities_model.Action{RepoID: repo.ID}, - &repo_model.Collaboration{RepoID: repoID}, - &issues_model.Comment{RefRepoID: repoID}, - &git_model.CommitStatus{RepoID: repoID}, - &git_model.Branch{RepoID: repoID}, - &git_model.LFSLock{RepoID: repoID}, - &repo_model.LanguageStat{RepoID: repoID}, - &repo_model.RepoLicense{RepoID: repoID}, - &issues_model.Milestone{RepoID: repoID}, - &repo_model.Mirror{RepoID: repoID}, - &activities_model.Notification{RepoID: repoID}, - &git_model.ProtectedBranch{RepoID: repoID}, - &git_model.ProtectedTag{RepoID: repoID}, - &repo_model.PushMirror{RepoID: repoID}, - &repo_model.Release{RepoID: repoID}, - &repo_model.RepoIndexerStatus{RepoID: repoID}, - &repo_model.Redirect{RedirectRepoID: repoID}, - &repo_model.RepoUnit{RepoID: repoID}, - &repo_model.Star{RepoID: repoID}, - &admin_model.Task{RepoID: repoID}, - &repo_model.Watch{RepoID: repoID}, - &webhook.Webhook{RepoID: repoID}, - &secret_model.Secret{RepoID: repoID}, - &actions_model.ActionTaskStep{RepoID: repoID}, - &actions_model.ActionTask{RepoID: repoID}, - &actions_model.ActionRunJob{RepoID: repoID}, - &actions_model.ActionRun{RepoID: repoID}, - &actions_model.ActionRunner{RepoID: repoID}, - &actions_model.ActionScheduleSpec{RepoID: repoID}, - &actions_model.ActionSchedule{RepoID: repoID}, - &actions_model.ActionArtifact{RepoID: repoID}, - &actions_model.ActionRunnerToken{RepoID: repoID}, - &issues_model.IssuePin{RepoID: repoID}, - ); err != nil { - return fmt.Errorf("deleteBeans: %w", err) - } + // CleanupEphemeralRunnersByPickedTaskOfRepo deletes ephemeral global/org/user that have started any task of this repo + // The cannot pick a second task hardening for ephemeral runners expect that task objects remain available until runner deletion + // This method will delete affected ephemeral global/org/user runners + // &actions_model.ActionRunner{RepoID: repoID} does only handle ephemeral repository runners + if err := actions_service.CleanupEphemeralRunnersByPickedTaskOfRepo(ctx, repoID); err != nil { + return fmt.Errorf("cleanupEphemeralRunners: %w", err) + } - // Delete Labels and related objects - if err := issues_model.DeleteLabelsByRepoID(ctx, repoID); err != nil { - return err - } + if err := db.DeleteBeans(ctx, + &access_model.Access{RepoID: repo.ID}, + &activities_model.Action{RepoID: repo.ID}, + &repo_model.Collaboration{RepoID: repoID}, + &issues_model.Comment{RefRepoID: repoID}, + &git_model.CommitStatus{RepoID: repoID}, + &git_model.Branch{RepoID: repoID}, + &git_model.LFSLock{RepoID: repoID}, + &repo_model.LanguageStat{RepoID: repoID}, + &repo_model.RepoLicense{RepoID: repoID}, + &issues_model.Milestone{RepoID: repoID}, + &repo_model.Mirror{RepoID: repoID}, + &activities_model.Notification{RepoID: repoID}, + &git_model.ProtectedBranch{RepoID: repoID}, + &git_model.ProtectedTag{RepoID: repoID}, + &repo_model.PushMirror{RepoID: repoID}, + &repo_model.Release{RepoID: repoID}, + &repo_model.RepoIndexerStatus{RepoID: repoID}, + &repo_model.Redirect{RedirectRepoID: repoID}, + &repo_model.RepoUnit{RepoID: repoID}, + &repo_model.Star{RepoID: repoID}, + &admin_model.Task{RepoID: repoID}, + &repo_model.Watch{RepoID: repoID}, + &webhook.Webhook{RepoID: repoID}, + &secret_model.Secret{RepoID: repoID}, + &actions_model.ActionTaskStep{RepoID: repoID}, + &actions_model.ActionTask{RepoID: repoID}, + &actions_model.ActionRunJob{RepoID: repoID}, + &actions_model.ActionRun{RepoID: repoID}, + &actions_model.ActionRunner{RepoID: repoID}, + &actions_model.ActionScheduleSpec{RepoID: repoID}, + &actions_model.ActionSchedule{RepoID: repoID}, + &actions_model.ActionArtifact{RepoID: repoID}, + &actions_model.ActionRunnerToken{RepoID: repoID}, + &issues_model.IssuePin{RepoID: repoID}, + ); err != nil { + return fmt.Errorf("deleteBeans: %w", err) + } - // Delete Pulls and related objects - if err := issues_model.DeletePullsByBaseRepoID(ctx, repoID); err != nil { - return err - } + // Delete Labels and related objects + if err := issues_model.DeleteLabelsByRepoID(ctx, repoID); err != nil { + return err + } - // Delete Issues and related objects - // attachments will be deleted later with repo_id, so we don't need to delete them here - if _, err := issue_service.DeleteIssuesByRepoID(ctx, repoID, false); err != nil { - return err - } + // Delete Pulls and related objects + if err := issues_model.DeletePullsByBaseRepoID(ctx, repoID); err != nil { + return err + } - // Delete issue index - if err := db.DeleteResourceIndex(ctx, "issue_index", repoID); err != nil { - return err - } + // Delete Issues and related objects + // attachments will be deleted later with repo_id, so we don't need to delete them here + if _, err := issue_service.DeleteIssuesByRepoID(ctx, repoID, false); err != nil { + return err + } - if repo.IsFork { - if _, err := db.Exec(ctx, "UPDATE `repository` SET num_forks=num_forks-1 WHERE id=?", repo.ForkID); err != nil { - return fmt.Errorf("decrease fork count: %w", err) + // Delete issue index + if err := db.DeleteResourceIndex(ctx, "issue_index", repoID); err != nil { + return err } - } - if _, err := db.Exec(ctx, "UPDATE `user` SET num_repos=num_repos-1 WHERE id=?", repo.OwnerID); err != nil { - return err - } + if repo.IsFork { + if _, err := db.Exec(ctx, "UPDATE `repository` SET num_forks=num_forks-1 WHERE id=?", repo.ForkID); err != nil { + return fmt.Errorf("decrease fork count: %w", err) + } + } - if len(repo.Topics) > 0 { - if err := repo_model.RemoveTopicsFromRepo(ctx, repo.ID); err != nil { + if _, err := db.Exec(ctx, "UPDATE `user` SET num_repos=num_repos-1 WHERE id=?", repo.OwnerID); err != nil { return err } - } - if err := project_model.DeleteProjectByRepoID(ctx, repoID); err != nil { - return fmt.Errorf("unable to delete projects for repo[%d]: %w", repoID, err) - } + if len(repo.Topics) > 0 { + if err := repo_model.RemoveTopicsFromRepo(ctx, repo.ID); err != nil { + return err + } + } - // Remove LFS objects - var lfsObjects []*git_model.LFSMetaObject - if err = sess.Where("repository_id=?", repoID).Find(&lfsObjects); err != nil { - return err - } + if err := project_model.DeleteProjectByRepoID(ctx, repoID); err != nil { + return fmt.Errorf("unable to delete projects for repo[%d]: %w", repoID, err) + } - lfsPaths := make([]string, 0, len(lfsObjects)) - for _, v := range lfsObjects { - count, err := db.CountByBean(ctx, &git_model.LFSMetaObject{Pointer: lfs.Pointer{Oid: v.Oid}}) - if err != nil { + // Remove LFS objects + var lfsObjects []*git_model.LFSMetaObject + if err = db.GetEngine(ctx).Where("repository_id=?", repoID).Find(&lfsObjects); err != nil { return err } - if count > 1 { - continue - } - lfsPaths = append(lfsPaths, v.RelativePath()) - } + lfsPaths = make([]string, 0, len(lfsObjects)) + for _, v := range lfsObjects { + count, err := db.CountByBean(ctx, &git_model.LFSMetaObject{Pointer: lfs.Pointer{Oid: v.Oid}}) + if err != nil { + return err + } + if count > 1 { + continue + } - if _, err := db.DeleteByBean(ctx, &git_model.LFSMetaObject{RepositoryID: repoID}); err != nil { - return err - } + lfsPaths = append(lfsPaths, v.RelativePath()) + } - // Remove archives - var archives []*repo_model.RepoArchiver - if err = sess.Where("repo_id=?", repoID).Find(&archives); err != nil { - return err - } + if _, err := db.DeleteByBean(ctx, &git_model.LFSMetaObject{RepositoryID: repoID}); err != nil { + return err + } - archivePaths := make([]string, 0, len(archives)) - for _, v := range archives { - archivePaths = append(archivePaths, v.RelativePath()) - } + // Remove archives + var archives []*repo_model.RepoArchiver + if err = db.GetEngine(ctx).Where("repo_id=?", repoID).Find(&archives); err != nil { + return err + } - if _, err := db.DeleteByBean(ctx, &repo_model.RepoArchiver{RepoID: repoID}); err != nil { - return err - } + archivePaths = make([]string, 0, len(archives)) + for _, v := range archives { + archivePaths = append(archivePaths, v.RelativePath()) + } - if repo.NumForks > 0 { - if _, err = sess.Exec("UPDATE `repository` SET fork_id=0,is_fork=? WHERE fork_id=?", false, repo.ID); err != nil { - log.Error("reset 'fork_id' and 'is_fork': %v", err) + if _, err := db.DeleteByBean(ctx, &repo_model.RepoArchiver{RepoID: repoID}); err != nil { + return err } - } - // Get all attachments with repo_id = repo.ID. some release attachments have repo_id = 0 should be deleted before - var repoAttachments []*repo_model.Attachment - if err := sess.Where(builder.Eq{ - "repo_id": repo.ID, - }).Find(&repoAttachments); err != nil { - return err - } - if _, err := repo_model.MarkAttachmentsDeleted(ctx, repoAttachments); err != nil { - return err - } + if repo.NumForks > 0 { + if _, err = db.GetEngine(ctx).Exec("UPDATE `repository` SET fork_id=0,is_fork=? WHERE fork_id=?", false, repo.ID); err != nil { + log.Error("reset 'fork_id' and 'is_fork': %v", err) + } + } - // unlink packages linked to this repository - if err = packages_model.UnlinkRepositoryFromAllPackages(ctx, repoID); err != nil { - return err - } + // Get all attachments with repo_id = repo.ID. some release attachments have repo_id = 0 should be deleted before + if err := db.GetEngine(ctx).Where(builder.Eq{ + "repo_id": repo.ID, + }).Find(&repoAttachments); err != nil { + return err + } + if _, err := repo_model.MarkAttachmentsDeleted(ctx, repoAttachments); err != nil { + return err + } - if err = committer.Commit(); err != nil { + // unlink packages linked to this repository + return packages_model.UnlinkRepositoryFromAllPackages(ctx, repoID) + }) + if err != nil { return err } - committer.Close() - if needRewriteKeysFile { if err := asymkey_service.RewriteAllPublicKeys(ctx); err != nil { log.Error("RewriteAllPublicKeys failed: %v", err) diff --git a/services/user/user.go b/services/user/user.go index ad17732df1595..ef2d27fffd528 100644 --- a/services/user/user.go +++ b/services/user/user.go @@ -211,48 +211,43 @@ func DeleteUser(ctx context.Context, u *user_model.User, purge bool) error { } } - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err - } - defer committer.Close() + toBeCleanedAttachments, err := db.WithTx2(ctx, func(ctx context.Context) ([]*repo_model.Attachment, error) { + // Note: A user owns any repository or belongs to any organization + // cannot perform delete operation. This causes a race with the purge above + // however consistency requires that we ensure that this is the case - // Note: A user owns any repository or belongs to any organization - // cannot perform delete operation. This causes a race with the purge above - // however consistency requires that we ensure that this is the case - - // Check ownership of repository. - count, err := repo_model.CountRepositories(ctx, repo_model.CountRepositoryOptions{OwnerID: u.ID}) - if err != nil { - return fmt.Errorf("GetRepositoryCount: %w", err) - } else if count > 0 { - return repo_model.ErrUserOwnRepos{UID: u.ID} - } + // Check ownership of repository. + count, err := repo_model.CountRepositories(ctx, repo_model.CountRepositoryOptions{OwnerID: u.ID}) + if err != nil { + return nil, fmt.Errorf("GetRepositoryCount: %w", err) + } else if count > 0 { + return nil, repo_model.ErrUserOwnRepos{UID: u.ID} + } - // Check membership of organization. - count, err = organization.GetOrganizationCount(ctx, u) - if err != nil { - return fmt.Errorf("GetOrganizationCount: %w", err) - } else if count > 0 { - return organization.ErrUserHasOrgs{UID: u.ID} - } + // Check membership of organization. + count, err = organization.GetOrganizationCount(ctx, u) + if err != nil { + return nil, fmt.Errorf("GetOrganizationCount: %w", err) + } else if count > 0 { + return nil, organization.ErrUserHasOrgs{UID: u.ID} + } - // Check ownership of packages. - if ownsPackages, err := packages_model.HasOwnerPackages(ctx, u.ID); err != nil { - return fmt.Errorf("HasOwnerPackages: %w", err) - } else if ownsPackages { - return packages_model.ErrUserOwnPackages{UID: u.ID} - } + // Check ownership of packages. + if ownsPackages, err := packages_model.HasOwnerPackages(ctx, u.ID); err != nil { + return nil, fmt.Errorf("HasOwnerPackages: %w", err) + } else if ownsPackages { + return nil, packages_model.ErrUserOwnPackages{UID: u.ID} + } - toBeCleanedAttachments, err := deleteUser(ctx, u, purge) + toBeCleanedAttachments, err := deleteUser(ctx, u, purge) + if err != nil { + return nil, fmt.Errorf("DeleteUser: %w", err) + } + return toBeCleanedAttachments, nil + }) if err != nil { - return fmt.Errorf("DeleteUser: %w", err) - } - - if err := committer.Commit(); err != nil { return err } - _ = committer.Close() attachment_service.AddAttachmentsToCleanQueue(ctx, toBeCleanedAttachments) From 0295c7bf0167a4d76adfeade214aae97610e1bde Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 19 Jul 2025 11:13:25 -0700 Subject: [PATCH 21/36] improve the comment --- services/org/org.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/services/org/org.go b/services/org/org.go index 13221e2895816..43aa32485c875 100644 --- a/services/org/org.go +++ b/services/org/org.go @@ -48,11 +48,14 @@ func deleteOrganization(ctx context.Context, org *org_model.Organization) error // DeleteOrganization completely and permanently deletes everything of organization. func DeleteOrganization(ctx context.Context, org *org_model.Organization, purge bool) error { - // The repositories deletion of the organization cannot be under a transaction, - // because it cannot be rolled back because the content in disk will be deleted - // in the DeleteOwnerRepositoriesDirectly function. - // Even not all repositories deleted successfully, we still delete the organization again. - // TODO: We should mark all the repositories as deleted and delete them in a background job. + // Deleting repositories under the organization cannot be wrapped in a transaction at the moment, + // because the associated disk content is permanently deleted by the DeleteOwnerRepositoriesDirectly function, + // which cannot be rolled back. + // + // Even if some repositories fail to delete, the organization will still be deleted. + // + // TODO: Consider marking repositories as "deleted" first, + // and handling the actual deletion in a background job for better reliability and rollback support. if purge { err := repo_service.DeleteOwnerRepositoriesDirectly(ctx, org.AsUser()) if err != nil { From 551f8ba25ed3846ba8dc4b22d9a65aba1fc9d6e3 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 19 Jul 2025 14:05:55 -0700 Subject: [PATCH 22/36] Fix test --- tests/integration/api_admin_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/api_admin_test.go b/tests/integration/api_admin_test.go index d28a103e596f8..1cceacefbad98 100644 --- a/tests/integration/api_admin_test.go +++ b/tests/integration/api_admin_test.go @@ -304,11 +304,11 @@ func TestAPICron(t *testing.T) { AddTokenAuth(token) resp := MakeRequest(t, req, http.StatusOK) - assert.Equal(t, "29", resp.Header().Get("X-Total-Count")) + assert.Equal(t, "30", resp.Header().Get("X-Total-Count")) var crons []api.Cron DecodeJSON(t, resp, &crons) - assert.Len(t, crons, 29) + assert.Len(t, crons, 30) }) t.Run("Execute", func(t *testing.T) { From 9c251fdb563ad26f4b3965c81fecb0edc0a40e8b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 20 Jul 2025 12:17:04 -0700 Subject: [PATCH 23/36] Fix some problems --- models/db/file_status.go | 4 +- models/issues/comment.go | 34 +++++++---- models/issues/comment_list.go | 6 +- models/issues/issue_list.go | 1 + models/migrations/v1_25/v321.go | 82 +++++++++++++++++++-------- models/repo/attachment.go | 45 ++++++++++++++- services/attachment/attachment.go | 9 ++- services/cron/tasks_extended.go | 2 +- services/issue/comments.go | 1 + services/migrations/gitea_uploader.go | 1 + 10 files changed, 139 insertions(+), 46 deletions(-) diff --git a/models/db/file_status.go b/models/db/file_status.go index 6def378a0e5ee..4ed1186fb56b5 100644 --- a/models/db/file_status.go +++ b/models/db/file_status.go @@ -7,6 +7,6 @@ package db type FileStatus int const ( - FileStatusNormal FileStatus = iota // FileStatusNormal indicates the file is normal and exists on disk. - FileStatusToBeDeleted // FileStatusToBeDeleted indicates the file is marked for deletion but still exists on disk. + FileStatusNormal FileStatus = iota + 1 // FileStatusNormal indicates the file is normal and exists on disk. + FileStatusToBeDeleted // FileStatusToBeDeleted indicates the file is marked for deletion but still exists on disk. ) diff --git a/models/issues/comment.go b/models/issues/comment.go index 1ffe1b650ef43..a69d805399852 100644 --- a/models/issues/comment.go +++ b/models/issues/comment.go @@ -1141,26 +1141,36 @@ func DeleteComment(ctx context.Context, comment *Comment) (*Comment, error) { // delete review & review comment if the code comment is the last comment of the review if comment.Type == CommentTypeCode && comment.ReviewID > 0 { - res, err := db.GetEngine(ctx).ID(comment.ReviewID). - Where("NOT EXISTS (SELECT 1 FROM comment WHERE review_id = ? AND `type` = ?)", comment.ReviewID, CommentTypeCode). - Delete(new(Review)) - if err != nil { + if err := comment.LoadReview(ctx); err != nil { return nil, err } - if res > 0 { - var reviewComment Comment - has, err := db.GetEngine(ctx).Where("review_id = ?", comment.ReviewID). - And("type = ?", CommentTypeReview).Get(&reviewComment) + if comment.Review != nil && comment.Review.Type == ReviewTypeComment { + res, err := db.GetEngine(ctx).ID(comment.ReviewID). + Where("NOT EXISTS (SELECT 1 FROM comment WHERE review_id = ? AND `type` = ?)", comment.ReviewID, CommentTypeCode). + Delete(new(Review)) if err != nil { return nil, err } - if has && reviewComment.Content == "" { - if _, err := db.GetEngine(ctx).ID(reviewComment.ID).Delete(new(Comment)); err != nil { + if res > 0 { + var reviewComment Comment + has, err := db.GetEngine(ctx).Where("review_id = ?", comment.ReviewID). + And("`type` = ?", CommentTypeReview).Get(&reviewComment) + if err != nil { return nil, err } - deletedReviewComment = &reviewComment + if has && reviewComment.Content == "" { + if err := reviewComment.LoadAttachments(ctx); err != nil { + return nil, err + } + if len(reviewComment.Attachments) == 0 { + if _, err := db.GetEngine(ctx).ID(reviewComment.ID).Delete(new(Comment)); err != nil { + return nil, err + } + deletedReviewComment = &reviewComment + } + } + comment.ReviewID = 0 // reset review ID to 0 for the notification } - comment.ReviewID = 0 // reset review ID to 0 for the notification } } diff --git a/models/issues/comment_list.go b/models/issues/comment_list.go index f6c485449f60b..bb95da7710bf5 100644 --- a/models/issues/comment_list.go +++ b/models/issues/comment_list.go @@ -349,7 +349,10 @@ func (comments CommentList) LoadAttachmentsByIssue(ctx context.Context) error { } attachments := make([]*repo_model.Attachment, 0, len(comments)/2) - if err := db.GetEngine(ctx).Where("issue_id=? AND comment_id>0", comments[0].IssueID).Find(&attachments); err != nil { + if err := db.GetEngine(ctx). + Where("issue_id=? AND comment_id>0", comments[0].IssueID). + And("status = ?", db.FileStatusNormal). + Find(&attachments); err != nil { return err } @@ -377,6 +380,7 @@ func (comments CommentList) LoadAttachments(ctx context.Context) (err error) { limit := min(left, db.DefaultMaxInSize) rows, err := db.GetEngine(ctx). In("comment_id", commentsIDs[:limit]). + And("status = ?", db.FileStatusNormal). Rows(new(repo_model.Attachment)) if err != nil { return err diff --git a/models/issues/issue_list.go b/models/issues/issue_list.go index 26b93189b8bed..98b0becafd1b5 100644 --- a/models/issues/issue_list.go +++ b/models/issues/issue_list.go @@ -339,6 +339,7 @@ func (issues IssueList) LoadAttachments(ctx context.Context) (err error) { limit := min(left, db.DefaultMaxInSize) rows, err := db.GetEngine(ctx). In("issue_id", issuesIDs[:limit]). + And("status = ?", db.FileStatusNormal). Rows(new(repo_model.Attachment)) if err != nil { return err diff --git a/models/migrations/v1_25/v321.go b/models/migrations/v1_25/v321.go index 174e6c40c914a..0084b99eacb2b 100644 --- a/models/migrations/v1_25/v321.go +++ b/models/migrations/v1_25/v321.go @@ -8,34 +8,68 @@ import ( "code.gitea.io/gitea/modules/timeutil" "xorm.io/xorm" + "xorm.io/xorm/schemas" ) -func AddFileStatusToAttachment(x *xorm.Engine) error { - type Attachment struct { - ID int64 `xorm:"pk autoincr"` - UUID string `xorm:"uuid UNIQUE"` - RepoID int64 `xorm:"INDEX"` // this should not be zero - IssueID int64 `xorm:"INDEX"` // maybe zero when creating - ReleaseID int64 `xorm:"INDEX"` // maybe zero when creating - UploaderID int64 `xorm:"INDEX DEFAULT 0"` // Notice: will be zero before this column added - CommentID int64 `xorm:"INDEX"` - Name string - DownloadCount int64 `xorm:"DEFAULT 0"` - Status db.FileStatus `xorm:"INDEX DEFAULT 0"` - DeleteFailedCount int `xorm:"DEFAULT 0"` // Number of times the deletion failed, used to prevent infinite loop - LastDeleteFailedTime timeutil.TimeStamp // Last time the deletion failed, used to prevent infinite loop - Size int64 `xorm:"DEFAULT 0"` - CreatedUnix timeutil.TimeStamp `xorm:"created"` - CustomDownloadURL string `xorm:"-"` - } +type Attachment321 struct { + ID int64 `xorm:"pk autoincr"` + UUID string `xorm:"uuid UNIQUE"` + RepoID int64 `xorm:"INDEX"` // this should not be zero + IssueID int64 `xorm:"INDEX"` // maybe zero when creating + ReleaseID int64 `xorm:"INDEX"` // maybe zero when creating + UploaderID int64 `xorm:"INDEX DEFAULT 0"` // Notice: will be zero before this column added + CommentID int64 `xorm:"INDEX"` + Name string + DownloadCount int64 `xorm:"DEFAULT 0"` + Status db.FileStatus `xorm:"INDEX DEFAULT 1 NOT NULL"` // 1 = normal, 2 = to be deleted + DeleteFailedCount int `xorm:"DEFAULT 0"` // Number of times the deletion failed, used to prevent infinite loop + LastDeleteFailedTime timeutil.TimeStamp // Last time the deletion failed, used to prevent infinite loop + Size int64 `xorm:"DEFAULT 0"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` +} - if err := x.Sync(new(Attachment)); err != nil { - return err - } +func (a *Attachment321) TableName() string { + return "attachment" +} + +// TableIndices implements xorm's TableIndices interface +func (a *Attachment321) TableIndices() []*schemas.Index { + uuidIndex := schemas.NewIndex("attachment_uuid", schemas.UniqueType) + uuidIndex.AddColumn("uuid") + + repoIndex := schemas.NewIndex("attachment_repo_id", schemas.IndexType) + repoIndex.AddColumn("repo_id") + + issueIndex := schemas.NewIndex("attachment_issue_id", schemas.IndexType) + issueIndex.AddColumn("issue_id") - if _, err := x.Exec("UPDATE `attachment` SET status = ? WHERE status IS NULL", db.FileStatusNormal); err != nil { - return err + releaseIndex := schemas.NewIndex("attachment_release_id", schemas.IndexType) + releaseIndex.AddColumn("release_id") + + uploaderIndex := schemas.NewIndex("attachment_uploader_id", schemas.IndexType) + uploaderIndex.AddColumn("uploader_id") + + commentIndex := schemas.NewIndex("attachment_comment_id", schemas.IndexType) + commentIndex.AddColumn("comment_id") + + statusIndex := schemas.NewIndex("attachment_status", schemas.IndexType) + statusIndex.AddColumn("status") + + statusIDIndex := schemas.NewIndex("attachment_status_id", schemas.IndexType) + statusIDIndex.AddColumn("status_id", "id") // For status = ? AND id > ? query + + return []*schemas.Index{ + uuidIndex, + repoIndex, + issueIndex, + releaseIndex, + uploaderIndex, + commentIndex, + statusIndex, + statusIDIndex, } +} - return nil +func AddFileStatusToAttachment(x *xorm.Engine) error { + return x.Sync(new(Attachment321)) } diff --git a/models/repo/attachment.go b/models/repo/attachment.go index af891384e0933..aab656ea74a05 100644 --- a/models/repo/attachment.go +++ b/models/repo/attachment.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" + "xorm.io/xorm/schemas" ) // Attachment represent a attachment of issue/comment/release. @@ -27,15 +28,53 @@ type Attachment struct { CommentID int64 `xorm:"INDEX"` Name string DownloadCount int64 `xorm:"DEFAULT 0"` - Status db.FileStatus `xorm:"INDEX DEFAULT 0"` - DeleteFailedCount int `xorm:"DEFAULT 0"` // Number of times the deletion failed, used to prevent infinite loop - LastDeleteFailedReason string `xorm:"TEXT"` // Last reason the deletion failed, used to prevent infinite loop + Status db.FileStatus `xorm:"INDEX DEFAULT 1 NOT NULL"` // 1 = normal, 2 = to be deleted + DeleteFailedCount int `xorm:"DEFAULT 0"` // Number of times the deletion failed, used to prevent infinite loop + LastDeleteFailedReason string `xorm:"TEXT"` // Last reason the deletion failed, used to prevent infinite loop LastDeleteFailedTime timeutil.TimeStamp // Last time the deletion failed, used to prevent infinite loop Size int64 `xorm:"DEFAULT 0"` CreatedUnix timeutil.TimeStamp `xorm:"created"` CustomDownloadURL string `xorm:"-"` } +// TableIndices implements xorm's TableIndices interface +func (a *Attachment) TableIndices() []*schemas.Index { + uuidIndex := schemas.NewIndex("attachment_uuid", schemas.UniqueType) + uuidIndex.AddColumn("uuid") + + repoIndex := schemas.NewIndex("attachment_repo_id", schemas.IndexType) + repoIndex.AddColumn("repo_id") + + issueIndex := schemas.NewIndex("attachment_issue_id", schemas.IndexType) + issueIndex.AddColumn("issue_id") + + releaseIndex := schemas.NewIndex("attachment_release_id", schemas.IndexType) + releaseIndex.AddColumn("release_id") + + uploaderIndex := schemas.NewIndex("attachment_uploader_id", schemas.IndexType) + uploaderIndex.AddColumn("uploader_id") + + commentIndex := schemas.NewIndex("attachment_comment_id", schemas.IndexType) + commentIndex.AddColumn("comment_id") + + statusIndex := schemas.NewIndex("attachment_status", schemas.IndexType) + statusIndex.AddColumn("status") + + statusIDIndex := schemas.NewIndex("attachment_status_id", schemas.IndexType) + statusIDIndex.AddColumn("status_id", "id") // For status = ? AND id > ? query + + return []*schemas.Index{ + uuidIndex, + repoIndex, + issueIndex, + releaseIndex, + uploaderIndex, + commentIndex, + statusIndex, + statusIDIndex, + } +} + func init() { db.RegisterModel(new(Attachment)) } diff --git a/services/attachment/attachment.go b/services/attachment/attachment.go index 8a214ea2f53e5..840ad2fd91178 100644 --- a/services/attachment/attachment.go +++ b/services/attachment/attachment.go @@ -37,6 +37,7 @@ func NewAttachment(ctx context.Context, attach *repo_model.Attachment, file io.R return fmt.Errorf("Create: %w", err) } attach.Size = size + attach.Status = db.FileStatusNormal return db.Insert(ctx, attach) }) @@ -147,14 +148,16 @@ func cleanAttachments(ctx context.Context, attachmentIDs []int64) []int64 { return failed } -// ScanTobeDeletedAttachments scans for attachments that are marked as to be deleted and send to +// ScanToBeDeletedAttachments scans for attachments that are marked as to be deleted and send to // clean queue -func ScanTobeDeletedAttachments(ctx context.Context) error { +func ScanToBeDeletedAttachments(ctx context.Context) error { attachments := make([]*repo_model.Attachment, 0, 10) lastID := int64(0) for { if err := db.GetEngine(ctx). - Where("id > ? AND status = ?", lastID, db.FileStatusToBeDeleted). + // use the status and id index to speed up the query + Where("status = ? AND id > ?", db.FileStatusToBeDeleted, lastID). + Asc("id"). Limit(100). Find(&attachments); err != nil { return fmt.Errorf("scan to-be-deleted attachments: %w", err) diff --git a/services/cron/tasks_extended.go b/services/cron/tasks_extended.go index 7c41ff399f471..f9383a3093fb3 100644 --- a/services/cron/tasks_extended.go +++ b/services/cron/tasks_extended.go @@ -230,7 +230,7 @@ func registerCleanAttachments() { RunAtStart: false, Schedule: "@every 24h", }, func(ctx context.Context, _ *user_model.User, _ Config) error { - return attachment_service.ScanTobeDeletedAttachments(ctx) + return attachment_service.ScanToBeDeletedAttachments(ctx) }) } diff --git a/services/issue/comments.go b/services/issue/comments.go index 3b6a7af2cd672..13e5811975dfc 100644 --- a/services/issue/comments.go +++ b/services/issue/comments.go @@ -141,6 +141,7 @@ func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAtt } } + // deletedReviewComment should be a review comment with no content and no attachments deletedReviewComment, err := issues_model.DeleteComment(ctx, comment) if err != nil { return nil, err diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 75eb06d01fa3f..076e494b4c37f 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -323,6 +323,7 @@ func (g *GiteaLocalUploader) CreateReleases(ctx context.Context, releases ...*ba DownloadCount: int64(*asset.DownloadCount), Size: int64(*asset.Size), CreatedUnix: timeutil.TimeStamp(asset.Created.Unix()), + Status: db.FileStatusNormal, } // SECURITY: We cannot check the DownloadURL and DownloadFunc are safe here From 0c9f37dbde6d594ba4addfeb1bc804989215f8c7 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 20 Jul 2025 12:39:24 -0700 Subject: [PATCH 24/36] revert unnecessary change in this pull request --- models/fixtures/comment.yml | 9 ----- models/issues/comment.go | 54 ++++------------------------ routers/api/v1/repo/issue_comment.go | 2 +- routers/web/repo/issue_comment.go | 10 ++---- services/issue/comments.go | 24 ++++++------- services/issue/comments_test.go | 40 --------------------- services/issue/issue.go | 3 +- services/user/delete.go | 2 +- 8 files changed, 23 insertions(+), 121 deletions(-) delete mode 100644 services/issue/comments_test.go diff --git a/models/fixtures/comment.yml b/models/fixtures/comment.yml index 7d472cdea409a..8fde386e226d4 100644 --- a/models/fixtures/comment.yml +++ b/models/fixtures/comment.yml @@ -102,12 +102,3 @@ review_id: 22 assignee_id: 5 created_unix: 946684817 - -- - id: 12 - type: 22 # review - poster_id: 100 - issue_id: 3 - content: "" - review_id: 10 - created_unix: 946684812 diff --git a/models/issues/comment.go b/models/issues/comment.go index a69d805399852..a1e3a7dff2a88 100644 --- a/models/issues/comment.go +++ b/models/issues/comment.go @@ -1113,20 +1113,20 @@ func UpdateComment(ctx context.Context, c *Comment, contentVersion int, doer *us } // DeleteComment deletes the comment -func DeleteComment(ctx context.Context, comment *Comment) (*Comment, error) { +func DeleteComment(ctx context.Context, comment *Comment) error { if _, err := db.GetEngine(ctx).ID(comment.ID).NoAutoCondition().Delete(comment); err != nil { - return nil, err + return err } if _, err := db.DeleteByBean(ctx, &ContentHistory{ CommentID: comment.ID, }); err != nil { - return nil, err + return err } if comment.Type.CountedAsConversation() { if err := UpdateIssueNumComments(ctx, comment.IssueID); err != nil { - return nil, err + return err } } if _, err := db.GetEngine(ctx).Table("action"). @@ -1134,54 +1134,14 @@ func DeleteComment(ctx context.Context, comment *Comment) (*Comment, error) { Update(map[string]any{ "is_deleted": true, }); err != nil { - return nil, err - } - - var deletedReviewComment *Comment - - // delete review & review comment if the code comment is the last comment of the review - if comment.Type == CommentTypeCode && comment.ReviewID > 0 { - if err := comment.LoadReview(ctx); err != nil { - return nil, err - } - if comment.Review != nil && comment.Review.Type == ReviewTypeComment { - res, err := db.GetEngine(ctx).ID(comment.ReviewID). - Where("NOT EXISTS (SELECT 1 FROM comment WHERE review_id = ? AND `type` = ?)", comment.ReviewID, CommentTypeCode). - Delete(new(Review)) - if err != nil { - return nil, err - } - if res > 0 { - var reviewComment Comment - has, err := db.GetEngine(ctx).Where("review_id = ?", comment.ReviewID). - And("`type` = ?", CommentTypeReview).Get(&reviewComment) - if err != nil { - return nil, err - } - if has && reviewComment.Content == "" { - if err := reviewComment.LoadAttachments(ctx); err != nil { - return nil, err - } - if len(reviewComment.Attachments) == 0 { - if _, err := db.GetEngine(ctx).ID(reviewComment.ID).Delete(new(Comment)); err != nil { - return nil, err - } - deletedReviewComment = &reviewComment - } - } - comment.ReviewID = 0 // reset review ID to 0 for the notification - } - } + return err } if err := comment.neuterCrossReferences(ctx); err != nil { - return nil, err + return err } - if err := DeleteReaction(ctx, &ReactionOptions{CommentID: comment.ID}); err != nil { - return nil, err - } - return deletedReviewComment, nil + return DeleteReaction(ctx, &ReactionOptions{CommentID: comment.ID}) } // UpdateCommentsMigrationsByType updates comments' migrations information via given git service type and original id and poster id diff --git a/routers/api/v1/repo/issue_comment.go b/routers/api/v1/repo/issue_comment.go index feb9f1da64cfe..66a5f0412926b 100644 --- a/routers/api/v1/repo/issue_comment.go +++ b/routers/api/v1/repo/issue_comment.go @@ -726,7 +726,7 @@ func deleteIssueComment(ctx *context.APIContext) { return } - if _, err = issue_service.DeleteComment(ctx, ctx.Doer, comment); err != nil { + if err = issue_service.DeleteComment(ctx, ctx.Doer, comment); err != nil { ctx.APIErrorInternal(err) return } diff --git a/routers/web/repo/issue_comment.go b/routers/web/repo/issue_comment.go index 1ad6c588a7798..ecaa9f05e471e 100644 --- a/routers/web/repo/issue_comment.go +++ b/routers/web/repo/issue_comment.go @@ -325,18 +325,12 @@ func DeleteComment(ctx *context.Context) { return } - deletedReviewComment, err := issue_service.DeleteComment(ctx, ctx.Doer, comment) - if err != nil { + if err := issue_service.DeleteComment(ctx, ctx.Doer, comment); err != nil { ctx.ServerError("DeleteComment", err) return } - res := map[string]any{} - if deletedReviewComment != nil { - res["deletedReviewCommentHashTag"] = deletedReviewComment.HashTag() - } - - ctx.JSON(http.StatusOK, res) + ctx.Status(http.StatusOK) } // ChangeCommentReaction create a reaction for comment diff --git a/services/issue/comments.go b/services/issue/comments.go index 13e5811975dfc..153d2ebbd60ac 100644 --- a/services/issue/comments.go +++ b/services/issue/comments.go @@ -132,42 +132,40 @@ func UpdateComment(ctx context.Context, c *issues_model.Comment, contentVersion } // deleteComment deletes the comment -func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAttachments bool) (*issues_model.Comment, error) { - return db.WithTx2(ctx, func(ctx context.Context) (*issues_model.Comment, error) { +func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAttachments bool) error { + return db.WithTx(ctx, func(ctx context.Context) error { if removeAttachments { // load attachments before deleting the comment if err := comment.LoadAttachments(ctx); err != nil { - return nil, err + return err } } // deletedReviewComment should be a review comment with no content and no attachments - deletedReviewComment, err := issues_model.DeleteComment(ctx, comment) - if err != nil { - return nil, err + if err := issues_model.DeleteComment(ctx, comment); err != nil { + return err } if removeAttachments { // mark comment attachments as deleted if _, err := repo_model.MarkAttachmentsDeleted(ctx, comment.Attachments); err != nil { - return nil, err + return err } } - return deletedReviewComment, nil + return nil }) } -func DeleteComment(ctx context.Context, doer *user_model.User, comment *issues_model.Comment) (*issues_model.Comment, error) { - deletedReviewComment, err := deleteComment(ctx, comment, true) - if err != nil { - return nil, err +func DeleteComment(ctx context.Context, doer *user_model.User, comment *issues_model.Comment) error { + if err := deleteComment(ctx, comment, true); err != nil { + return err } attachment.AddAttachmentsToCleanQueue(ctx, comment.Attachments) notify_service.DeleteComment(ctx, doer, comment) - return deletedReviewComment, nil + return nil } // LoadCommentPushCommits Load push commits diff --git a/services/issue/comments_test.go b/services/issue/comments_test.go deleted file mode 100644 index 4088bd1020d25..0000000000000 --- a/services/issue/comments_test.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2025 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package issue - -import ( - "testing" - - "code.gitea.io/gitea/models/db" - issues_model "code.gitea.io/gitea/models/issues" - repo_model "code.gitea.io/gitea/models/repo" - "code.gitea.io/gitea/models/unittest" - user_model "code.gitea.io/gitea/models/user" - - "github.com/stretchr/testify/assert" -) - -func Test_DeleteCommentWithReview(t *testing.T) { - assert.NoError(t, unittest.PrepareTestDatabase()) - - comment := unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{ID: 7}) - assert.NoError(t, comment.LoadAttachments(t.Context())) - assert.Len(t, comment.Attachments, 1) - assert.Equal(t, int64(13), comment.Attachments[0].ID) - assert.Equal(t, int64(10), comment.ReviewID) - review := unittest.AssertExistsAndLoadBean(t, &issues_model.Review{ID: comment.ReviewID}) - user1 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) - - // since this is the last comment of the review, it should be deleted when the comment is deleted - deletedReviewComment, err := DeleteComment(db.DefaultContext, user1, comment) - assert.NoError(t, err) - assert.NotNil(t, deletedReviewComment) - - // the review should be deleted as well - unittest.AssertNotExistsBean(t, &issues_model.Review{ID: review.ID}) - // the attachment should be deleted as well - newAttachment, err := repo_model.GetAttachmentByID(t.Context(), comment.Attachments[0].ID) - assert.Error(t, err) - assert.Nil(t, newAttachment) -} diff --git a/services/issue/issue.go b/services/issue/issue.go index fe369fe1e68ed..967e0376749d7 100644 --- a/services/issue/issue.go +++ b/services/issue/issue.go @@ -315,8 +315,7 @@ func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachmen } for _, comment := range issue.Comments { - _, err := deleteComment(ctx, comment, deleteAttachments) - if err != nil { + if err := deleteComment(ctx, comment, deleteAttachments); err != nil { return nil, fmt.Errorf("deleteComment [comment_id: %d]: %w", comment.ID, err) } toBeCleanedAttachments = append(toBeCleanedAttachments, comment.Attachments...) diff --git a/services/user/delete.go b/services/user/delete.go index af287a074a354..fb8ea392f94f7 100644 --- a/services/user/delete.go +++ b/services/user/delete.go @@ -122,7 +122,7 @@ func deleteUser(ctx context.Context, u *user_model.User, purge bool) (toBeCleane return nil, err } - if _, err = issues_model.DeleteComment(ctx, comment); err != nil { + if err = issues_model.DeleteComment(ctx, comment); err != nil { return nil, err } From 70d2960677ebc2ccca2bfb75fc24b8e5bb2b05a1 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 20 Jul 2025 12:43:44 -0700 Subject: [PATCH 25/36] revert unnecessary change in this pull request --- routers/api/v1/repo/issue_comment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/api/v1/repo/issue_comment.go b/routers/api/v1/repo/issue_comment.go index 66a5f0412926b..cc342a9313c71 100644 --- a/routers/api/v1/repo/issue_comment.go +++ b/routers/api/v1/repo/issue_comment.go @@ -721,7 +721,7 @@ func deleteIssueComment(ctx *context.APIContext) { if !ctx.IsSigned || (ctx.Doer.ID != comment.PosterID && !ctx.Repo.CanWriteIssuesOrPulls(comment.Issue.IsPull)) { ctx.Status(http.StatusForbidden) return - } else if comment.Type != issues_model.CommentTypeComment && comment.Type != issues_model.CommentTypeCode { + } else if comment.Type != issues_model.CommentTypeComment { ctx.Status(http.StatusNoContent) return } From f24dc5e65881a57594f0b266ce03334a45b2c850 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 20 Jul 2025 12:44:51 -0700 Subject: [PATCH 26/36] revert unnecessary change in this pull request --- routers/web/repo/issue_comment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/web/repo/issue_comment.go b/routers/web/repo/issue_comment.go index ecaa9f05e471e..cb5b2d801952d 100644 --- a/routers/web/repo/issue_comment.go +++ b/routers/web/repo/issue_comment.go @@ -325,7 +325,7 @@ func DeleteComment(ctx *context.Context) { return } - if err := issue_service.DeleteComment(ctx, ctx.Doer, comment); err != nil { + if err = issue_service.DeleteComment(ctx, ctx.Doer, comment); err != nil { ctx.ServerError("DeleteComment", err) return } From 9faf99d8ed6d2c739ac29b1181e29919f577dadc Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 20 Jul 2025 13:14:29 -0700 Subject: [PATCH 27/36] Fix bug --- models/issues/pull_list_test.go | 5 +++-- models/migrations/v1_25/main_test.go | 2 +- models/migrations/v1_25/v321_test.go | 23 +++++++++++------------ services/attachment/attachment.go | 17 ++++++++++++----- services/issue/issue.go | 4 +++- 5 files changed, 30 insertions(+), 21 deletions(-) diff --git a/models/issues/pull_list_test.go b/models/issues/pull_list_test.go index feb59df216045..eb2de006d60a4 100644 --- a/models/issues/pull_list_test.go +++ b/models/issues/pull_list_test.go @@ -39,8 +39,9 @@ func TestPullRequestList_LoadReviewCommentsCounts(t *testing.T) { reviewComments, err := prs.LoadReviewCommentsCounts(db.DefaultContext) assert.NoError(t, err) assert.Len(t, reviewComments, 2) - assert.Equal(t, 1, reviewComments[prs[0].IssueID]) - assert.Equal(t, 2, reviewComments[prs[1].IssueID]) + for _, pr := range prs { + assert.Equal(t, 1, reviewComments[pr.IssueID]) + } } func TestPullRequestList_LoadReviews(t *testing.T) { diff --git a/models/migrations/v1_25/main_test.go b/models/migrations/v1_25/main_test.go index 8ac213c908f78..d2c4a4105d3a8 100644 --- a/models/migrations/v1_25/main_test.go +++ b/models/migrations/v1_25/main_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. +// Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package v1_25 diff --git a/models/migrations/v1_25/v321_test.go b/models/migrations/v1_25/v321_test.go index 0febbe861fab8..47bb9ea242975 100644 --- a/models/migrations/v1_25/v321_test.go +++ b/models/migrations/v1_25/v321_test.go @@ -14,18 +14,17 @@ import ( func Test_AddFileStatusToAttachment(t *testing.T) { type Attachment struct { - ID int64 `xorm:"pk autoincr"` - UUID string `xorm:"uuid UNIQUE"` - RepoID int64 `xorm:"INDEX"` // this should not be zero - IssueID int64 `xorm:"INDEX"` // maybe zero when creating - ReleaseID int64 `xorm:"INDEX"` // maybe zero when creating - UploaderID int64 `xorm:"INDEX DEFAULT 0"` // Notice: will be zero before this column added - CommentID int64 `xorm:"INDEX"` - Name string - DownloadCount int64 `xorm:"DEFAULT 0"` - Size int64 `xorm:"DEFAULT 0"` - CreatedUnix timeutil.TimeStamp `xorm:"created"` - CustomDownloadURL string `xorm:"-"` + ID int64 `xorm:"pk autoincr"` + UUID string `xorm:"uuid UNIQUE"` + RepoID int64 `xorm:"INDEX"` // this should not be zero + IssueID int64 `xorm:"INDEX"` // maybe zero when creating + ReleaseID int64 `xorm:"INDEX"` // maybe zero when creating + UploaderID int64 `xorm:"INDEX DEFAULT 0"` // Notice: will be zero before this column added + CommentID int64 `xorm:"INDEX"` + Name string + DownloadCount int64 `xorm:"DEFAULT 0"` + Size int64 `xorm:"DEFAULT 0"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` } // Prepare and load the testing database diff --git a/services/attachment/attachment.go b/services/attachment/attachment.go index 840ad2fd91178..6f57c55c4e6d6 100644 --- a/services/attachment/attachment.go +++ b/services/attachment/attachment.go @@ -151,24 +151,31 @@ func cleanAttachments(ctx context.Context, attachmentIDs []int64) []int64 { // ScanToBeDeletedAttachments scans for attachments that are marked as to be deleted and send to // clean queue func ScanToBeDeletedAttachments(ctx context.Context) error { - attachments := make([]*repo_model.Attachment, 0, 10) + attachmentIDs := make([]int64, 0, 100) lastID := int64(0) for { if err := db.GetEngine(ctx). + Select("id"). // use the status and id index to speed up the query Where("status = ? AND id > ?", db.FileStatusToBeDeleted, lastID). Asc("id"). Limit(100). - Find(&attachments); err != nil { + Find(&attachmentIDs); err != nil { return fmt.Errorf("scan to-be-deleted attachments: %w", err) } - if len(attachments) == 0 { + if len(attachmentIDs) == 0 { log.Trace("No more attachments to be deleted") break } - AddAttachmentsToCleanQueue(ctx, attachments) - lastID = attachments[len(attachments)-1].ID + for _, id := range attachmentIDs { + if err := cleanQueue.Push(id); err != nil { + log.Error("Failed to push attachment ID %d to clean queue: %v", id, err) + } + } + + lastID = attachmentIDs[len(attachmentIDs)-1] + attachmentIDs = attachmentIDs[0:0] } return nil diff --git a/services/issue/issue.go b/services/issue/issue.go index 967e0376749d7..9b3b0c66b2145 100644 --- a/services/issue/issue.go +++ b/services/issue/issue.go @@ -318,7 +318,9 @@ func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachmen if err := deleteComment(ctx, comment, deleteAttachments); err != nil { return nil, fmt.Errorf("deleteComment [comment_id: %d]: %w", comment.ID, err) } - toBeCleanedAttachments = append(toBeCleanedAttachments, comment.Attachments...) + if deleteAttachments { + toBeCleanedAttachments = append(toBeCleanedAttachments, comment.Attachments...) + } } if deleteAttachments { From df456dd7f8021f77ecef106a02a2fe733db18752 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 20 Jul 2025 13:38:24 -0700 Subject: [PATCH 28/36] fix lint --- models/repo/attachment.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/repo/attachment.go b/models/repo/attachment.go index aab656ea74a05..1a03dbe8bf7d1 100644 --- a/models/repo/attachment.go +++ b/models/repo/attachment.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" + "xorm.io/xorm/schemas" ) From fa5fc02cb8537b35a5255bebba99809539ce0dcf Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 20 Jul 2025 14:50:04 -0700 Subject: [PATCH 29/36] Revert unnecessary changes --- web_src/js/features/repo-issue.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/web_src/js/features/repo-issue.ts b/web_src/js/features/repo-issue.ts index 693ec4f68765e..49e8fc40a23a2 100644 --- a/web_src/js/features/repo-issue.ts +++ b/web_src/js/features/repo-issue.ts @@ -150,12 +150,6 @@ export function initRepoIssueCommentDelete() { counter.textContent = String(num); } - const json: Record = await response.json(); - if (json.errorMessage) throw new Error(json.errorMessage); - - if (json.deletedReviewCommentHashTag) { - document.querySelector(`#${json.deletedReviewCommentHashTag}`)?.remove(); - } document.querySelector(`#${deleteButton.getAttribute('data-comment-id')}`)?.remove(); if (conversationHolder && !conversationHolder.querySelector('.comment')) { From 7134ad90519fef97c8a4d69a6e0562ac7c1eecdf Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 20 Jul 2025 16:47:10 -0700 Subject: [PATCH 30/36] Fix test --- models/fixtures/attachment.yml | 13 +++++++++++++ models/migrations/v1_25/v321.go | 18 +++++++++--------- models/repo/attachment.go | 20 ++++++++++---------- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/models/fixtures/attachment.yml b/models/fixtures/attachment.yml index b86a15b28269f..06f578f7b7b32 100644 --- a/models/fixtures/attachment.yml +++ b/models/fixtures/attachment.yml @@ -8,6 +8,7 @@ comment_id: 0 name: attach1 download_count: 0 + status: 1 size: 0 created_unix: 946684800 @@ -21,6 +22,7 @@ comment_id: 0 name: attach2 download_count: 1 + status: 1 size: 0 created_unix: 946684800 @@ -34,6 +36,7 @@ comment_id: 1 name: attach1 download_count: 0 + status: 1 size: 0 created_unix: 946684800 @@ -47,6 +50,7 @@ comment_id: 1 name: attach2 download_count: 1 + status: 1 size: 0 created_unix: 946684800 @@ -60,6 +64,7 @@ comment_id: 0 name: attach1 download_count: 0 + status: 1 size: 0 created_unix: 946684800 @@ -73,6 +78,7 @@ comment_id: 2 name: attach1 download_count: 0 + status: 1 size: 0 created_unix: 946684800 @@ -86,6 +92,7 @@ comment_id: 2 name: attach1 download_count: 0 + status: 1 size: 0 created_unix: 946684800 @@ -99,6 +106,7 @@ comment_id: 0 name: attach1 download_count: 0 + status: 1 size: 0 created_unix: 946684800 @@ -112,6 +120,7 @@ comment_id: 0 name: attach1 download_count: 0 + status: 1 size: 0 created_unix: 946684800 @@ -125,6 +134,7 @@ comment_id: 0 name: attach1 download_count: 0 + status: 1 size: 0 created_unix: 946684800 @@ -138,6 +148,7 @@ comment_id: 0 name: attach1 download_count: 0 + status: 1 size: 0 created_unix: 946684800 @@ -151,6 +162,7 @@ comment_id: 0 name: README.md download_count: 0 + status: 1 size: 0 created_unix: 946684800 @@ -164,5 +176,6 @@ comment_id: 7 name: code_comment_uploaded_attachment.png download_count: 0 + status: 1 size: 0 created_unix: 946684812 diff --git a/models/migrations/v1_25/v321.go b/models/migrations/v1_25/v321.go index 0084b99eacb2b..0eb556259d167 100644 --- a/models/migrations/v1_25/v321.go +++ b/models/migrations/v1_25/v321.go @@ -13,16 +13,16 @@ import ( type Attachment321 struct { ID int64 `xorm:"pk autoincr"` - UUID string `xorm:"uuid UNIQUE"` - RepoID int64 `xorm:"INDEX"` // this should not be zero - IssueID int64 `xorm:"INDEX"` // maybe zero when creating - ReleaseID int64 `xorm:"INDEX"` // maybe zero when creating - UploaderID int64 `xorm:"INDEX DEFAULT 0"` // Notice: will be zero before this column added - CommentID int64 `xorm:"INDEX"` + UUID string `xorm:"uuid"` + RepoID int64 // this should not be zero + IssueID int64 // maybe zero when creating + ReleaseID int64 // maybe zero when creating + UploaderID int64 `xorm:"DEFAULT 0"` // Notice: will be zero before this column added + CommentID int64 Name string DownloadCount int64 `xorm:"DEFAULT 0"` - Status db.FileStatus `xorm:"INDEX DEFAULT 1 NOT NULL"` // 1 = normal, 2 = to be deleted - DeleteFailedCount int `xorm:"DEFAULT 0"` // Number of times the deletion failed, used to prevent infinite loop + Status db.FileStatus `xorm:"DEFAULT 1 NOT NULL"` // 1 = normal, 2 = to be deleted + DeleteFailedCount int `xorm:"DEFAULT 0"` // Number of times the deletion failed, used to prevent infinite loop LastDeleteFailedTime timeutil.TimeStamp // Last time the deletion failed, used to prevent infinite loop Size int64 `xorm:"DEFAULT 0"` CreatedUnix timeutil.TimeStamp `xorm:"created"` @@ -56,7 +56,7 @@ func (a *Attachment321) TableIndices() []*schemas.Index { statusIndex.AddColumn("status") statusIDIndex := schemas.NewIndex("attachment_status_id", schemas.IndexType) - statusIDIndex.AddColumn("status_id", "id") // For status = ? AND id > ? query + statusIDIndex.AddColumn("status", "id") // For status = ? AND id > ? query return []*schemas.Index{ uuidIndex, diff --git a/models/repo/attachment.go b/models/repo/attachment.go index 1a03dbe8bf7d1..98f97d01e0b9f 100644 --- a/models/repo/attachment.go +++ b/models/repo/attachment.go @@ -21,17 +21,17 @@ import ( // Attachment represent a attachment of issue/comment/release. type Attachment struct { ID int64 `xorm:"pk autoincr"` - UUID string `xorm:"uuid UNIQUE"` - RepoID int64 `xorm:"INDEX"` // this should not be zero - IssueID int64 `xorm:"INDEX"` // maybe zero when creating - ReleaseID int64 `xorm:"INDEX"` // maybe zero when creating - UploaderID int64 `xorm:"INDEX DEFAULT 0"` // Notice: will be zero before this column added - CommentID int64 `xorm:"INDEX"` + UUID string `xorm:"uuid"` + RepoID int64 // this should not be zero + IssueID int64 // maybe zero when creating + ReleaseID int64 // maybe zero when creating + UploaderID int64 `xorm:"DEFAULT 0"` // Notice: will be zero before this column added + CommentID int64 Name string DownloadCount int64 `xorm:"DEFAULT 0"` - Status db.FileStatus `xorm:"INDEX DEFAULT 1 NOT NULL"` // 1 = normal, 2 = to be deleted - DeleteFailedCount int `xorm:"DEFAULT 0"` // Number of times the deletion failed, used to prevent infinite loop - LastDeleteFailedReason string `xorm:"TEXT"` // Last reason the deletion failed, used to prevent infinite loop + Status db.FileStatus `xorm:"DEFAULT 1 NOT NULL"` // 1 = normal, 2 = to be deleted + DeleteFailedCount int `xorm:"DEFAULT 0"` // Number of times the deletion failed, used to prevent infinite loop + LastDeleteFailedReason string `xorm:"TEXT"` // Last reason the deletion failed, used to prevent infinite loop LastDeleteFailedTime timeutil.TimeStamp // Last time the deletion failed, used to prevent infinite loop Size int64 `xorm:"DEFAULT 0"` CreatedUnix timeutil.TimeStamp `xorm:"created"` @@ -62,7 +62,7 @@ func (a *Attachment) TableIndices() []*schemas.Index { statusIndex.AddColumn("status") statusIDIndex := schemas.NewIndex("attachment_status_id", schemas.IndexType) - statusIDIndex.AddColumn("status_id", "id") // For status = ? AND id > ? query + statusIDIndex.AddColumn("status", "id") // For status = ? AND id > ? query return []*schemas.Index{ uuidIndex, From e50adecc7b2bbdbfb4cd522c4a09d63ce292b72a Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 20 Jul 2025 17:32:41 -0700 Subject: [PATCH 31/36] Fix --- models/issues/comment.go | 4 +++- models/issues/issue_update.go | 4 +++- models/migrations/migrations.go | 4 ++++ models/migrations/v1_25/v321.go | 18 +++++++++--------- models/repo/attachment.go | 18 +++++++++--------- services/attachment/attachment.go | 1 + 6 files changed, 29 insertions(+), 20 deletions(-) diff --git a/models/issues/comment.go b/models/issues/comment.go index a1e3a7dff2a88..d230b2594578c 100644 --- a/models/issues/comment.go +++ b/models/issues/comment.go @@ -599,7 +599,9 @@ func UpdateCommentAttachments(ctx context.Context, c *Comment, uuids []string) e return fmt.Errorf("getAttachmentsByUUIDs [uuids: %v]: %w", uuids, err) } for i := range attachments { - if attachments[i].IssueID != 0 || attachments[i].CommentID != 0 { + if attachments[i].CommentID == c.ID && attachments[i].IssueID == c.IssueID { + continue + } else if attachments[i].IssueID != 0 || attachments[i].CommentID != 0 { return util.NewPermissionDeniedErrorf("update comment attachments permission denied") } attachments[i].IssueID = c.IssueID diff --git a/models/issues/issue_update.go b/models/issues/issue_update.go index bb49470dd56f4..4fdd36fae1fe0 100644 --- a/models/issues/issue_update.go +++ b/models/issues/issue_update.go @@ -305,7 +305,9 @@ func UpdateIssueAttachments(ctx context.Context, issueID int64, uuids []string) return fmt.Errorf("getAttachmentsByUUIDs [uuids: %v]: %w", uuids, err) } for i := range attachments { - if attachments[i].IssueID != 0 { + if attachments[i].IssueID == issueID { + continue + } else if attachments[i].IssueID != 0 { return util.NewPermissionDeniedErrorf("update issue attachments permission denied") } attachments[i].IssueID = issueID diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 176372486e8f6..1b6ec04d36e8b 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -24,6 +24,7 @@ import ( "code.gitea.io/gitea/models/migrations/v1_22" "code.gitea.io/gitea/models/migrations/v1_23" "code.gitea.io/gitea/models/migrations/v1_24" + "code.gitea.io/gitea/models/migrations/v1_25" "code.gitea.io/gitea/models/migrations/v1_6" "code.gitea.io/gitea/models/migrations/v1_7" "code.gitea.io/gitea/models/migrations/v1_8" @@ -382,6 +383,9 @@ func prepareMigrationTasks() []*migration { newMigration(318, "Add anonymous_access_mode for repo_unit", v1_24.AddRepoUnitAnonymousAccessMode), newMigration(319, "Add ExclusiveOrder to Label table", v1_24.AddExclusiveOrderColumnToLabelTable), newMigration(320, "Migrate two_factor_policy to login_source table", v1_24.MigrateSkipTwoFactor), + + // Gitea 1.24.0-rc0 ends at migration ID number 320 (database version 321) + newMigration(321, "Add file status columns to attachment table", v1_25.AddFileStatusToAttachment), } return preparedMigrations } diff --git a/models/migrations/v1_25/v321.go b/models/migrations/v1_25/v321.go index 0eb556259d167..47096f5ad9e2c 100644 --- a/models/migrations/v1_25/v321.go +++ b/models/migrations/v1_25/v321.go @@ -22,7 +22,7 @@ type Attachment321 struct { Name string DownloadCount int64 `xorm:"DEFAULT 0"` Status db.FileStatus `xorm:"DEFAULT 1 NOT NULL"` // 1 = normal, 2 = to be deleted - DeleteFailedCount int `xorm:"DEFAULT 0"` // Number of times the deletion failed, used to prevent infinite loop + DeleteFailedCount int `xorm:"DEFAULT 0 NOT NULL"` // Number of times the deletion failed, used to prevent infinite loop LastDeleteFailedTime timeutil.TimeStamp // Last time the deletion failed, used to prevent infinite loop Size int64 `xorm:"DEFAULT 0"` CreatedUnix timeutil.TimeStamp `xorm:"created"` @@ -34,28 +34,28 @@ func (a *Attachment321) TableName() string { // TableIndices implements xorm's TableIndices interface func (a *Attachment321) TableIndices() []*schemas.Index { - uuidIndex := schemas.NewIndex("attachment_uuid", schemas.UniqueType) + uuidIndex := schemas.NewIndex("uuid", schemas.UniqueType) uuidIndex.AddColumn("uuid") - repoIndex := schemas.NewIndex("attachment_repo_id", schemas.IndexType) + repoIndex := schemas.NewIndex("repo_id", schemas.IndexType) repoIndex.AddColumn("repo_id") - issueIndex := schemas.NewIndex("attachment_issue_id", schemas.IndexType) + issueIndex := schemas.NewIndex("issue_id", schemas.IndexType) issueIndex.AddColumn("issue_id") - releaseIndex := schemas.NewIndex("attachment_release_id", schemas.IndexType) + releaseIndex := schemas.NewIndex("release_id", schemas.IndexType) releaseIndex.AddColumn("release_id") - uploaderIndex := schemas.NewIndex("attachment_uploader_id", schemas.IndexType) + uploaderIndex := schemas.NewIndex("uploader_id", schemas.IndexType) uploaderIndex.AddColumn("uploader_id") - commentIndex := schemas.NewIndex("attachment_comment_id", schemas.IndexType) + commentIndex := schemas.NewIndex("comment_id", schemas.IndexType) commentIndex.AddColumn("comment_id") - statusIndex := schemas.NewIndex("attachment_status", schemas.IndexType) + statusIndex := schemas.NewIndex("status", schemas.IndexType) statusIndex.AddColumn("status") - statusIDIndex := schemas.NewIndex("attachment_status_id", schemas.IndexType) + statusIDIndex := schemas.NewIndex("status_id", schemas.IndexType) statusIDIndex.AddColumn("status", "id") // For status = ? AND id > ? query return []*schemas.Index{ diff --git a/models/repo/attachment.go b/models/repo/attachment.go index 98f97d01e0b9f..86dfea1c2da99 100644 --- a/models/repo/attachment.go +++ b/models/repo/attachment.go @@ -30,7 +30,7 @@ type Attachment struct { Name string DownloadCount int64 `xorm:"DEFAULT 0"` Status db.FileStatus `xorm:"DEFAULT 1 NOT NULL"` // 1 = normal, 2 = to be deleted - DeleteFailedCount int `xorm:"DEFAULT 0"` // Number of times the deletion failed, used to prevent infinite loop + DeleteFailedCount int `xorm:"DEFAULT 0 NOT NULL"` // Number of times the deletion failed, used to prevent infinite loop LastDeleteFailedReason string `xorm:"TEXT"` // Last reason the deletion failed, used to prevent infinite loop LastDeleteFailedTime timeutil.TimeStamp // Last time the deletion failed, used to prevent infinite loop Size int64 `xorm:"DEFAULT 0"` @@ -40,28 +40,28 @@ type Attachment struct { // TableIndices implements xorm's TableIndices interface func (a *Attachment) TableIndices() []*schemas.Index { - uuidIndex := schemas.NewIndex("attachment_uuid", schemas.UniqueType) + uuidIndex := schemas.NewIndex("uuid", schemas.UniqueType) uuidIndex.AddColumn("uuid") - repoIndex := schemas.NewIndex("attachment_repo_id", schemas.IndexType) + repoIndex := schemas.NewIndex("repo_id", schemas.IndexType) repoIndex.AddColumn("repo_id") - issueIndex := schemas.NewIndex("attachment_issue_id", schemas.IndexType) + issueIndex := schemas.NewIndex("issue_id", schemas.IndexType) issueIndex.AddColumn("issue_id") - releaseIndex := schemas.NewIndex("attachment_release_id", schemas.IndexType) + releaseIndex := schemas.NewIndex("release_id", schemas.IndexType) releaseIndex.AddColumn("release_id") - uploaderIndex := schemas.NewIndex("attachment_uploader_id", schemas.IndexType) + uploaderIndex := schemas.NewIndex("uploader_id", schemas.IndexType) uploaderIndex.AddColumn("uploader_id") - commentIndex := schemas.NewIndex("attachment_comment_id", schemas.IndexType) + commentIndex := schemas.NewIndex("comment_id", schemas.IndexType) commentIndex.AddColumn("comment_id") - statusIndex := schemas.NewIndex("attachment_status", schemas.IndexType) + statusIndex := schemas.NewIndex("status", schemas.IndexType) statusIndex.AddColumn("status") - statusIDIndex := schemas.NewIndex("attachment_status_id", schemas.IndexType) + statusIDIndex := schemas.NewIndex("status_id", schemas.IndexType) statusIDIndex.AddColumn("status", "id") // For status = ? AND id > ? query return []*schemas.Index{ diff --git a/services/attachment/attachment.go b/services/attachment/attachment.go index 6f57c55c4e6d6..f14daf20e8ecf 100644 --- a/services/attachment/attachment.go +++ b/services/attachment/attachment.go @@ -92,6 +92,7 @@ func Init() error { if cleanQueue == nil { return errors.New("Unable to create attachments-clean queue") } + go graceful.GetManager().RunWithCancel(cleanQueue) return nil } From 8f8dd8cae1c058b194e929e8841d5248abbf47c6 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 20 Jul 2025 17:36:19 -0700 Subject: [PATCH 32/36] Fix test --- models/issues/comment_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/models/issues/comment_test.go b/models/issues/comment_test.go index c08e3b970d3b2..610a75aea6086 100644 --- a/models/issues/comment_test.go +++ b/models/issues/comment_test.go @@ -50,7 +50,9 @@ func Test_UpdateCommentAttachment(t *testing.T) { comment := unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{ID: 1}) attachment := repo_model.Attachment{ - Name: "test.txt", + Name: "test.txt", + Status: db.FileStatusNormal, + UUID: "test-uuid", } assert.NoError(t, db.Insert(db.DefaultContext, &attachment)) From 3b2e424820d0f6c85d8cb265795820fbcf5a9f73 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 20 Jul 2025 22:02:54 -0700 Subject: [PATCH 33/36] Use a standalone table to store deletion files so that all kinds of storage could reuse the deletion infrastructure --- models/db/file_status.go | 12 -- models/fixtures/attachment.yml | 13 -- models/issues/comment.go | 5 +- models/issues/comment_list.go | 6 +- models/issues/comment_test.go | 4 +- models/issues/issue_list.go | 1 - models/migrations/migrations.go | 2 +- models/migrations/v1_25/main_test.go | 14 -- models/migrations/v1_25/v321.go | 73 ++------- models/migrations/v1_25/v321_test.go | 35 ----- models/repo/attachment.go | 181 +++++++--------------- models/repo/release.go | 1 - models/system/storage_cleanup.go | 42 +++++ models/user/main_test.go | 4 +- modules/storage/storage.go | 34 ++++ options/locale/locale_en-US.ini | 2 +- routers/init.go | 4 +- services/attachment/attachment.go | 111 +------------ services/attachment/attachment_test.go | 3 +- services/cron/tasks_extended.go | 10 +- services/doctor/repository.go | 3 +- services/issue/comments.go | 21 ++- services/issue/issue.go | 42 ++--- services/issue/issue_test.go | 14 +- services/issue/main_test.go | 4 +- services/migrations/gitea_uploader.go | 1 - services/release/release.go | 16 +- services/release/release_test.go | 3 +- services/repository/delete.go | 18 ++- services/repository/main_test.go | 4 +- services/storagecleanup/storagecleanup.go | 116 ++++++++++++++ services/user/delete.go | 11 +- services/user/user.go | 10 +- services/user/user_test.go | 4 +- 34 files changed, 365 insertions(+), 459 deletions(-) delete mode 100644 models/db/file_status.go delete mode 100644 models/migrations/v1_25/main_test.go delete mode 100644 models/migrations/v1_25/v321_test.go create mode 100644 models/system/storage_cleanup.go create mode 100644 services/storagecleanup/storagecleanup.go diff --git a/models/db/file_status.go b/models/db/file_status.go deleted file mode 100644 index 4ed1186fb56b5..0000000000000 --- a/models/db/file_status.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2025 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package db - -// FileStatus represents the status of a file in the disk. -type FileStatus int - -const ( - FileStatusNormal FileStatus = iota + 1 // FileStatusNormal indicates the file is normal and exists on disk. - FileStatusToBeDeleted // FileStatusToBeDeleted indicates the file is marked for deletion but still exists on disk. -) diff --git a/models/fixtures/attachment.yml b/models/fixtures/attachment.yml index 06f578f7b7b32..b86a15b28269f 100644 --- a/models/fixtures/attachment.yml +++ b/models/fixtures/attachment.yml @@ -8,7 +8,6 @@ comment_id: 0 name: attach1 download_count: 0 - status: 1 size: 0 created_unix: 946684800 @@ -22,7 +21,6 @@ comment_id: 0 name: attach2 download_count: 1 - status: 1 size: 0 created_unix: 946684800 @@ -36,7 +34,6 @@ comment_id: 1 name: attach1 download_count: 0 - status: 1 size: 0 created_unix: 946684800 @@ -50,7 +47,6 @@ comment_id: 1 name: attach2 download_count: 1 - status: 1 size: 0 created_unix: 946684800 @@ -64,7 +60,6 @@ comment_id: 0 name: attach1 download_count: 0 - status: 1 size: 0 created_unix: 946684800 @@ -78,7 +73,6 @@ comment_id: 2 name: attach1 download_count: 0 - status: 1 size: 0 created_unix: 946684800 @@ -92,7 +86,6 @@ comment_id: 2 name: attach1 download_count: 0 - status: 1 size: 0 created_unix: 946684800 @@ -106,7 +99,6 @@ comment_id: 0 name: attach1 download_count: 0 - status: 1 size: 0 created_unix: 946684800 @@ -120,7 +112,6 @@ comment_id: 0 name: attach1 download_count: 0 - status: 1 size: 0 created_unix: 946684800 @@ -134,7 +125,6 @@ comment_id: 0 name: attach1 download_count: 0 - status: 1 size: 0 created_unix: 946684800 @@ -148,7 +138,6 @@ comment_id: 0 name: attach1 download_count: 0 - status: 1 size: 0 created_unix: 946684800 @@ -162,7 +151,6 @@ comment_id: 0 name: README.md download_count: 0 - status: 1 size: 0 created_unix: 946684800 @@ -176,6 +164,5 @@ comment_id: 7 name: code_comment_uploaded_attachment.png download_count: 0 - status: 1 size: 0 created_unix: 946684812 diff --git a/models/issues/comment.go b/models/issues/comment.go index d230b2594578c..db48e4ffac185 100644 --- a/models/issues/comment.go +++ b/models/issues/comment.go @@ -1116,7 +1116,8 @@ func UpdateComment(ctx context.Context, c *Comment, contentVersion int, doer *us // DeleteComment deletes the comment func DeleteComment(ctx context.Context, comment *Comment) error { - if _, err := db.GetEngine(ctx).ID(comment.ID).NoAutoCondition().Delete(comment); err != nil { + e := db.GetEngine(ctx) + if _, err := e.ID(comment.ID).NoAutoCondition().Delete(comment); err != nil { return err } @@ -1131,7 +1132,7 @@ func DeleteComment(ctx context.Context, comment *Comment) error { return err } } - if _, err := db.GetEngine(ctx).Table("action"). + if _, err := e.Table("action"). Where("comment_id = ?", comment.ID). Update(map[string]any{ "is_deleted": true, diff --git a/models/issues/comment_list.go b/models/issues/comment_list.go index bb95da7710bf5..f6c485449f60b 100644 --- a/models/issues/comment_list.go +++ b/models/issues/comment_list.go @@ -349,10 +349,7 @@ func (comments CommentList) LoadAttachmentsByIssue(ctx context.Context) error { } attachments := make([]*repo_model.Attachment, 0, len(comments)/2) - if err := db.GetEngine(ctx). - Where("issue_id=? AND comment_id>0", comments[0].IssueID). - And("status = ?", db.FileStatusNormal). - Find(&attachments); err != nil { + if err := db.GetEngine(ctx).Where("issue_id=? AND comment_id>0", comments[0].IssueID).Find(&attachments); err != nil { return err } @@ -380,7 +377,6 @@ func (comments CommentList) LoadAttachments(ctx context.Context) (err error) { limit := min(left, db.DefaultMaxInSize) rows, err := db.GetEngine(ctx). In("comment_id", commentsIDs[:limit]). - And("status = ?", db.FileStatusNormal). Rows(new(repo_model.Attachment)) if err != nil { return err diff --git a/models/issues/comment_test.go b/models/issues/comment_test.go index 610a75aea6086..c08e3b970d3b2 100644 --- a/models/issues/comment_test.go +++ b/models/issues/comment_test.go @@ -50,9 +50,7 @@ func Test_UpdateCommentAttachment(t *testing.T) { comment := unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{ID: 1}) attachment := repo_model.Attachment{ - Name: "test.txt", - Status: db.FileStatusNormal, - UUID: "test-uuid", + Name: "test.txt", } assert.NoError(t, db.Insert(db.DefaultContext, &attachment)) diff --git a/models/issues/issue_list.go b/models/issues/issue_list.go index 98b0becafd1b5..26b93189b8bed 100644 --- a/models/issues/issue_list.go +++ b/models/issues/issue_list.go @@ -339,7 +339,6 @@ func (issues IssueList) LoadAttachments(ctx context.Context) (err error) { limit := min(left, db.DefaultMaxInSize) rows, err := db.GetEngine(ctx). In("issue_id", issuesIDs[:limit]). - And("status = ?", db.FileStatusNormal). Rows(new(repo_model.Attachment)) if err != nil { return err diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 1b6ec04d36e8b..f6a6c9b49aa0f 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -385,7 +385,7 @@ func prepareMigrationTasks() []*migration { newMigration(320, "Migrate two_factor_policy to login_source table", v1_24.MigrateSkipTwoFactor), // Gitea 1.24.0-rc0 ends at migration ID number 320 (database version 321) - newMigration(321, "Add file status columns to attachment table", v1_25.AddFileStatusToAttachment), + newMigration(321, "Add storage_path_deletion table", v1_25.AddStoragePathDeletion), } return preparedMigrations } diff --git a/models/migrations/v1_25/main_test.go b/models/migrations/v1_25/main_test.go deleted file mode 100644 index d2c4a4105d3a8..0000000000000 --- a/models/migrations/v1_25/main_test.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2025 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package v1_25 - -import ( - "testing" - - "code.gitea.io/gitea/models/migrations/base" -) - -func TestMain(m *testing.M) { - base.MainTest(m) -} diff --git a/models/migrations/v1_25/v321.go b/models/migrations/v1_25/v321.go index 47096f5ad9e2c..a804f4ea7bf5a 100644 --- a/models/migrations/v1_25/v321.go +++ b/models/migrations/v1_25/v321.go @@ -4,72 +4,23 @@ package v1_25 import ( - "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/timeutil" "xorm.io/xorm" - "xorm.io/xorm/schemas" ) -type Attachment321 struct { - ID int64 `xorm:"pk autoincr"` - UUID string `xorm:"uuid"` - RepoID int64 // this should not be zero - IssueID int64 // maybe zero when creating - ReleaseID int64 // maybe zero when creating - UploaderID int64 `xorm:"DEFAULT 0"` // Notice: will be zero before this column added - CommentID int64 - Name string - DownloadCount int64 `xorm:"DEFAULT 0"` - Status db.FileStatus `xorm:"DEFAULT 1 NOT NULL"` // 1 = normal, 2 = to be deleted - DeleteFailedCount int `xorm:"DEFAULT 0 NOT NULL"` // Number of times the deletion failed, used to prevent infinite loop - LastDeleteFailedTime timeutil.TimeStamp // Last time the deletion failed, used to prevent infinite loop - Size int64 `xorm:"DEFAULT 0"` - CreatedUnix timeutil.TimeStamp `xorm:"created"` -} - -func (a *Attachment321) TableName() string { - return "attachment" -} - -// TableIndices implements xorm's TableIndices interface -func (a *Attachment321) TableIndices() []*schemas.Index { - uuidIndex := schemas.NewIndex("uuid", schemas.UniqueType) - uuidIndex.AddColumn("uuid") - - repoIndex := schemas.NewIndex("repo_id", schemas.IndexType) - repoIndex.AddColumn("repo_id") - - issueIndex := schemas.NewIndex("issue_id", schemas.IndexType) - issueIndex.AddColumn("issue_id") - - releaseIndex := schemas.NewIndex("release_id", schemas.IndexType) - releaseIndex.AddColumn("release_id") - - uploaderIndex := schemas.NewIndex("uploader_id", schemas.IndexType) - uploaderIndex.AddColumn("uploader_id") - - commentIndex := schemas.NewIndex("comment_id", schemas.IndexType) - commentIndex.AddColumn("comment_id") - - statusIndex := schemas.NewIndex("status", schemas.IndexType) - statusIndex.AddColumn("status") - - statusIDIndex := schemas.NewIndex("status_id", schemas.IndexType) - statusIDIndex.AddColumn("status", "id") // For status = ? AND id > ? query - - return []*schemas.Index{ - uuidIndex, - repoIndex, - issueIndex, - releaseIndex, - uploaderIndex, - commentIndex, - statusIndex, - statusIDIndex, +func AddStoragePathDeletion(x *xorm.Engine) error { + // StoragePathDeletion represents a file or directory that is pending deletion. + type StoragePathDeletion struct { + ID int64 + StorageName string // storage name defines in storage module + PathType int // 1 for file, 2 for directory + RelativePath string `xorm:"TEXT"` + DeleteFailedCount int `xorm:"DEFAULT 0 NOT NULL"` // Number of times the deletion failed, used to prevent infinite loop + LastDeleteFailedReason string `xorm:"TEXT"` // Last reason the deletion failed, used to prevent infinite loop + LastDeleteFailedTime timeutil.TimeStamp // Last time the deletion failed, used to prevent infinite loop + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` } -} -func AddFileStatusToAttachment(x *xorm.Engine) error { - return x.Sync(new(Attachment321)) + return x.Sync(new(StoragePathDeletion)) } diff --git a/models/migrations/v1_25/v321_test.go b/models/migrations/v1_25/v321_test.go deleted file mode 100644 index 47bb9ea242975..0000000000000 --- a/models/migrations/v1_25/v321_test.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2025 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package v1_25 - -import ( - "testing" - - "code.gitea.io/gitea/models/migrations/base" - "code.gitea.io/gitea/modules/timeutil" - - "github.com/stretchr/testify/assert" -) - -func Test_AddFileStatusToAttachment(t *testing.T) { - type Attachment struct { - ID int64 `xorm:"pk autoincr"` - UUID string `xorm:"uuid UNIQUE"` - RepoID int64 `xorm:"INDEX"` // this should not be zero - IssueID int64 `xorm:"INDEX"` // maybe zero when creating - ReleaseID int64 `xorm:"INDEX"` // maybe zero when creating - UploaderID int64 `xorm:"INDEX DEFAULT 0"` // Notice: will be zero before this column added - CommentID int64 `xorm:"INDEX"` - Name string - DownloadCount int64 `xorm:"DEFAULT 0"` - Size int64 `xorm:"DEFAULT 0"` - CreatedUnix timeutil.TimeStamp `xorm:"created"` - } - - // Prepare and load the testing database - x, deferable := base.PrepareTestEnv(t, 0, new(Attachment)) - defer deferable() - - assert.NoError(t, AddFileStatusToAttachment(x)) -} diff --git a/models/repo/attachment.go b/models/repo/attachment.go index 86dfea1c2da99..b82317d47bab4 100644 --- a/models/repo/attachment.go +++ b/models/repo/attachment.go @@ -11,69 +11,27 @@ import ( "path" "code.gitea.io/gitea/models/db" + system_model "code.gitea.io/gitea/models/system" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" - - "xorm.io/xorm/schemas" ) // Attachment represent a attachment of issue/comment/release. type Attachment struct { - ID int64 `xorm:"pk autoincr"` - UUID string `xorm:"uuid"` - RepoID int64 // this should not be zero - IssueID int64 // maybe zero when creating - ReleaseID int64 // maybe zero when creating - UploaderID int64 `xorm:"DEFAULT 0"` // Notice: will be zero before this column added - CommentID int64 - Name string - DownloadCount int64 `xorm:"DEFAULT 0"` - Status db.FileStatus `xorm:"DEFAULT 1 NOT NULL"` // 1 = normal, 2 = to be deleted - DeleteFailedCount int `xorm:"DEFAULT 0 NOT NULL"` // Number of times the deletion failed, used to prevent infinite loop - LastDeleteFailedReason string `xorm:"TEXT"` // Last reason the deletion failed, used to prevent infinite loop - LastDeleteFailedTime timeutil.TimeStamp // Last time the deletion failed, used to prevent infinite loop - Size int64 `xorm:"DEFAULT 0"` - CreatedUnix timeutil.TimeStamp `xorm:"created"` - CustomDownloadURL string `xorm:"-"` -} - -// TableIndices implements xorm's TableIndices interface -func (a *Attachment) TableIndices() []*schemas.Index { - uuidIndex := schemas.NewIndex("uuid", schemas.UniqueType) - uuidIndex.AddColumn("uuid") - - repoIndex := schemas.NewIndex("repo_id", schemas.IndexType) - repoIndex.AddColumn("repo_id") - - issueIndex := schemas.NewIndex("issue_id", schemas.IndexType) - issueIndex.AddColumn("issue_id") - - releaseIndex := schemas.NewIndex("release_id", schemas.IndexType) - releaseIndex.AddColumn("release_id") - - uploaderIndex := schemas.NewIndex("uploader_id", schemas.IndexType) - uploaderIndex.AddColumn("uploader_id") - - commentIndex := schemas.NewIndex("comment_id", schemas.IndexType) - commentIndex.AddColumn("comment_id") - - statusIndex := schemas.NewIndex("status", schemas.IndexType) - statusIndex.AddColumn("status") - - statusIDIndex := schemas.NewIndex("status_id", schemas.IndexType) - statusIDIndex.AddColumn("status", "id") // For status = ? AND id > ? query - - return []*schemas.Index{ - uuidIndex, - repoIndex, - issueIndex, - releaseIndex, - uploaderIndex, - commentIndex, - statusIndex, - statusIDIndex, - } + ID int64 `xorm:"pk autoincr"` + UUID string `xorm:"uuid UNIQUE"` + RepoID int64 `xorm:"INDEX"` // this should not be zero + IssueID int64 `xorm:"INDEX"` // maybe zero when creating + ReleaseID int64 `xorm:"INDEX"` // maybe zero when creating + UploaderID int64 `xorm:"INDEX DEFAULT 0"` // Notice: will be zero before this column added + CommentID int64 `xorm:"INDEX"` + Name string + DownloadCount int64 `xorm:"DEFAULT 0"` + Size int64 `xorm:"DEFAULT 0"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` + CustomDownloadURL string `xorm:"-"` } func init() { @@ -132,9 +90,7 @@ func (err ErrAttachmentNotExist) Unwrap() error { // GetAttachmentByID returns attachment by given id func GetAttachmentByID(ctx context.Context, id int64) (*Attachment, error) { attach := &Attachment{} - if has, err := db.GetEngine(ctx).ID(id). - And("status = ?", db.FileStatusNormal). - Get(attach); err != nil { + if has, err := db.GetEngine(ctx).ID(id).Get(attach); err != nil { return nil, err } else if !has { return nil, ErrAttachmentNotExist{ID: id, UUID: ""} @@ -145,9 +101,7 @@ func GetAttachmentByID(ctx context.Context, id int64) (*Attachment, error) { // GetAttachmentByUUID returns attachment by given UUID. func GetAttachmentByUUID(ctx context.Context, uuid string) (*Attachment, error) { attach := &Attachment{} - has, err := db.GetEngine(ctx).Where("uuid=?", uuid). - And("status = ?", db.FileStatusNormal). - Get(attach) + has, err := db.GetEngine(ctx).Where("uuid=?", uuid).Get(attach) if err != nil { return nil, err } else if !has { @@ -164,24 +118,18 @@ func GetAttachmentsByUUIDs(ctx context.Context, uuids []string) ([]*Attachment, // Silently drop invalid uuids. attachments := make([]*Attachment, 0, len(uuids)) - return attachments, db.GetEngine(ctx).In("uuid", uuids). - And("status = ?", db.FileStatusNormal). - Find(&attachments) + return attachments, db.GetEngine(ctx).In("uuid", uuids).Find(&attachments) } // ExistAttachmentsByUUID returns true if attachment exists with the given UUID func ExistAttachmentsByUUID(ctx context.Context, uuid string) (bool, error) { - return db.GetEngine(ctx).Where("`uuid`=?", uuid). - And("status = ?", db.FileStatusNormal). - Exist(new(Attachment)) + return db.GetEngine(ctx).Where("`uuid`=?", uuid).Exist(new(Attachment)) } // GetAttachmentsByIssueID returns all attachments of an issue. func GetAttachmentsByIssueID(ctx context.Context, issueID int64) ([]*Attachment, error) { attachments := make([]*Attachment, 0, 10) - return attachments, db.GetEngine(ctx).Where("issue_id = ? AND comment_id = 0", issueID). - And("status = ?", db.FileStatusNormal). - Find(&attachments) + return attachments, db.GetEngine(ctx).Where("issue_id = ? AND comment_id = 0", issueID).Find(&attachments) } // GetAttachmentsByIssueIDImagesLatest returns the latest image attachments of an issue. @@ -196,23 +144,19 @@ func GetAttachmentsByIssueIDImagesLatest(ctx context.Context, issueID int64) ([] OR name like '%.jxl' OR name like '%.png' OR name like '%.svg' - OR name like '%.webp')`, issueID). - And("status = ?", db.FileStatusNormal). - Desc("comment_id").Limit(5).Find(&attachments) + OR name like '%.webp')`, issueID).Desc("comment_id").Limit(5).Find(&attachments) } // GetAttachmentsByCommentID returns all attachments if comment by given ID. func GetAttachmentsByCommentID(ctx context.Context, commentID int64) ([]*Attachment, error) { attachments := make([]*Attachment, 0, 10) - return attachments, db.GetEngine(ctx).Where("comment_id=?", commentID). - And("status = ?", db.FileStatusNormal). - Find(&attachments) + return attachments, db.GetEngine(ctx).Where("comment_id=?", commentID).Find(&attachments) } // GetAttachmentByReleaseIDFileName returns attachment by given releaseId and fileName. func GetAttachmentByReleaseIDFileName(ctx context.Context, releaseID int64, fileName string) (*Attachment, error) { attach := &Attachment{ReleaseID: releaseID, Name: fileName} - has, err := db.GetEngine(ctx).Where("status = ?", db.FileStatusNormal).Get(attach) + has, err := db.GetEngine(ctx).Get(attach) if err != nil { return nil, err } else if !has { @@ -221,6 +165,41 @@ func GetAttachmentByReleaseIDFileName(ctx context.Context, releaseID int64, file return attach, nil } +// DeleteAttachments delete the given attachments and add disk files to pending deletion +func DeleteAttachments(ctx context.Context, attachments []*Attachment) ([]int64, error) { + if len(attachments) == 0 { + return nil, nil + } + + ids := make([]int64, 0, len(attachments)) + for _, a := range attachments { + ids = append(ids, a.ID) + } + + return db.WithTx2(ctx, func(ctx context.Context) ([]int64, error) { + // delete attachments from database + if _, err := db.GetEngine(ctx).Table("attachment").In("id", ids).Delete(); err != nil { + return nil, err + } + + // add disk files to pending deletion table as well + var deletionIDs []int64 + for _, a := range attachments { + pendingDeletion := &system_model.StoragePathDeletion{ + StorageName: storage.AttachmentStorageName, + PathType: system_model.PathFile, + RelativePath: a.RelativePath(), + } + if err := db.Insert(ctx, pendingDeletion); err != nil { + return nil, fmt.Errorf("insert pending deletion: %w", err) + } + + deletionIDs = append(deletionIDs, pendingDeletion.ID) // Collect pending deletions + } + return deletionIDs, nil + }) +} + // UpdateAttachmentByUUID Updates attachment via uuid func UpdateAttachmentByUUID(ctx context.Context, attach *Attachment, cols ...string) error { if attach.UUID == "" { @@ -243,52 +222,6 @@ func UpdateAttachment(ctx context.Context, atta *Attachment) error { return err } -// MarkAttachmentsDeleted marks the given attachments as deleted -func MarkAttachmentsDeleted(ctx context.Context, attachments []*Attachment) (int64, error) { - if len(attachments) == 0 { - return 0, nil - } - - ids := make([]int64, 0, len(attachments)) - for _, a := range attachments { - ids = append(ids, a.ID) - } - - return db.GetEngine(ctx).Table("attachment").In("id", ids).Update(map[string]any{ - "status": db.FileStatusToBeDeleted, - }) -} - -// MarkAttachmentsDeletedByRelease marks all attachments associated with the given release as deleted. -func MarkAttachmentsDeletedByRelease(ctx context.Context, releaseID int64) error { - _, err := db.GetEngine(ctx).Table("attachment").Where("release_id = ?", releaseID).Update(map[string]any{ - "status": db.FileStatusToBeDeleted, - }) - return err -} - -// DeleteMarkedAttachmentByID deletes the attachment which has been marked as deleted by given id -func DeleteMarkedAttachmentByID(ctx context.Context, id int64) error { - cnt, err := db.GetEngine(ctx).ID(id).Where("status = ?", db.FileStatusToBeDeleted).Delete(new(Attachment)) - if err != nil { - return fmt.Errorf("delete attachment by id: %w", err) - } - if cnt != 1 { - return fmt.Errorf("the attachment with id %d was not found or is not marked for deletion", id) - } - return nil -} - -func UpdateMarkedAttachmentFailure(ctx context.Context, attachment *Attachment, err error) error { - attachment.DeleteFailedCount++ - _, updateErr := db.GetEngine(ctx).Table("attachment").ID(attachment.ID).Update(map[string]any{ - "delete_failed_count": attachment.DeleteFailedCount, - "last_delete_failed_reason": err.Error(), - "last_delete_failed_time": timeutil.TimeStampNow(), - }) - return updateErr -} - // CountOrphanedAttachments returns the number of bad attachments func CountOrphanedAttachments(ctx context.Context) (int64, error) { return db.GetEngine(ctx).Where("(issue_id > 0 and issue_id not in (select id from issue)) or (release_id > 0 and release_id not in (select id from `release`))"). diff --git a/models/repo/release.go b/models/repo/release.go index b3aae97560c16..59f4caf5aa9e0 100644 --- a/models/repo/release.go +++ b/models/repo/release.go @@ -378,7 +378,6 @@ func GetReleaseAttachments(ctx context.Context, rels ...*Release) (err error) { err = db.GetEngine(ctx). Asc("release_id", "name"). In("release_id", sortedRels.ID). - And("status = ?", db.FileStatusNormal). Find(&attachments) if err != nil { return err diff --git a/models/system/storage_cleanup.go b/models/system/storage_cleanup.go new file mode 100644 index 0000000000000..3151b0a62ae52 --- /dev/null +++ b/models/system/storage_cleanup.go @@ -0,0 +1,42 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package system + +import ( + "context" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/timeutil" +) + +const ( + PathFile = 1 // PathTypeFile represents a file + PathDir = 2 // PathTypeDir represents a directory +) + +// StoragePathDeletion represents a file or directory that is pending deletion. +type StoragePathDeletion struct { + ID int64 + StorageName string // storage name defines in storage module + PathType int // 1 for file, 2 for directory + RelativePath string `xorm:"TEXT"` + DeleteFailedCount int `xorm:"DEFAULT 0 NOT NULL"` // Number of times the deletion failed, used to prevent infinite loop + LastDeleteFailedReason string `xorm:"TEXT"` // Last reason the deletion failed, used to prevent infinite loop + LastDeleteFailedTime timeutil.TimeStamp // Last time the deletion failed, used to prevent infinite loop + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` +} + +func init() { + db.RegisterModel(new(StoragePathDeletion)) +} + +func UpdateDeletionFailure(ctx context.Context, deletion *StoragePathDeletion, err error) error { + deletion.DeleteFailedCount++ + _, updateErr := db.GetEngine(ctx).Table("storage_path_deletion").ID(deletion.ID).Update(map[string]any{ + "delete_failed_count": deletion.DeleteFailedCount, + "last_delete_failed_reason": err.Error(), + "last_delete_failed_time": timeutil.TimeStampNow(), + }) + return updateErr +} diff --git a/models/user/main_test.go b/models/user/main_test.go index 2ca502bbeaabd..db60a28146543 100644 --- a/models/user/main_test.go +++ b/models/user/main_test.go @@ -8,7 +8,7 @@ import ( "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/services/attachment" + "code.gitea.io/gitea/services/storagecleanup" _ "code.gitea.io/gitea/models" _ "code.gitea.io/gitea/models/actions" @@ -20,7 +20,7 @@ func TestMain(m *testing.M) { unittest.MainTest(m, &unittest.TestOptions{ SetUp: func() error { setting.LoadQueueSettings() - return attachment.Init() + return storagecleanup.Init() }, }) } diff --git a/modules/storage/storage.go b/modules/storage/storage.go index 1868817c057cf..c017838be1b64 100644 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -166,6 +166,40 @@ func NewStorage(typStr Type, cfg *setting.Storage) (ObjectStorage, error) { return fn(context.Background(), cfg) } +const ( + AttachmentStorageName = "attachment" + AvatarStorageName = "avatar" + RepoAvatarStorageName = "repo_avatar" + LFSStorageName = "lfs" + RepoArchiveStorageName = "repo_archive" + PackagesStorageName = "packages" + ActionsLogStorageName = "actions_logs" + ActionsArtifactsStorageName = "actions_artifacts" +) + +func GetStorageByName(name string) (ObjectStorage, error) { + switch name { + case AttachmentStorageName: + return Attachments, nil + case AvatarStorageName: + return Avatars, nil + case RepoAvatarStorageName: + return RepoAvatars, nil + case LFSStorageName: + return LFS, nil + case RepoArchiveStorageName: + return RepoArchives, nil + case PackagesStorageName: + return Packages, nil + case ActionsLogStorageName: + return Actions, nil + case ActionsArtifactsStorageName: + return ActionsArtifacts, nil + default: + return nil, fmt.Errorf("Unknown storage name: %s", name) + } +} + func initAvatars() (err error) { log.Info("Initialising Avatar storage with type: %s", setting.Avatar.Storage.Type) Avatars, err = NewStorage(setting.Avatar.Storage.Type, setting.Avatar.Storage) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index ab925669a56a9..e11a8477fe5ec 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3065,7 +3065,7 @@ dashboard.sync_branch.started = Branches Sync started dashboard.sync_tag.started = Tags Sync started dashboard.rebuild_issue_indexer = Rebuild issue indexer dashboard.sync_repo_licenses = Sync repo licenses -dashboard.clean_attachments = Clean up deleted attachments +dashboard.cleanup_storage = Clean up deleted storage files users.user_manage_panel = User Account Management users.new_account = Create User Account diff --git a/routers/init.go b/routers/init.go index b8bcd937bf90d..39c34bf3a377a 100644 --- a/routers/init.go +++ b/routers/init.go @@ -36,7 +36,6 @@ import ( web_routers "code.gitea.io/gitea/routers/web" actions_service "code.gitea.io/gitea/services/actions" asymkey_service "code.gitea.io/gitea/services/asymkey" - attachment_service "code.gitea.io/gitea/services/attachment" "code.gitea.io/gitea/services/auth" "code.gitea.io/gitea/services/auth/source/oauth2" "code.gitea.io/gitea/services/automerge" @@ -53,6 +52,7 @@ import ( release_service "code.gitea.io/gitea/services/release" repo_service "code.gitea.io/gitea/services/repository" "code.gitea.io/gitea/services/repository/archiver" + "code.gitea.io/gitea/services/storagecleanup" "code.gitea.io/gitea/services/task" "code.gitea.io/gitea/services/uinotification" "code.gitea.io/gitea/services/webhook" @@ -175,7 +175,7 @@ func InitWebInstalled(ctx context.Context) { mustInitCtx(ctx, actions_service.Init) mustInit(repo_service.InitLicenseClassifier) - mustInit(attachment_service.Init) + mustInit(storagecleanup.Init) // Finally start up the cron cron.NewContext(ctx) diff --git a/services/attachment/attachment.go b/services/attachment/attachment.go index f14daf20e8ecf..65ff3629ec3a1 100644 --- a/services/attachment/attachment.go +++ b/services/attachment/attachment.go @@ -6,20 +6,15 @@ package attachment import ( "bytes" "context" - "errors" "fmt" "io" - "os" "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" - "code.gitea.io/gitea/models/system" - "code.gitea.io/gitea/modules/graceful" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/queue" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/context/upload" + "code.gitea.io/gitea/services/storagecleanup" "github.com/google/uuid" ) @@ -37,7 +32,6 @@ func NewAttachment(ctx context.Context, attach *repo_model.Attachment, file io.R return fmt.Errorf("Create: %w", err) } attach.Size = size - attach.Status = db.FileStatusNormal return db.Insert(ctx, attach) }) @@ -75,109 +69,12 @@ func DeleteAttachment(ctx context.Context, a *repo_model.Attachment) error { // DeleteAttachments deletes the given attachments and optionally the associated files. func DeleteAttachments(ctx context.Context, attachments []*repo_model.Attachment) (int, error) { - cnt, err := repo_model.MarkAttachmentsDeleted(ctx, attachments) + deletions, err := repo_model.DeleteAttachments(ctx, attachments) if err != nil { return 0, err } - AddAttachmentsToCleanQueue(ctx, attachments) + storagecleanup.AddDeletionsToCleanQueue(ctx, deletions) - return int(cnt), nil -} - -var cleanQueue *queue.WorkerPoolQueue[int64] - -func Init() error { - cleanQueue = queue.CreateSimpleQueue(graceful.GetManager().ShutdownContext(), "attachments-clean", handler) - if cleanQueue == nil { - return errors.New("Unable to create attachments-clean queue") - } - go graceful.GetManager().RunWithCancel(cleanQueue) - return nil -} - -// AddAttachmentsToCleanQueue adds the attachments to the clean queue for deletion. -func AddAttachmentsToCleanQueue(ctx context.Context, attachments []*repo_model.Attachment) { - for _, a := range attachments { - if err := cleanQueue.Push(a.ID); err != nil { - log.Error("Failed to push attachment ID %d to clean queue: %v", a.ID, err) - continue - } - } -} - -func handler(attachmentIDs ...int64) []int64 { - return cleanAttachments(graceful.GetManager().ShutdownContext(), attachmentIDs) -} - -func cleanAttachments(ctx context.Context, attachmentIDs []int64) []int64 { - var failed []int64 - for _, attachmentID := range attachmentIDs { - attachment, exist, err := db.GetByID[repo_model.Attachment](ctx, attachmentID) - if err != nil { - log.Error("Failed to get attachment by ID %d: %v", attachmentID, err) - continue - } - if !exist { - continue - } - if attachment.Status != db.FileStatusToBeDeleted { - log.Trace("Attachment %s is not marked for deletion, skipping", attachment.RelativePath()) - continue - } - - if err := storage.Attachments.Delete(attachment.RelativePath()); err != nil { - if !errors.Is(err, os.ErrNotExist) { - log.Error("delete attachment[uuid: %s] failed: %v", attachment.UUID, err) - failed = append(failed, attachment.ID) - if attachment.DeleteFailedCount%3 == 0 { - _ = system.CreateNotice(ctx, system.NoticeRepository, fmt.Sprintf("Failed to delete attachment %s (%d times): %v", attachment.RelativePath(), attachment.DeleteFailedCount+1, err)) - } - if err := repo_model.UpdateMarkedAttachmentFailure(ctx, attachment, err); err != nil { - log.Error("Failed to update attachment failure for ID %d: %v", attachment.ID, err) - } - continue - } - } - if err := repo_model.DeleteMarkedAttachmentByID(ctx, attachment.ID); err != nil { - log.Error("Failed to delete attachment by ID %d(will be tried later): %v", attachment.ID, err) - failed = append(failed, attachment.ID) - } else { - log.Trace("Attachment %s deleted from database", attachment.RelativePath()) - } - } - return failed -} - -// ScanToBeDeletedAttachments scans for attachments that are marked as to be deleted and send to -// clean queue -func ScanToBeDeletedAttachments(ctx context.Context) error { - attachmentIDs := make([]int64, 0, 100) - lastID := int64(0) - for { - if err := db.GetEngine(ctx). - Select("id"). - // use the status and id index to speed up the query - Where("status = ? AND id > ?", db.FileStatusToBeDeleted, lastID). - Asc("id"). - Limit(100). - Find(&attachmentIDs); err != nil { - return fmt.Errorf("scan to-be-deleted attachments: %w", err) - } - - if len(attachmentIDs) == 0 { - log.Trace("No more attachments to be deleted") - break - } - for _, id := range attachmentIDs { - if err := cleanQueue.Push(id); err != nil { - log.Error("Failed to push attachment ID %d to clean queue: %v", id, err) - } - } - - lastID = attachmentIDs[len(attachmentIDs)-1] - attachmentIDs = attachmentIDs[0:0] - } - - return nil + return len(deletions), nil } diff --git a/services/attachment/attachment_test.go b/services/attachment/attachment_test.go index f4e178c3623ec..937d4cba7d812 100644 --- a/services/attachment/attachment_test.go +++ b/services/attachment/attachment_test.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/services/storagecleanup" _ "code.gitea.io/gitea/models/actions" @@ -23,7 +24,7 @@ func TestMain(m *testing.M) { unittest.MainTest(m, &unittest.TestOptions{ SetUp: func() error { setting.LoadQueueSettings() - return Init() + return storagecleanup.Init() }, }) } diff --git a/services/cron/tasks_extended.go b/services/cron/tasks_extended.go index f9383a3093fb3..4252d98b402fb 100644 --- a/services/cron/tasks_extended.go +++ b/services/cron/tasks_extended.go @@ -15,9 +15,9 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/updatechecker" asymkey_service "code.gitea.io/gitea/services/asymkey" - attachment_service "code.gitea.io/gitea/services/attachment" repo_service "code.gitea.io/gitea/services/repository" archiver_service "code.gitea.io/gitea/services/repository/archiver" + "code.gitea.io/gitea/services/storagecleanup" user_service "code.gitea.io/gitea/services/user" ) @@ -224,13 +224,13 @@ func registerRebuildIssueIndexer() { }) } -func registerCleanAttachments() { - RegisterTaskFatal("clean_attachments", &BaseConfig{ +func registerCleanStorage() { + RegisterTaskFatal("cleanup_storage", &BaseConfig{ Enabled: false, RunAtStart: false, Schedule: "@every 24h", }, func(ctx context.Context, _ *user_model.User, _ Config) error { - return attachment_service.ScanToBeDeletedAttachments(ctx) + return storagecleanup.ScanToBeDeletedFilesOrDir(ctx) }) } @@ -249,5 +249,5 @@ func initExtendedTasks() { registerDeleteOldSystemNotices() registerGCLFS() registerRebuildIssueIndexer() - registerCleanAttachments() + registerCleanStorage() } diff --git a/services/doctor/repository.go b/services/doctor/repository.go index 4a8d00b5716e5..359c4a17e0d82 100644 --- a/services/doctor/repository.go +++ b/services/doctor/repository.go @@ -36,6 +36,7 @@ func deleteOrphanedRepos(ctx context.Context) (int64, error) { } batchSize := db.MaxBatchInsertSize("repository") + e := db.GetEngine(ctx) var deleted int64 for { @@ -44,7 +45,7 @@ func deleteOrphanedRepos(ctx context.Context) (int64, error) { return deleted, ctx.Err() default: var ids []int64 - if err := db.GetEngine(ctx).Table("`repository`"). + if err := e.Table("`repository`"). Join("LEFT", "`user`", "repository.owner_id=`user`.id"). Where(builder.IsNull{"`user`.id"}). Select("`repository`.id").Limit(batchSize).Find(&ids); err != nil { diff --git a/services/issue/comments.go b/services/issue/comments.go index 153d2ebbd60ac..cfcfe78dafa0b 100644 --- a/services/issue/comments.go +++ b/services/issue/comments.go @@ -16,9 +16,9 @@ import ( "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/services/attachment" git_service "code.gitea.io/gitea/services/git" notify_service "code.gitea.io/gitea/services/notify" + "code.gitea.io/gitea/services/storagecleanup" ) // CreateRefComment creates a commit reference comment to issue. @@ -132,36 +132,35 @@ func UpdateComment(ctx context.Context, c *issues_model.Comment, contentVersion } // deleteComment deletes the comment -func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAttachments bool) error { - return db.WithTx(ctx, func(ctx context.Context) error { +func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAttachments bool) ([]int64, error) { + return db.WithTx2(ctx, func(ctx context.Context) ([]int64, error) { if removeAttachments { // load attachments before deleting the comment if err := comment.LoadAttachments(ctx); err != nil { - return err + return nil, err } } // deletedReviewComment should be a review comment with no content and no attachments if err := issues_model.DeleteComment(ctx, comment); err != nil { - return err + return nil, err } if removeAttachments { // mark comment attachments as deleted - if _, err := repo_model.MarkAttachmentsDeleted(ctx, comment.Attachments); err != nil { - return err - } + return repo_model.DeleteAttachments(ctx, comment.Attachments) } - return nil + return nil, nil }) } func DeleteComment(ctx context.Context, doer *user_model.User, comment *issues_model.Comment) error { - if err := deleteComment(ctx, comment, true); err != nil { + deletions, err := deleteComment(ctx, comment, true) + if err != nil { return err } - attachment.AddAttachmentsToCleanQueue(ctx, comment.Attachments) + storagecleanup.AddDeletionsToCleanQueue(ctx, deletions) notify_service.DeleteComment(ctx, doer, comment) diff --git a/services/issue/issue.go b/services/issue/issue.go index 9b3b0c66b2145..c0f61859805a9 100644 --- a/services/issue/issue.go +++ b/services/issue/issue.go @@ -17,8 +17,8 @@ import ( "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" - attachment_service "code.gitea.io/gitea/services/attachment" notify_service "code.gitea.io/gitea/services/notify" + "code.gitea.io/gitea/services/storagecleanup" ) // NewIssue creates new issue with labels for repository. @@ -189,12 +189,12 @@ func DeleteIssue(ctx context.Context, doer *user_model.User, gitRepo *git.Reposi } // delete entries in database - toBeCleanedAttachments, err := deleteIssue(ctx, issue, true) + toBeCleanedDeletions, err := deleteIssue(ctx, issue, true) if err != nil { return err } - attachment_service.AddAttachmentsToCleanQueue(ctx, toBeCleanedAttachments) + storagecleanup.AddDeletionsToCleanQueue(ctx, toBeCleanedDeletions) // delete pull request related git data if issue.IsPull && gitRepo != nil { @@ -258,9 +258,9 @@ func GetRefEndNamesAndURLs(issues []*issues_model.Issue, repoLink string) (map[i } // deleteIssue deletes the issue -func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachments bool) ([]*repo_model.Attachment, error) { - return db.WithTx2(ctx, func(ctx context.Context) ([]*repo_model.Attachment, error) { - toBeCleanedAttachments := make([]*repo_model.Attachment, 0) +func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachments bool) ([]int64, error) { + return db.WithTx2(ctx, func(ctx context.Context) ([]int64, error) { + toBeCleanedDeletions := make([]int64, 0) if _, err := db.GetEngine(ctx).ID(issue.ID).NoAutoCondition().Delete(issue); err != nil { return nil, err } @@ -315,11 +315,12 @@ func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachmen } for _, comment := range issue.Comments { - if err := deleteComment(ctx, comment, deleteAttachments); err != nil { + deletions, err := deleteComment(ctx, comment, deleteAttachments) + if err != nil { return nil, fmt.Errorf("deleteComment [comment_id: %d]: %w", comment.ID, err) } if deleteAttachments { - toBeCleanedAttachments = append(toBeCleanedAttachments, comment.Attachments...) + toBeCleanedDeletions = append(toBeCleanedDeletions, deletions...) } } @@ -328,41 +329,42 @@ func deleteIssue(ctx context.Context, issue *issues_model.Issue, deleteAttachmen if err := issue.LoadAttachments(ctx); err != nil { return nil, err } - if _, err := repo_model.MarkAttachmentsDeleted(ctx, issue.Attachments); err != nil { + deletions, err := repo_model.DeleteAttachments(ctx, issue.Attachments) + if err != nil { return nil, err } - toBeCleanedAttachments = append(toBeCleanedAttachments, issue.Attachments...) + toBeCleanedDeletions = append(toBeCleanedDeletions, deletions...) } - return toBeCleanedAttachments, nil + return toBeCleanedDeletions, nil }) } // DeleteOrphanedIssues delete issues without a repo func DeleteOrphanedIssues(ctx context.Context) error { - toBeCleanedAttachments := make([]*repo_model.Attachment, 0) + toBeCleanedDeletions := make([]int64, 0) if err := db.WithTx(ctx, func(ctx context.Context) error { repoIDs, err := issues_model.GetOrphanedIssueRepoIDs(ctx) if err != nil { return err } for i := range repoIDs { - toBeCleanedIssueAttachments, err := DeleteIssuesByRepoID(ctx, repoIDs[i], true) + deletions, err := DeleteIssuesByRepoID(ctx, repoIDs[i], true) if err != nil { return err } - toBeCleanedAttachments = append(toBeCleanedAttachments, toBeCleanedIssueAttachments...) + toBeCleanedDeletions = append(toBeCleanedDeletions, deletions...) } return nil }); err != nil { return err } - attachment_service.AddAttachmentsToCleanQueue(ctx, toBeCleanedAttachments) + storagecleanup.AddDeletionsToCleanQueue(ctx, toBeCleanedDeletions) return nil } // DeleteIssuesByRepoID deletes issues by repositories id -func DeleteIssuesByRepoID(ctx context.Context, repoID int64, deleteAttachments bool) ([]*repo_model.Attachment, error) { - toBeCleanedAttachments := make([]*repo_model.Attachment, 0) +func DeleteIssuesByRepoID(ctx context.Context, repoID int64, deleteAttachments bool) ([]int64, error) { + toBeCleanedDeletions := make([]int64, 0) for { issues := make([]*issues_model.Issue, 0, db.DefaultMaxInSize) if err := db.GetEngine(ctx). @@ -378,13 +380,13 @@ func DeleteIssuesByRepoID(ctx context.Context, repoID int64, deleteAttachments b } for _, issue := range issues { - toBeCleanedIssueAttachments, err := deleteIssue(ctx, issue, deleteAttachments) + deletions, err := deleteIssue(ctx, issue, deleteAttachments) if err != nil { return nil, fmt.Errorf("deleteIssue [issue_id: %d]: %w", issue.ID, err) } - toBeCleanedAttachments = append(toBeCleanedAttachments, toBeCleanedIssueAttachments...) + toBeCleanedDeletions = append(toBeCleanedDeletions, deletions...) } } - return toBeCleanedAttachments, nil + return toBeCleanedDeletions, nil } diff --git a/services/issue/issue_test.go b/services/issue/issue_test.go index 5bf8426d7b982..780f25ad0092b 100644 --- a/services/issue/issue_test.go +++ b/services/issue/issue_test.go @@ -11,7 +11,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" - attachment_service "code.gitea.io/gitea/services/attachment" + "code.gitea.io/gitea/services/storagecleanup" "github.com/stretchr/testify/assert" ) @@ -45,9 +45,9 @@ func TestIssue_DeleteIssue(t *testing.T) { ID: issueIDs[2], } - toBeCleanedAttachments, err := deleteIssue(db.DefaultContext, issue, true) + toBeCleanedDeletions, err := deleteIssue(db.DefaultContext, issue, true) assert.NoError(t, err) - attachment_service.AddAttachmentsToCleanQueue(db.DefaultContext, toBeCleanedAttachments) + storagecleanup.AddDeletionsToCleanQueue(db.DefaultContext, toBeCleanedDeletions) issueIDs, err = issues_model.GetIssueIDsByRepoID(db.DefaultContext, 1) assert.NoError(t, err) assert.Len(t, issueIDs, 4) @@ -57,9 +57,9 @@ func TestIssue_DeleteIssue(t *testing.T) { assert.NoError(t, err) issue, err = issues_model.GetIssueByID(db.DefaultContext, 4) assert.NoError(t, err) - toBeCleanedAttachments, err = deleteIssue(db.DefaultContext, issue, true) + toBeCleanedDeletions, err = deleteIssue(db.DefaultContext, issue, true) assert.NoError(t, err) - attachment_service.AddAttachmentsToCleanQueue(db.DefaultContext, toBeCleanedAttachments) + storagecleanup.AddDeletionsToCleanQueue(db.DefaultContext, toBeCleanedDeletions) assert.Len(t, attachments, 2) for i := range attachments { attachment, err := repo_model.GetAttachmentByUUID(db.DefaultContext, attachments[i].UUID) @@ -81,9 +81,9 @@ func TestIssue_DeleteIssue(t *testing.T) { assert.NoError(t, err) assert.False(t, left) - toBeCleanedAttachments, err = deleteIssue(db.DefaultContext, issue2, true) + toBeCleanedDeletions, err = deleteIssue(db.DefaultContext, issue2, true) assert.NoError(t, err) - attachment_service.AddAttachmentsToCleanQueue(db.DefaultContext, toBeCleanedAttachments) + storagecleanup.AddDeletionsToCleanQueue(db.DefaultContext, toBeCleanedDeletions) left, err = issues_model.IssueNoDependenciesLeft(db.DefaultContext, issue1) assert.NoError(t, err) assert.True(t, left) diff --git a/services/issue/main_test.go b/services/issue/main_test.go index 544cf2aad7eb4..2a57a09a87906 100644 --- a/services/issue/main_test.go +++ b/services/issue/main_test.go @@ -8,7 +8,7 @@ import ( "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/services/attachment" + "code.gitea.io/gitea/services/storagecleanup" _ "code.gitea.io/gitea/models" _ "code.gitea.io/gitea/models/actions" @@ -18,7 +18,7 @@ func TestMain(m *testing.M) { unittest.MainTest(m, &unittest.TestOptions{ SetUp: func() error { setting.LoadQueueSettings() - return attachment.Init() + return storagecleanup.Init() }, }) } diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 076e494b4c37f..75eb06d01fa3f 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -323,7 +323,6 @@ func (g *GiteaLocalUploader) CreateReleases(ctx context.Context, releases ...*ba DownloadCount: int64(*asset.DownloadCount), Size: int64(*asset.Size), CreatedUnix: timeutil.TimeStamp(asset.Created.Unix()), - Status: db.FileStatusNormal, } // SECURITY: We cannot check the DownloadURL and DownloadFunc are safe here diff --git a/services/release/release.go b/services/release/release.go index 417b16ce51e9d..f6502a307678f 100644 --- a/services/release/release.go +++ b/services/release/release.go @@ -21,8 +21,8 @@ import ( "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" - attachment_service "code.gitea.io/gitea/services/attachment" notify_service "code.gitea.io/gitea/services/notify" + "code.gitea.io/gitea/services/storagecleanup" ) // ErrInvalidTagName represents a "InvalidTagName" kind of error. @@ -289,6 +289,7 @@ func UpdateRelease(ctx context.Context, doer *user_model.User, gitRepo *git.Repo deletedUUIDs := make(container.Set[string]) deletedAttachments := make([]*repo_model.Attachment, 0, len(delAttachmentUUIDs)) + toBeCleanedDeletions := make([]int64, 0, len(delAttachmentUUIDs)) if len(delAttachmentUUIDs) > 0 { // Check attachments attachments, err := repo_model.GetAttachmentsByUUIDs(ctx, delAttachmentUUIDs) @@ -303,9 +304,11 @@ func UpdateRelease(ctx context.Context, doer *user_model.User, gitRepo *git.Repo deletedAttachments = append(deletedAttachments, attach) } - if _, err := repo_model.MarkAttachmentsDeleted(ctx, deletedAttachments); err != nil { + deletions, err := repo_model.DeleteAttachments(ctx, deletedAttachments) + if err != nil { return fmt.Errorf("DeleteAttachments [uuids: %v]: %w", deletedUUIDs.Values(), err) } + toBeCleanedDeletions = append(toBeCleanedDeletions, deletions...) // files will be deleted after database transaction is committed successfully } @@ -341,7 +344,7 @@ func UpdateRelease(ctx context.Context, doer *user_model.User, gitRepo *git.Repo return err } - attachment_service.AddAttachmentsToCleanQueue(ctx, deletedAttachments) + storagecleanup.AddDeletionsToCleanQueue(ctx, toBeCleanedDeletions) if !rel.IsDraft { if !isTagCreated && !isConvertedFromTag { @@ -355,6 +358,7 @@ func UpdateRelease(ctx context.Context, doer *user_model.User, gitRepo *git.Repo // DeleteReleaseByID deletes a release and corresponding Git tag by given ID. func DeleteReleaseByID(ctx context.Context, repo *repo_model.Repository, rel *repo_model.Release, doer *user_model.User, delTag bool) error { + var toBeCleanedDeletions []int64 if err := db.WithTx(ctx, func(ctx context.Context) error { if delTag { protectedTags, err := git_model.GetProtectedTags(ctx, rel.RepoID) @@ -404,15 +408,17 @@ func DeleteReleaseByID(ctx context.Context, repo *repo_model.Repository, rel *re return fmt.Errorf("LoadAttributes: %w", err) } - if err := repo_model.MarkAttachmentsDeletedByRelease(ctx, rel.ID); err != nil { + deletions, err := repo_model.DeleteAttachments(ctx, rel.Attachments) + if err != nil { return fmt.Errorf("DeleteAttachments: %w", err) } + toBeCleanedDeletions = append(toBeCleanedDeletions, deletions...) return nil }); err != nil { return err } - attachment_service.AddAttachmentsToCleanQueue(ctx, rel.Attachments) + storagecleanup.AddDeletionsToCleanQueue(ctx, toBeCleanedDeletions) if !rel.IsDraft { notify_service.DeleteRelease(ctx, doer, rel) diff --git a/services/release/release_test.go b/services/release/release_test.go index 50da93446a35e..69b8384d1e37b 100644 --- a/services/release/release_test.go +++ b/services/release/release_test.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/services/attachment" + "code.gitea.io/gitea/services/storagecleanup" _ "code.gitea.io/gitea/models/actions" @@ -26,7 +27,7 @@ func TestMain(m *testing.M) { unittest.MainTest(m, &unittest.TestOptions{ SetUp: func() error { setting.LoadQueueSettings() - return attachment.Init() + return storagecleanup.Init() }, }) } diff --git a/services/repository/delete.go b/services/repository/delete.go index 5647843e4a752..ea281c4420800 100644 --- a/services/repository/delete.go +++ b/services/repository/delete.go @@ -29,8 +29,8 @@ import ( "code.gitea.io/gitea/modules/storage" actions_service "code.gitea.io/gitea/services/actions" asymkey_service "code.gitea.io/gitea/services/asymkey" - attachment_service "code.gitea.io/gitea/services/attachment" issue_service "code.gitea.io/gitea/services/issue" + "code.gitea.io/gitea/services/storagecleanup" "xorm.io/builder" ) @@ -76,10 +76,9 @@ func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams } var needRewriteKeysFile bool - releaseAttachments := make([]*repo_model.Attachment, 0, 20) - var repoAttachments []*repo_model.Attachment var archivePaths []string var lfsPaths []string + toBeCleanedDeletions := make([]int64, 0, 20) err = db.WithTx(ctx, func(ctx context.Context) error { // In case owner is a organization, we have to change repo specific teams @@ -116,6 +115,7 @@ func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams } } + releaseAttachments := make([]*repo_model.Attachment, 0, 20) // some attachments have release_id but repo_id = 0 if err = db.GetEngine(ctx).Join("INNER", "`release`", "`release`.id = `attachment`.release_id"). Where("`release`.repo_id = ?", repoID). @@ -123,9 +123,11 @@ func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams return err } - if _, err := repo_model.MarkAttachmentsDeleted(ctx, releaseAttachments); err != nil { + deletions, err := repo_model.DeleteAttachments(ctx, releaseAttachments) + if err != nil { return fmt.Errorf("delete release attachments: %w", err) } + toBeCleanedDeletions = append(toBeCleanedDeletions, deletions...) if _, err := db.Exec(ctx, "UPDATE `user` SET num_stars=num_stars-1 WHERE id IN (SELECT `uid` FROM `star` WHERE repo_id = ?)", repo.ID); err != nil { return err @@ -268,15 +270,18 @@ func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams } } + var repoAttachments []*repo_model.Attachment // Get all attachments with repo_id = repo.ID. some release attachments have repo_id = 0 should be deleted before if err := db.GetEngine(ctx).Where(builder.Eq{ "repo_id": repo.ID, }).Find(&repoAttachments); err != nil { return err } - if _, err := repo_model.MarkAttachmentsDeleted(ctx, repoAttachments); err != nil { + deletions, err = repo_model.DeleteAttachments(ctx, repoAttachments) + if err != nil { return err } + toBeCleanedDeletions = append(toBeCleanedDeletions, deletions...) // unlink packages linked to this repository return packages_model.UnlinkRepositoryFromAllPackages(ctx, repoID) @@ -318,8 +323,7 @@ func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams system_model.RemoveStorageWithNotice(ctx, storage.LFS, "Delete orphaned LFS file", lfsObj) } - attachment_service.AddAttachmentsToCleanQueue(ctx, releaseAttachments) - attachment_service.AddAttachmentsToCleanQueue(ctx, repoAttachments) + storagecleanup.AddDeletionsToCleanQueue(ctx, toBeCleanedDeletions) if len(repo.Avatar) > 0 { if err := storage.RepoAvatars.Delete(repo.CustomAvatarRelativePath()); err != nil { diff --git a/services/repository/main_test.go b/services/repository/main_test.go index 01d04cc10d4f8..a3a9e8774ecd2 100644 --- a/services/repository/main_test.go +++ b/services/repository/main_test.go @@ -8,14 +8,14 @@ import ( "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/services/attachment" + "code.gitea.io/gitea/services/storagecleanup" ) func TestMain(m *testing.M) { unittest.MainTest(m, &unittest.TestOptions{ SetUp: func() error { setting.LoadQueueSettings() - return attachment.Init() + return storagecleanup.Init() }, }) } diff --git a/services/storagecleanup/storagecleanup.go b/services/storagecleanup/storagecleanup.go new file mode 100644 index 0000000000000..c9419b826fef3 --- /dev/null +++ b/services/storagecleanup/storagecleanup.go @@ -0,0 +1,116 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package storagecleanup + +import ( + "context" + "errors" + "fmt" + "os" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/system" + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/queue" + "code.gitea.io/gitea/modules/storage" +) + +var cleanQueue *queue.WorkerPoolQueue[int64] + +func Init() error { + cleanQueue = queue.CreateSimpleQueue(graceful.GetManager().ShutdownContext(), "storage-cleanup", handler) + if cleanQueue == nil { + return errors.New("Unable to create storage-cleanup queue") + } + go graceful.GetManager().RunWithCancel(cleanQueue) + return nil +} + +// AddDeletionsToCleanQueue adds the attachments to the clean queue for deletion. +func AddDeletionsToCleanQueue(ctx context.Context, deletionIDs []int64) { + for _, id := range deletionIDs { + if err := cleanQueue.Push(id); err != nil { + log.Error("Failed to push deletion ID %d to clean queue: %v", id, err) + continue + } + } +} + +func handler(deletionIDs ...int64) []int64 { + return cleanupDeletions(graceful.GetManager().ShutdownContext(), deletionIDs) +} + +func cleanupDeletions(ctx context.Context, deletionIDs []int64) []int64 { + var failed []int64 + for _, deletionID := range deletionIDs { + deletion, exist, err := db.GetByID[system.StoragePathDeletion](ctx, deletionID) + if err != nil { + log.Error("Failed to get deletion by ID %d: %v", deletionID, err) + continue + } + if !exist { + continue + } + + theStorage, err := storage.GetStorageByName(deletion.StorageName) + if err != nil { + log.Error("Failed to get storage by name %s: %v", deletion.StorageName, err) + continue + } + if err := theStorage.Delete(deletion.RelativePath); err != nil { + if !errors.Is(err, os.ErrNotExist) { + log.Error("delete pending deletion[relative path: %s] failed: %v", deletion.RelativePath, err) + failed = append(failed, deletion.ID) + if deletion.DeleteFailedCount%3 == 0 { + _ = system.CreateNotice(ctx, system.NoticeRepository, fmt.Sprintf("Failed to delete pending deletion %s (%d times): %v", deletion.RelativePath, deletion.DeleteFailedCount+1, err)) + } + if err := system.UpdateDeletionFailure(ctx, deletion, err); err != nil { + log.Error("Failed to update deletion failure for ID %d: %v", deletion.ID, err) + } + continue + } + } + if _, err := db.DeleteByID[system.StoragePathDeletion](ctx, deletion.ID); err != nil { + log.Error("Failed to delete pending deletion by ID %d(will be tried later): %v", deletion.ID, err) + failed = append(failed, deletion.ID) + } else { + log.Trace("Pending deletion %s deleted from database", deletion.RelativePath) + } + } + return failed +} + +// ScanToBeDeletedFilesOrDir scans for files or directories that are marked as to be deleted and send to +// clean queue +func ScanToBeDeletedFilesOrDir(ctx context.Context) error { + deletionIDs := make([]int64, 0, 100) + lastID := int64(0) + for { + if err := db.GetEngine(ctx). + Select("id"). + // use the status and id index to speed up the query + Where("id > ?", lastID). + Asc("id"). + Limit(100). + Find(&deletionIDs); err != nil { + return fmt.Errorf("scan to-be-deleted files or directories: %w", err) + } + + if len(deletionIDs) == 0 { + log.Trace("No more files or directories to be deleted") + break + } + for _, id := range deletionIDs { + if err := cleanQueue.Push(id); err != nil { + log.Error("Failed to push deletion ID %d to clean queue: %v", id, err) + } + } + + lastID = deletionIDs[len(deletionIDs)-1] + deletionIDs = deletionIDs[0:0] + } + + return nil +} diff --git a/services/user/delete.go b/services/user/delete.go index fb8ea392f94f7..89a4a3e43edea 100644 --- a/services/user/delete.go +++ b/services/user/delete.go @@ -28,8 +28,8 @@ import ( ) // deleteUser deletes models associated to an user. -func deleteUser(ctx context.Context, u *user_model.User, purge bool) (toBeCleanedAttachments []*repo_model.Attachment, err error) { - toBeCleanedAttachments = make([]*repo_model.Attachment, 0) +func deleteUser(ctx context.Context, u *user_model.User, purge bool) (toBeCleanedDeletions []int64, err error) { + toBeCleanedDeletions = make([]int64, 0) // ***** START: Watch ***** watchedRepoIDs, err := db.FindIDs(ctx, "watch", "watch.repo_id", @@ -126,10 +126,11 @@ func deleteUser(ctx context.Context, u *user_model.User, purge bool) (toBeCleane return nil, err } - if _, err := repo_model.MarkAttachmentsDeleted(ctx, comment.Attachments); err != nil { + pendingDeletions, err := repo_model.DeleteAttachments(ctx, comment.Attachments) + if err != nil { return nil, err } - toBeCleanedAttachments = append(toBeCleanedAttachments, comment.Attachments...) + toBeCleanedDeletions = append(toBeCleanedDeletions, pendingDeletions...) } } @@ -207,5 +208,5 @@ func deleteUser(ctx context.Context, u *user_model.User, purge bool) (toBeCleane return nil, fmt.Errorf("delete: %w", err) } - return toBeCleanedAttachments, nil + return toBeCleanedDeletions, nil } diff --git a/services/user/user.go b/services/user/user.go index ef2d27fffd528..c9aab51ecd961 100644 --- a/services/user/user.go +++ b/services/user/user.go @@ -24,11 +24,11 @@ import ( "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/agit" asymkey_service "code.gitea.io/gitea/services/asymkey" - attachment_service "code.gitea.io/gitea/services/attachment" org_service "code.gitea.io/gitea/services/org" "code.gitea.io/gitea/services/packages" container_service "code.gitea.io/gitea/services/packages/container" repo_service "code.gitea.io/gitea/services/repository" + "code.gitea.io/gitea/services/storagecleanup" ) // RenameUser renames a user @@ -211,7 +211,7 @@ func DeleteUser(ctx context.Context, u *user_model.User, purge bool) error { } } - toBeCleanedAttachments, err := db.WithTx2(ctx, func(ctx context.Context) ([]*repo_model.Attachment, error) { + toBeCleanedDeletions, err := db.WithTx2(ctx, func(ctx context.Context) ([]int64, error) { // Note: A user owns any repository or belongs to any organization // cannot perform delete operation. This causes a race with the purge above // however consistency requires that we ensure that this is the case @@ -239,17 +239,17 @@ func DeleteUser(ctx context.Context, u *user_model.User, purge bool) error { return nil, packages_model.ErrUserOwnPackages{UID: u.ID} } - toBeCleanedAttachments, err := deleteUser(ctx, u, purge) + toBeCleanedDeletions, err := deleteUser(ctx, u, purge) if err != nil { return nil, fmt.Errorf("DeleteUser: %w", err) } - return toBeCleanedAttachments, nil + return toBeCleanedDeletions, nil }) if err != nil { return err } - attachment_service.AddAttachmentsToCleanQueue(ctx, toBeCleanedAttachments) + storagecleanup.AddDeletionsToCleanQueue(ctx, toBeCleanedDeletions) if err = asymkey_service.RewriteAllPublicKeys(ctx); err != nil { return err diff --git a/services/user/user_test.go b/services/user/user_test.go index f687095135e18..868cd08c4ea00 100644 --- a/services/user/user_test.go +++ b/services/user/user_test.go @@ -17,8 +17,8 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/services/attachment" org_service "code.gitea.io/gitea/services/org" + "code.gitea.io/gitea/services/storagecleanup" "github.com/stretchr/testify/assert" ) @@ -27,7 +27,7 @@ func TestMain(m *testing.M) { unittest.MainTest(m, &unittest.TestOptions{ SetUp: func() error { setting.LoadQueueSettings() - return attachment.Init() + return storagecleanup.Init() }, }) } From a15917611c360a1af52fe3e394317794aa29eb03 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 20 Jul 2025 22:52:34 -0700 Subject: [PATCH 34/36] Some improvements --- models/fixtures/attachment.yml | 15 +-------------- services/attachment/attachment_test.go | 21 +++++++++++++++++++-- services/issue/comments.go | 1 - services/storagecleanup/storagecleanup.go | 3 ++- services/user/delete.go | 4 ++-- 5 files changed, 24 insertions(+), 20 deletions(-) diff --git a/models/fixtures/attachment.yml b/models/fixtures/attachment.yml index b86a15b28269f..1c1eb402befc8 100644 --- a/models/fixtures/attachment.yml +++ b/models/fixtures/attachment.yml @@ -99,7 +99,7 @@ comment_id: 0 name: attach1 download_count: 0 - size: 0 + size: 29 created_unix: 946684800 - @@ -153,16 +153,3 @@ download_count: 0 size: 0 created_unix: 946684800 - -- - id: 13 - uuid: a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a23 - repo_id: 1 - issue_id: 3 - release_id: 0 - uploader_id: 0 - comment_id: 7 - name: code_comment_uploaded_attachment.png - download_count: 0 - size: 0 - created_unix: 946684812 diff --git a/services/attachment/attachment_test.go b/services/attachment/attachment_test.go index 937d4cba7d812..da45a8079f38b 100644 --- a/services/attachment/attachment_test.go +++ b/services/attachment/attachment_test.go @@ -6,13 +6,16 @@ package attachment import ( "os" "path/filepath" + "strings" "testing" + "time" "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/services/storagecleanup" _ "code.gitea.io/gitea/models/actions" @@ -55,12 +58,26 @@ func TestUploadAttachment(t *testing.T) { func TestDeleteAttachments(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) attachment8 := unittest.AssertExistsAndLoadBean(t, &repo_model.Attachment{ID: 8}) + const attachment8Content = "test content for attachment 8" // 29 bytes + _, err := storage.Attachments.Save(attachment8.RelativePath(), strings.NewReader(attachment8Content), int64(len(attachment8Content))) + assert.NoError(t, err) + + fileInfo, err := storage.Attachments.Stat(attachment8.RelativePath()) + assert.NoError(t, err) + assert.Equal(t, attachment8.Size, fileInfo.Size()) - err := DeleteAttachment(db.DefaultContext, attachment8) + err = DeleteAttachment(db.DefaultContext, attachment8) assert.NoError(t, err) - attachment, err := repo_model.GetAttachmentByUUID(db.DefaultContext, "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a18") + attachment, err := repo_model.GetAttachmentByUUID(db.DefaultContext, attachment8.UUID) assert.Error(t, err) assert.True(t, repo_model.IsErrAttachmentNotExist(err)) assert.Nil(t, attachment) + + // allow the queue to process the deletion + time.Sleep(1 * time.Second) + + _, err = storage.Attachments.Stat(attachment8.RelativePath()) + assert.Error(t, err) + assert.True(t, os.IsNotExist(err)) } diff --git a/services/issue/comments.go b/services/issue/comments.go index cfcfe78dafa0b..611e3ed4c3b39 100644 --- a/services/issue/comments.go +++ b/services/issue/comments.go @@ -141,7 +141,6 @@ func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAtt } } - // deletedReviewComment should be a review comment with no content and no attachments if err := issues_model.DeleteComment(ctx, comment); err != nil { return nil, err } diff --git a/services/storagecleanup/storagecleanup.go b/services/storagecleanup/storagecleanup.go index c9419b826fef3..ffa9f2164bf9d 100644 --- a/services/storagecleanup/storagecleanup.go +++ b/services/storagecleanup/storagecleanup.go @@ -48,6 +48,7 @@ func cleanupDeletions(ctx context.Context, deletionIDs []int64) []int64 { deletion, exist, err := db.GetByID[system.StoragePathDeletion](ctx, deletionID) if err != nil { log.Error("Failed to get deletion by ID %d: %v", deletionID, err) + failed = append(failed, deletionID) continue } if !exist { @@ -57,6 +58,7 @@ func cleanupDeletions(ctx context.Context, deletionIDs []int64) []int64 { theStorage, err := storage.GetStorageByName(deletion.StorageName) if err != nil { log.Error("Failed to get storage by name %s: %v", deletion.StorageName, err) + failed = append(failed, deletionID) continue } if err := theStorage.Delete(deletion.RelativePath); err != nil { @@ -90,7 +92,6 @@ func ScanToBeDeletedFilesOrDir(ctx context.Context) error { for { if err := db.GetEngine(ctx). Select("id"). - // use the status and id index to speed up the query Where("id > ?", lastID). Asc("id"). Limit(100). diff --git a/services/user/delete.go b/services/user/delete.go index 89a4a3e43edea..ef58db1ed383b 100644 --- a/services/user/delete.go +++ b/services/user/delete.go @@ -100,12 +100,12 @@ func deleteUser(ctx context.Context, u *user_model.User, purge bool) (toBeCleane } if err := auth_model.DeleteOAuth2RelictsByUserID(ctx, u.ID); err != nil { - return nil, fmt.Errorf("deleteOAuth2RelictsByUserID: %w", err) + return nil, err } if purge || (setting.Service.UserDeleteWithCommentsMaxTime != 0 && u.CreatedUnix.AsTime().Add(setting.Service.UserDeleteWithCommentsMaxTime).After(time.Now())) { - // Delete Comments with attachments + // Delete Comments const batchSize = 50 for { comments := make([]*issues_model.Comment, 0, batchSize) From 9b164e0724c8bd67a475e4534a49eeb5aef9fa53 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 21 Jul 2025 08:44:21 -0700 Subject: [PATCH 35/36] Fix comment --- services/issue/comments.go | 1 - 1 file changed, 1 deletion(-) diff --git a/services/issue/comments.go b/services/issue/comments.go index 611e3ed4c3b39..b9213a10fa4c2 100644 --- a/services/issue/comments.go +++ b/services/issue/comments.go @@ -146,7 +146,6 @@ func deleteComment(ctx context.Context, comment *issues_model.Comment, removeAtt } if removeAttachments { - // mark comment attachments as deleted return repo_model.DeleteAttachments(ctx, comment.Attachments) } return nil, nil From 9671f2e7a7ffaddf7d56084833dd3e597091738b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 26 Jul 2025 11:55:43 -0700 Subject: [PATCH 36/36] revert unrelated change --- models/issues/issue_update.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/models/issues/issue_update.go b/models/issues/issue_update.go index 0c2e61e828099..1c16817491826 100644 --- a/models/issues/issue_update.go +++ b/models/issues/issue_update.go @@ -266,11 +266,6 @@ func UpdateIssueAttachments(ctx context.Context, issueID int64, uuids []string) return fmt.Errorf("getAttachmentsByUUIDs [uuids: %v]: %w", uuids, err) } for i := range attachments { - if attachments[i].IssueID == issueID { - continue - } else if attachments[i].IssueID != 0 { - return util.NewPermissionDeniedErrorf("update issue attachments permission denied") - } attachments[i].IssueID = issueID if err := repo_model.UpdateAttachment(ctx, attachments[i]); err != nil { return fmt.Errorf("update attachment [id: %d]: %w", attachments[i].ID, err)