diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 31b88a7981..07fad18452 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -425,6 +425,8 @@ var migrations = []Migration{ NewMigration("Add ConfidentialClient column (default true) to OAuth2Application table", addConfidentialClientColumnToOAuth2ApplicationTable), // v231 -> v232 NewMigration("Add index for hook_task", addIndexForHookTask), + // v232 -> v233 + NewMigration("Add package cleanup rule table", createPackageCleanupRuleTable), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v232.go b/models/migrations/v232.go new file mode 100644 index 0000000000..894703c43e --- /dev/null +++ b/models/migrations/v232.go @@ -0,0 +1,29 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/xorm" +) + +func createPackageCleanupRuleTable(x *xorm.Engine) error { + type PackageCleanupRule struct { + ID int64 `xorm:"pk autoincr"` + Enabled bool `xorm:"INDEX NOT NULL DEFAULT false"` + OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL DEFAULT 0"` + Type string `xorm:"UNIQUE(s) INDEX NOT NULL"` + KeepCount int `xorm:"NOT NULL DEFAULT 0"` + KeepPattern string `xorm:"NOT NULL DEFAULT ''"` + RemoveDays int `xorm:"NOT NULL DEFAULT 0"` + RemovePattern string `xorm:"NOT NULL DEFAULT ''"` + MatchFullName bool `xorm:"NOT NULL DEFAULT false"` + CreatedUnix timeutil.TimeStamp `xorm:"created NOT NULL DEFAULT 0"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated NOT NULL DEFAULT 0"` + } + + return x.Sync2(new(PackageCleanupRule)) +} diff --git a/models/packages/package.go b/models/packages/package.go index e39a7c4e41..cea04a0957 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -45,6 +45,21 @@ const ( TypeVagrant Type = "vagrant" ) +var TypeList = []Type{ + TypeComposer, + TypeConan, + TypeContainer, + TypeGeneric, + TypeHelm, + TypeMaven, + TypeNpm, + TypeNuGet, + TypePub, + TypePyPI, + TypeRubyGems, + TypeVagrant, +} + // Name gets the name of the package type func (pt Type) Name() string { switch pt { diff --git a/models/packages/package_cleanup_rule.go b/models/packages/package_cleanup_rule.go new file mode 100644 index 0000000000..dbed7545b8 --- /dev/null +++ b/models/packages/package_cleanup_rule.go @@ -0,0 +1,111 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "context" + "errors" + "fmt" + "regexp" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/builder" +) + +var ErrPackageCleanupRuleNotExist = errors.New("Package blob does not exist") + +func init() { + db.RegisterModel(new(PackageCleanupRule)) +} + +// PackageCleanupRule represents a rule which describes when to clean up package versions +type PackageCleanupRule struct { + ID int64 `xorm:"pk autoincr"` + Enabled bool `xorm:"INDEX NOT NULL DEFAULT false"` + OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL DEFAULT 0"` + Type Type `xorm:"UNIQUE(s) INDEX NOT NULL"` + KeepCount int `xorm:"NOT NULL DEFAULT 0"` + KeepPattern string `xorm:"NOT NULL DEFAULT ''"` + KeepPatternMatcher *regexp.Regexp `xorm:"-"` + RemoveDays int `xorm:"NOT NULL DEFAULT 0"` + RemovePattern string `xorm:"NOT NULL DEFAULT ''"` + RemovePatternMatcher *regexp.Regexp `xorm:"-"` + MatchFullName bool `xorm:"NOT NULL DEFAULT false"` + CreatedUnix timeutil.TimeStamp `xorm:"created NOT NULL DEFAULT 0"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated NOT NULL DEFAULT 0"` +} + +func (pcr *PackageCleanupRule) CompiledPattern() error { + if pcr.KeepPatternMatcher != nil || pcr.RemovePatternMatcher != nil { + return nil + } + + if pcr.KeepPattern != "" { + var err error + pcr.KeepPatternMatcher, err = regexp.Compile(fmt.Sprintf(`(?i)\A%s\z`, pcr.KeepPattern)) + if err != nil { + return err + } + } + + if pcr.RemovePattern != "" { + var err error + pcr.RemovePatternMatcher, err = regexp.Compile(fmt.Sprintf(`(?i)\A%s\z`, pcr.RemovePattern)) + if err != nil { + return err + } + } + + return nil +} + +func InsertCleanupRule(ctx context.Context, pcr *PackageCleanupRule) (*PackageCleanupRule, error) { + _, err := db.GetEngine(ctx).Insert(pcr) + return pcr, err +} + +func GetCleanupRuleByID(ctx context.Context, id int64) (*PackageCleanupRule, error) { + pcr := &PackageCleanupRule{} + + has, err := db.GetEngine(ctx).ID(id).Get(pcr) + if err != nil { + return nil, err + } + if !has { + return nil, ErrPackageCleanupRuleNotExist + } + return pcr, nil +} + +func UpdateCleanupRule(ctx context.Context, pcr *PackageCleanupRule) error { + _, err := db.GetEngine(ctx).ID(pcr.ID).AllCols().Update(pcr) + return err +} + +func GetCleanupRulesByOwner(ctx context.Context, ownerID int64) ([]*PackageCleanupRule, error) { + pcrs := make([]*PackageCleanupRule, 0, 10) + return pcrs, db.GetEngine(ctx).Where("owner_id = ?", ownerID).Find(&pcrs) +} + +func DeleteCleanupRuleByID(ctx context.Context, ruleID int64) error { + _, err := db.GetEngine(ctx).ID(ruleID).Delete(&PackageCleanupRule{}) + return err +} + +func HasOwnerCleanupRuleForPackageType(ctx context.Context, ownerID int64, packageType Type) (bool, error) { + return db.GetEngine(ctx). + Where("owner_id = ? AND type = ?", ownerID, packageType). + Exist(&PackageCleanupRule{}) +} + +func IterateEnabledCleanupRules(ctx context.Context, callback func(context.Context, *PackageCleanupRule) error) error { + return db.Iterate( + ctx, + builder.Eq{"enabled": true}, + callback, + ) +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 1566dfc97d..58b073ca3c 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -86,6 +86,9 @@ remove = Remove remove_all = Remove All edit = Edit +enabled = Enabled +disabled = Disabled + copy = Copy copy_url = Copy URL copy_branch = Copy branch name @@ -3177,3 +3180,23 @@ settings.delete.description = Deleting a package is permanent and cannot be undo settings.delete.notice = You are about to delete %s (%s). This operation is irreversible, are you sure? settings.delete.success = The package has been deleted. settings.delete.error = Failed to delete the package. +owner.settings.cleanuprules.title = Manage Cleanup Rules +owner.settings.cleanuprules.add = Add Cleanup Rule +owner.settings.cleanuprules.edit = Edit Cleanup Rule +owner.settings.cleanuprules.none = No cleanup rules available. Read the docs to learn more. +owner.settings.cleanuprules.preview = Cleanup Rule Preview +owner.settings.cleanuprules.preview.overview = %d packages are scheduled to be removed. +owner.settings.cleanuprules.preview.none = Cleanup rule does not match any packages. +owner.settings.cleanuprules.enabled = Enabled +owner.settings.cleanuprules.pattern_full_match = Apply pattern to full package name +owner.settings.cleanuprules.keep.title = Versions that match these rules are kept, even if they match a removal rule below. +owner.settings.cleanuprules.keep.count = Keep the most recent +owner.settings.cleanuprules.keep.count.1 = 1 version per package +owner.settings.cleanuprules.keep.count.n = %d versions per package +owner.settings.cleanuprules.keep.pattern = Keep versions matching +owner.settings.cleanuprules.keep.pattern.container = The latest version is always kept for Container packages. +owner.settings.cleanuprules.remove.title = Versions that match these rules are removed, unless a rule above says to keep them. +owner.settings.cleanuprules.remove.days = Remove versions older than +owner.settings.cleanuprules.remove.pattern = Remove versions matching +owner.settings.cleanuprules.success.update = Cleanup rule has been updated. +owner.settings.cleanuprules.success.delete = Cleanup rule has been deleted. diff --git a/routers/web/org/setting_packages.go b/routers/web/org/setting_packages.go new file mode 100644 index 0000000000..c7edf4a185 --- /dev/null +++ b/routers/web/org/setting_packages.go @@ -0,0 +1,87 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package org + +import ( + "fmt" + "net/http" + + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/setting" + shared "code.gitea.io/gitea/routers/web/shared/packages" +) + +const ( + tplSettingsPackages base.TplName = "org/settings/packages" + tplSettingsPackagesRuleEdit base.TplName = "org/settings/packages_cleanup_rules_edit" + tplSettingsPackagesRulePreview base.TplName = "org/settings/packages_cleanup_rules_preview" +) + +func Packages(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetPackagesContext(ctx, ctx.ContextUser) + + ctx.HTML(http.StatusOK, tplSettingsPackages) +} + +func PackagesRuleAdd(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetRuleAddContext(ctx) + + ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit) +} + +func PackagesRuleEdit(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetRuleEditContext(ctx, ctx.ContextUser) + + ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit) +} + +func PackagesRuleAddPost(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.PerformRuleAddPost( + ctx, + ctx.ContextUser, + fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name), + tplSettingsPackagesRuleEdit, + ) +} + +func PackagesRuleEditPost(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.PerformRuleEditPost( + ctx, + ctx.ContextUser, + fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name), + tplSettingsPackagesRuleEdit, + ) +} + +func PackagesRulePreview(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetRulePreviewContext(ctx, ctx.ContextUser) + + ctx.HTML(http.StatusOK, tplSettingsPackagesRulePreview) +} diff --git a/routers/web/shared/packages/packages.go b/routers/web/shared/packages/packages.go new file mode 100644 index 0000000000..82cb4d37c6 --- /dev/null +++ b/routers/web/shared/packages/packages.go @@ -0,0 +1,232 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package packages + +import ( + "fmt" + "net/http" + "time" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/web" + "code.gitea.io/gitea/services/forms" + container_service "code.gitea.io/gitea/services/packages/container" +) + +const ( + tplSettingsPackages base.TplName = "user/settings/packages" + tplSettingsPackagesRuleEdit base.TplName = "user/settings/packages_cleanup_rules_edit" + tplSettingsPackagesRulePreview base.TplName = "user/settings/packages_cleanup_rules_preview" +) + +func SetPackagesContext(ctx *context.Context, owner *user_model.User) { + pcrs, err := packages_model.GetCleanupRulesByOwner(ctx, owner.ID) + if err != nil { + ctx.ServerError("GetCleanupRulesByOwner", err) + return + } + + ctx.Data["CleanupRules"] = pcrs +} + +func SetRuleAddContext(ctx *context.Context) { + setRuleEditContext(ctx, nil) +} + +func SetRuleEditContext(ctx *context.Context, owner *user_model.User) { + pcr := getCleanupRuleByContext(ctx, owner) + if pcr == nil { + return + } + + setRuleEditContext(ctx, pcr) +} + +func setRuleEditContext(ctx *context.Context, pcr *packages_model.PackageCleanupRule) { + ctx.Data["IsEditRule"] = pcr != nil + + if pcr == nil { + pcr = &packages_model.PackageCleanupRule{} + } + ctx.Data["CleanupRule"] = pcr + ctx.Data["AvailableTypes"] = packages_model.TypeList +} + +func PerformRuleAddPost(ctx *context.Context, owner *user_model.User, redirectURL string, template base.TplName) { + performRuleEditPost(ctx, owner, nil, redirectURL, template) +} + +func PerformRuleEditPost(ctx *context.Context, owner *user_model.User, redirectURL string, template base.TplName) { + pcr := getCleanupRuleByContext(ctx, owner) + if pcr == nil { + return + } + + form := web.GetForm(ctx).(*forms.PackageCleanupRuleForm) + + if form.Action == "remove" { + if err := packages_model.DeleteCleanupRuleByID(ctx, pcr.ID); err != nil { + ctx.ServerError("DeleteCleanupRuleByID", err) + return + } + + ctx.Flash.Success(ctx.Tr("packages.owner.settings.cleanuprules.success.delete")) + ctx.Redirect(redirectURL) + } else { + performRuleEditPost(ctx, owner, pcr, redirectURL, template) + } +} + +func performRuleEditPost(ctx *context.Context, owner *user_model.User, pcr *packages_model.PackageCleanupRule, redirectURL string, template base.TplName) { + isEditRule := pcr != nil + + if pcr == nil { + pcr = &packages_model.PackageCleanupRule{} + } + + form := web.GetForm(ctx).(*forms.PackageCleanupRuleForm) + + pcr.Enabled = form.Enabled + pcr.OwnerID = owner.ID + pcr.KeepCount = form.KeepCount + pcr.KeepPattern = form.KeepPattern + pcr.RemoveDays = form.RemoveDays + pcr.RemovePattern = form.RemovePattern + pcr.MatchFullName = form.MatchFullName + + ctx.Data["IsEditRule"] = isEditRule + ctx.Data["CleanupRule"] = pcr + ctx.Data["AvailableTypes"] = packages_model.TypeList + + if ctx.HasError() { + ctx.HTML(http.StatusOK, template) + return + } + + if isEditRule { + if err := packages_model.UpdateCleanupRule(ctx, pcr); err != nil { + ctx.ServerError("UpdateCleanupRule", err) + return + } + } else { + pcr.Type = packages_model.Type(form.Type) + + if has, err := packages_model.HasOwnerCleanupRuleForPackageType(ctx, owner.ID, pcr.Type); err != nil { + ctx.ServerError("HasOwnerCleanupRuleForPackageType", err) + return + } else if has { + ctx.Data["Err_Type"] = true + ctx.HTML(http.StatusOK, template) + return + } + + var err error + if pcr, err = packages_model.InsertCleanupRule(ctx, pcr); err != nil { + ctx.ServerError("InsertCleanupRule", err) + return + } + } + + ctx.Flash.Success(ctx.Tr("packages.owner.settings.cleanuprules.success.update")) + ctx.Redirect(fmt.Sprintf("%s/rules/%d", redirectURL, pcr.ID)) +} + +func SetRulePreviewContext(ctx *context.Context, owner *user_model.User) { + pcr := getCleanupRuleByContext(ctx, owner) + if pcr == nil { + return + } + + if err := pcr.CompiledPattern(); err != nil { + ctx.ServerError("CompiledPattern", err) + return + } + + olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays) + + packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type) + if err != nil { + ctx.ServerError("GetPackagesByType", err) + return + } + + versionsToRemove := make([]*packages_model.PackageDescriptor, 0, 10) + + for _, p := range packages { + pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ + PackageID: p.ID, + IsInternal: util.OptionalBoolFalse, + Sort: packages_model.SortCreatedDesc, + Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200), + }) + if err != nil { + ctx.ServerError("SearchVersions", err) + return + } + for _, pv := range pvs { + if skip, err := container_service.ShouldBeSkipped(pcr, p, pv); err != nil { + ctx.ServerError("ShouldBeSkipped", err) + return + } else if skip { + continue + } + + toMatch := pv.LowerVersion + if pcr.MatchFullName { + toMatch = p.LowerName + "/" + pv.LowerVersion + } + + if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) { + continue + } + if pv.CreatedUnix.AsLocalTime().After(olderThan) { + continue + } + if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) { + continue + } + + pd, err := packages_model.GetPackageDescriptor(ctx, pv) + if err != nil { + ctx.ServerError("GetPackageDescriptor", err) + return + } + versionsToRemove = append(versionsToRemove, pd) + } + } + + ctx.Data["CleanupRule"] = pcr + ctx.Data["VersionsToRemove"] = versionsToRemove +} + +func getCleanupRuleByContext(ctx *context.Context, owner *user_model.User) *packages_model.PackageCleanupRule { + id := ctx.FormInt64("id") + if id == 0 { + id = ctx.ParamsInt64("id") + } + + pcr, err := packages_model.GetCleanupRuleByID(ctx, id) + if err != nil { + if err == packages_model.ErrPackageCleanupRuleNotExist { + ctx.NotFound("", err) + } else { + ctx.ServerError("GetCleanupRuleByID", err) + } + return nil + } + + if pcr != nil && pcr.OwnerID == owner.ID { + return pcr + } + + ctx.NotFound("", fmt.Errorf("PackageCleanupRule[%v] not associated to owner %v", id, owner)) + + return nil +} diff --git a/routers/web/user/setting/packages.go b/routers/web/user/setting/packages.go new file mode 100644 index 0000000000..d44e904556 --- /dev/null +++ b/routers/web/user/setting/packages.go @@ -0,0 +1,80 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package setting + +import ( + "net/http" + + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/setting" + shared "code.gitea.io/gitea/routers/web/shared/packages" +) + +const ( + tplSettingsPackages base.TplName = "user/settings/packages" + tplSettingsPackagesRuleEdit base.TplName = "user/settings/packages_cleanup_rules_edit" + tplSettingsPackagesRulePreview base.TplName = "user/settings/packages_cleanup_rules_preview" +) + +func Packages(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetPackagesContext(ctx, ctx.Doer) + + ctx.HTML(http.StatusOK, tplSettingsPackages) +} + +func PackagesRuleAdd(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetRuleAddContext(ctx) + + ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit) +} + +func PackagesRuleEdit(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetRuleEditContext(ctx, ctx.Doer) + + ctx.HTML(http.StatusOK, tplSettingsPackagesRuleEdit) +} + +func PackagesRuleAddPost(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("settings") + ctx.Data["PageIsSettingsPackages"] = true + + shared.PerformRuleAddPost( + ctx, + ctx.Doer, + setting.AppSubURL+"/user/settings/packages", + tplSettingsPackagesRuleEdit, + ) +} + +func PackagesRuleEditPost(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsSettingsPackages"] = true + + shared.PerformRuleEditPost( + ctx, + ctx.Doer, + setting.AppSubURL+"/user/settings/packages", + tplSettingsPackagesRuleEdit, + ) +} + +func PackagesRulePreview(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsSettingsPackages"] = true + + shared.SetRulePreviewContext(ctx, ctx.Doer) + + ctx.HTML(http.StatusOK, tplSettingsPackagesRulePreview) +} diff --git a/routers/web/web.go b/routers/web/web.go index 0b16e756e1..4aff835498 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -443,6 +443,20 @@ func RegisterRoutes(m *web.Route) { m.Combo("/keys").Get(user_setting.Keys). Post(bindIgnErr(forms.AddKeyForm{}), user_setting.KeysPost) m.Post("/keys/delete", user_setting.DeleteKey) + m.Group("/packages", func() { + m.Get("", user_setting.Packages) + m.Group("/rules", func() { + m.Group("/add", func() { + m.Get("", user_setting.PackagesRuleAdd) + m.Post("", bindIgnErr(forms.PackageCleanupRuleForm{}), user_setting.PackagesRuleAddPost) + }) + m.Group("/{id}", func() { + m.Get("", user_setting.PackagesRuleEdit) + m.Post("", bindIgnErr(forms.PackageCleanupRuleForm{}), user_setting.PackagesRuleEditPost) + m.Get("/preview", user_setting.PackagesRulePreview) + }) + }) + }) m.Get("/organization", user_setting.Organization) m.Get("/repos", user_setting.Repos) m.Post("/repos/unadopted", user_setting.AdoptOrDeleteRepository) @@ -751,6 +765,21 @@ func RegisterRoutes(m *web.Route) { }) m.Route("/delete", "GET,POST", org.SettingsDelete) + + m.Group("/packages", func() { + m.Get("", org.Packages) + m.Group("/rules", func() { + m.Group("/add", func() { + m.Get("", org.PackagesRuleAdd) + m.Post("", bindIgnErr(forms.PackageCleanupRuleForm{}), org.PackagesRuleAddPost) + }) + m.Group("/{id}", func() { + m.Get("", org.PackagesRuleEdit) + m.Post("", bindIgnErr(forms.PackageCleanupRuleForm{}), org.PackagesRuleEditPost) + m.Get("/preview", org.PackagesRulePreview) + }) + }) + }) }, func(ctx *context.Context) { ctx.Data["EnableOAuth2"] = setting.OAuth2.Enable }) diff --git a/services/forms/package_form.go b/services/forms/package_form.go new file mode 100644 index 0000000000..6c3ff52a9c --- /dev/null +++ b/services/forms/package_form.go @@ -0,0 +1,31 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package forms + +import ( + "net/http" + + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/web/middleware" + + "gitea.com/go-chi/binding" +) + +type PackageCleanupRuleForm struct { + ID int64 + Enabled bool + Type string `binding:"Required;In(composer,conan,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,vagrant)"` + KeepCount int `binding:"In(0,1,5,10,25,50,100)"` + KeepPattern string `binding:"RegexPattern"` + RemoveDays int `binding:"In(0,7,14,30,60,90,180)"` + RemovePattern string `binding:"RegexPattern"` + MatchFullName bool + Action string `binding:"Required;In(save,remove)"` +} + +func (f *PackageCleanupRuleForm) Validate(req *http.Request, errs binding.Errors) binding.Errors { + ctx := context.GetContext(req) + return middleware.Validate(errs, ctx.Data, f, ctx.Locale) +} diff --git a/services/packages/container/cleanup.go b/services/packages/container/cleanup.go index d23a481f27..e5c8565617 100644 --- a/services/packages/container/cleanup.go +++ b/services/packages/container/cleanup.go @@ -82,6 +82,10 @@ func cleanupExpiredUploadedBlobs(ctx context.Context, olderThan time.Duration) e return nil } +func ShouldBeSkipped(pcr *packages_model.PackageCleanupRule, p *packages_model.Package, pv *packages_model.PackageVersion) (bool, error) { + return pv.LowerVersion == "latest", nil +} + // UpdateRepositoryNames updates the repository name property for all packages of the specific owner func UpdateRepositoryNames(ctx context.Context, owner *user_model.User, newOwnerName string) error { ps, err := packages_model.GetPackagesByType(ctx, owner.ID, packages_model.TypeContainer) diff --git a/services/packages/packages.go b/services/packages/packages.go index 96132eac09..0c72be0902 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -352,13 +352,75 @@ func DeletePackageFile(ctx context.Context, pf *packages_model.PackageFile) erro } // Cleanup removes expired package data -func Cleanup(unused context.Context, olderThan time.Duration) error { +func Cleanup(_ context.Context, olderThan time.Duration) error { ctx, committer, err := db.TxContext() if err != nil { return err } defer committer.Close() + err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error { + if err := pcr.CompiledPattern(); err != nil { + return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err) + } + + olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays) + + packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type) + if err != nil { + return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err) + } + + for _, p := range packages { + pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ + PackageID: p.ID, + IsInternal: util.OptionalBoolFalse, + Sort: packages_model.SortCreatedDesc, + Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200), + }) + if err != nil { + return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err) + } + for _, pv := range pvs { + if skip, err := container_service.ShouldBeSkipped(pcr, p, pv); err != nil { + return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err) + } else if skip { + log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version) + continue + } + + toMatch := pv.LowerVersion + if pcr.MatchFullName { + toMatch = p.LowerName + "/" + pv.LowerVersion + } + + if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) { + log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version) + continue + } + if pv.CreatedUnix.AsLocalTime().After(olderThan) { + log.Debug("Rule[%d]: keep '%s/%s' (remove days)", pcr.ID, p.Name, pv.Version) + continue + } + if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) { + log.Debug("Rule[%d]: do not remove '%s/%s'", pcr.ID, p.Name, pv.Version) + continue + } + + log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version) + + if err := DeletePackageVersionAndReferences(ctx, pv); err != nil { + return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err) + } + } + } + return nil + }) + if err != nil { + log.Error("%#v", err) + return err + } + if err := container_service.Cleanup(ctx, olderThan); err != nil { return err } diff --git a/templates/org/settings/navbar.tmpl b/templates/org/settings/navbar.tmpl index e7cbb87344..dfd2c024ff 100644 --- a/templates/org/settings/navbar.tmpl +++ b/templates/org/settings/navbar.tmpl @@ -17,6 +17,9 @@ {{.locale.Tr "settings.applications"}} {{end}} + + {{.locale.Tr "packages.title"}} + {{.locale.Tr "org.settings.delete"}} diff --git a/templates/org/settings/packages.tmpl b/templates/org/settings/packages.tmpl new file mode 100644 index 0000000000..bb5d95e107 --- /dev/null +++ b/templates/org/settings/packages.tmpl @@ -0,0 +1,14 @@ +{{template "base/head" .}} +
+ {{template "org/header" .}} +
+
+ {{template "org/settings/navbar" .}} +
+ {{template "base/alert" .}} + {{template "package/shared/cleanup_rules/list" .}} +
+
+
+
+{{template "base/footer" .}} diff --git a/templates/org/settings/packages_cleanup_rules_edit.tmpl b/templates/org/settings/packages_cleanup_rules_edit.tmpl new file mode 100644 index 0000000000..8c3725f4d7 --- /dev/null +++ b/templates/org/settings/packages_cleanup_rules_edit.tmpl @@ -0,0 +1,14 @@ +{{template "base/head" .}} +
+ {{template "org/header" .}} +
+
+ {{template "org/settings/navbar" .}} +
+ {{template "base/alert" .}} + {{template "package/shared/cleanup_rules/edit" .}} +
+
+
+
+{{template "base/footer" .}} diff --git a/templates/org/settings/packages_cleanup_rules_preview.tmpl b/templates/org/settings/packages_cleanup_rules_preview.tmpl new file mode 100644 index 0000000000..e0e4652c36 --- /dev/null +++ b/templates/org/settings/packages_cleanup_rules_preview.tmpl @@ -0,0 +1,13 @@ +{{template "base/head" .}} +
+ {{template "org/header" .}} +
+
+ {{template "org/settings/navbar" .}} +
+ {{template "package/shared/cleanup_rules/preview" .}} +
+
+
+
+{{template "base/footer" .}} diff --git a/templates/package/shared/cleanup_rules/edit.tmpl b/templates/package/shared/cleanup_rules/edit.tmpl new file mode 100644 index 0000000000..f8525afb70 --- /dev/null +++ b/templates/package/shared/cleanup_rules/edit.tmpl @@ -0,0 +1,73 @@ +

{{if .IsEditRule}}{{.locale.Tr "packages.owner.settings.cleanuprules.edit"}}{{else}}{{.locale.Tr "packages.owner.settings.cleanuprules.add"}}{{end}}

+
+
+ {{.CsrfTokenHtml}} + +
+
+ + +
+
+
+ + +
+
+
+ + +
+
+
+

{{.locale.Tr "packages.owner.settings.cleanuprules.keep.title"}}

+
+ + +
+
+ + +

{{.locale.Tr "packages.owner.settings.cleanuprules.keep.pattern.container" | Safe}}

+
+
+

{{.locale.Tr "packages.owner.settings.cleanuprules.remove.title"}}

+
+ + +
+
+ + +
+
+ {{if .IsEditRule}} + + + {{.locale.Tr "packages.owner.settings.cleanuprules.preview"}} + {{else}} + + {{end}} +
+
+
diff --git a/templates/package/shared/cleanup_rules/list.tmpl b/templates/package/shared/cleanup_rules/list.tmpl new file mode 100644 index 0000000000..9475a4fabb --- /dev/null +++ b/templates/package/shared/cleanup_rules/list.tmpl @@ -0,0 +1,34 @@ +

+ {{.locale.Tr "packages.owner.settings.cleanuprules.title"}} +
+ {{.locale.Tr "packages.owner.settings.cleanuprules.add"}} +
+

+
+
+ {{range .CleanupRules}} +
+ + {{svg .Type.SVGName 36}} +
+ {{.Type.Name}} +
{{if .Enabled}}{{$.locale.Tr "enabled"}}{{else}}{{$.locale.Tr "disabled"}}{{end}}
+ {{if .KeepCount}}
{{$.locale.Tr "packages.owner.settings.cleanuprules.keep.count"}}: {{if eq .KeepCount 1}}{{$.locale.Tr "packages.owner.settings.cleanuprules.keep.count.1"}}{{else}}{{$.locale.Tr "packages.owner.settings.cleanuprules.keep.count.n" .KeepCount}}{{end}}
{{end}} + {{if .KeepPattern}}
{{$.locale.Tr "packages.owner.settings.cleanuprules.keep.pattern"}}: {{EllipsisString .KeepPattern 100}}
{{end}} + {{if .RemoveDays}}
{{$.locale.Tr "packages.owner.settings.cleanuprules.remove.days"}}: {{$.locale.Tr "tool.days" .RemoveDays}}
{{end}} + {{if .RemovePattern}}
{{$.locale.Tr "packages.owner.settings.cleanuprules.remove.pattern"}}: {{EllipsisString .RemovePattern 100}}
{{end}} +
+
+ {{else}} +
{{.locale.Tr "packages.owner.settings.cleanuprules.none"}}
+ {{end}} +
+
\ No newline at end of file diff --git a/templates/package/shared/cleanup_rules/preview.tmpl b/templates/package/shared/cleanup_rules/preview.tmpl new file mode 100644 index 0000000000..c59ad67f77 --- /dev/null +++ b/templates/package/shared/cleanup_rules/preview.tmpl @@ -0,0 +1,34 @@ +

{{.locale.Tr "packages.owner.settings.cleanuprules.preview"}}

+
+

{{.locale.Tr "packages.owner.settings.cleanuprules.preview.overview" (len .VersionsToRemove)}}

+
+
+ + + + + + + + + + + + + {{range .VersionsToRemove}} + + + + + + + + + {{else}} + + + + {{end}} + +
{{.locale.Tr "admin.packages.type"}}{{.locale.Tr "admin.packages.name"}}{{.locale.Tr "admin.packages.version"}}{{.locale.Tr "admin.packages.creator"}}{{.locale.Tr "admin.packages.size"}}{{.locale.Tr "admin.packages.published"}}
{{.Package.Type.Name}}{{.Package.Name}}{{.Version.Version}}{{.Creator.Name}}{{FileSize .CalculateBlobSize}}
{{.locale.Tr "packages.owner.settings.cleanuprules.preview.none"}}
+
diff --git a/templates/user/settings/navbar.tmpl b/templates/user/settings/navbar.tmpl index 01ae055d79..84a74514bc 100644 --- a/templates/user/settings/navbar.tmpl +++ b/templates/user/settings/navbar.tmpl @@ -18,6 +18,9 @@ {{.locale.Tr "settings.ssh_gpg_keys"}} + + {{.locale.Tr "packages.title"}} + {{.locale.Tr "settings.organization"}} diff --git a/templates/user/settings/packages.tmpl b/templates/user/settings/packages.tmpl new file mode 100644 index 0000000000..2612313454 --- /dev/null +++ b/templates/user/settings/packages.tmpl @@ -0,0 +1,9 @@ +{{template "base/head" .}} +
+ {{template "user/settings/navbar" .}} +
+ {{template "base/alert" .}} + {{template "package/shared/cleanup_rules/list" .}} +
+
+{{template "base/footer" .}} diff --git a/templates/user/settings/packages_cleanup_rules_edit.tmpl b/templates/user/settings/packages_cleanup_rules_edit.tmpl new file mode 100644 index 0000000000..4cf642b7e1 --- /dev/null +++ b/templates/user/settings/packages_cleanup_rules_edit.tmpl @@ -0,0 +1,9 @@ +{{template "base/head" .}} +
+ {{template "user/settings/navbar" .}} +
+ {{template "base/alert" .}} + {{template "package/shared/cleanup_rules/edit" .}} +
+
+{{template "base/footer" .}} diff --git a/templates/user/settings/packages_cleanup_rules_preview.tmpl b/templates/user/settings/packages_cleanup_rules_preview.tmpl new file mode 100644 index 0000000000..20041f9a42 --- /dev/null +++ b/templates/user/settings/packages_cleanup_rules_preview.tmpl @@ -0,0 +1,8 @@ +{{template "base/head" .}} +
+ {{template "user/settings/navbar" .}} +
+ {{template "package/shared/cleanup_rules/preview" .}} +
+
+{{template "base/footer" .}}