nonshy/website
nonshy/website
1
0
Fork 0
Code Issues 21 Pull Requests 1 Packages Projects Releases Wiki Activity

Database cleanup tasks

Browse Source
This commit is contained in:
Noah Petherbridge 2024-07-25 22:39:11 -07:00
parent 188e2e147c
commit 40b1f2f57a
6 changed files with 150 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
cmd/nonshy/main.go
View File

@ -242,6 +242,20 @@ func main() {
},
},
},
{
Name: "vacuum",
Usage: "Run database maintenance tasks (clean up broken links, remove orphaned comment photos, etc.) for data consistency.",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "dryrun",
Usage: "don't actually delete anything",
},
},
Action: func(c *cli.Context) error {
initdb(c)
return worker.Vacuum(c.Bool("dryrun"))
},
},
},
}

31
pkg/controller/api/orphaned_comment_photos.go
View File

@ -1,12 +1,10 @@
package api
import (
"fmt"
"net/http"
"code.nonshy.com/nonshy/website/pkg/config"
"code.nonshy.com/nonshy/website/pkg/models"
"code.nonshy.com/nonshy/website/pkg/photo"
"code.nonshy.com/nonshy/website/pkg/worker"
)
// RemoveOrphanedCommentPhotos API.
@ -24,7 +22,7 @@ func RemoveOrphanedCommentPhotos() http.HandlerFunc {
OK bool `json:"OK"`
Error string `json:"error,omitempty"`
Total int64 `json:"total"`
Removed int `json:"removed"`
Removed int64 `json:"removed"`
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -56,38 +54,19 @@ func RemoveOrphanedCommentPhotos() http.HandlerFunc {
}
// Do the needful.
photos, total, err := models.GetOrphanedCommentPhotos()
total, err := worker.VacuumOrphanedCommentPhotos(false)
if err != nil {
SendJSON(w, http.StatusInternalServerError, Response{
OK: false,
Error: fmt.Sprintf("GetOrphanedCommentPhotos: %s", err),
Error: err.Error(),
})
return
}
for _, row := range photos {
if err := photo.Delete(row.Filename); err != nil {
SendJSON(w, http.StatusInternalServerError, Response{
OK: false,
Error: fmt.Sprintf("Photo ID %d: removing file %s: %s", row.ID, row.Filename, err),
})
return
}
if err := row.Delete(); err != nil {
SendJSON(w, http.StatusInternalServerError, Response{
OK: false,
Error: fmt.Sprintf("DeleteOrphanedCommentPhotos(%d): %s", row.ID, err),
})
return
}
}
// Send success response.
SendJSON(w, http.StatusOK, Response{
OK: true,
Total: total,
Removed: len(photos),
Removed: total,
})
})
}

8
pkg/controller/photo/site_gallery.go
View File

@ -64,13 +64,18 @@ func SiteGallery() http.HandlerFunc {
// They didn't post a "Whose photos" filter, restore it from their last saved default.
who = currentUser.GetProfileField("site_gallery_default")
}
if who != "friends" && who != "everybody" && who != "friends+private" && who != "likes" {
if who != "friends" && who != "everybody" && who != "friends+private" && who != "likes" && who != "uncertified" {
// Default Who setting should be Friends-only, unless you have no friends.
if myFriendCount > 0 {
who = "friends"
} else {
who = "everybody"
}
// Admin only who option.
if who == "uncertified" && !currentUser.HasAdminScope(config.ScopePhotoModerator) {
who = "friends"
}
}
// Store their "Whose photos" filter on their page to default it for next time.
@ -95,6 +100,7 @@ func SiteGallery() http.HandlerFunc {
FriendsOnly: who == "friends",
IsShy: isShy || who == "friends+private",
MyLikes: who == "likes",
Uncertified: who == "uncertified",
}, pager)
// Bulk load the users associated with these photos.

34
pkg/models/photo.go
View File

@ -174,6 +174,28 @@ func CountPhotos(userID uint64) int64 {
return count
}
// GetOrphanedPhotos gets all photos having no user ID associated.
func GetOrphanedPhotos() ([]*Photo, int64, error) {
var (
count int64
ps = []*Photo{}
)
query := DB.Model(&Photo{}).Where(`
NOT EXISTS (
SELECT 1 FROM users WHERE users.id = photos.user_id
)
OR photos.user_id = 0
`)
query.Count(&count)
res := query.Find(&ps)
if res.Error != nil {
return nil, 0, res.Error
}
return ps, count, res.Error
}
/*
IsSiteGalleryThrottled returns whether the user is throttled from marking additional pictures for the Site Gallery.
@ -477,6 +499,7 @@ type Gallery struct {
IsShy bool // Current user is like a Shy Account (or: show self/friends and private photo grants only)
FriendsOnly bool // Only show self/friends instead of everybody's pics
MyLikes bool // Filter to photos I have liked
Uncertified bool // Filter for non-certified members only
}
/*
@ -627,9 +650,15 @@ func PaginateGalleryPhotos(user *User, conf Gallery, pager *Pagination) ([]*Phot
}
// Only certified (and not banned) user photos.
if conf.Uncertified {
wheres = append(wheres,
"EXISTS (SELECT 1 FROM users WHERE id = photos.user_id AND certified IS NOT true AND status='active')",
)
} else {
wheres = append(wheres,
"EXISTS (SELECT 1 FROM users WHERE id = photos.user_id AND certified = true AND status='active')",
)
}
// Exclude private users' photos.
wheres = append(wheres,
@ -647,6 +676,11 @@ func PaginateGalleryPhotos(user *User, conf Gallery, pager *Pagination) ([]*Phot
if filterExplicit != "" {
query = query.Where("explicit = ?", filterExplicit == "true")
}
if conf.Uncertified {
query = query.Where(
"EXISTS (SELECT 1 FROM users WHERE id = photos.user_id AND certified IS NOT true AND status='active')",
)
}
} else {
query = DB.Where(
strings.Join(wheres, " AND "),

84
pkg/worker/vacuum.go Normal file
View File

@ -0,0 +1,84 @@
package worker
import (
"fmt"
"code.nonshy.com/nonshy/website/pkg/log"
"code.nonshy.com/nonshy/website/pkg/models"
"code.nonshy.com/nonshy/website/pkg/photo"
)
// Vacuum runs database cleanup tasks for data consistency. Run it like `nonshy vacuum` from the CLI.
func Vacuum(dryrun bool) error {
log.Warn("Vacuum: Orphaned Comment Photos")
if total, err := VacuumOrphanedCommentPhotos(dryrun); err != nil {
log.Error("Orphaned Comment Photos: %s", err)
} else {
log.Info("Removed %d photo(s)", total)
}
log.Warn("Vacuum: Orphaned Gallery Photos")
if total, err := VacuumOrphanedPhotos(dryrun); err != nil {
log.Error("Orphaned Gallery Photos: %s", err)
} else {
log.Info("Removed %d photo(s)", total)
}
return nil
}
// VacuumOrphanedPhotos removes any lingering photo from failed account deletion.
func VacuumOrphanedPhotos(dryrun bool) (int64, error) {
photos, count, err := models.GetOrphanedPhotos()
if err != nil {
return count, err
}
if dryrun {
return count, nil
}
for _, row := range photos {
log.Info(" #%d: %s", row.ID, row.Filename)
if err := photo.Delete(row.Filename); err != nil {
return count, fmt.Errorf("photo ID %d: removing file %s: %s", row.ID, row.Filename, err)
}
if row.CroppedFilename != "" {
if err := photo.Delete(row.CroppedFilename); err != nil {
return count, fmt.Errorf("photo ID %d: removing file %s: %s", row.ID, row.Filename, err)
}
}
if err := row.Delete(); err != nil {
return count, fmt.Errorf("deleting orphaned photo (%d): %s", row.ID, err)
}
}
return count, nil
}
// VacuumOrphanedCommentPhotos cleans up comment photos that weren't associated to a post, returning the count removed.
func VacuumOrphanedCommentPhotos(dryrun bool) (int64, error) {
// Do the needful.
photos, total, err := models.GetOrphanedCommentPhotos()
if err != nil {
return total, err
}
if dryrun {
return total, nil
}
for _, row := range photos {
if err := photo.Delete(row.Filename); err != nil {
return total, fmt.Errorf("photo ID %d: removing file %s: %s", row.ID, row.Filename, err)
}
if err := row.Delete(); err != nil {
return total, fmt.Errorf("deleting orphaned comment photo (%d): %s", row.ID, err)
}
}
return total, nil
}

3
web/templates/photo/gallery.html
View File

@ -378,6 +378,9 @@
<option value="friends+private"{{if eq .FilterWho "friends+private"}} selected{{end}}>Myself, friends, &amp; private photo grants</option>
<option value="likes"{{if eq .FilterWho "likes"}} selected{{end}}>Photos I have 'liked'</option>
<option value="everybody"{{if eq .FilterWho "everybody"}} selected{{end}}>All certified members</option>
{{if .CurrentUser.HasAdminScope "social.moderator.photo"}}
<option value="uncertified"{{if eq .FilterWho "uncertified"}} selected{{end}}>☮ Non-certified members</option>
{{end}}
</select>
</div>
</div>