diff --git a/.gitignore b/.gitignore index f0947c1..7edf15e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /web/static/photos database.sqlite settings.json +pgdump.sql diff --git a/cmd/nonshy/main.go b/cmd/nonshy/main.go index df72fd1..a8ab1a4 100644 --- a/cmd/nonshy/main.go +++ b/cmd/nonshy/main.go @@ -8,6 +8,7 @@ import ( "code.nonshy.com/nonshy/website/pkg/config" "code.nonshy.com/nonshy/website/pkg/log" "code.nonshy.com/nonshy/website/pkg/models" + "code.nonshy.com/nonshy/website/pkg/models/backfill" "code.nonshy.com/nonshy/website/pkg/redis" "github.com/urfave/cli/v2" "gorm.io/driver/postgres" @@ -133,6 +134,27 @@ func main() { }, }, }, + { + Name: "backfill", + Usage: "One-off maintenance tasks and data backfills for database migrations", + Subcommands: []*cli.Command{ + { + Name: "filesizes", + Usage: "repopulate Filesizes on all photos and comment_photos which have a zero stored in the DB", + Action: func(c *cli.Context) error { + initdb(c) + + log.Info("Running BackfillFilesizes()") + err := backfill.BackfillFilesizes() + if err != nil { + return err + } + + return nil + }, + }, + }, + }, }, } diff --git a/pkg/config/config.go b/pkg/config/config.go index 6f92b64..a4500c9 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -86,7 +86,7 @@ const ( // Quotas for uploaded photos. PhotoQuotaUncertified = 6 - PhotoQuotaCertified = 24 + PhotoQuotaCertified = 100 ) // Forum settings diff --git a/pkg/models/backfill/backfill_filesize.go b/pkg/models/backfill/backfill_filesize.go new file mode 100644 index 0000000..ec92997 --- /dev/null +++ b/pkg/models/backfill/backfill_filesize.go @@ -0,0 +1,64 @@ +package backfill + +import ( + "os" + + "code.nonshy.com/nonshy/website/pkg/log" + "code.nonshy.com/nonshy/website/pkg/models" + "code.nonshy.com/nonshy/website/pkg/photo" +) + +// BackfillFilesizes finds Photos and CommentPhotos which have a zero Filesize and recomputes them. +func BackfillFilesizes() error { + // Find Photos with a filesize of zero. + var ( + photos []*models.Photo + commentPhotos []*models.CommentPhoto + ) + + // Query for photos w/ zero filesize. + result := models.DB.Model(&models.Photo{}).Where("filesize = 0").Find(&photos) + if result.Error != nil { + return result.Error + } + + // And CommentPhotos too. + result = models.DB.Model(&models.CommentPhoto{}).Where("filesize = 0").Find(&commentPhotos) + if result.Error != nil { + return result.Error + } + + // Do the Photos. + for i, row := range photos { + // Set the filesize from disk. + if stat, err := os.Stat(photo.DiskPath(row.Filename)); err == nil { + row.Filesize = stat.Size() + } + + log.Info("[%d of %d] Update photo %d: %s filesize=%d", + i+1, len(photos), row.ID, row.Filename, row.Filesize, + ) + + if err := row.Save(); err != nil { + return err + } + } + + // And same for the CommentPhotos. + for i, row := range commentPhotos { + // Set the filesize from disk. + if stat, err := os.Stat(photo.DiskPath(row.Filename)); err == nil { + row.Filesize = stat.Size() + } + + log.Info("[%d of %d] Update comment_photo %d: %s filesize=%d", + i+1, len(photos), row.ID, row.Filename, row.Filesize, + ) + + if err := row.Save(); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/models/comment_photo.go b/pkg/models/comment_photo.go index 09512bb..7bbbf98 100644 --- a/pkg/models/comment_photo.go +++ b/pkg/models/comment_photo.go @@ -22,6 +22,7 @@ func CreateCommentPhoto(tmpl CommentPhoto) (*CommentPhoto, error) { CommentID: tmpl.CommentID, UserID: tmpl.UserID, Filename: tmpl.Filename, + Filesize: tmpl.Filesize, } result := DB.Create(p) diff --git a/pkg/models/data_backfills.go b/pkg/models/data_backfills.go new file mode 100644 index 0000000..2640e7f --- /dev/null +++ b/pkg/models/data_backfills.go @@ -0,0 +1 @@ +package models diff --git a/pkg/models/photo.go b/pkg/models/photo.go index a46200b..e0dd5ae 100644 --- a/pkg/models/photo.go +++ b/pkg/models/photo.go @@ -60,6 +60,7 @@ func CreatePhoto(tmpl Photo) (*Photo, error) { UserID: tmpl.UserID, Filename: tmpl.Filename, CroppedFilename: tmpl.CroppedFilename, + Filesize: tmpl.Filesize, Caption: tmpl.Caption, Visibility: tmpl.Visibility, Gallery: tmpl.Gallery,