diff --git a/cmd/nonshy/main.go b/cmd/nonshy/main.go index e0526b5..9c96f15 100644 --- a/cmd/nonshy/main.go +++ b/cmd/nonshy/main.go @@ -9,6 +9,7 @@ import ( "code.nonshy.com/nonshy/website/pkg/log" "code.nonshy.com/nonshy/website/pkg/models" "code.nonshy.com/nonshy/website/pkg/models/backfill" + "code.nonshy.com/nonshy/website/pkg/models/exporting" "code.nonshy.com/nonshy/website/pkg/redis" "code.nonshy.com/nonshy/website/pkg/worker" "github.com/urfave/cli/v2" @@ -136,6 +137,36 @@ func main() { return nil }, }, + { + Name: "export", + Usage: "create a data export ZIP from a user's account", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "username", + Aliases: []string{"u"}, + Required: true, + Usage: "username or e-mail, case insensitive", + }, + &cli.StringFlag{ + Name: "output", + Aliases: []string{"o"}, + Required: true, + Usage: "output file (.zip extension)", + }, + }, + Action: func(c *cli.Context) error { + initdb(c) + + log.Info("Creating data export for user account: %s", c.String("username")) + user, err := models.FindUser(c.String("username")) + if err != nil { + return err + } + + err = exporting.ExportUser(user, c.String("output")) + return err + }, + }, }, }, { diff --git a/pkg/models/deletion/delete_user.go b/pkg/models/deletion/delete_user.go index d0b36e5..18b469f 100644 --- a/pkg/models/deletion/delete_user.go +++ b/pkg/models/deletion/delete_user.go @@ -21,6 +21,8 @@ func DeleteUser(user *models.User) error { // Blank out the user's profile photo ID to avoid conflict removing their picture. user.RemoveProfilePhoto() + // Tables to remove. In case of any unexpected DB errors, these tables are ordered + // to remove the "safest" fields first. var todo = []remover{ {"Notifications", DeleteNotifications}, {"Likes", DeleteLikes}, @@ -34,6 +36,9 @@ func DeleteUser(user *models.User) error { {"Comment Photos", DeleteUserCommentPhotos}, {"Messages", DeleteUserMessages}, {"Friends", DeleteFriends}, + {"Blocks", DeleteBlocks}, + {"Feedbacks", DeleteFeedbacks}, + {"Two Factor", DeleteTwoFactor}, {"Profile Fields", DeleteProfile}, {"User Notes", DeleteUserNotes}, } @@ -143,6 +148,16 @@ func DeleteCertification(userID uint64) error { return nil } +// DeleteTwoFactor scrubs data for deleting a user. +func DeleteTwoFactor(userID uint64) error { + log.Error("DeleteUser: DeleteTwoFactor(%d)", userID) + result := models.DB.Where( + "user_id = ?", + userID, + ).Delete(&models.TwoFactor{}) + return result.Error +} + // DeleteUserLocation scrubs data for deleting a user. func DeleteUserLocation(userID uint64) error { log.Error("DeleteUser: DeleteUserLocation(%d)", userID) @@ -173,6 +188,26 @@ func DeleteFriends(userID uint64) error { return result.Error } +// DeleteBlocks scrubs data for deleting a user. +func DeleteBlocks(userID uint64) error { + log.Error("DeleteUser: DeleteBlocks(%d)", userID) + result := models.DB.Where( + "source_user_id = ? OR target_user_id = ?", + userID, userID, + ).Delete(&models.Block{}) + return result.Error +} + +// DeleteFeedbacks scrubs data for deleting a user. +func DeleteFeedbacks(userID uint64) error { + log.Error("DeleteUser: DeleteFeedbacks(%d)", userID) + result := models.DB.Where( + "user_id = ? OR (table_name='users' AND table_id=?)", + userID, userID, + ).Delete(&models.Feedback{}) + return result.Error +} + // DeletePrivateGrants scrubs data for deleting a user. func DeletePrivateGrants(userID uint64) error { log.Error("DeleteUser: DeletePrivateGrants(%d)", userID) diff --git a/pkg/models/exporting/exporting.go b/pkg/models/exporting/exporting.go new file mode 100644 index 0000000..75dc4e2 --- /dev/null +++ b/pkg/models/exporting/exporting.go @@ -0,0 +1,73 @@ +package exporting + +import ( + "archive/zip" + "encoding/json" + "errors" + "fmt" + "os" + "path" + "strings" + + "code.nonshy.com/nonshy/website/pkg/log" + "code.nonshy.com/nonshy/website/pkg/models" + "code.nonshy.com/nonshy/website/pkg/photo" +) + +// ExportUser creates a data export archive of ALL data stored about a user account.. +func ExportUser(user *models.User, filename string) error { + if !strings.HasSuffix(filename, ".zip") { + return errors.New("output file should be a .zip file") + } + + // Prepare the output zip writer. + fh, err := os.Create(filename) + if err != nil { + return fmt.Errorf("creating output file (%s): %s", filename, err) + } + + zw := zip.NewWriter(fh) + defer zw.Close() + + // Export all their database tables into the zip. + return ExportModels(zw, user) +} + +// ZipJson serializes a JSON file into the zipfile. +func ZipJson(zw *zip.Writer, filename string, v any) error { + fh, err := zw.Create(filename) + if err != nil { + return err + } + + encoder := json.NewEncoder(fh) + encoder.SetIndent("", " ") + + return encoder.Encode(v) +} + +// ZipPhoto copies a user photo into the ZIP archive. +func ZipPhoto(zw *zip.Writer, prefix, filename string) error { + var ( + diskPath = photo.DiskPath(filename) + data, err = os.ReadFile(diskPath) + ) + if err != nil { + if os.IsNotExist(err) { + // Not fatal but log it. + log.Error("ZipPhoto(%s): read from disk: %s", diskPath, err) + return nil + } + return fmt.Errorf("ZipPhoto(%s): read from disk: %s", diskPath, err) + } + + outfh, err := zw.Create(path.Join(prefix, filename)) + if err != nil { + return fmt.Errorf("ZipPhoto(%s): create in zip: %s", filename, err) + } + + log.Info("Add photo to zip: %s", filename) + + _, err = outfh.Write(data) + return err +} diff --git a/pkg/models/exporting/models.go b/pkg/models/exporting/models.go new file mode 100644 index 0000000..fc751fb --- /dev/null +++ b/pkg/models/exporting/models.go @@ -0,0 +1,414 @@ +package exporting + +import ( + "archive/zip" + "fmt" + + "code.nonshy.com/nonshy/website/pkg/log" + "code.nonshy.com/nonshy/website/pkg/models" + "gorm.io/gorm" +) + +// ExportModels is the entry point function to export all data tables about a user. +func ExportModels(zw *zip.Writer, user *models.User) error { + type task struct { + Step string + Fn func(*zip.Writer, *models.User) error + } + + // List of tables to export. Keep the ordering in sync with + // the AutoMigrate() calls in ../models.go + var todo = []task{ + {"User", ExportUserTable}, + {"ProfileField", ExportProfileFieldTable}, + {"Photo", ExportPhotoTable}, + {"PrivatePhoto", ExportPrivatePhotoTable}, + {"CertificationPhoto", ExportCertificationPhotoTable}, + {"Message", ExportMessageTable}, + {"Friend", ExportFriendTable}, + {"Block", ExportBlockTable}, + {"Feedback", ExportFeedbackTable}, + {"Forum", ExportForumTable}, + {"Thread", ExportThreadTable}, + {"Comment", ExportCommentTable}, + {"Like", ExportLikeTable}, + {"Notification", ExportNotificationTable}, + {"Subscription", ExportSubscriptionTable}, + {"CommentPhoto", ExportCommentPhotoTable}, + // Note: Poll table is eager-loaded in Thread export + {"PollVote", ExportPollVoteTable}, + // Note: AdminGroup info is eager-loaded in User export + {"UserLocation", ExportUserLocationTable}, + {"UserNote", ExportUserNoteTable}, + {"TwoFactor", ExportTwoFactorTable}, + } + for _, item := range todo { + log.Info("Exporting data model: %s", item.Step) + if err := item.Fn(zw, user); err != nil { + return fmt.Errorf("%s: %s", item.Step, err) + } + } + + return nil +} + +func ExportUserTable(zw *zip.Writer, user *models.User) error { + return ZipJson(zw, "user.json", user) +} + +func ExportProfileFieldTable(zw *zip.Writer, user *models.User) error { + var ( + items = []*models.ProfileField{} + query = models.DB.Model(&models.ProfileField{}).Where( + "user_id = ?", + user.ID, + ).Find(&items) + ) + if query.Error != nil { + return query.Error + } + + return ZipJson(zw, "profile_fields.json", items) +} + +func ExportPhotoTable(zw *zip.Writer, user *models.User) error { + var ( + items = []*models.Photo{} + query = models.DB.Model(&models.Photo{}).Where( + "user_id = ?", + user.ID, + ).Find(&items) + ) + if query.Error != nil { + return query.Error + } + + // Copy all the images into the ZIP. + for _, row := range items { + if row.Filename != "" { + if err := ZipPhoto(zw, "photos", row.Filename); err != nil { + return err + } + } + if row.CroppedFilename != "" { + if err := ZipPhoto(zw, "profile_photos", row.CroppedFilename); err != nil { + return err + } + } + } + + return ZipJson(zw, "photos.json", items) +} + +func ExportPrivatePhotoTable(zw *zip.Writer, user *models.User) error { + var ( + items = []*models.PrivatePhoto{} + query = models.DB.Model(&models.PrivatePhoto{}).Where( + "source_user_id = ? OR target_user_id = ?", + user.ID, user.ID, + ).Find(&items) + ) + if query.Error != nil { + return query.Error + } + + return ZipJson(zw, "private_photos.json", items) +} + +func ExportCertificationPhotoTable(zw *zip.Writer, user *models.User) error { + var ( + items = []*models.CertificationPhoto{} + query = models.DB.Model(&models.CertificationPhoto{}).Where( + "user_id = ?", + user.ID, + ).Find(&items) + ) + if query.Error != nil { + return query.Error + } + + // Copy all the images into the ZIP. + for _, row := range items { + if row.Filename != "" { + if err := ZipPhoto(zw, "certification_photo", row.Filename); err != nil { + return err + } + } + } + + return ZipJson(zw, "certification_photo.json", items) +} + +func ExportMessageTable(zw *zip.Writer, user *models.User) error { + var ( + items = []*models.Message{} + query = models.DB.Model(&models.Message{}).Where( + "source_user_id = ? OR target_user_id = ?", + user.ID, user.ID, + ).Find(&items) + ) + if query.Error != nil { + return query.Error + } + + return ZipJson(zw, "messages.json", items) +} + +func ExportFriendTable(zw *zip.Writer, user *models.User) error { + var ( + items = []*models.Friend{} + query = models.DB.Model(&models.Friend{}).Where( + "source_user_id = ? OR target_user_id = ?", + user.ID, user.ID, + ).Find(&items) + ) + if query.Error != nil { + return query.Error + } + + return ZipJson(zw, "friends.json", items) +} + +func ExportBlockTable(zw *zip.Writer, user *models.User) error { + var ( + items = []*models.Block{} + query = models.DB.Model(&models.Block{}).Where( + "source_user_id = ? OR target_user_id = ?", + user.ID, user.ID, + ).Find(&items) + ) + if query.Error != nil { + return query.Error + } + + return ZipJson(zw, "blocks.json", items) +} + +func ExportFeedbackTable(zw *zip.Writer, user *models.User) error { + var ( + items = []*models.Feedback{} + photoIDs, _ = user.AllPhotoIDs() + query *gorm.DB + ) + + // If they have photos, query on those. + if len(photoIDs) > 0 { + query = models.DB.Model(&models.Feedback{}).Where( + "user_id = ? OR (table_name = 'users' AND table_id = ?) OR (table_name = 'photos' AND table_id IN ?)", + user.ID, user.ID, photoIDs, + ).Find(&items) + } else { + // Only reports about their user. + query = models.DB.Model(&models.Feedback{}).Where( + "user_id = ? OR (table_name = 'users' AND table_id = ?)", + user.ID, user.ID, + ).Find(&items) + } + + if query.Error != nil { + return query.Error + } + + return ZipJson(zw, "feedback.json", items) +} + +func ExportForumTable(zw *zip.Writer, user *models.User) error { + var ( + items = []*models.Forum{} + query = models.DB.Model(&models.Forum{}).Where( + "owner_id = ?", + user.ID, + ).Find(&items) + ) + if query.Error != nil { + return query.Error + } + + return ZipJson(zw, "forums.json", items) +} + +func ExportThreadTable(zw *zip.Writer, user *models.User) error { + var ( + items = []*models.Thread{} + query = (&models.Thread{}).Preload().Joins( + "JOIN comments ON (comments.id = threads.comment_id)", + ).Where( + "comments.user_id = ?", + user.ID, + ).Find(&items) + ) + if query.Error != nil { + return query.Error + } + + return ZipJson(zw, "threads.json", items) +} + +func ExportCommentTable(zw *zip.Writer, user *models.User) error { + var ( + items = []*models.Comment{} + photoIDs, _ = user.AllPhotoIDs() + query *gorm.DB + ) + + // If they have photos, query on those. + if len(photoIDs) > 0 { + query = models.DB.Model(&models.Comment{}).Where( + "user_id = ? OR (table_name = 'users' AND table_id = ?) OR (table_name = 'photos' AND table_id IN ?)", + user.ID, user.ID, photoIDs, + ).Find(&items) + } else { + query = models.DB.Model(&models.Comment{}).Where( + "user_id = ? OR (table_name = 'users' AND table_id = ?)", + user.ID, user.ID, + ).Find(&items) + } + + if query.Error != nil { + return query.Error + } + + return ZipJson(zw, "comments.json", items) +} + +func ExportLikeTable(zw *zip.Writer, user *models.User) error { + var ( + items = []*models.Like{} + photoIDs, _ = user.AllPhotoIDs() + query *gorm.DB + ) + + // If they have photos, query on those. + if len(photoIDs) > 0 { + query = models.DB.Model(&models.Like{}).Where( + "user_id = ? OR (table_name = 'users' AND table_id = ?) OR (table_name = 'photos' AND table_id IN ?)", + user.ID, user.ID, photoIDs, + ).Find(&items) + } else { + // Only reports about their user. + query = models.DB.Model(&models.Like{}).Where( + "user_id = ? OR (table_name = 'users' AND table_id = ?)", + user.ID, user.ID, + ).Find(&items) + } + + if query.Error != nil { + return query.Error + } + + return ZipJson(zw, "likes.json", items) +} + +func ExportNotificationTable(zw *zip.Writer, user *models.User) error { + var ( + items = []*models.Notification{} + query = models.DB.Model(&models.Notification{}).Where( + "user_id = ?", + user.ID, + ).Find(&items) + ) + if query.Error != nil { + return query.Error + } + + return ZipJson(zw, "notifications.json", items) +} + +func ExportSubscriptionTable(zw *zip.Writer, user *models.User) error { + var ( + items = []*models.Subscription{} + query = models.DB.Model(&models.Subscription{}).Where( + "user_id = ?", + user.ID, + ).Find(&items) + ) + if query.Error != nil { + return query.Error + } + + return ZipJson(zw, "subscriptions.json", items) +} + +func ExportCommentPhotoTable(zw *zip.Writer, user *models.User) error { + var ( + items = []*models.CommentPhoto{} + query = models.DB.Model(&models.CommentPhoto{}).Where( + "user_id = ?", + user.ID, + ).Find(&items) + ) + if query.Error != nil { + return query.Error + } + + // Copy all the images into the ZIP. + for _, row := range items { + if row.Filename != "" { + if err := ZipPhoto(zw, "comment_photos", row.Filename); err != nil { + return err + } + } + } + + return ZipJson(zw, "comment_photos.json", items) +} + +func ExportPollVoteTable(zw *zip.Writer, user *models.User) error { + var ( + items = []*models.PollVote{} + query = (&models.PollVote{}).Preload().Where( + "user_id = ?", + user.ID, + ).Find(&items) + ) + if query.Error != nil { + return query.Error + } + + return ZipJson(zw, "poll_votes.json", items) +} + +func ExportUserNoteTable(zw *zip.Writer, user *models.User) error { + var ( + items = []*models.UserNote{} + query = models.DB.Model(&models.UserNote{}).Where( + "user_id = ? OR about_user_id = ?", + user.ID, user.ID, + ).Find(&items) + ) + if query.Error != nil { + return query.Error + } + + return ZipJson(zw, "user_notes.json", items) +} + +func ExportUserLocationTable(zw *zip.Writer, user *models.User) error { + var ( + items = []*models.UserLocation{} + query = models.DB.Model(&models.UserLocation{}).Where( + "user_id = ?", + user.ID, + ).Find(&items) + ) + if query.Error != nil { + return query.Error + } + + return ZipJson(zw, "user_location.json", items) +} + +func ExportTwoFactorTable(zw *zip.Writer, user *models.User) error { + var ( + items = []*models.TwoFactor{} + query = models.DB.Model(&models.TwoFactor{}).Where( + "user_id = ?", + user.ID, + ).Find(&items) + ) + if query.Error != nil { + return query.Error + } + + return ZipJson(zw, "two_factor.json", items) +} diff --git a/pkg/models/poll_votes.go b/pkg/models/poll_votes.go index 1a0afa4..bae4698 100644 --- a/pkg/models/poll_votes.go +++ b/pkg/models/poll_votes.go @@ -5,6 +5,7 @@ import ( "time" "code.nonshy.com/nonshy/website/pkg/log" + "gorm.io/gorm" ) // PollVote table records answers to polls. @@ -18,6 +19,11 @@ type PollVote struct { UpdatedAt time.Time } +// Preload related tables for the poll (classmethod). +func (u *PollVote) Preload() *gorm.DB { + return DB.Preload("Poll") +} + // CastVote on a poll. Multiple answers OK for multiple choice polls. func (p *Poll) CastVote(user *User, answers []string) error { if len(answers) > 1 && !p.MultipleChoice { diff --git a/pkg/models/private_photo.go b/pkg/models/private_photo.go index d8ad346..c237bd0 100644 --- a/pkg/models/private_photo.go +++ b/pkg/models/private_photo.go @@ -102,6 +102,10 @@ func (u *User) AllPrivatePhotoIDs() ([]uint64, error) { // AllPhotoIDs returns the listing of all IDs of the user's photos. func (u *User) AllPhotoIDs() ([]uint64, error) { + if u.cachePhotoIDs != nil { + return u.cachePhotoIDs, nil + } + var photoIDs = []uint64{} err := DB.Table( "photos", @@ -116,6 +120,8 @@ func (u *User) AllPhotoIDs() ([]uint64, error) { return photoIDs, fmt.Errorf("AllPhotoIDs(%s): %s", u.Username, err.Error) } + u.cachePhotoIDs = photoIDs + return photoIDs, nil } diff --git a/pkg/models/user.go b/pkg/models/user.go index 2a596af..2492b8e 100644 --- a/pkg/models/user.go +++ b/pkg/models/user.go @@ -45,6 +45,7 @@ type User struct { // Caches cachePhotoTypes map[PhotoVisibility]struct{} cacheBlockedUserIDs []uint64 + cachePhotoIDs []uint64 } type UserVisibility string