Add command to export user data for privacy regulations

cmd/nonshy/main.go

@@ -9,6 +9,7 @@ import (
 	"code.nonshy.com/nonshy/website/pkg/log"
 	"code.nonshy.com/nonshy/website/pkg/models"
 	"code.nonshy.com/nonshy/website/pkg/models/backfill"
+	"code.nonshy.com/nonshy/website/pkg/models/exporting"
 	"code.nonshy.com/nonshy/website/pkg/redis"
 	"code.nonshy.com/nonshy/website/pkg/worker"
 	"github.com/urfave/cli/v2"
@@ -136,6 +137,36 @@ func main() {
 							return nil
 						},
 					},
+					{
+						Name:  "export",
+						Usage: "create a data export ZIP from a user's account",
+						Flags: []cli.Flag{
+							&cli.StringFlag{
+								Name:     "username",
+								Aliases:  []string{"u"},
+								Required: true,
+								Usage:    "username or e-mail, case insensitive",
+							},
+							&cli.StringFlag{
+								Name:     "output",
+								Aliases:  []string{"o"},
+								Required: true,
+								Usage:    "output file (.zip extension)",
+							},
+						},
+						Action: func(c *cli.Context) error {
+							initdb(c)
+
+							log.Info("Creating data export for user account: %s", c.String("username"))
+							user, err := models.FindUser(c.String("username"))
+							if err != nil {
+								return err
+							}
+
+							err = exporting.ExportUser(user, c.String("output"))
+							return err
+						},
+					},
 				},
 			},
 			{

pkg/models/deletion/delete_user.go

@@ -21,6 +21,8 @@ func DeleteUser(user *models.User) error {
 	// Blank out the user's profile photo ID to avoid conflict removing their picture.
 	user.RemoveProfilePhoto()
 
+	// Tables to remove. In case of any unexpected DB errors, these tables are ordered
+	// to remove the "safest" fields first.
 	var todo = []remover{
 		{"Notifications", DeleteNotifications},
 		{"Likes", DeleteLikes},
@@ -34,6 +36,9 @@ func DeleteUser(user *models.User) error {
 		{"Comment Photos", DeleteUserCommentPhotos},
 		{"Messages", DeleteUserMessages},
 		{"Friends", DeleteFriends},
+		{"Blocks", DeleteBlocks},
+		{"Feedbacks", DeleteFeedbacks},
+		{"Two Factor", DeleteTwoFactor},
 		{"Profile Fields", DeleteProfile},
 		{"User Notes", DeleteUserNotes},
 	}
@@ -143,6 +148,16 @@ func DeleteCertification(userID uint64) error {
 	return nil
 }
 
+// DeleteTwoFactor scrubs data for deleting a user.
+func DeleteTwoFactor(userID uint64) error {
+	log.Error("DeleteUser: DeleteTwoFactor(%d)", userID)
+	result := models.DB.Where(
+		"user_id = ?",
+		userID,
+	).Delete(&models.TwoFactor{})
+	return result.Error
+}
+
 // DeleteUserLocation scrubs data for deleting a user.
 func DeleteUserLocation(userID uint64) error {
 	log.Error("DeleteUser: DeleteUserLocation(%d)", userID)
@@ -173,6 +188,26 @@ func DeleteFriends(userID uint64) error {
 	return result.Error
 }
 
+// DeleteBlocks scrubs data for deleting a user.
+func DeleteBlocks(userID uint64) error {
+	log.Error("DeleteUser: DeleteBlocks(%d)", userID)
+	result := models.DB.Where(
+		"source_user_id = ? OR target_user_id = ?",
+		userID, userID,
+	).Delete(&models.Block{})
+	return result.Error
+}
+
+// DeleteFeedbacks scrubs data for deleting a user.
+func DeleteFeedbacks(userID uint64) error {
+	log.Error("DeleteUser: DeleteFeedbacks(%d)", userID)
+	result := models.DB.Where(
+		"user_id = ? OR (table_name='users' AND table_id=?)",
+		userID, userID,
+	).Delete(&models.Feedback{})
+	return result.Error
+}
+
 // DeletePrivateGrants scrubs data for deleting a user.
 func DeletePrivateGrants(userID uint64) error {
 	log.Error("DeleteUser: DeletePrivateGrants(%d)", userID)

pkg/models/exporting/exporting.go

@@ -0,0 +1,73 @@
+package exporting
+
+import (
+	"archive/zip"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"code.nonshy.com/nonshy/website/pkg/log"
+	"code.nonshy.com/nonshy/website/pkg/models"
+	"code.nonshy.com/nonshy/website/pkg/photo"
+)
+
+// ExportUser creates a data export archive of ALL data stored about a user account..
+func ExportUser(user *models.User, filename string) error {
+	if !strings.HasSuffix(filename, ".zip") {
+		return errors.New("output file should be a .zip file")
+	}
+
+	// Prepare the output zip writer.
+	fh, err := os.Create(filename)
+	if err != nil {
+		return fmt.Errorf("creating output file (%s): %s", filename, err)
+	}
+
+	zw := zip.NewWriter(fh)
+	defer zw.Close()
+
+	// Export all their database tables into the zip.
+	return ExportModels(zw, user)
+}
+
+// ZipJson serializes a JSON file into the zipfile.
+func ZipJson(zw *zip.Writer, filename string, v any) error {
+	fh, err := zw.Create(filename)
+	if err != nil {
+		return err
+	}
+
+	encoder := json.NewEncoder(fh)
+	encoder.SetIndent("", " ")
+
+	return encoder.Encode(v)
+}
+
+// ZipPhoto copies a user photo into the ZIP archive.
+func ZipPhoto(zw *zip.Writer, prefix, filename string) error {
+	var (
+		diskPath  = photo.DiskPath(filename)
+		data, err = os.ReadFile(diskPath)
+	)
+	if err != nil {
+		if os.IsNotExist(err) {
+			// Not fatal but log it.
+			log.Error("ZipPhoto(%s): read from disk: %s", diskPath, err)
+			return nil
+		}
+		return fmt.Errorf("ZipPhoto(%s): read from disk: %s", diskPath, err)
+	}
+
+	outfh, err := zw.Create(path.Join(prefix, filename))
+	if err != nil {
+		return fmt.Errorf("ZipPhoto(%s): create in zip: %s", filename, err)
+	}
+
+	log.Info("Add photo to zip: %s", filename)
+
+	_, err = outfh.Write(data)
+	return err
+}

pkg/models/exporting/models.go

@@ -0,0 +1,414 @@
+package exporting
+
+import (
+	"archive/zip"
+	"fmt"
+
+	"code.nonshy.com/nonshy/website/pkg/log"
+	"code.nonshy.com/nonshy/website/pkg/models"
+	"gorm.io/gorm"
+)
+
+// ExportModels is the entry point function to export all data tables about a user.
+func ExportModels(zw *zip.Writer, user *models.User) error {
+	type task struct {
+		Step string
+		Fn   func(*zip.Writer, *models.User) error
+	}
+
+	// List of tables to export. Keep the ordering in sync with
+	// the AutoMigrate() calls in ../models.go
+	var todo = []task{
+		{"User", ExportUserTable},
+		{"ProfileField", ExportProfileFieldTable},
+		{"Photo", ExportPhotoTable},
+		{"PrivatePhoto", ExportPrivatePhotoTable},
+		{"CertificationPhoto", ExportCertificationPhotoTable},
+		{"Message", ExportMessageTable},
+		{"Friend", ExportFriendTable},
+		{"Block", ExportBlockTable},
+		{"Feedback", ExportFeedbackTable},
+		{"Forum", ExportForumTable},
+		{"Thread", ExportThreadTable},
+		{"Comment", ExportCommentTable},
+		{"Like", ExportLikeTable},
+		{"Notification", ExportNotificationTable},
+		{"Subscription", ExportSubscriptionTable},
+		{"CommentPhoto", ExportCommentPhotoTable},
+		// Note: Poll table is eager-loaded in Thread export
+		{"PollVote", ExportPollVoteTable},
+		// Note: AdminGroup info is eager-loaded in User export
+		{"UserLocation", ExportUserLocationTable},
+		{"UserNote", ExportUserNoteTable},
+		{"TwoFactor", ExportTwoFactorTable},
+	}
+	for _, item := range todo {
+		log.Info("Exporting data model: %s", item.Step)
+		if err := item.Fn(zw, user); err != nil {
+			return fmt.Errorf("%s: %s", item.Step, err)
+		}
+	}
+
+	return nil
+}
+
+func ExportUserTable(zw *zip.Writer, user *models.User) error {
+	return ZipJson(zw, "user.json", user)
+}
+
+func ExportProfileFieldTable(zw *zip.Writer, user *models.User) error {
+	var (
+		items = []*models.ProfileField{}
+		query = models.DB.Model(&models.ProfileField{}).Where(
+			"user_id = ?",
+			user.ID,
+		).Find(&items)
+	)
+	if query.Error != nil {
+		return query.Error
+	}
+
+	return ZipJson(zw, "profile_fields.json", items)
+}
+
+func ExportPhotoTable(zw *zip.Writer, user *models.User) error {
+	var (
+		items = []*models.Photo{}
+		query = models.DB.Model(&models.Photo{}).Where(
+			"user_id = ?",
+			user.ID,
+		).Find(&items)
+	)
+	if query.Error != nil {
+		return query.Error
+	}
+
+	// Copy all the images into the ZIP.
+	for _, row := range items {
+		if row.Filename != "" {
+			if err := ZipPhoto(zw, "photos", row.Filename); err != nil {
+				return err
+			}
+		}
+		if row.CroppedFilename != "" {
+			if err := ZipPhoto(zw, "profile_photos", row.CroppedFilename); err != nil {
+				return err
+			}
+		}
+	}
+
+	return ZipJson(zw, "photos.json", items)
+}
+
+func ExportPrivatePhotoTable(zw *zip.Writer, user *models.User) error {
+	var (
+		items = []*models.PrivatePhoto{}
+		query = models.DB.Model(&models.PrivatePhoto{}).Where(
+			"source_user_id = ? OR target_user_id = ?",
+			user.ID, user.ID,
+		).Find(&items)
+	)
+	if query.Error != nil {
+		return query.Error
+	}
+
+	return ZipJson(zw, "private_photos.json", items)
+}
+
+func ExportCertificationPhotoTable(zw *zip.Writer, user *models.User) error {
+	var (
+		items = []*models.CertificationPhoto{}
+		query = models.DB.Model(&models.CertificationPhoto{}).Where(
+			"user_id = ?",
+			user.ID,
+		).Find(&items)
+	)
+	if query.Error != nil {
+		return query.Error
+	}
+
+	// Copy all the images into the ZIP.
+	for _, row := range items {
+		if row.Filename != "" {
+			if err := ZipPhoto(zw, "certification_photo", row.Filename); err != nil {
+				return err
+			}
+		}
+	}
+
+	return ZipJson(zw, "certification_photo.json", items)
+}
+
+func ExportMessageTable(zw *zip.Writer, user *models.User) error {
+	var (
+		items = []*models.Message{}
+		query = models.DB.Model(&models.Message{}).Where(
+			"source_user_id = ? OR target_user_id = ?",
+			user.ID, user.ID,
+		).Find(&items)
+	)
+	if query.Error != nil {
+		return query.Error
+	}
+
+	return ZipJson(zw, "messages.json", items)
+}
+
+func ExportFriendTable(zw *zip.Writer, user *models.User) error {
+	var (
+		items = []*models.Friend{}
+		query = models.DB.Model(&models.Friend{}).Where(
+			"source_user_id = ? OR target_user_id = ?",
+			user.ID, user.ID,
+		).Find(&items)
+	)
+	if query.Error != nil {
+		return query.Error
+	}
+
+	return ZipJson(zw, "friends.json", items)
+}
+
+func ExportBlockTable(zw *zip.Writer, user *models.User) error {
+	var (
+		items = []*models.Block{}
+		query = models.DB.Model(&models.Block{}).Where(
+			"source_user_id = ? OR target_user_id = ?",
+			user.ID, user.ID,
+		).Find(&items)
+	)
+	if query.Error != nil {
+		return query.Error
+	}
+
+	return ZipJson(zw, "blocks.json", items)
+}
+
+func ExportFeedbackTable(zw *zip.Writer, user *models.User) error {
+	var (
+		items       = []*models.Feedback{}
+		photoIDs, _ = user.AllPhotoIDs()
+		query       *gorm.DB
+	)
+
+	// If they have photos, query on those.
+	if len(photoIDs) > 0 {
+		query = models.DB.Model(&models.Feedback{}).Where(
+			"user_id = ? OR (table_name = 'users' AND table_id = ?) OR (table_name = 'photos' AND table_id IN ?)",
+			user.ID, user.ID, photoIDs,
+		).Find(&items)
+	} else {
+		// Only reports about their user.
+		query = models.DB.Model(&models.Feedback{}).Where(
+			"user_id = ? OR (table_name = 'users' AND table_id = ?)",
+			user.ID, user.ID,
+		).Find(&items)
+	}
+
+	if query.Error != nil {
+		return query.Error
+	}
+
+	return ZipJson(zw, "feedback.json", items)
+}
+
+func ExportForumTable(zw *zip.Writer, user *models.User) error {
+	var (
+		items = []*models.Forum{}
+		query = models.DB.Model(&models.Forum{}).Where(
+			"owner_id = ?",
+			user.ID,
+		).Find(&items)
+	)
+	if query.Error != nil {
+		return query.Error
+	}
+
+	return ZipJson(zw, "forums.json", items)
+}
+
+func ExportThreadTable(zw *zip.Writer, user *models.User) error {
+	var (
+		items = []*models.Thread{}
+		query = (&models.Thread{}).Preload().Joins(
+			"JOIN comments ON (comments.id = threads.comment_id)",
+		).Where(
+			"comments.user_id = ?",
+			user.ID,
+		).Find(&items)
+	)
+	if query.Error != nil {
+		return query.Error
+	}
+
+	return ZipJson(zw, "threads.json", items)
+}
+
+func ExportCommentTable(zw *zip.Writer, user *models.User) error {
+	var (
+		items       = []*models.Comment{}
+		photoIDs, _ = user.AllPhotoIDs()
+		query       *gorm.DB
+	)
+
+	// If they have photos, query on those.
+	if len(photoIDs) > 0 {
+		query = models.DB.Model(&models.Comment{}).Where(
+			"user_id = ? OR (table_name = 'users' AND table_id = ?) OR (table_name = 'photos' AND table_id IN ?)",
+			user.ID, user.ID, photoIDs,
+		).Find(&items)
+	} else {
+		query = models.DB.Model(&models.Comment{}).Where(
+			"user_id = ? OR (table_name = 'users' AND table_id = ?)",
+			user.ID, user.ID,
+		).Find(&items)
+	}
+
+	if query.Error != nil {
+		return query.Error
+	}
+
+	return ZipJson(zw, "comments.json", items)
+}
+
+func ExportLikeTable(zw *zip.Writer, user *models.User) error {
+	var (
+		items       = []*models.Like{}
+		photoIDs, _ = user.AllPhotoIDs()
+		query       *gorm.DB
+	)
+
+	// If they have photos, query on those.
+	if len(photoIDs) > 0 {
+		query = models.DB.Model(&models.Like{}).Where(
+			"user_id = ? OR (table_name = 'users' AND table_id = ?) OR (table_name = 'photos' AND table_id IN ?)",
+			user.ID, user.ID, photoIDs,
+		).Find(&items)
+	} else {
+		// Only reports about their user.
+		query = models.DB.Model(&models.Like{}).Where(
+			"user_id = ? OR (table_name = 'users' AND table_id = ?)",
+			user.ID, user.ID,
+		).Find(&items)
+	}
+
+	if query.Error != nil {
+		return query.Error
+	}
+
+	return ZipJson(zw, "likes.json", items)
+}
+
+func ExportNotificationTable(zw *zip.Writer, user *models.User) error {
+	var (
+		items = []*models.Notification{}
+		query = models.DB.Model(&models.Notification{}).Where(
+			"user_id = ?",
+			user.ID,
+		).Find(&items)
+	)
+	if query.Error != nil {
+		return query.Error
+	}
+
+	return ZipJson(zw, "notifications.json", items)
+}
+
+func ExportSubscriptionTable(zw *zip.Writer, user *models.User) error {
+	var (
+		items = []*models.Subscription{}
+		query = models.DB.Model(&models.Subscription{}).Where(
+			"user_id = ?",
+			user.ID,
+		).Find(&items)
+	)
+	if query.Error != nil {
+		return query.Error
+	}
+
+	return ZipJson(zw, "subscriptions.json", items)
+}
+
+func ExportCommentPhotoTable(zw *zip.Writer, user *models.User) error {
+	var (
+		items = []*models.CommentPhoto{}
+		query = models.DB.Model(&models.CommentPhoto{}).Where(
+			"user_id = ?",
+			user.ID,
+		).Find(&items)
+	)
+	if query.Error != nil {
+		return query.Error
+	}
+
+	// Copy all the images into the ZIP.
+	for _, row := range items {
+		if row.Filename != "" {
+			if err := ZipPhoto(zw, "comment_photos", row.Filename); err != nil {
+				return err
+			}
+		}
+	}
+
+	return ZipJson(zw, "comment_photos.json", items)
+}
+
+func ExportPollVoteTable(zw *zip.Writer, user *models.User) error {
+	var (
+		items = []*models.PollVote{}
+		query = (&models.PollVote{}).Preload().Where(
+			"user_id = ?",
+			user.ID,
+		).Find(&items)
+	)
+	if query.Error != nil {
+		return query.Error
+	}
+
+	return ZipJson(zw, "poll_votes.json", items)
+}
+
+func ExportUserNoteTable(zw *zip.Writer, user *models.User) error {
+	var (
+		items = []*models.UserNote{}
+		query = models.DB.Model(&models.UserNote{}).Where(
+			"user_id = ? OR about_user_id = ?",
+			user.ID, user.ID,
+		).Find(&items)
+	)
+	if query.Error != nil {
+		return query.Error
+	}
+
+	return ZipJson(zw, "user_notes.json", items)
+}
+
+func ExportUserLocationTable(zw *zip.Writer, user *models.User) error {
+	var (
+		items = []*models.UserLocation{}
+		query = models.DB.Model(&models.UserLocation{}).Where(
+			"user_id = ?",
+			user.ID,
+		).Find(&items)
+	)
+	if query.Error != nil {
+		return query.Error
+	}
+
+	return ZipJson(zw, "user_location.json", items)
+}
+
+func ExportTwoFactorTable(zw *zip.Writer, user *models.User) error {
+	var (
+		items = []*models.TwoFactor{}
+		query = models.DB.Model(&models.TwoFactor{}).Where(
+			"user_id = ?",
+			user.ID,
+		).Find(&items)
+	)
+	if query.Error != nil {
+		return query.Error
+	}
+
+	return ZipJson(zw, "two_factor.json", items)
+}

pkg/models/poll_votes.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"code.nonshy.com/nonshy/website/pkg/log"
+	"gorm.io/gorm"
 )
 
 // PollVote table records answers to polls.
@@ -18,6 +19,11 @@ type PollVote struct {
 	UpdatedAt time.Time
 }
 
+// Preload related tables for the poll (classmethod).
+func (u *PollVote) Preload() *gorm.DB {
+	return DB.Preload("Poll")
+}
+
 // CastVote on a poll. Multiple answers OK for multiple choice polls.
 func (p *Poll) CastVote(user *User, answers []string) error {
 	if len(answers) > 1 && !p.MultipleChoice {

pkg/models/private_photo.go

@@ -102,6 +102,10 @@ func (u *User) AllPrivatePhotoIDs() ([]uint64, error) {
 
 // AllPhotoIDs returns the listing of all IDs of the user's photos.
 func (u *User) AllPhotoIDs() ([]uint64, error) {
+	if u.cachePhotoIDs != nil {
+		return u.cachePhotoIDs, nil
+	}
+
 	var photoIDs = []uint64{}
 	err := DB.Table(
 		"photos",
@@ -116,6 +120,8 @@ func (u *User) AllPhotoIDs() ([]uint64, error) {
 		return photoIDs, fmt.Errorf("AllPhotoIDs(%s): %s", u.Username, err.Error)
 	}
 
+	u.cachePhotoIDs = photoIDs
+
 	return photoIDs, nil
 }
 

pkg/models/user.go

@@ -45,6 +45,7 @@ type User struct {
 	// Caches
 	cachePhotoTypes     map[PhotoVisibility]struct{}
 	cacheBlockedUserIDs []uint64
+	cachePhotoIDs       []uint64
 }
 
 type UserVisibility string