From a0df988ffa2f17ee08a1ee0b54ddb868a7ea312d Mon Sep 17 00:00:00 2001 From: Noah Petherbridge Date: Mon, 8 Jan 2024 20:10:12 -0800 Subject: [PATCH] WIP face detection experiment --- README.md | 13 ++++ cmd/nonshy/main.go | 4 ++ go.mod | 2 + go.sum | 9 +++ pkg/config/variable.go | 12 +++- pkg/controller/photo/edit_delete.go | 14 +++++ pkg/models/photo.go | 5 +- pkg/photo/face_score.go | 95 +++++++++++++++++++++++++++++ web/templates/photo/gallery.html | 8 +++ web/templates/photo/upload.html | 29 +++++++++ 10 files changed, 188 insertions(+), 3 deletions(-) create mode 100644 pkg/photo/face_score.go diff --git a/README.md b/README.md index a724809..37b6357 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,19 @@ create extension postgis; If you get errors like "Type geography not found" from Postgres when running distance based searches, this is the likely culprit. +### Face Detection + +Fedora: `dnf install python3 python3-opencv opencv-data` + +Debian: `apt install python3-opencv opencv-data` + +If you get an error like: + +> facedetect: error: cannot load HAAR_FRONTALFACE_ALT2 from /usr/share/opencv/haarcascades/haarcascade_frontalface_alt2.xml + +Check whether the correct path on disk is actually /usr/share/opencv4 instead of /usr/share/opencv. +One solution then is to symlink the path correctly. + ## Building the App This app is written in Go: [go.dev](https://go.dev). You can probably diff --git a/cmd/nonshy/main.go b/cmd/nonshy/main.go index 9c96f15..5f33b8a 100644 --- a/cmd/nonshy/main.go +++ b/cmd/nonshy/main.go @@ -10,6 +10,7 @@ import ( "code.nonshy.com/nonshy/website/pkg/models" "code.nonshy.com/nonshy/website/pkg/models/backfill" "code.nonshy.com/nonshy/website/pkg/models/exporting" + "code.nonshy.com/nonshy/website/pkg/photo" "code.nonshy.com/nonshy/website/pkg/redis" "code.nonshy.com/nonshy/website/pkg/worker" "github.com/urfave/cli/v2" @@ -229,6 +230,9 @@ func initdb(c *cli.Context) { // Auto-migrate the DB. models.AutoMigrate() + + // Initialize FaceScore face detection. + photo.InitFaceScore() } func initcache(c *cli.Context) { diff --git a/go.mod b/go.mod index 03dcd3c..28390a5 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( ) require ( + github.com/Kagami/go-face v0.0.0-20210630145111-0c14797b4d0e // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect @@ -21,6 +22,7 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/disintegration/imaging v1.6.2 // indirect github.com/edwvee/exiffix v0.0.0-20210922235313-0f6cbda5e58f // indirect + github.com/esimov/pigo v1.4.6 // indirect github.com/go-redis/redis v6.15.9+incompatible // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect github.com/gorilla/css v1.0.0 // indirect diff --git a/go.sum b/go.sum index 35d2b27..fa12d82 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ git.kirsle.net/go/log v0.0.0-20200902035305-70ac2848949b h1:TDxEEWOJqMzsu9JW8/QgmT1lgQ9WD2KWlb2lKN/Ql2o= git.kirsle.net/go/log v0.0.0-20200902035305-70ac2848949b/go.mod h1:jl+Qr58W3Op7OCxIYIT+b42jq8xFncJXzPufhrvza7Y= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Kagami/go-face v0.0.0-20210630145111-0c14797b4d0e h1:lqIUFzxaqyYqUn4MhzAvSAh4wIte/iLNcIEWxpT/qbc= +github.com/Kagami/go-face v0.0.0-20210630145111-0c14797b4d0e/go.mod h1:9wdDJkRgo3SGTcFwbQ7elVIQhIr2bbBjecuY7VoqmPU= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= @@ -22,6 +24,9 @@ github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1 github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/edwvee/exiffix v0.0.0-20210922235313-0f6cbda5e58f h1:RMnUwTnNR070mFAEIoqMYjNirHj8i0h79VXTYyBCyVA= github.com/edwvee/exiffix v0.0.0-20210922235313-0f6cbda5e58f/go.mod h1:KoE3Ti1qbQXCb3s/XGj0yApHnbnNnn1bXTtB5Auq/Vc= +github.com/esimov/pigo v1.4.6 h1:wpB9FstbqeGP/CZP+nTR52tUJe7XErq8buG+k4xCXlw= +github.com/esimov/pigo v1.4.6/go.mod h1:uqj9Y3+3IRYhFK071rxz1QYq0ePhA6+R9jrUZavi46M= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= @@ -32,6 +37,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -190,6 +196,7 @@ golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJ golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200927104501-e162460cd6b5/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20220722155232-062f8c9fd539 h1:/eM0PCrQI2xd471rI+snWuu251/+/jpBpZqir2mPdnU= golang.org/x/image v0.0.0-20220722155232-062f8c9fd539/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -215,6 +222,7 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201107080550-4d91cf3a1aaf/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -224,6 +232,7 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9w golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20191110171634-ad39bd3f0407/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/pkg/config/variable.go b/pkg/config/variable.go index 7f4ebe4..0abddc8 100644 --- a/pkg/config/variable.go +++ b/pkg/config/variable.go @@ -14,7 +14,7 @@ import ( // Version of the config format - when new fields are added, it will attempt // to write the settings.toml to disk so new defaults populate. -var currentVersion = 2 +var currentVersion = 3 // Current loaded settings.json var Current = DefaultVariable() @@ -31,6 +31,7 @@ type Variable struct { BareRTC BareRTC Maintenance Maintenance Encryption Encryption + FaceScore FaceScore UseXForwardedFor bool } @@ -52,6 +53,9 @@ func DefaultVariable() Variable { SQLite: "database.sqlite", Postgres: "host=localhost user=nonshy password=nonshy dbname=nonshy port=5679 sslmode=disable TimeZone=America/Los_Angeles", }, + FaceScore: FaceScore{ + CascadeFile: "/path/to/cascade/file", + }, CronAPIKey: uuid.New().String(), } } @@ -165,3 +169,9 @@ type Maintenance struct { type Encryption struct { AESKey []byte } + +// FaceScore settings for face detection in photos via esimov/pigo. +type FaceScore struct { + Enabled bool + CascadeFile string +} diff --git a/pkg/controller/photo/edit_delete.go b/pkg/controller/photo/edit_delete.go index 2f5211f..66808b3 100644 --- a/pkg/controller/photo/edit_delete.go +++ b/pkg/controller/photo/edit_delete.go @@ -75,6 +75,9 @@ func Edit() http.HandlerFunc { setProfilePic = r.FormValue("intent") == "profile-pic" crop = pphoto.ParseCropCoords(r.FormValue("crop")) + // Re-compute the face score (admin only) + recomputeFaceScore = r.FormValue("recompute_face_score") == "true" && currentUser.IsAdmin + // Are we GOING private or changing to Inner Circle? goingPrivate = visibility == models.PhotoPrivate && visibility != photo.Visibility goingCircle = visibility == models.PhotoInnerCircle && visibility != photo.Visibility @@ -118,6 +121,17 @@ func Edit() http.HandlerFunc { log.Error("SAVING PHOTO: %+v", photo) + // Are we re-computing the face score? + if recomputeFaceScore { + score, err := pphoto.ComputeFaceScore(pphoto.DiskPath(photo.Filename)) + if err != nil { + session.FlashError(w, r, "Face score: %s", err) + } else { + session.Flash(w, r, "Face score recomputed!") + photo.FaceScore = &score + } + } + if err := photo.Save(); err != nil { session.FlashError(w, r, "Couldn't save photo: %s", err) } diff --git a/pkg/models/photo.go b/pkg/models/photo.go index ba1b1c0..f50c40a 100644 --- a/pkg/models/photo.go +++ b/pkg/models/photo.go @@ -21,8 +21,9 @@ type Photo struct { Caption string Flagged bool // photo has been reported by the community Visibility PhotoVisibility - Gallery bool // photo appears in the public gallery (if public) - Explicit bool // is an explicit photo + Gallery bool // photo appears in the public gallery (if public) + Explicit bool // is an explicit photo + FaceScore *float64 // face detection score (best) CreatedAt time.Time UpdatedAt time.Time } diff --git a/pkg/photo/face_score.go b/pkg/photo/face_score.go new file mode 100644 index 0000000..1cd71ec --- /dev/null +++ b/pkg/photo/face_score.go @@ -0,0 +1,95 @@ +package photo + +import ( + "errors" + "os" + + "code.nonshy.com/nonshy/website/pkg/config" + "code.nonshy.com/nonshy/website/pkg/log" + pigo "github.com/esimov/pigo/core" +) + +// Functionality to do with face detection in pictures. + +var ( + faceScoreReady bool + faceClassifier *pigo.Pigo + cascadeFile []byte +) + +// InitFaceScore initializes the face recognition library (esimov/pigo). +func InitFaceScore() { + if faceScoreReady { + return + } + + if !config.Current.FaceScore.Enabled { + log.Error("InitFaceScore: not enabled in settings, face detection will not run") + return + } + + // Load the cascade file needed for face detection. + data, err := os.ReadFile(config.Current.FaceScore.CascadeFile) + if err != nil { + log.Error("InitFaceScore: could not load cascade file (%s): %s", config.Current.FaceScore.CascadeFile, err) + return + } + + cascadeFile = data + + log.Info("Initializing FaceScore with cascade file (%d bytes)", len(cascadeFile)) + + faceClassifier = pigo.NewPigo() + faceClassifier, err = faceClassifier.Unpack(cascadeFile) + if err != nil { + log.Error("InitFaceScore: could not unpack the cascade file: %s", err) + return + } + + faceScoreReady = true +} + +// ComputeFaceScore checks a photo on disk and returns the detected face score. +func ComputeFaceScore(filename string) (float64, error) { + if !faceScoreReady { + return 0, errors.New("face detection is not available") + } + + src, err := pigo.GetImage(filename) + if err != nil { + return 0, err + } + + var ( + pixels = pigo.RgbToGrayscale(src) + cols, rows = src.Bounds().Max.X, src.Bounds().Max.Y + cParams = pigo.CascadeParams{ + MinSize: 20, + MaxSize: 1000, + ShiftFactor: 0.1, + ScaleFactor: 1.1, + ImageParams: pigo.ImageParams{ + Pixels: pixels, + Rows: rows, + Cols: cols, + Dim: cols, + }, + } + ) + + // Run the classifier. + dets := faceClassifier.RunCascade(cParams, 0.0) + for _, row := range dets { + log.Warn("%+v", row) + } + + // Note: the classifier may return multiple matched faces, return the highest score. + var highest float32 + for _, row := range dets { + if row.Q > highest { + highest = row.Q + } + } + + return float64(highest), nil +} diff --git a/web/templates/photo/gallery.html b/web/templates/photo/gallery.html index cff1b83..c259907 100644 --- a/web/templates/photo/gallery.html +++ b/web/templates/photo/gallery.html @@ -64,6 +64,14 @@ Gallery {{end}} + + + {{if ne .FaceScore nil}} + + + {{.FaceScore}} + + {{end}} {{end}} diff --git a/web/templates/photo/upload.html b/web/templates/photo/upload.html index b8491f9..c3929a1 100644 --- a/web/templates/photo/upload.html +++ b/web/templates/photo/upload.html @@ -390,6 +390,35 @@ {{end}} + + {{if and .EditPhoto .CurrentUser.IsAdmin}} +
+ + + + + Face detection score: + {{if eq .EditPhoto.FaceScore nil}} + missing + {{else}} + {{.EditPhoto.FaceScore}} + {{end}} + + +
+ +
+
+ {{end}} + {{if not .EditPhoto}}