WIP face detection experiment

README.md

@@ -34,6 +34,19 @@ create extension postgis;
 If you get errors like "Type geography not found" from Postgres when
 running distance based searches, this is the likely culprit.
 
+### Face Detection
+
+Fedora: `dnf install python3 python3-opencv opencv-data`
+
+Debian: `apt install python3-opencv opencv-data`
+
+If you get an error like:
+
+> facedetect: error: cannot load HAAR_FRONTALFACE_ALT2 from /usr/share/opencv/haarcascades/haarcascade_frontalface_alt2.xml
+
+Check whether the correct path on disk is actually /usr/share/opencv4 instead of /usr/share/opencv.
+One solution then is to symlink the path correctly.
+
 ## Building the App
 
 This app is written in Go: [go.dev](https://go.dev). You can probably

cmd/nonshy/main.go

@@ -10,6 +10,7 @@ import (
 	"code.nonshy.com/nonshy/website/pkg/models"
 	"code.nonshy.com/nonshy/website/pkg/models/backfill"
 	"code.nonshy.com/nonshy/website/pkg/models/exporting"
+	"code.nonshy.com/nonshy/website/pkg/photo"
 	"code.nonshy.com/nonshy/website/pkg/redis"
 	"code.nonshy.com/nonshy/website/pkg/worker"
 	"github.com/urfave/cli/v2"
@@ -229,6 +230,9 @@ func initdb(c *cli.Context) {
 
 	// Auto-migrate the DB.
 	models.AutoMigrate()
+
+	// Initialize FaceScore face detection.
+	photo.InitFaceScore()
 }
 
 func initcache(c *cli.Context) {

go.mod

@@ -14,6 +14,7 @@ require (
 )
 
 require (
+	github.com/Kagami/go-face v0.0.0-20210630145111-0c14797b4d0e // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
@@ -21,6 +22,7 @@ require (
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/disintegration/imaging v1.6.2 // indirect
 	github.com/edwvee/exiffix v0.0.0-20210922235313-0f6cbda5e58f // indirect
+	github.com/esimov/pigo v1.4.6 // indirect
 	github.com/go-redis/redis v6.15.9+incompatible // indirect
 	github.com/golang-jwt/jwt/v4 v4.4.3 // indirect
 	github.com/gorilla/css v1.0.0 // indirect

go.sum

@@ -1,6 +1,8 @@
 git.kirsle.net/go/log v0.0.0-20200902035305-70ac2848949b h1:TDxEEWOJqMzsu9JW8/QgmT1lgQ9WD2KWlb2lKN/Ql2o=
 git.kirsle.net/go/log v0.0.0-20200902035305-70ac2848949b/go.mod h1:jl+Qr58W3Op7OCxIYIT+b42jq8xFncJXzPufhrvza7Y=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Kagami/go-face v0.0.0-20210630145111-0c14797b4d0e h1:lqIUFzxaqyYqUn4MhzAvSAh4wIte/iLNcIEWxpT/qbc=
+github.com/Kagami/go-face v0.0.0-20210630145111-0c14797b4d0e/go.mod h1:9wdDJkRgo3SGTcFwbQ7elVIQhIr2bbBjecuY7VoqmPU=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
@@ -22,6 +24,9 @@ github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1
 github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
 github.com/edwvee/exiffix v0.0.0-20210922235313-0f6cbda5e58f h1:RMnUwTnNR070mFAEIoqMYjNirHj8i0h79VXTYyBCyVA=
 github.com/edwvee/exiffix v0.0.0-20210922235313-0f6cbda5e58f/go.mod h1:KoE3Ti1qbQXCb3s/XGj0yApHnbnNnn1bXTtB5Auq/Vc=
+github.com/esimov/pigo v1.4.6 h1:wpB9FstbqeGP/CZP+nTR52tUJe7XErq8buG+k4xCXlw=
+github.com/esimov/pigo v1.4.6/go.mod h1:uqj9Y3+3IRYhFK071rxz1QYq0ePhA6+R9jrUZavi46M=
+github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
@@ -32,6 +37,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
 github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU=
 github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -190,6 +196,7 @@ golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJ
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U=
 golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20200927104501-e162460cd6b5/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/image v0.0.0-20220722155232-062f8c9fd539 h1:/eM0PCrQI2xd471rI+snWuu251/+/jpBpZqir2mPdnU=
 golang.org/x/image v0.0.0-20220722155232-062f8c9fd539/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@@ -215,6 +222,7 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201107080550-4d91cf3a1aaf/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -224,6 +232,7 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9w
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
 golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20191110171634-ad39bd3f0407/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

pkg/config/variable.go

@@ -14,7 +14,7 @@ import (
 
 // Version of the config format - when new fields are added, it will attempt
 // to write the settings.toml to disk so new defaults populate.
-var currentVersion = 2
+var currentVersion = 3
 
 // Current loaded settings.json
 var Current = DefaultVariable()
@@ -31,6 +31,7 @@ type Variable struct {
 	BareRTC          BareRTC
 	Maintenance      Maintenance
 	Encryption       Encryption
+	FaceScore        FaceScore
 	UseXForwardedFor bool
 }
 
@@ -52,6 +53,9 @@ func DefaultVariable() Variable {
 			SQLite:   "database.sqlite",
 			Postgres: "host=localhost user=nonshy password=nonshy dbname=nonshy port=5679 sslmode=disable TimeZone=America/Los_Angeles",
 		},
+		FaceScore: FaceScore{
+			CascadeFile: "/path/to/cascade/file",
+		},
 		CronAPIKey: uuid.New().String(),
 	}
 }
@@ -165,3 +169,9 @@ type Maintenance struct {
 type Encryption struct {
 	AESKey []byte
 }
+
+// FaceScore settings for face detection in photos via esimov/pigo.
+type FaceScore struct {
+	Enabled     bool
+	CascadeFile string
+}

pkg/controller/photo/edit_delete.go

@@ -75,6 +75,9 @@ func Edit() http.HandlerFunc {
 				setProfilePic = r.FormValue("intent") == "profile-pic"
 				crop          = pphoto.ParseCropCoords(r.FormValue("crop"))
 
+				// Re-compute the face score (admin only)
+				recomputeFaceScore = r.FormValue("recompute_face_score") == "true" && currentUser.IsAdmin
+
 				// Are we GOING private or changing to Inner Circle?
 				goingPrivate = visibility == models.PhotoPrivate && visibility != photo.Visibility
 				goingCircle  = visibility == models.PhotoInnerCircle && visibility != photo.Visibility
@@ -118,6 +121,17 @@ func Edit() http.HandlerFunc {
 
 			log.Error("SAVING PHOTO: %+v", photo)
 
+			// Are we re-computing the face score?
+			if recomputeFaceScore {
+				score, err := pphoto.ComputeFaceScore(pphoto.DiskPath(photo.Filename))
+				if err != nil {
+					session.FlashError(w, r, "Face score: %s", err)
+				} else {
+					session.Flash(w, r, "Face score recomputed!")
+					photo.FaceScore = &score
+				}
+			}
+
 			if err := photo.Save(); err != nil {
 				session.FlashError(w, r, "Couldn't save photo: %s", err)
 			}

pkg/models/photo.go

@@ -23,6 +23,7 @@ type Photo struct {
 	Visibility      PhotoVisibility
 	Gallery         bool     // photo appears in the public gallery (if public)
 	Explicit        bool     // is an explicit photo
+	FaceScore       *float64 // face detection score (best)
 	CreatedAt       time.Time
 	UpdatedAt       time.Time
 }

pkg/photo/face_score.go

@@ -0,0 +1,95 @@
+package photo
+
+import (
+	"errors"
+	"os"
+
+	"code.nonshy.com/nonshy/website/pkg/config"
+	"code.nonshy.com/nonshy/website/pkg/log"
+	pigo "github.com/esimov/pigo/core"
+)
+
+// Functionality to do with face detection in pictures.
+
+var (
+	faceScoreReady bool
+	faceClassifier *pigo.Pigo
+	cascadeFile    []byte
+)
+
+// InitFaceScore initializes the face recognition library (esimov/pigo).
+func InitFaceScore() {
+	if faceScoreReady {
+		return
+	}
+
+	if !config.Current.FaceScore.Enabled {
+		log.Error("InitFaceScore: not enabled in settings, face detection will not run")
+		return
+	}
+
+	// Load the cascade file needed for face detection.
+	data, err := os.ReadFile(config.Current.FaceScore.CascadeFile)
+	if err != nil {
+		log.Error("InitFaceScore: could not load cascade file (%s): %s", config.Current.FaceScore.CascadeFile, err)
+		return
+	}
+
+	cascadeFile = data
+
+	log.Info("Initializing FaceScore with cascade file (%d bytes)", len(cascadeFile))
+
+	faceClassifier = pigo.NewPigo()
+	faceClassifier, err = faceClassifier.Unpack(cascadeFile)
+	if err != nil {
+		log.Error("InitFaceScore: could not unpack the cascade file: %s", err)
+		return
+	}
+
+	faceScoreReady = true
+}
+
+// ComputeFaceScore checks a photo on disk and returns the detected face score.
+func ComputeFaceScore(filename string) (float64, error) {
+	if !faceScoreReady {
+		return 0, errors.New("face detection is not available")
+	}
+
+	src, err := pigo.GetImage(filename)
+	if err != nil {
+		return 0, err
+	}
+
+	var (
+		pixels     = pigo.RgbToGrayscale(src)
+		cols, rows = src.Bounds().Max.X, src.Bounds().Max.Y
+		cParams    = pigo.CascadeParams{
+			MinSize:     20,
+			MaxSize:     1000,
+			ShiftFactor: 0.1,
+			ScaleFactor: 1.1,
+			ImageParams: pigo.ImageParams{
+				Pixels: pixels,
+				Rows:   rows,
+				Cols:   cols,
+				Dim:    cols,
+			},
+		}
+	)
+
+	// Run the classifier.
+	dets := faceClassifier.RunCascade(cParams, 0.0)
+	for _, row := range dets {
+		log.Warn("%+v", row)
+	}
+
+	// Note: the classifier may return multiple matched faces, return the highest score.
+	var highest float32
+	for _, row := range dets {
+		if row.Q > highest {
+			highest = row.Q
+		}
+	}
+
+	return float64(highest), nil
+}

web/templates/photo/gallery.html

@@ -64,6 +64,14 @@
         <span>Gallery</span>
     </span>
     {{end}}
+
+    <!-- Face detection score, TODO: admin only -->
+    {{if ne .FaceScore nil}}
+    <span class="tag is-success is-light">
+        <span class="icon"><i class="fa fa-smile"></i></span>
+        <span>{{.FaceScore}}</span>
+    </span>
+    {{end}}
 </div>
 {{end}}
 

web/templates/photo/upload.html

@@ -390,6 +390,35 @@
                                 {{end}}
                             </div>
 
+                            <!-- Admin controls -->
+                            {{if and .EditPhoto .CurrentUser.IsAdmin}}
+                            <div class="field mb-5">
+                                <label class="label">
+                                    <span>Admin Options</span>
+                                    <span class="icon"><i class="fa fa-peace"></i></span>
+                                </label>
+
+                                <span class="tag is-info">
+                                    <i class="fa fa-smile mr-2"></i>
+                                    Face detection score:
+                                    {{if eq .EditPhoto.FaceScore nil}}
+                                        <em>missing</em>
+                                    {{else}}
+                                        {{.EditPhoto.FaceScore}}
+                                    {{end}}
+                                </span>
+
+                                <div class="mt-2">
+                                    <label class="checkbox">
+                                        <input type="checkbox"
+                                            name="recompute_face_score"
+                                            value="true">
+                                            Re-compute the face score
+                                    </label>
+                                </div>
+                            </div>
+                            {{end}}
+
                             {{if not .EditPhoto}}
                             <div class="field">
                                 <label class="label">Confirm Upload</label>