WIP: Deduplicate threads on Newest forum tab

pkg/controller/forum/newest.go

@@ -14,6 +14,11 @@ import (
 func Newest() http.HandlerFunc {
 	tmpl := templates.Must("forum/newest.html")
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Query parameters.
+		var (
+			allComments = r.FormValue("all") == "true"
+		)
+
 		// Get the current user.
 		currentUser, err := session.CurrentUser(r)
 		if err != nil {
@@ -29,7 +34,7 @@ func Newest() http.HandlerFunc {
 		}
 		pager.ParsePage(r)
 
-		posts, err := models.PaginateRecentPosts(currentUser, config.ForumCategories, pager)
+		posts, err := models.PaginateRecentPosts(currentUser, config.ForumCategories, allComments, pager)
 		if err != nil {
 			session.FlashError(w, r, "Couldn't paginate forums: %s", err)
 			templates.Redirect(w, "/")
@@ -50,6 +55,7 @@ func Newest() http.HandlerFunc {
 			"Pager":       pager,
 			"RecentPosts": posts,
 			"PhotoMap":    photos,
+			"AllComments": allComments,
 		}
 		if err := tmpl.Execute(w, r, vars); err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)

pkg/models/forum_recent.go

@@ -1,9 +1,11 @@
 package models
 
 import (
+	"fmt"
 	"strings"
 	"time"
 
+	"code.nonshy.com/nonshy/website/pkg/config"
 	"code.nonshy.com/nonshy/website/pkg/log"
 )
 
@@ -20,7 +22,7 @@ type RecentPost struct {
 }
 
 // PaginateRecentPosts returns all of the comments on a forum paginated.
-func PaginateRecentPosts(user *User, categories []string, pager *Pagination) ([]*RecentPost, error) {
+func PaginateRecentPosts(user *User, categories []string, allComments bool, pager *Pagination) ([]*RecentPost, error) {
 	var (
 		result         = []*RecentPost{}
 		query          = (&Comment{}).Preload()
@@ -65,24 +67,73 @@ func PaginateRecentPosts(user *User, categories []string, pager *Pagination) ([]
 		CommentID uint64
 		ThreadID  *uint64
 		ForumID   *uint64
+		UpdatedAt time.Time
 	}
 	var scan []scanner
-	query = DB.Table("comments").Select(
+
-		`comments.id AS comment_id,
+	// Deduplicate forum threads: if one thread is BLOWING UP with replies, we should only
-		 threads.id AS thread_id,
+	// mention the thread once and show the newest comment so it doesn't spam the whole page.
-		 forums.id AS forum_id`,
+	if config.Current.Database.IsPostgres && !allComments {
-	).Joins(
+		// Deduplicating is supported in Postgres but not SQLite. We also need a custom
-		"LEFT OUTER JOIN threads ON (table_name = 'threads' AND table_id = threads.id)",
+		// total count query which is 99% the same as the select query except for the
-	).Joins(
+		// SELECT and ORDER BY bookends.
-		"LEFT OUTER JOIN forums ON (threads.forum_id = forums.id)",
+		subquery := fmt.Sprintf(`
-	).Where(
+			FROM (
-		strings.Join(wheres, " AND "),
+				SELECT DISTINCT ON (threads.id)
-		placeholders...,
+					comments.id AS comment_id,
-	).Order("comments.updated_at desc")
+					threads.id AS thread_id,
+					forums.id AS forum_id,
+					comments.updated_at AS updated_at
+				FROM comments
+				LEFT OUTER JOIN threads ON (table_name='threads' AND table_id=threads.id)
+				LEFT OUTER JOIN forums ON (threads.forum_id=forums.id)
+				WHERE %s
+				ORDER BY threads.id
+			) AS subquery
+		`, strings.Join(wheres, " AND "))
+
+		query = DB.Raw(fmt.Sprintf(`
+				SELECT comment_id, thread_id, forum_id, updated_at
+				%s
+				ORDER BY subquery.updated_at DESC
+				OFFSET %d LIMIT %d
+			`, subquery, pager.GetOffset(), pager.PerPage), placeholders...)
+
+		// Get a count of records.
+		DB.Raw(fmt.Sprintf("SELECT count(*) %s", subquery), placeholders...).Count(&pager.Total)
+
+		query = query.Find(&scan)
+		if query.Error != nil {
+			return nil, query.Error
+		}
+	} else {
+		// SQLite/non-Postgres doesn't support DISTINCT ON, this is the old query which
+		// shows objectively all comments and a popular thread may dominate the page.
+		query = DB.Table("comments").Select(
+			`comments.id AS comment_id,
+			 threads.id AS thread_id,
+			 forums.id AS forum_id,
+			 comments.updated_at AS updated_at`,
+		).Joins(
+			"LEFT OUTER JOIN threads ON (table_name = 'threads' AND table_id = threads.id)",
+		).Joins(
+			"LEFT OUTER JOIN forums ON (threads.forum_id = forums.id)",
+		).Where(
+			strings.Join(wheres, " AND "),
+			placeholders...,
+		).Order("comments.updated_at desc")
+		query.Model(&Comment{}).Count(&pager.Total)
+
+		// Execute the query.
+		query = query.Offset(pager.GetOffset()).Limit(pager.PerPage).Find(&scan)
+		if query.Error != nil {
+			return nil, query.Error
+		}
+	}
 
 	// Get the total for the pager and scan the page of ID sets.
-	query.Model(&Comment{}).Count(&pager.Total)
+	// query.Model(&Comment{}).Count(&pager.Total)
-	query = query.Offset(pager.GetOffset()).Limit(pager.PerPage).Find(&scan)
+	// query = query.Offset(pager.GetOffset()).Limit(pager.PerPage).Find(&scan)
 	if query.Error != nil {
 		return nil, query.Error
 	}

web/templates/account/inner_circle.html

@@ -99,7 +99,7 @@
                 </li>
                 <li>
                     On the
-                    <a href="/members"><strong><i class="fa fa-comments mr-1"></i> Forums</strong></a>
+                    <a href="/f/circle"><strong><i class="fa fa-comments mr-1"></i> Forums</strong></a>
                     you can access exclusive inner circle-only boards.
                 </li>
                 <li>

web/templates/forum/newest.html

@@ -39,7 +39,16 @@
 </div>
 
 <div class="p-4">
-    Found {{FormatNumberCommas .Pager.Total}} posts (page {{.Pager.Page}} of {{.Pager.Pages}})
+    Found {{FormatNumberCommas .Pager.Total}} {{if .AllComments}}posts{{else}}threads{{end}} (page {{.Pager.Page}} of {{.Pager.Pages}})
+
+    <div class="mt-2">
+        {{if not .AllComments}}
+            <!-- Default view is to deduplicate and show only threads and their newest comment -->
+            Showing only the latest comment per thread. <a href="?{{QueryPlus "all" "true"}}">Show all comments instead?</a>
+        {{else}}
+            Showing <strong>all</strong> forum posts by most recent. <a href="{{.Request.URL.Path}}">Deduplicate by thread?</a>
+        {{end}}
+    </div>
 </div>
 
 <div class="p-4">