Merge remote-tracking branch 'upstream/master' into relay-pinned-stre…

…amer-image-link

app/controllers/Coach.scala

@@ -37,7 +37,6 @@ final class Coach(env: Env) extends LilaController(env):
           studies <- env.study.pager.withChaptersAndLiking(ctx.me, 4)(stu)
           posts   <- env.ublog.api.latestPosts(lila.ublog.UblogBlog.Id.User(c.user.id), 4)
           page    <- renderPage(views.coach.show(c, studies, posts))
-          _ = lila.mon.coach.pageView.profile(c.coach.id.value).increment()
         yield Ok(page)
 
   private def WithVisibleCoach(c: CoachModel.WithUser)(f: Fu[Result])(using ctx: Context) =

app/controllers/Streamer.scala

@@ -102,29 +102,28 @@ final class Streamer(env: Env, apiC: => Api) extends LilaController(env):
             modData(s.streamer).flatMap: forMod =>
               BadRequest.page(views.streamer.edit(sws, error, forMod)),
           data =>
-            api.update(sws.streamer, data, isGranted(_.Streamers)).flatMap { change =>
-              if change.decline then logApi.streamerDecline(s.user.id)
-              change.list.foreach { logApi.streamerList(s.user.id, _) }
-              change.tier.foreach { logApi.streamerTier(s.user.id, _) }
-              if data.approval.flatMap(_.quick).isDefined
-              then
-                env.streamer.pager.nextRequestId.map: nextId =>
-                  Redirect:
-                    nextId.fold(s"${routes.Streamer.index()}?requests=1"): id =>
-                      s"${routes.Streamer.edit.url}?u=$id"
-              else
-                val next = if sws.streamer.is(me) then "" else s"?u=${sws.user.id}"
-                Redirect(s"${routes.Streamer.edit.url}$next")
-            }
+            api
+              .update(sws.streamer, data, isGranted(_.Streamers))
+              .flatMap:
+                case Some(change) =>
+                  if change.decline then logApi.streamerDecline(s.user.id)
+                  change.list.foreach { logApi.streamerList(s.user.id, _) }
+                  change.tier.foreach { logApi.streamerTier(s.user.id, _) }
+                  if data.approval.flatMap(_.quick).isDefined
+                  then
+                    env.streamer.pager.nextRequestId.map: nextId =>
+                      Redirect:
+                        nextId.fold(s"${routes.Streamer.index()}?requests=1"): id =>
+                          s"${routes.Streamer.edit.url}?u=$id"
+                  else
+                    val next = if sws.streamer.is(me) then "" else s"?u=${sws.user.id}"
+                    Redirect(s"${routes.Streamer.edit.url}$next")
+                case _ =>
+                  Redirect(routes.Streamer.edit)
         )
       }
   }
 
-  def approvalRequest = AuthBody { _ ?=> me ?=>
-    NoBot:
-      api.approval.request(me).inject(Redirect(routes.Streamer.edit))
-  }
-
   def pictureApply = AuthBody(parse.multipartFormData) { ctx ?=> me ?=>
     AsStreamer: s =>
       ctx.body.body.file("picture") match
@@ -135,7 +134,7 @@ final class Streamer(env: Env, apiC: => Api) extends LilaController(env):
               .recoverWith { case e: Exception =>
                 Redirect(routes.Streamer.edit).flashFailure
               }
-              .inject(Redirect(routes.Streamer.edit))
+              .inject(Ok)
         case None => Redirect(routes.Streamer.edit).flashFailure
   }
 

app/controllers/Team.scala

@@ -439,9 +439,7 @@ final class Team(env: Env) extends LilaController(env):
                     env.team.memberStream.subscribedIds(team, MaxPerSecond(50)),
                     full
                   )
-                  .addEffect: nb =>
-                    if nb > 100
-                    then lila.mon.msg.teamBulk(team.id).record(nb)
+                  .addEffect(lila.mon.msg.teamBulk.record(_))
                 // we don't wait for the stream to complete, it would make lichess time out
                 fuccess(Result.Through)
               }(Result.Limited)

app/controllers/User.scala

@@ -553,24 +553,21 @@ final class User(
   }
 
   def perfStat(username: UserStr, perfKey: PerfKey) = Open:
-    PerfType
-      .isLeaderboardable(perfKey)
-      .so:
-        Found(env.perfStat.api.data(username, perfKey)): data =>
-          negotiate(
-            Ok.async:
-              env.history
-                .ratingChartApi(data.user.user)
-                .map:
-                  views.user.perfStatPage(data, _)
-            ,
-            JsonOk:
-              getBool("graph")
-                .soFu:
-                  env.history.ratingChartApi.singlePerf(data.user.user, data.stat.perfType.key)
-                .map: graph =>
-                  env.perfStat.jsonView(data).add("graph", graph)
-          )
+    Found(env.perfStat.api.data(username, perfKey)): data =>
+      negotiate(
+        Ok.async:
+          env.history
+            .ratingChartApi(data.user.user)
+            .map:
+              views.user.perfStatPage(data, _)
+        ,
+        JsonOk:
+          getBool("graph")
+            .soFu:
+              env.history.ratingChartApi.singlePerf(data.user.user, data.stat.perfType.key)
+            .map: graph =>
+              env.perfStat.jsonView(data).add("graph", graph)
+      )
 
   def autocomplete = OpenOrScoped(): ctx ?=>
     NoTor:

bin/dependency-graph.py

@@ -93,7 +93,7 @@ def pick(essentials, providers_dict):
     print()
 
 pydot_graph = nx.drawing.nx_pydot.to_pydot(G)
-pydot_graph.set_rankdir('LR')
+# pydot_graph.set_rankdir('LR')
 # pydot_graph.set_ratio(1)
 
 sink_nodes = pydot.Subgraph(rank="same")
@@ -120,6 +120,6 @@ def pick(essentials, providers_dict):
             pydot_graph.add_edge(edge)
 
         edge.set_color('red')
-        edge.set_penwidth(2)
+        edge.set_penwidth(1)
 
 pydot_graph.write_png(output_path)

build.sbt

@@ -42,11 +42,6 @@ Compile / sourceDirectory                  := baseDirectory.value / "app"
 Compile / scalaSource                      := baseDirectory.value / "app"
 Universal / sourceDirectory                := baseDirectory.value / "dist"
 
-// cats-parse v1.0.0 is the same as v0.3.1, so this is safe
-ThisBuild / libraryDependencySchemes ++= Seq(
-  "org.typelevel" %% "cats-parse" % VersionScheme.Always
-)
-
 // format: off
 libraryDependencies ++= akka.bundle ++ playWs.bundle ++ macwire.bundle ++ scalalib.bundle ++ chess.bundle ++ Seq(
   play.json, play.logback, compression, hasher,
@@ -482,7 +477,7 @@ lazy val notifyModule = module("notify",
 )
 
 lazy val recap = module("recap",
-  Seq(memo, ui, user, game, puzzle),
+  Seq(user, game, puzzle),
   Seq()
 )
 

conf/base.conf

@@ -215,7 +215,7 @@ security {
     enabled = false
     url = "http://ip2proxy.lichess.ovh:1929"
   }
-  pwned.url = ""
+  pwned.range_url = "https://api.pwnedpasswords.com/range/"
   hcaptcha = ${hcaptcha}
   lame_name_check = true
   password.bpass.secret = ${user.password.bpass.secret}
@@ -449,6 +449,7 @@ kamon {
     process-metrics.enabled = yes
     host-metrics.enabled = no
     prometheus-reporter.enabled = yes
+    prometheus-reporter.factory = "lila.web.PrometheusReporter$Factory"
   }
 }
 # Don't let play manage its own PID file

conf/routes

@@ -340,7 +340,6 @@ GET   /streamer/live                   controllers.Main.movedPermanently(to = "/
 GET   /streamer/edit                   controllers.Streamer.edit
 POST  /streamer/new                    controllers.Streamer.create
 POST  /streamer/edit                   controllers.Streamer.editApply
-POST  /streamer/approval/request       controllers.Streamer.approvalRequest
 POST  /streamer/subscribe/:streamer    controllers.Streamer.subscribe(streamer: UserStr, set: Boolean ?= true)
 POST  /upload/image/streamer           controllers.Streamer.pictureApply
 GET   /streamer/:username              controllers.Streamer.show(username: UserStr)

modules/api/src/main/LinkCheck.scala

@@ -59,10 +59,9 @@ final private class LinkCheck(
   private def tourLink(tourId: String, source: FullSource): Fu[Boolean] =
     tournamentRepo.byId(TourId(tourId)).flatMapz { tour =>
       fuccess(tour.isScheduled) >>| {
-        source.teamId.so { sourceTeamId =>
+        source.teamId.so: sourceTeamId =>
           fuccess(tour.conditions.teamMember.exists(_.teamId == sourceTeamId)) >>|
             tournamentRepo.isForTeam(tour.id, sourceTeamId)
-        }
       }
     }
 

modules/clas/src/main/ClasApi.scala

@@ -260,7 +260,11 @@ final class ClasApi(
 
     def move(s: Student.WithUser, toClas: Clas)(using teacher: Me): Fu[Option[Student]] = for
       _ <- closeAccount(s)
-      stu = s.student.copy(id = Student.makeId(s.user.id, toClas.id), clasId = toClas.id)
+      stu = s.student.copy(
+        id = Student.makeId(s.user.id, toClas.id),
+        clasId = toClas.id,
+        created = Clas.Recorded(by = teacher.userId, at = nowInstant)
+      )
       moved <- colls.student.insert
         .one(stu)
         .inject(stu.some)

modules/common/src/main/mon.scala

@@ -289,7 +289,8 @@ object mon:
       timer("relay.sync.time").withTags(relay(official, id, slug))
     def httpGet(host: String, proxy: Option[String]) =
       future("relay.http.get", tags("host" -> host, "proxy" -> proxy.getOrElse("none")))
-    val dedup = counter("relay.fetch.dedup").withoutTags()
+    val dedup              = counter("relay.fetch.dedup").withoutTags()
+    def etag(hit: Boolean) = counter("relay.fetch.etag").withTag("hit", hit)
 
   object bot:
     def moves(username: String)   = counter("bot.moves").withTag("name", username)
@@ -368,9 +369,6 @@ object mon:
     val unfollow  = c.withTag("type", "unfollow")
     val block     = c.withTag("type", "block")
     val unblock   = c.withTag("type", "unblock")
-  object coach:
-    object pageView:
-      def profile(coachId: String) = counter("coach.pageView").withTag("name", coachId)
   object clas:
     object student:
       def create(teacher: String) = counter("clas.student.create").withTag("teacher", teacher)
@@ -457,17 +455,17 @@ object mon:
     def post(verdict: String, isNew: Boolean, multi: Boolean) = counter("msg.post").withTags(
       tags("verdict" -> verdict, "isNew" -> isNew, "multi" -> multi)
     )
-    def teamBulk(teamId: TeamId) = histogram("msg.bulk.team").withTag("id", teamId.value)
+    val teamBulk                 = histogram("msg.bulk.team").withoutTags()
     def clasBulk(clasId: ClasId) = histogram("msg.bulk.clas").withTag("id", clasId.value)
   object puzzle:
     object selector:
       object user:
-        def time(theme: String)    = timer("puzzle.selector.user.puzzle").withTag("theme", theme)
-        def retries(theme: String) = histogram("puzzle.selector.user.retries").withTag("theme", theme)
+        def time(categ: String)    = timer("puzzle.selector.user.puzzle").withTag("categ", categ)
+        def retries(categ: String) = histogram("puzzle.selector.user.retries").withTag("categ", categ)
         val vote                   = histogram("puzzle.selector.user.vote").withoutTags()
         def tier(t: String, categ: String, difficulty: String) =
           counter("puzzle.selector.user.tier").withTags:
-            tags("tier" -> t, "theme" -> categ, "difficulty" -> difficulty)
+            tags("tier" -> t, "categ" -> categ, "difficulty" -> difficulty)
         def batch(nb: Int) = timer("puzzle.selector.user.batch").withTag("nb", nb)
       object anon:
         val time           = timer("puzzle.selector.anon.puzzle").withoutTags()
@@ -476,7 +474,7 @@ object mon:
       def nextPuzzleResult(result: String) =
         timer("puzzle.selector.user.puzzleResult").withTag("result", result)
     object path:
-      def nextFor(categ: String) = timer("puzzle.path.nextFor").withTag("theme", categ)
+      def nextFor(categ: String) = timer("puzzle.path.nextFor").withTag("categ", categ)
 
     object batch:
       object selector:

modules/coreI18n/src/main/key.scala

@@ -2331,10 +2331,11 @@ object I18nKey:
     val `pendingReview`: I18nKey = "streamer:pendingReview"
     val `pleaseFillIn`: I18nKey = "streamer:pleaseFillIn"
     val `whenReady`: I18nKey = "streamer:whenReady"
-    val `requestReview`: I18nKey = "streamer:requestReview"
+    val `submitForReview`: I18nKey = "streamer:submitForReview"
     val `streamerLanguageSettings`: I18nKey = "streamer:streamerLanguageSettings"
     val `twitchUsername`: I18nKey = "streamer:twitchUsername"
-    val `optionalOrEmpty`: I18nKey = "streamer:optionalOrEmpty"
+    val `twitchOrYouTubeRequired`: I18nKey = "streamer:twitchOrYouTubeRequired"
+    val `twitchOrYouTubeMustBeVerified`: I18nKey = "streamer:twitchOrYouTubeMustBeVerified"
     val `youTubeChannelId`: I18nKey = "streamer:youTubeChannelId"
     val `streamerName`: I18nKey = "streamer:streamerName"
     val `visibility`: I18nKey = "streamer:visibility"

modules/msg/src/main/MsgSecurity.scala

@@ -140,7 +140,8 @@ final private class MsgSecurity(
     def post(contacts: Contacts, isNew: Boolean): Fu[Boolean] =
       (
         !contacts.dest.isLichess &&
-          (!contacts.any(_.marks.exists(_.isolate)) || contacts.any(_.isGranted(_.Shadowban)))
+          (!contacts.any(_.marks.exists(_.isolate)) ||
+            contacts.any(c => c.isGranted(_.Shadowban) && c.isGranted(_.PublicMod)))
       ).so:
         fuccess(contacts.orig.isGranted(_.PublicMod)) >>| {
           relationApi.fetchBlocks(contacts.dest.id, contacts.orig.id).not >>&

modules/push/src/main/WebPush.scala

@@ -2,6 +2,7 @@ package lila.push
 
 import play.api.ConfigLoader
 import play.api.libs.json.*
+import play.api.libs.ws.JsonBodyReadables.*
 import play.api.libs.ws.JsonBodyWritables.*
 import play.api.libs.ws.StandaloneWSClient
 
@@ -17,16 +18,18 @@ final private class WebPush(
 )(using Executor):
 
   def apply(userId: UserId, data: LazyFu[PushApi.Data]): Funit =
-    webSubscriptionApi.getSubscriptions(5)(userId).flatMap(sendTo(data))
+    webSubscriptionApi.getSubscriptions(5)(userId).flatMap(sendTo(data, List(userId)))
 
   def apply(userIds: Iterable[UserId], data: LazyFu[PushApi.Data]): Funit =
-    webSubscriptionApi.getSubscriptions(userIds, 5).flatMap(sendTo(data))
+    webSubscriptionApi.getSubscriptions(userIds, 5).flatMap(sendTo(data, userIds))
 
-  private def sendTo(data: LazyFu[PushApi.Data])(subs: List[WebSubscription]): Funit =
+  private def sendTo(data: LazyFu[PushApi.Data], to: Iterable[UserId])(subs: List[WebSubscription]): Funit =
     subs.toNel.so: subs =>
-      data.value.flatMap(send(subs))
+      data.value.flatMap(send(subs, to))
 
-  private def send(subscriptions: NonEmptyList[WebSubscription])(data: PushApi.Data): Funit =
+  private def send(subscriptions: NonEmptyList[WebSubscription], to: Iterable[UserId])(
+      data: PushApi.Data
+  ): Funit =
     ws.url(config.url)
       .withHttpHeaders("ContentType" -> "application/json")
       .post(
@@ -56,8 +59,20 @@ final private class WebPush(
         )
       )
       .flatMap {
-        case res if res.status == 200 => funit
-        case res                      => fufail(s"[push] web: ${res.status} ${res.body}")
+        case res if res.status == 200 =>
+          res
+            .body[JsValue]
+            .asOpt[JsObject]
+            .map:
+              _.fields.collect:
+                case (endpoint, JsString("endpoint_not_valid" | "endpoint_not_found")) => endpoint
+            .filter(_.nonEmpty)
+            .so: staleEndpoints =>
+              webSubscriptionApi
+                .unsubscribeByEndpoints(staleEndpoints, to)
+                .map: n =>
+                  logger.info(s"[push] web: $n/${staleEndpoints.size} stale endpoints unsubscribed")
+        case res => fufail(s"[push] web: ${res.status} ${res.body}")
       }
 
 private object WebPush:

modules/push/src/main/WebSubscriptionApi.scala

@@ -29,6 +29,11 @@ final class WebSubscriptionApi(coll: Coll)(using Executor):
   def unsubscribeByUser(user: User): Funit =
     coll.delete.one($doc("userId" -> user.id)).void
 
+  // userIds is necessary to match the mongodb index
+  def unsubscribeByEndpoints(endpoints: Iterable[String], userIds: Iterable[UserId]): Fu[Int] =
+    endpoints.nonEmpty.so:
+      coll.delete.one($doc("userId".$in(userIds), "endpoint".$in(endpoints))).map(_.n)
+
   private[push] def getSubscriptions(max: Int)(userId: UserId): Fu[List[WebSubscription]] =
     coll
       .find($doc("userId" -> userId), $doc("endpoint" -> true, "auth" -> true, "p256dh" -> true).some)

modules/puzzle/src/main/PuzzleHistory.scala

@@ -14,13 +14,7 @@ object PuzzleHistory:
   case class PuzzleSession(
       theme: PuzzleTheme.Key,
       puzzles: NonEmptyList[SessionRound] // chronological order, oldest first
-  ) {
-    // val nb              = puzzles.size
-    // val firstWins       = puzzles.toList.count(_.round.firstWin)
-    // val fails           = nb - firstWins
-    // def puzzleRatingAvg = puzzles.toList.foldLeft(0)(_ + _.puzzle.glicko.intRating)
-    // def performance     = puzzleRatingAvg - 500 + math.round(1000 * (firstWins.toFloat / nb))
-  }
+  )
 
   final class HistoryAdapter(user: WithPerf, colls: PuzzleColls)(using Executor)
       extends AdapterLike[PuzzleSession]:

modules/puzzle/src/main/PuzzleSelector.scala

@@ -27,7 +27,7 @@ final class PuzzleSelector(
         ,
         some
       )
-      .mon(_.puzzle.selector.user.time(angle.key))
+      .mon(_.puzzle.selector.user.time(angle.categ))
 
   private def findNextPuzzleFor(angle: PuzzleAngle, retries: Int)(using me: Me, perf: Perf): Fu[Puzzle] =
     sessionApi
@@ -47,7 +47,7 @@ final class PuzzleSelector(
 
         def serveAndMonitor(puzzle: Puzzle) =
           val mon = lila.mon.puzzle.selector.user
-          mon.retries(angle.key).record(retries)
+          mon.retries(angle.categ).record(retries)
           mon.vote.record(100 + math.round(puzzle.vote * 100))
           mon.tier(session.path.tier.key, angle.categ, session.settings.difficulty.key).increment()
           puzzle

modules/recap/src/main/RecapBuilder.scala

@@ -1,6 +1,7 @@
 package lila.recap
 
 import reactivemongo.akkastream.{ AkkaStreamCursor, cursorProducer }
+import reactivemongo.api.bson.BSONNull
 import chess.ByColor
 import chess.opening.OpeningDb
 import chess.format.pgn.SanStr
@@ -9,11 +10,8 @@ import scalalib.model.Days
 import lila.common.SimpleOpening
 import lila.db.dsl.{ *, given }
 import lila.game.Query
-import lila.puzzle.PuzzleRound
 import lila.common.LichessDay
 import lila.core.game.Source
-import java.time.LocalDate
-import reactivemongo.api.bson.BSONNull
 
 private final class RecapBuilder(
     repo: RecapRepo,