diff --git a/common/app/agents/DeeplyReadAgent.scala b/common/app/agents/DeeplyReadAgent.scala index 8cd908104e8e..b5ef29e1d6e1 100644 --- a/common/app/agents/DeeplyReadAgent.scala +++ b/common/app/agents/DeeplyReadAgent.scala @@ -3,25 +3,98 @@ package agents import com.gu.contentapi.client.model.v1.{Content, ElementType} import com.gu.contentapi.client.utils.CapiModelEnrichment.RenderingFormat import common._ +import conf.Configuration +import conf.switches.Switches.UseTrailsFromS3 +import model.dotcomrendering.Trail +import services.{FaciaContentConvert, OphanApi, S3Async} + +import scala.concurrent.{ExecutionContext, Future} +import scala.util.control.NonFatal import contentapi.ContentApiClient import layout.DiscussionSettings import model.ContentFormat -import services.{FaciaContentConvert, OphanApi} -import scala.concurrent.{ExecutionContext, Future} -import scala.util.control.NonFatal -import model.dotcomrendering.Trail +object DeeplyReadS3Agent extends S3Async { + override lazy val bucket = Configuration.cache.bucket; + lazy val stage: String = Configuration.environment.stage.toUpperCase +} class DeeplyReadAgent(contentApiClient: ContentApiClient, ophanApi: OphanApi) extends GuLogging { private val deeplyReadItems = Box[Map[Edition, Seq[Trail]]](Map.empty) + def getTrailsFromS3(edition: Edition)(implicit ec: ExecutionContext): Future[Unit] = { + Future + .sequence(Edition.allEditions.map { edition => + DeeplyReadS3Agent + .getObjectAsJson[Seq[Trail]]( + s"${DeeplyReadS3Agent.stage}/deeply-read/${edition.id.toLowerCase()}.json", + ) + .map(trailsList => { + edition -> trailsList.take(10) + }) + }) + .map(trailsList => { + val map = trailsList.toMap + for { + (edition, list) <- map + } yield log.debug(s"Deeply Read in ${edition.displayName}, ${list.size} items: ${list.map(_.url).toString()}") + + val mapWithTenItems = map.filter { case (_, list) => list.size == 10 } + log.debug( + s"Updating the following ${mapWithTenItems.size} editions: ${mapWithTenItems.keys.map(_.id).toList.sorted.toString()}", + ) + + deeplyReadItems.alter(deeplyReadItems.get() ++ mapWithTenItems) + }) + } + def removeStartingSlash(path: String): String = { if (path.startsWith("/")) path.stripPrefix("/") else path } - def refresh()(implicit ec: ExecutionContext): Future[Unit] = { - log.debug(s"Deeply Read Agent refresh()") + def correctPillar(pillar: String): String = if (pillar == "arts") "culture" else pillar + + def deeplyReadUrlToTrail(content: Content): Option[Trail] = { + + val contentFormat: ContentFormat = ContentFormat(content.design, content.theme, content.display) + + for { + webPublicationDate <- content.webPublicationDate + fields <- content.fields + linkText <- fields.trailText + pillar <- content.pillarName + headline <- fields.headline + shortUrl <- fields.shortUrl + } yield Trail( + url = content.webUrl, + linkText = linkText, + showByline = false, + byline = fields.byline, + masterImage = None, + image = fields.thumbnail, + carouselImages = Map.empty, + ageWarning = None, + isLiveBlog = fields.liveBloggingNow.getOrElse(false), + pillar = correctPillar(pillar.toLowerCase), + designType = content.`type`.toString, + format = contentFormat, + webPublicationDate = webPublicationDate.toString(), + headline = headline, + mediaType = None, + shortUrl = shortUrl, + kickerText = None, + starRating = None, + avatarUrl = None, + branding = None, + discussion = DiscussionSettings.fromTrail(FaciaContentConvert.contentToFaciaContent(content)), + trailText = content.fields.flatMap(_.trailText), + galleryCount = + content.elements.map(_.count(el => el.`type` == ElementType.Image && el.relation == "gallery")).filter(_ > 0), + ) + } + + def getTrailsFromCAPI(edition: Edition)(implicit ec: ExecutionContext): Future[Unit] = { /* We query Ophan for the deeply read URLs and use them to queryCapi then use this information to create a sequence of trails that we cache @@ -84,45 +157,15 @@ class DeeplyReadAgent(contentApiClient: ContentApiClient, ophanApi: OphanApi) ex }) } - def correctPillar(pillar: String): String = if (pillar == "arts") "culture" else pillar - - def deeplyReadUrlToTrail(content: Content): Option[Trail] = { - - val contentFormat: ContentFormat = ContentFormat(content.design, content.theme, content.display) - - for { - webPublicationDate <- content.webPublicationDate - fields <- content.fields - linkText <- fields.trailText - pillar <- content.pillarName - headline <- fields.headline - shortUrl <- fields.shortUrl - } yield Trail( - url = content.webUrl, - linkText = linkText, - showByline = false, - byline = fields.byline, - masterImage = None, - image = fields.thumbnail, - carouselImages = Map.empty, - ageWarning = None, - isLiveBlog = fields.liveBloggingNow.getOrElse(false), - pillar = correctPillar(pillar.toLowerCase), - designType = content.`type`.toString, - format = contentFormat, - webPublicationDate = webPublicationDate.toString(), - headline = headline, - mediaType = None, - shortUrl = shortUrl, - kickerText = None, - starRating = None, - avatarUrl = None, - branding = None, - discussion = DiscussionSettings.fromTrail(FaciaContentConvert.contentToFaciaContent(content)), - trailText = content.fields.flatMap(_.trailText), - galleryCount = - content.elements.map(_.count(el => el.`type` == ElementType.Image && el.relation == "gallery")).filter(_ > 0), - ) + def refresh()(implicit ec: ExecutionContext): Future[Unit] = { + if (UseTrailsFromS3.isSwitchedOn) { + // TODO revert to log.debug once we switch to S3 permanently + log.info(s"Deeply Read Agent refresh() - Using S3") + getTrailsFromS3(Edition.defaultEdition) + } else { + log.info(s"Deeply Read Agent refresh() - Using CAPI") + getTrailsFromCAPI(Edition.defaultEdition) + } } def getTrails(edition: Edition)(implicit ec: ExecutionContext): Seq[Trail] = { @@ -132,5 +175,4 @@ class DeeplyReadAgent(contentApiClient: ContentApiClient, ophanApi: OphanApi) ex } updatedTrails } - } diff --git a/common/app/common/configuration.scala b/common/app/common/configuration.scala index 83de4f3a02d7..1ae42a61265a 100644 --- a/common/app/common/configuration.scala +++ b/common/app/common/configuration.scala @@ -705,6 +705,10 @@ class GuardianConfiguration extends GuLogging { lazy val host = configuration.getStringProperty("newsletterApi.host") lazy val origin = configuration.getStringProperty("newsletterApi.origin") } + + object cache { + lazy val bucket = configuration.getMandatoryStringProperty("cache.bucket") + } } object ManifestData { diff --git a/common/app/conf/switches/FeatureSwitches.scala b/common/app/conf/switches/FeatureSwitches.scala index db85fe11e765..d682cf5aa4e6 100644 --- a/common/app/conf/switches/FeatureSwitches.scala +++ b/common/app/conf/switches/FeatureSwitches.scala @@ -671,4 +671,15 @@ trait FeatureSwitches { exposeClientSide = true, highImpact = false, ) + + val UseTrailsFromS3 = Switch( + group = SwitchGroup.Feature, + name = "use-trails-from-s3", + description = "Use trails from S3", + owners = Seq(Owner.withEmail("dotcom.platform@theguardian.com")), + sellByDate = never, + safeState = Off, + exposeClientSide = false, + highImpact = false, + ) } diff --git a/common/app/model/dotcomrendering/Trail.scala b/common/app/model/dotcomrendering/Trail.scala index 184c1c6ed4d1..ea2db67325b5 100644 --- a/common/app/model/dotcomrendering/Trail.scala +++ b/common/app/model/dotcomrendering/Trail.scala @@ -1,14 +1,21 @@ package model.dotcomrendering import com.github.nscala_time.time.Imports.DateTimeZone -import com.gu.commercial.branding.{Branding, BrandingType, Dimensions, Logo => CommercialLogo} +import com.gu.commercial.branding.{ + Branding, + BrandingType, + Dimensions, + Foundation, + PaidContent, + Sponsored, + Logo => CommercialLogo, +} import common.{Edition, LinkTo} import implicits.FaciaContentFrontendHelpers.FaciaContentFrontendHelper import layout.DiscussionSettings -import model.dotcomrendering.DotcomRenderingUtils.withoutNull import model.{Article, ContentFormat, ImageMedia, Pillar} import model.pressed.PressedContent -import play.api.libs.json.{Json, OWrites, Writes} +import play.api.libs.json.{JsNull, JsObject, JsResult, JsValue, Json, OFormat, Reads, Writes} import play.api.mvc.RequestHeader import views.support.{ImageProfile, ImgSrc, Item300, Item460, RemoveOuterParaHtml} @@ -40,50 +47,116 @@ case class Trail( object Trail { - implicit val brandingTypeWrites: Writes[BrandingType] = new Writes[BrandingType] { - def writes(bt: BrandingType) = { - Json.obj( - "name" -> bt.name, - ) + implicit val carouselImagesReads: Reads[Map[String, Option[String]]] = + Reads[Map[String, Option[String]]] { json => + json.validate[JsObject].map { + _.fields.map { case (k, v) => k -> v.asOpt[String] }.toMap + } } + + implicit val brandingTypeFormat: OFormat[BrandingType] = new OFormat[BrandingType] { + def reads(json: JsValue): JsResult[BrandingType] = + (json \ "name").validate[String].map { + case PaidContent.name => PaidContent + case Foundation.name => Foundation + case _ => Sponsored + } + def writes(bt: BrandingType): JsObject = Json.obj("name" -> bt.name) } - implicit val dimensionsWrites: OWrites[Dimensions] = Json.writes[Dimensions] - - implicit val logoWrites: OWrites[CommercialLogo] = Json.writes[CommercialLogo] - - implicit val brandingWrites: OWrites[Branding] = Json.writes[Branding] - - implicit val discussionWrites: OWrites[DiscussionSettings] = Json.writes[DiscussionSettings] - - implicit val OnwardItemWrites: Writes[Trail] = Writes { trail => - val jsObject = Json.obj( - "url" -> trail.url, - "linkText" -> trail.linkText, - "showByline" -> trail.showByline, - "byline" -> trail.byline, - "masterImage" -> trail.masterImage, - "image" -> trail.image, - "carouselImages" -> trail.carouselImages, - "ageWarning" -> trail.ageWarning, - "isLiveBlog" -> trail.isLiveBlog, - "pillar" -> trail.pillar, - "designType" -> trail.designType, - "format" -> trail.format, - "webPublicationDate" -> trail.webPublicationDate, - "headline" -> trail.headline, - "mediaType" -> trail.mediaType, - "shortUrl" -> trail.shortUrl, - "kickerText" -> trail.kickerText, - "starRating" -> trail.starRating, - "avatarUrl" -> trail.avatarUrl, - "branding" -> trail.branding, - "discussion" -> trail.discussion, - "trailText" -> trail.trailText, - "galleryCount" -> trail.galleryCount, - ) + implicit val dimensionsFormat: OFormat[Dimensions] = Json.format[Dimensions] + + implicit val logoFormat: OFormat[CommercialLogo] = Json.format[CommercialLogo] + + implicit val brandingFormat: OFormat[Branding] = Json.format[Branding] - withoutNull(jsObject) + implicit val discussionSettingsFormat: OFormat[DiscussionSettings] = Json.format[DiscussionSettings] + + implicit val contentFormatFormat: OFormat[ContentFormat] = new OFormat[ContentFormat] { + def reads(json: JsValue): JsResult[ContentFormat] = ContentFormat.contentFormatReads.reads(json) + def writes(cf: ContentFormat): JsObject = ContentFormat.contentFormatWrites.writes(cf).as[JsObject] + } + + implicit val trailFormat: OFormat[Trail] = new OFormat[Trail] { + override def reads(json: JsValue): JsResult[Trail] = + for { + url <- (json \ "url").validate[String] + linkText <- (json \ "linkText").validate[String] + showByline <- (json \ "showByline").validate[Boolean] + byline <- (json \ "byline").validateOpt[String] + masterImage <- (json \ "masterImage").validateOpt[String] + image <- (json \ "image").validateOpt[String] + carouselImages <- (json \ "carouselImages").validate[Map[String, Option[String]]] + ageWarning <- (json \ "ageWarning").validateOpt[String] + isLiveBlog <- (json \ "isLiveBlog").validate[Boolean] + pillar <- (json \ "pillar").validate[String] + designType <- (json \ "designType").validate[String] + format <- (json \ "format").validate[ContentFormat] + webPublicationDate <- (json \ "webPublicationDate").validate[String] + headline <- (json \ "headline").validate[String] + mediaType <- (json \ "mediaType").validateOpt[String] + shortUrl <- (json \ "shortUrl").validate[String] + kickerText <- (json \ "kickerText").validateOpt[String] + starRating <- (json \ "starRating").validateOpt[Int] + avatarUrl <- (json \ "avatarUrl").validateOpt[String] + branding <- (json \ "branding").validateOpt[Branding] + discussion <- (json \ "discussion").validate[DiscussionSettings] + trailText <- (json \ "trailText").validateOpt[String] + galleryCount <- (json \ "galleryCount").validateOpt[Int] + } yield Trail( + url, + linkText, + showByline, + byline, + masterImage, + image, + carouselImages, + ageWarning, + isLiveBlog, + pillar, + designType, + format, + webPublicationDate, + headline, + mediaType, + shortUrl, + kickerText, + starRating, + avatarUrl, + branding, + discussion, + trailText, + galleryCount, + ) + + override def writes(trail: Trail): JsObject = { + val obj = Json.obj( + "url" -> trail.url, + "linkText" -> trail.linkText, + "showByline" -> trail.showByline, + "byline" -> trail.byline, + "masterImage" -> trail.masterImage, + "image" -> trail.image, + "carouselImages" -> trail.carouselImages, + "ageWarning" -> trail.ageWarning, + "isLiveBlog" -> trail.isLiveBlog, + "pillar" -> trail.pillar, + "designType" -> trail.designType, + "format" -> trail.format, + "webPublicationDate" -> trail.webPublicationDate, + "headline" -> trail.headline, + "mediaType" -> trail.mediaType, + "shortUrl" -> trail.shortUrl, + "kickerText" -> trail.kickerText, + "starRating" -> trail.starRating, + "avatarUrl" -> trail.avatarUrl, + "branding" -> trail.branding, + "discussion" -> trail.discussion, + "trailText" -> trail.trailText, + "galleryCount" -> trail.galleryCount, + ) + JsObject(obj.fields.filterNot(_._2 == JsNull)) + } } // We ideally want this to be replaced by something else in the near future. Probably diff --git a/common/app/services/S3Async.scala b/common/app/services/S3Async.scala new file mode 100644 index 000000000000..10e794b20868 --- /dev/null +++ b/common/app/services/S3Async.scala @@ -0,0 +1,60 @@ +package services + +import com.gu.etagcaching.aws.s3.ObjectId +import common.GuLogging +import conf.Configuration +import play.api.libs.json.{JsError, JsSuccess, Json, Reads} +import services.S3.logS3ExceptionWithDevHint +import software.amazon.awssdk.core.async.AsyncResponseTransformer +import software.amazon.awssdk.services.s3.model.{GetObjectRequest, GetObjectResponse, NoSuchKeyException, S3Exception} +import utils.AWSv2 + +import scala.concurrent.{ExecutionContext, Future} +import scala.jdk.FutureConverters._ +import scala.io.Codec + +trait S3Async extends GuLogging { + + lazy val bucket: String = Configuration.aws.frontendStoreBucket + + lazy private val client = AWSv2.S3Async + + def handleS3Errors[T](key: String)(future: Future[T])(implicit ec: ExecutionContext): Future[T] = { + val objectId = ObjectId(bucket, key) + future.recoverWith { + case e: NoSuchKeyException => + log.warn(s"not found at ${objectId.s3Uri}") + Future.failed(e) + case e: S3Exception => + logS3ExceptionWithDevHint(objectId, e) + Future.failed(e) + } + } + + private def getResponse( + key: String, + )(implicit codec: Codec, ec: ExecutionContext): Future[(GetObjectResponse, String)] = { + val request = GetObjectRequest.builder().bucket(bucket).key(key).build() + val responseFutureJava = client.getObject(request, AsyncResponseTransformer.toBytes[GetObjectResponse]()) + + responseFutureJava.asScala.map { responseBytes => + val objectResponse = responseBytes.response() + log.debug(s"S3 got ${objectResponse.contentLength} bytes from $key") + val content = new String(responseBytes.asByteArray(), codec.charSet) + (objectResponse, content) + } + } + + def getObjectAsJson[T: Reads](key: String)(implicit ec: ExecutionContext): Future[T] = { + val futureResponse = getResponse(key)(Codec.UTF8, ec).map(_._2).flatMap { jsonString => + val parsedJson = Json.parse(jsonString) + parsedJson.validate[T] match { + case JsSuccess(parsedObject, _) => + Future.successful(parsedObject) + case JsError(errors) => + Future.failed(new RuntimeException(s"Failed to parse JSON for key $key. Errors: $errors")) + } + } + handleS3Errors(key)(futureResponse) + } +} diff --git a/common/conf/env/DEVINFRA.properties b/common/conf/env/DEVINFRA.properties index f2353bc7f4e5..035a43ab2343 100644 --- a/common/conf/env/DEVINFRA.properties +++ b/common/conf/env/DEVINFRA.properties @@ -9,6 +9,7 @@ content.api.host=https://content.guardianapis.com pa.api.key=none aws.bucket=none +cache.bucket=none ophan.api.host=http://api.ophan.co.uk/api ophan.api.key=none diff --git a/onward/test/MostPopularControllerTest.scala b/onward/test/MostPopularControllerTest.scala index 6c914b0c3e01..dfcd5d31c7db 100644 --- a/onward/test/MostPopularControllerTest.scala +++ b/onward/test/MostPopularControllerTest.scala @@ -27,7 +27,7 @@ import agents.DeeplyReadAgent testContentApiClient, new GeoMostPopularAgent(testContentApiClient, ophanApi), new MostPopularAgent(testContentApiClient), - new DeeplyReadAgent(testContentApiClient, ophanApi), + new DeeplyReadAgent(testContentApiClient, new OphanApi(wsClient)), play.api.test.Helpers.stubControllerComponents(), )