add novelty timeseries to disease and target

app/esecuele/Functions.scala

@@ -36,6 +36,8 @@ object Functions {
 
   def any(col: Column): Column = f("any", col)
 
+  def anyIf(col: Column, cond: Column): Column = f("anyIf", col, cond)
+
   def lower(col: Column): Column = f("lower", col)
 
   def upper(col: Column): Column = f("upper", col)

app/models/Backend.scala

@@ -24,13 +24,14 @@ import models.entities.Interactions.*
 import models.entities.Loci.*
 import models.entities.MechanismsOfAction.*
 import models.entities.MousePhenotypes.*
+import models.entities.AssociationTimeSeriesResults.*
 import models.entities.Pharmacogenomics.*
 import models.entities.SearchFacetsResults.*
 import models.entities.Studies.*
 import models.entities.Evidences.*
 import models.entities.SequenceOntologyTerm.*
 import models.entities.*
-import models.gql.{Fetchers, StudyTypeEnum, InteractionSourceEnum}
+import models.gql.{Fetchers, AggregationTypeEnum, StudyTypeEnum, InteractionSourceEnum}
 import models.entities.Violations.{DateFilterError, InputParameterCheckError}
 import org.apache.http.impl.nio.reactor.IOReactorConfig
 import play.api.cache.AsyncCacheApi
@@ -744,6 +745,37 @@ class Backend @Inject() (implicit
     dbRetriever.executeQuery[MechanismsOfAction, Query](query.query)
   }
 
+  def getAssociationTimeSeries(diseaseId: String,
+                               targetId: String,
+                               isDirect: Boolean,
+                               aggregationTypes: Option[Seq[AggregationTypeEnum.Value]],
+                               startYear: Option[Int],
+                               endYear: Option[Int],
+                               pagination: Option[Pagination]
+  ): Future[AssociationTimeSeriesResults] = {
+    val tableName = getTableWithPrefixOrDefault(
+      defaultOTSettings.clickhouse.associationTimeSeries.name
+    )
+    val pag = pagination.getOrElse(Pagination.mkDefault).offsetLimit
+    val query = AssociationTimeSeriesQuery(diseaseId,
+                                           targetId,
+                                           isDirect,
+                                           tableName,
+                                           pag._1,
+                                           pag._2,
+                                           aggregationTypes,
+                                           startYear,
+                                           endYear
+    )
+    dbRetriever.executeQuery[AssociationTimeSeries, Query](query.query).map { timeSeriesSeq =>
+      if (timeSeriesSeq.isEmpty) {
+        AssociationTimeSeriesResults(0, timeSeriesSeq)
+      } else {
+        AssociationTimeSeriesResults(timeSeriesSeq.head.meta_total, timeSeriesSeq)
+      }
+    }
+  }
+
   def getDrugWarnings(ids: Seq[String]): Future[IndexedSeq[DrugWarnings]] = {
     val tableName = getTableWithPrefixOrDefault(defaultOTSettings.clickhouse.drugWarnings.name)
     val localMarkerContext = markerContext.fromExistingContext(append("table", tableName))
@@ -871,6 +903,7 @@ class Backend @Inject() (implicit
       tableName: String,
       datasources: Option[Seq[DatasourceSettings]],
       fixedEntityId: String,
+      indirect: Boolean,
       indirectIds: Set[String],
       bIds: Set[String],
       columnFilters: Seq[(String, Any)],
@@ -900,6 +933,7 @@ class Backend @Inject() (implicit
       weights,
       _,
       dontPropagate,
+      indirect,
       page.offset,
       page.size
     )
@@ -950,6 +984,7 @@ class Backend @Inject() (implicit
       getTableWithPrefixOrDefault(defaultOTSettings.clickhouse.disease.associations.name),
       datasources,
       disease.id,
+      indirect,
       indirectIDs,
       targetIds,
       Seq.empty,
@@ -996,6 +1031,7 @@ class Backend @Inject() (implicit
       getTableWithPrefixOrDefault(defaultOTSettings.clickhouse.target.associations.name),
       datasources,
       target.id,
+      indirect,
       indirectIDs,
       diseaseIds,
       columnFilters,

app/models/db/AssociationTimeSeriesQuery.scala

@@ -0,0 +1,79 @@
+package models.db
+
+import esecuele.Column.column
+import esecuele.Column.literal
+import esecuele.*
+import utils.OTLogging
+import models.gql.AggregationTypeEnum
+
+case class AssociationTimeSeriesQuery(
+    diseaseId: String,
+    targetId: String,
+    isDirect: Boolean,
+    tableName: String,
+    offset: Int,
+    size: Int,
+    aggregationTypeInclude: Option[Seq[AggregationTypeEnum.Value]] = None,
+    yearFrom: Option[Int] = None,
+    yearTo: Option[Int] = None
+) extends Queryable
+    with OTLogging {
+
+  private val aggregationTypes: Seq[AggregationTypeEnum.Value] = aggregationTypeInclude.getOrElse(
+    Seq(AggregationTypeEnum.overall, AggregationTypeEnum.datasourceId)
+  )
+
+  private val aggregationTypeFilter =
+    Functions.in(column("aggregationType"),
+                 Functions.set(aggregationTypes.map(t => literal(t.toString)))
+    )
+
+  private val yearFromFilter: Option[Column] = yearFrom match {
+    case Some(year) => Some(Functions.greaterOrEquals(column("year"), literal(year)))
+    case None       => None
+  }
+  private val yearToFilter: Option[Column] = yearTo match {
+    case Some(year) => Some(Functions.lessOrEquals(column("year"), literal(year)))
+    case None       => None
+  }
+  private val yearFilter: Column = (yearFromFilter, yearToFilter) match {
+    case (Some(from), Some(to)) => Functions.and(from, to)
+    case (Some(filter), None)   => filter
+    case (None, Some(filter))   => filter
+    case (None, None)           => literal(true)
+  }
+
+  private val positionalQuery =
+    Functions.and(
+      Functions.equals(column("diseaseId"), literal(diseaseId)),
+      Functions.equals(column("targetId"), literal(targetId)),
+      Functions.equals(column("isDirect"), literal(isDirect))
+    )
+
+  private val queryWithFilters = Where(
+    Functions.and(
+      positionalQuery,
+      aggregationTypeFilter,
+      yearFilter
+    )
+  )
+
+  val totals: Query =
+    Query(
+      Select(Functions.count(Column.star) :: Nil),
+      From(column(tableName)),
+      queryWithFilters
+    )
+
+  override val query: Query =
+    Query(
+      Select(
+        Column.star :: Functions.countOver("meta_total") :: Nil
+      ),
+      From(column(tableName)),
+      queryWithFilters,
+      OrderBy(column("year").asc :: Nil),
+      Limit(offset, size),
+      Format("JSONEachRow")
+    )
+}

app/models/db/CredibleSetQuery.scala

@@ -4,7 +4,6 @@ import esecuele.Column.column
 import esecuele.Column.literal
 import esecuele._
 import utils.OTLogging
-import models.entities.StudyQueryArgs
 import models.gql.StudyTypeEnum
 import models.entities.CredibleSetQueryArgs
 

app/models/db/InteractionSourcesQuery.scala

@@ -1,10 +1,8 @@
 package models.db
 
 import esecuele.Column.column
-import esecuele.Column.literal
 import esecuele._
 import utils.OTLogging
-import play.libs.F
 import models.gql.InteractionSourceEnum
 
 case class InteractionSourcesQuery(

app/models/db/QAOTF.scala

@@ -47,6 +47,7 @@ case class QAOTF(
     datasourceWeights: Seq[(String, Double)],
     mustIncludeDatasources: Set[String],
     nonPropagatedDatasources: Set[String],
+    indirect: Boolean,
     offset: Int,
     size: Int
 ) extends Queryable
@@ -61,8 +62,7 @@ case class QAOTF(
   val T: Column = column(tableName)
   val RowID: Column = column("rowId")
   val RowScore: Column = column("rowScore")
-  val NoveltyDirect: Column = column("noveltyDirect")
-  val NoveltyIndirect: Column = column("noveltyIndirect")
+  val Novelty: Column = if indirect then column("noveltyIndirect") else column("noveltyDirect")
   val maxHS: Column = literal(Harmonic.maxValue(100000, pExponentDefault, 1.0))
     .as(Some("max_hs_score"))
 
@@ -152,8 +152,9 @@ case class QAOTF(
       .as(Some("weightPair"))
     val DSFieldWC = F.tupleElement(WC.name, literal(1)).as(Some("datasourceId"))
     val WFieldWC = F.toNullable(F.tupleElement(WC.name, literal(2))).as(Some("weight"))
-    val NoveltyDirectAny: Column = F.any(NoveltyDirect).as(Some("noveltyDirectAny"))
-    val NoveltyIndirectAny: Column = F.any(NoveltyIndirect).as(Some("noveltyIndirectAny"))
+    // novelty where A is is the AId, cannot use any, it must be the novelty where A is AId.
+    val NoveltyWhereA: Column =
+      F.anyIf(Novelty, F.equals(A, literal(AId))).as(Some("noveltyWhereA"))
 
     // transform weights vector into a table to extract each value of each tuple
     val q = Q(
@@ -163,7 +164,7 @@ case class QAOTF(
     )
     val withDT = With(DSScore :: DTAny :: DSW :: Nil)
     val selectDSScores = Select(
-      B :: DSW.name :: DTAny.name :: DS :: DSScore.name :: NoveltyDirectAny :: NoveltyIndirectAny :: Nil
+      B :: DSW.name :: DTAny.name :: DS :: DSScore.name :: NoveltyWhereA :: Nil
     )
     val fromT = From(T, Some("l"))
     val joinWeights =
@@ -242,8 +243,8 @@ case class QAOTF(
       F.arrayMap("x -> (x.3, x.1)", collectedDScored.name).as(Some("score_datasources"))
     val scoreDTs = F.arrayMap("x -> (x.4, x.1)", collectedDScored.name).as(Some("score_dt"))
     val uniqDTs = F.groupUniqArray(DT).as(Some("datatypes_v"))
-    val NoveltyDirectAny: Column = F.any(NoveltyDirect).as(Some("noveltyDirectAny"))
-    val NoveltyIndirectAny: Column = F.any(NoveltyIndirect).as(Some("noveltyIndirectAny"))
+    val NoveltyWhereA: Column =
+      F.anyIf(Novelty, F.equals(A, literal(AId))).as(Some("noveltyWhereA"))
 
     val mappedDTs = F
       .arrayMap(
@@ -258,8 +259,7 @@ case class QAOTF(
         mappedDTs.name
       )
       .as(Some("score_datatypes"))
-    val noveltyDirectScore: Column = F.any(NoveltyDirectAny).as(Some("noveltyDirect"))
-    val noveltyIndirectScore: Column = F.any(NoveltyIndirectAny).as(Some("noveltyIndirect"))
+    val noveltyScore: Column = F.any(NoveltyWhereA).as(Some("novelty"))
 
     val orderColumn = orderScoreBy.getOrElse((scoreOverall.name.rep, "desc"))
     val jointColumns = F.concat(scoredDTs.name, scoreDSs.name)
@@ -291,12 +291,13 @@ case class QAOTF(
         mappedDTs,
         scoredDTs,
         scoreOverall,
+        noveltyScore,
         jointColumns,
         orderByC
       )
     )
     val selectScores = Select(
-      B :: scoreOverall.name :: scoredDTs.name :: scoreDSs.name :: noveltyDirectScore :: noveltyIndirectScore :: Nil
+      B :: scoreOverall.name :: scoredDTs.name :: scoreDSs.name :: noveltyScore.name :: Nil
     ) // :: scoreDTs.name :: collectedDScored :: Nil)
     val fromAgg = From(queryGroupByDS.toColumn(None))
     val groupByB = GroupBy(B :: Nil)
@@ -306,6 +307,11 @@ case class QAOTF(
           (if (order == "desc") scoreOverall.name.desc
            else scoreOverall.name.asc) :: Nil
         )
+      case ("novelty", order) =>
+        OrderBy(
+          (if (order == "desc") noveltyScore.name.desc
+           else noveltyScore.name.asc) :: Nil
+        )
       case (_, order) =>
         OrderBy(
           (if (order == "desc") orderByC.name.desc

app/models/entities/AssociationTimeSeries.scala

@@ -0,0 +1,34 @@
+package models.entities
+
+import play.api.libs.json.{Json, OFormat}
+import slick.jdbc.GetResult
+import utils.db.DbJsonParser.fromPositionedResult
+import models.gql.AggregationTypeEnum
+
+case class AssociationTimeSeries(
+    diseaseId: String,
+    targetId: String,
+    aggregationType: AggregationTypeEnum.AggregationType,
+    aggregationValue: String,
+    year: Option[Int],
+    associationScore: Double,
+    novelty: Option[Double],
+    yearlyEvidenceCount: Option[Int],
+    isDirect: Boolean,
+    meta_total: Long
+)
+
+case class AssociationTimeSeriesResults(
+    count: Long,
+    rows: Vector[AssociationTimeSeries]
+)
+
+object AssociationTimeSeriesResults {
+  val empty: AssociationTimeSeriesResults = AssociationTimeSeriesResults(0, Vector.empty)
+  implicit val getAssociationTimeSeriesRowFromDB: GetResult[AssociationTimeSeries] =
+    GetResult(fromPositionedResult[AssociationTimeSeries])
+  implicit val AssociationTimeSeriesImp: OFormat[AssociationTimeSeries] =
+    Json.format[AssociationTimeSeries]
+  implicit val AssociationTimeSeriesResultsImp: OFormat[AssociationTimeSeriesResults] =
+    Json.format[AssociationTimeSeriesResults]
+}

app/models/entities/Associations.scala

@@ -17,8 +17,7 @@ case class Association(
     score: Double,
     datatypeScores: Vector[ScoredComponent],
     datasourceScores: Vector[ScoredComponent],
-    noveltyDirect: Option[Double] = None,
-    noveltyIndirect: Option[Double] = None
+    novelty: Option[Double] = None
 )
 
 case class Associations(
@@ -38,8 +37,7 @@ object Associations {
       val score: Double = r.<<
       val tuples1: String = r.<<
       val tuples2: String = r.<<
-      val noveltyDirect: Option[Double] = r.<<
-      val noveltyIndirect: Option[Double] = r.<<
+      val novelty: Option[Double] = r.<<
 
       Association(
         id,
@@ -62,8 +60,7 @@ object Associations {
             ScoredComponent(left, right)
           }
         ).rep,
-        noveltyDirect,
-        noveltyIndirect
+        novelty
       )
     }
 

app/models/entities/Configuration.scala

@@ -111,6 +111,7 @@ object Configuration {
       clinicalTarget: DbTableSettings,
       mechanismOfAction: DbTableSettings,
       mousePhenotypes: DbTableSettings,
+      associationTimeSeries: DbTableSettings,
       otarProjects: DbTableSettings,
       pharmacogenomics: PharmacogenomicsSettings,
       proteinCodingCoordinates: ProteinCodingCoordinatesSettings,