From 5a9103442e12661e1b37342bf05638ebca4bdf85 Mon Sep 17 00:00:00 2001 From: Sam Guymer Date: Sun, 28 Sep 2025 15:59:01 +1000 Subject: [PATCH 1/2] More PostgreSQL array instances Add array instances for Short, LocalDate and Instant. --- .../bench/PostgresConnectionState.scala | 32 ++++ .../src/main/scala/doobie/bench/README.md | 34 ++++ .../src/main/scala/doobie/bench/insert.scala | 152 ++++++++++++++++++ .../src/main/scala/doobie/bench/select.scala | 63 +++----- .../src/main/scala/doobie/bench/text.scala | 55 ++++--- .../doobie/postgres/instances/array.scala | 128 +++++++++++---- .../doobie/postgres/PostgresCheckSuite.scala | 3 +- .../doobie/postgres/PostgresTypesSuite.scala | 8 +- .../util/generators/SQLGenerators.scala | 2 + .../util/generators/TimeGenerators.scala | 11 ++ project/build.properties | 2 +- 11 files changed, 389 insertions(+), 101 deletions(-) create mode 100644 modules/bench/src/main/scala/doobie/bench/PostgresConnectionState.scala create mode 100644 modules/bench/src/main/scala/doobie/bench/README.md create mode 100644 modules/bench/src/main/scala/doobie/bench/insert.scala diff --git a/modules/bench/src/main/scala/doobie/bench/PostgresConnectionState.scala b/modules/bench/src/main/scala/doobie/bench/PostgresConnectionState.scala new file mode 100644 index 000000000..8ae0ccb5e --- /dev/null +++ b/modules/bench/src/main/scala/doobie/bench/PostgresConnectionState.scala @@ -0,0 +1,32 @@ +package doobie.bench + +import cats.effect.IO +import doobie.free.connection.ConnectionIO +import doobie.syntax.connectionio.* +import doobie.util.transactor.Transactor +import org.openjdk.jmh.annotations.* + +import java.sql.Connection +import java.sql.DriverManager + +@State(Scope.Thread) +class PostgresConnectionState { + + var connection: Connection = _ + var xa: Transactor[IO] = _ + + @Setup() + def setup(): Unit = { + connection = DriverManager.getConnection("jdbc:postgresql:world", "postgres", "password") + xa = Transactor.fromConnection[IO](connection) + } + + @TearDown() + def tearDown(): Unit = { + connection.close() + } + + def transact[A](io: ConnectionIO[A]): A = { + io.transact(xa).unsafeRunSync()(cats.effect.unsafe.implicits.global) + } +} diff --git a/modules/bench/src/main/scala/doobie/bench/README.md b/modules/bench/src/main/scala/doobie/bench/README.md new file mode 100644 index 000000000..fbf7ae2f1 --- /dev/null +++ b/modules/bench/src/main/scala/doobie/bench/README.md @@ -0,0 +1,34 @@ + +`bench/Jmh/run -wi 2 -f 1 -t 2 doobie.bench.select` +``` +# JMH version: 1.37 +# VM version: JDK 24.0.2, OpenJDK 64-Bit Server VM, 24.0.2 +Benchmark Mode Cnt Score Error Units +d.b.s.list_accum_1000_jdbc avgt 5 184.697 ± 3.700 ns/op +d.b.s.list_accum_1000 avgt 5 225.033 ± 2.429 ns/op +d.b.s.stream_accum_1000 avgt 5 268.893 ± 2.624 ns/op +``` + +`bench/Jmh/run -wi 3 -f 1 -t 2 doobie.bench.text` +``` +# JMH version: 1.37 +# VM version: JDK 24.0.2, OpenJDK 64-Bit Server VM, 24.0.2 +``` + +`bench/Jmh/run -wi 500 -i 1500 -f 1 doobie.bench.insert` +``` +d.b.i.batch_256 ss 1500 6139.577 ± 854.028 ns/op +d.b.i.batch_512 ss 1500 4093.105 ± 62.703 ns/op +d.b.i.batch_1024 ss 1500 3281.137 ± 45.282 ns/op +d.b.i.batch_2048 ss 1500 2904.488 ± 35.369 ns/op + +d.b.i.values_256 ss 1500 4801.105 ± 140.525 ns/op +d.b.i.values_512 ss 1500 3134.149 ± 32.245 ns/op +d.b.i.values_1024 ss 1500 2468.134 ± 60.528 ns/op +d.b.i.values_2048 ss 1500 2515.974 ± 43.229 ns/op + +d.b.i.array_256 ss 1500 4015.970 ± 67.103 ns/op +d.b.i.array_512 ss 1500 2521.262 ± 67.015 ns/op +d.b.i.array_1024 ss 1500 1686.773 ± 18.812 ns/op +d.b.i.array_2048 ss 1500 1249.931 ± 10.294 ns/op +``` diff --git a/modules/bench/src/main/scala/doobie/bench/insert.scala b/modules/bench/src/main/scala/doobie/bench/insert.scala new file mode 100644 index 000000000..9bacd5381 --- /dev/null +++ b/modules/bench/src/main/scala/doobie/bench/insert.scala @@ -0,0 +1,152 @@ +// Copyright (c) 2013-2020 Rob Norris and Contributors +// This software is licensed under the MIT License (MIT). +// For more information see LICENSE or https://opensource.org/licenses/MIT + +package doobie.bench + +import cats.syntax.apply.* +import cats.syntax.foldable.* +import cats.syntax.functor.* +import doobie.free.connection.ConnectionIO +import doobie.syntax.string.* +import doobie.util.Write +import doobie.util.update.Update +import org.openjdk.jmh.annotations.* + +import java.time.Instant +import java.util.concurrent.TimeUnit + +@BenchmarkMode(Array(Mode.SingleShotTime)) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +class insert { + import insert.* + + private def batch(n: Int) = { + val widgets = Widget.generate(n) + insertBatch.updateMany(widgets) + } + + private def values(n: Int) = { + val widgets = Widget.generate(n) + insertValues(widgets).run + } + + private def array(n: Int) = { + val widgets = Widget.generate(n) + insertArrayValues(widgets).run + } + + @Benchmark + @OperationsPerInvocation(512) + def batch_512(state: state): Int = state.transact(batch(512)) + + @Benchmark + @OperationsPerInvocation(1024) + def batch_1024(state: state): Int = state.transact(batch(1024)) + + @Benchmark + @OperationsPerInvocation(2048) + def batch_2048(state: state): Int = state.transact(batch(2048)) + + @Benchmark + @OperationsPerInvocation(512) + def values_512(state: state): Int = state.transact(values(512)) + + @Benchmark + @OperationsPerInvocation(1024) + def values_1024(state: state): Int = state.transact(values(1024)) + + @Benchmark + @OperationsPerInvocation(2048) + def values_2048(state: state): Int = state.transact(values(2048)) + + @Benchmark + @OperationsPerInvocation(512) + def array_512(state: state): Int = state.transact(array(512)) + + @Benchmark + @OperationsPerInvocation(1024) + def array_1024(state: state): Int = state.transact(array(1024)) + + @Benchmark + @OperationsPerInvocation(2048) + def array_2048(state: state): Int = state.transact(array(2048)) +} +object insert { + import doobie.postgres.instances.array.* + + final case class Widget(name: String, extensions: Int, produced: Instant) + object Widget { + implicit val write: Write[Widget] = Write.derived + + val now = Instant.now() + + def generate(n: Int) = List.fill(n)(Widget("widget", n, now.plusMillis(n.toLong))) + } + + private val insertBatch = { + val sql = """ + INSERT INTO bench_widget + (name, extensions, produced) + VALUES + (?, ?, ?) + """ + Update[Widget](sql) + } + + private def insertValues(widgets: List[Widget]) = { + val sql = fr""" + INSERT INTO bench_widget + (name, extensions, produced) + VALUES + ${widgets.map(w => fr"(${w.name}, ${w.extensions}, ${w.produced})").intercalate(fr",")} + """ + sql.update + } + + private def insertArrayValues(widgets: List[Widget]) = { + val n = widgets.length + val names = Array.ofDim[String](n) + val extensions = Array.ofDim[Int](n) + val produced = Array.ofDim[Instant](n) + var i = 0 + widgets.foreach { w => + names(i) = w.name + extensions(i) = w.extensions + produced(i) = w.produced + i = i + 1 + } + + val sql = fr""" + INSERT INTO bench_widget + (name, extensions, produced) + SELECT * FROM unnest( + $names::text[], + $extensions::int4[], + $produced::timestamptz[] + ) + """ + sql.update + } + + private val ddl: ConnectionIO[Unit] = + sql"drop table if exists bench_widget".update.run *> + sql"""create table bench_widget ( + name text not null, + extensions integer not null, + produced timestamptz not null + )""".update.run.void + + @State(Scope.Thread) + class state extends PostgresConnectionState { + + @Setup() + override def setup(): Unit = { + super.setup() + transact(ddl) + } + + @TearDown() + override def tearDown(): Unit = super.tearDown() + } +} diff --git a/modules/bench/src/main/scala/doobie/bench/select.scala b/modules/bench/src/main/scala/doobie/bench/select.scala index 52b29410a..cc1b76b46 100644 --- a/modules/bench/src/main/scala/doobie/bench/select.scala +++ b/modules/bench/src/main/scala/doobie/bench/select.scala @@ -4,30 +4,21 @@ package doobie.bench -import cats.effect.IO import cats.syntax.apply.* -import doobie.syntax.connectionio.* import doobie.syntax.string.* import doobie.util.Read -import doobie.util.transactor.Transactor import org.openjdk.jmh.annotations.* -import java.sql.DriverManager +import java.sql.Connection +import java.util.concurrent.TimeUnit -object shared { - - @State(Scope.Benchmark) - val xa = Transactor.fromDriverManager[IO]("org.postgresql.Driver", "jdbc:postgresql:world", "postgres", "password") -} - -class bench { - import cats.effect.unsafe.implicits.global - import shared.* +@BenchmarkMode(Array(Mode.AverageTime)) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +class select { + import select.* // Baseline hand-written JDBC code - def jdbcBench(n: Int): Int = { - Class.forName("org.postgresql.Driver") - val co = DriverManager.getConnection("jdbc:postgresql:world", "postgres", "password") + private def jdbcBench(co: Connection, n: Int): Int = { try { co.setAutoCommit(false) val ps = co.prepareStatement("select a.name, b.name, co.name from country a, country b, country co limit ?") @@ -47,58 +38,42 @@ class bench { } finally ps.close } finally { co.commit() - co.close() } } - implicit val read3Strings: Read[(String, String, String)] = ( - Read[String], - Read[String], - Read[String], - ).tupled - // Reading via .stream, which adds a fair amount of overhead - def doobieBenchP(n: Int): Int = + private def doobieBenchP(n: Int) = sql"select a.name, b.name, c.name from country a, country b, country c limit $n" .query[(String, String, String)] .stream .compile.toList - .transact(xa) .map(_.length) - .unsafeRunSync() // Reading via .list, which uses a lower-level collector - def doobieBench(n: Int): Int = + private def doobieBench(n: Int) = sql"select a.name, b.name, c.name from country a, country b, country c limit $n" .query[(String, String, String)] .to[List] - .transact(xa) .map(_.length) - .unsafeRunSync() - - // Reading via .vector, which uses a lower-level collector - def doobieBenchV(n: Int): Int = - sql"select a.name, b.name, c.name from country a, country b, country c limit $n" - .query[(String, String, String)] - .to[Vector] - .transact(xa) - .map(_.length) - .unsafeRunSync() @Benchmark @OperationsPerInvocation(1000) - def list_accum_1000_jdbc: Int = jdbcBench(1000) + def list_accum_1000_jdbc(state: PostgresConnectionState): Int = jdbcBench(state.connection, 1000) @Benchmark @OperationsPerInvocation(1000) - def list_accum_1000: Int = doobieBench(1000) + def list_accum_1000(state: PostgresConnectionState): Int = state.transact(doobieBench(1000)) @Benchmark @OperationsPerInvocation(1000) - def vector_accum_1000: Int = doobieBenchV(1000) + def stream_accum_1000(state: PostgresConnectionState): Int = state.transact(doobieBenchP(1000)) - @Benchmark - @OperationsPerInvocation(1000) - def stream_accum_1000: Int = doobieBenchP(1000) +} +object select { + implicit val read3Strings: Read[(String, String, String)] = ( + Read[String], + Read[String], + Read[String], + ).tupled } diff --git a/modules/bench/src/main/scala/doobie/bench/text.scala b/modules/bench/src/main/scala/doobie/bench/text.scala index c967164a1..acfd8faf1 100644 --- a/modules/bench/src/main/scala/doobie/bench/text.scala +++ b/modules/bench/src/main/scala/doobie/bench/text.scala @@ -12,35 +12,28 @@ import doobie.HC import doobie.HPS import doobie.free.connection.ConnectionIO import doobie.postgres.syntax.fragment.* -import doobie.syntax.connectionio.* import doobie.syntax.string.* import doobie.util.Write import fs2.Stream import org.openjdk.jmh.annotations.* -final case class Person(name: String, age: Int) -object Person { - implicit val write: Write[Person] = Write.derived -} +import java.util.concurrent.TimeUnit +@BenchmarkMode(Array(Mode.AverageTime)) +@OutputTimeUnit(TimeUnit.NANOSECONDS) class text { - import cats.effect.unsafe.implicits.global - import shared.* + import text.* def people(n: Int): List[Person] = List.fill(n)(Person("Bob", 42)) - def ddl: ConnectionIO[Unit] = - sql"drop table if exists bench_person".update.run *> - sql"create table bench_person (name varchar not null, age integer not null)".update.run.void - def naive(n: Int): ConnectionIO[Int] = - ddl *> HC.prepareStatement("insert into bench_person (name, age) values (?, ?)")( + HC.prepareStatement("insert into bench_person (name, age) values (?, ?)")( people(n).foldRight(HPS.executeBatch)((p, k) => HPS.set(p) *> HPS.addBatch *> k), ).map(_.combineAll) def optimized(n: Int): ConnectionIO[Int] = - ddl *> HC.prepareStatement("insert into bench_person (name, age) values (?, ?)")( + HC.prepareStatement("insert into bench_person (name, age) values (?, ?)")( FPS.raw { ps => people(n).foreach { p => ps.setString(1, p.name) @@ -52,24 +45,48 @@ class text { ) def copyin_stream(n: Int): ConnectionIO[Long] = - ddl *> sql"COPY bench_person (name, age) FROM STDIN".copyIn(Stream.emits[ConnectionIO, Person](people(n)), 10000) + sql"COPY bench_person (name, age) FROM STDIN".copyIn(Stream.emits[ConnectionIO, Person](people(n)), 10000) def copyin_foldable(n: Int): ConnectionIO[Long] = - ddl *> sql"COPY bench_person (name, age) FROM STDIN".copyIn(people(n)) + sql"COPY bench_person (name, age) FROM STDIN".copyIn(people(n)) @Benchmark @OperationsPerInvocation(10000) - def naive_copyin: Int = naive(10000).transact(xa).unsafeRunSync() + def batch(state: state): Int = state.transact(naive(10000)) @Benchmark @OperationsPerInvocation(10000) - def jdbc_copyin: Int = optimized(10000).transact(xa).unsafeRunSync() + def batch_optimized(state: state): Int = state.transact(optimized(10000)) @Benchmark @OperationsPerInvocation(10000) - def fast_copyin_stream: Long = copyin_stream(10000).transact(xa).unsafeRunSync() + def copy_stream(state: state): Long = state.transact(copyin_stream(10000)) @Benchmark @OperationsPerInvocation(10000) - def fast_copyin_foldable: Long = copyin_foldable(10000).transact(xa).unsafeRunSync() + def copy_foldable(state: state): Long = state.transact(copyin_foldable(10000)) +} +object text { + + final case class Person(name: String, age: Int) + object Person { + implicit val write: Write[Person] = Write.derived + } + + private val ddl: ConnectionIO[Unit] = + sql"drop table if exists bench_person".update.run *> + sql"create table bench_person (name varchar not null, age integer not null)".update.run.void + + @State(Scope.Thread) + class state extends PostgresConnectionState { + + @Setup() + override def setup(): Unit = { + super.setup() + transact(ddl) + } + + @TearDown() + override def tearDown(): Unit = super.tearDown() + } } diff --git a/modules/postgres/src/main/scala/doobie/postgres/instances/array.scala b/modules/postgres/src/main/scala/doobie/postgres/instances/array.scala index 89e198201..d1ef192df 100644 --- a/modules/postgres/src/main/scala/doobie/postgres/instances/array.scala +++ b/modules/postgres/src/main/scala/doobie/postgres/instances/array.scala @@ -4,11 +4,20 @@ package doobie.postgres.instances +import cats.data.NonEmptyList +import doobie.enumerated.JdbcType +import doobie.util.Get +import doobie.util.Put import doobie.util.invariant.* import doobie.util.meta.Meta +import org.postgresql.jdbc.PgArray +import java.time.Instant +import java.time.LocalDate +import java.time.ZoneOffset import scala.reflect.ClassTag +@SuppressWarnings(Array("org.wartremover.warts.AutoUnboxing")) object array { // java.sql.Array::getArray returns an Object that may be of primitive type or of boxed type, @@ -27,32 +36,40 @@ object array { // automatic lifting to Meta will give us lifted and unlifted arrays, for a total of four variants // of each 1-d array type. In the non-nullable case we simply check for nulls and perform a cast; // in the nullable case we must copy the array in both directions to lift/unlift Option. - @SuppressWarnings(Array("org.wartremover.warts.Null", "org.wartremover.warts.Throw")) private def boxedPair[A >: Null <: AnyRef: ClassTag]( elemType: String, arrayType: String, arrayTypeT: String*, ): (Meta[Array[A]], Meta[Array[Option[A]]]) = { val raw = Meta.Advanced.array[A](elemType, arrayType, arrayTypeT*) - // Ensure `a`, which may be null, which is ok, contains no null elements. - def checkNull[B >: Null](a: Array[B], e: => Exception): Array[B] = - if (a == null) null else if (a.exists(_ == null)) throw e else a - ( - raw.timap(checkNull(_, NullableCellRead()))(checkNull(_, NullableCellUpdate())), - raw.timap[Array[Option[A]]](_.map(Option(_)))(_.map(_.orNull).toArray), - ) + boxedPairMeta(raw) } - // Arrays of lifted (nullable) and unlifted (non-nullable) Java wrapped primitives. PostgreSQL - // does not seem to support tinyint[](use a bytea instead) and smallint[] always arrives as Int[] - // so you can xmap if you need Short[]. The type names provided here are what is reported by JDBC - // when metadata is requested; there are numerous aliases but these are the ones we need. Nothing - // about this is portable, sorry. (╯°□°)╯︵ ┻━┻ + @SuppressWarnings(Array("org.wartremover.warts.Null")) + private def boxedPairMeta[A >: Null <: AnyRef: ClassTag]( + raw: Meta[Array[A]], + ): (Meta[Array[A]], Meta[Array[Option[A]]]) = ( + raw.timap(checkNull(_, NullableCellRead()))(checkNull(_, NullableCellUpdate())), + raw.timap[Array[Option[A]]](_.map(Option(_)))(_.map(_.orNull).toArray), + ) + + // Ensure `a`, which may be null, which is ok, contains no null elements. + @SuppressWarnings(Array("org.wartremover.warts.Null", "org.wartremover.warts.Throw")) + private def checkNull[B >: Null](a: Array[B], e: => Exception): Array[B] = + if (a == null) null else if (a.exists(_ == null)) throw e else a + + // Arrays of lifted (nullable) and unlifted (non-nullable) Java wrapped primitives. + // The type names provided here are what is reported by JDBC when metadata is requested; there + // are numerous aliases but these are the ones we need. private val boxedPairBoolean = boxedPair[java.lang.Boolean]("bit", "_bit") implicit val unliftedBooleanArrayType: Meta[Array[java.lang.Boolean]] = boxedPairBoolean._1 implicit val liftedBooleanArrayType: Meta[Array[Option[java.lang.Boolean]]] = boxedPairBoolean._2 + private val boxedPairShort = boxedPair[java.lang.Short]("int2", "_int2") + implicit val unliftedShortArrayType: Meta[Array[java.lang.Short]] = boxedPairShort._1 + implicit val liftedShortArrayType: Meta[Array[Option[java.lang.Short]]] = boxedPairShort._2 + private val boxedPairInteger = boxedPair[java.lang.Integer]("int4", "_int4") implicit val unliftedIntegerArrayType: Meta[Array[java.lang.Integer]] = boxedPairInteger._1 implicit val liftedIntegerArrayType: Meta[Array[Option[java.lang.Integer]]] = boxedPairInteger._2 @@ -81,6 +98,62 @@ object array { implicit val unliftedBigDecimalArrayType: Meta[Array[java.math.BigDecimal]] = boxedPairBigDecimal._1 implicit val iftedBigDecimalArrayType: Meta[Array[Option[java.math.BigDecimal]]] = boxedPairBigDecimal._2 + private val boxedPairDate = boxedPair[java.sql.Date]("date", "_date") + implicit val unliftedDateArrayType: Meta[Array[java.sql.Date]] = boxedPairDate._1 + implicit val liftedDateArrayType: Meta[Array[Option[java.sql.Date]]] = boxedPairDate._2 + + implicit val unliftedLocalDateArrayType: Meta[Array[LocalDate]] = + unliftedDateArrayType.timap(_.map(_.toLocalDate))(_.map(java.sql.Date.valueOf)) + implicit val liftedLocalDateArrayType: Meta[Array[Option[LocalDate]]] = + liftedDateArrayType.timap(_.map(_.map(_.toLocalDate)))(_.map(_.map(java.sql.Date.valueOf))) + + private val boxedPairTimestamp = boxedPair[java.sql.Timestamp]("timestamp", "_timestamp") + implicit val unliftedTimestampArrayType: Meta[Array[java.sql.Timestamp]] = boxedPairTimestamp._1 + implicit val liftedTimestampArrayType: Meta[Array[Option[java.sql.Timestamp]]] = boxedPairTimestamp._2 + + private class InstantValidToString(val i: Instant) { + override def toString = i.toString.replace('T', ' ').stripSuffix("Z") + } + + @SuppressWarnings(Array("org.wartremover.warts.AsInstanceOf", "org.wartremover.warts.Null")) + private val boxedPairInstant = { + val schema = NonEmptyList.of("_timestamptz") + + val fieldStringField = classOf[PgArray].getDeclaredField("fieldString") + fieldStringField.setAccessible(true) + + val get = Get.Advanced.one( + JdbcType.Array, + schema, + (r, n) => { + val a = r.getArray(n) + if (a == null) { + null + } else { + a match { + case a: PgArray => + // the array string looks like "0001-01-01 20:03:28.658293+10:12:08" for some reason... + val fieldString = fieldStringField.get(a).asInstanceOf[String] + val newFieldString = fieldString.replaceAll("\\+[0-9]{2}(:[0-9]{2})?(:[0-9]{2})?", "") + fieldStringField.set(a, newFieldString) + case _ => () + } + // cant override, a timestamptz array always has java.sql.Timestamp elements + a.getArray.asInstanceOf[Array[java.sql.Timestamp]] + .map(_.toLocalDateTime.toInstant(ZoneOffset.UTC)) + } + }, + ) + + // postgres driver calls .toString() ... + val put = Put.Advanced.array[InstantValidToString](schema, "timestamptz") + .contramap[Array[Instant]](_.map(new InstantValidToString(_))) + + boxedPairMeta(new Meta(get, put)) + } + implicit val unliftedInstantArrayType: Meta[Array[Instant]] = boxedPairInstant._1 + implicit val liftedInstantArrayType: Meta[Array[Option[Instant]]] = boxedPairInstant._2 + // Unboxed equivalents (actually identical in the lifted case). We require that B is the unboxed // equivalent of A, otherwise this will fail in spectacular fashion, and we're using a cast in the // lifted case because the representation is identical, assuming no nulls. In the long run this @@ -91,31 +164,33 @@ object array { boxed: Meta[Array[A]], boxedLifted: Meta[Array[Option[A]]], ): (Meta[Array[B]], Meta[Array[Option[B]]]) = - // TODO: assert, somehow, that A is the boxed version of B so we catch errors on instance - // construction, which is somewhat better than at [logical] execution time. ( boxed.timap(a => if (a == null) null else a.map(f))(a => if (a == null) null else a.map(g)), boxedLifted.timap(_.asInstanceOf[Array[Option[B]]])(_.asInstanceOf[Array[Option[A]]]), ) // Arrays of lifted (nullable) and unlifted (non-nullable) AnyVals - private val unboxedPairBoolean = unboxedPair[java.lang.Boolean, Boolean](_.booleanValue, java.lang.Boolean.valueOf) + private val unboxedPairBoolean = unboxedPair[java.lang.Boolean, Boolean](Boolean2boolean, boolean2Boolean) implicit val unliftedUnboxedBooleanArrayType: Meta[Array[Boolean]] = unboxedPairBoolean._1 implicit val liftedUnboxedBooleanArrayType: Meta[Array[Option[Boolean]]] = unboxedPairBoolean._2 - private val unboxedPairInteger = unboxedPair[java.lang.Integer, Int](_.intValue, java.lang.Integer.valueOf) + private val unboxedPairShort = unboxedPair[java.lang.Short, Short](Short2short, short2Short) + implicit val unliftedUnboxedShortArrayType: Meta[Array[Short]] = unboxedPairShort._1 + implicit val liftedUnboxedShortArrayType: Meta[Array[Option[Short]]] = unboxedPairShort._2 + + private val unboxedPairInteger = unboxedPair[java.lang.Integer, Int](Integer2int, int2Integer) implicit val unliftedUnboxedIntegerArrayType: Meta[Array[Int]] = unboxedPairInteger._1 implicit val liftedUnboxedIntegerArrayType: Meta[Array[Option[Int]]] = unboxedPairInteger._2 - private val unboxedPairLong = unboxedPair[java.lang.Long, Long](_.longValue, java.lang.Long.valueOf) + private val unboxedPairLong = unboxedPair[java.lang.Long, Long](Long2long, long2Long) implicit val unliftedUnboxedLongArrayType: Meta[Array[Long]] = unboxedPairLong._1 implicit val liftedUnboxedLongArrayType: Meta[Array[Option[Long]]] = unboxedPairLong._2 - private val unboxedPairFloat = unboxedPair[java.lang.Float, Float](_.floatValue, java.lang.Float.valueOf) + private val unboxedPairFloat = unboxedPair[java.lang.Float, Float](Float2float, float2Float) implicit val unliftedUnboxedFloatArrayType: Meta[Array[Float]] = unboxedPairFloat._1 implicit val liftedUnboxedFloatArrayType: Meta[Array[Option[Float]]] = unboxedPairFloat._2 - private val unboxedPairDouble = unboxedPair[java.lang.Double, Double](_.doubleValue, java.lang.Double.valueOf) + private val unboxedPairDouble = unboxedPair[java.lang.Double, Double](Double2double, double2Double) implicit val unliftedUnboxedDoubleArrayType: Meta[Array[Double]] = unboxedPairDouble._1 implicit val liftedUnboxedDoubleArrayType: Meta[Array[Option[Double]]] = unboxedPairDouble._2 @@ -135,17 +210,4 @@ object array { )( _.map(_.map(a => if (a == null) null else a.bigDecimal)), ) - - // So, it turns out that arrays of structs don't work because something is missing from the - // implementation. So this means we will only be able to support primitive types for arrays. - // - // java.sql.SQLFeatureNotSupportedException: Method org.postgresql.jdbc4.Jdbc4Array.getArrayImpl(long,int,Map) is not yet implemented. - // at org.postgresql.Driver.notImplemented(Driver.java:729) - // at org.postgresql.jdbc2.AbstractJdbc2Array.buildArray(AbstractJdbc2Array.java:771) - // at org.postgresql.jdbc2.AbstractJdbc2Array.getArrayImpl(AbstractJdbc2Array.java:171) - // at org.postgresql.jdbc2.AbstractJdbc2Array.getArray(AbstractJdbc2Array.java:128) - - // TODO: multidimensional arrays; in the worst case it's just copy/paste of everything above but - // we can certainly do better than that. - } diff --git a/modules/postgres/src/test/scala/doobie/postgres/PostgresCheckSuite.scala b/modules/postgres/src/test/scala/doobie/postgres/PostgresCheckSuite.scala index 50193152c..fd77b22a0 100644 --- a/modules/postgres/src/test/scala/doobie/postgres/PostgresCheckSuite.scala +++ b/modules/postgres/src/test/scala/doobie/postgres/PostgresCheckSuite.scala @@ -159,8 +159,7 @@ object PostgresCheckSuite extends PostgresDatabaseSpec { } private def errorWrite[A: Put](value: A, dbType: String) = for { - analysisResult <- - fr"SELECT $value::${Fragment.const(dbType)}".update.analysis.transact + analysisResult <- fr"SELECT $value::${Fragment.const(dbType)}".update.analysis.transact } yield { val errorClasses = analysisResult.parameterAlignmentErrors.map(_.getClass) assertTrue(errorClasses == List(classOf[ParameterTypeError])) diff --git a/modules/postgres/src/test/scala/doobie/postgres/PostgresTypesSuite.scala b/modules/postgres/src/test/scala/doobie/postgres/PostgresTypesSuite.scala index 780fdd2b1..21768bb5f 100644 --- a/modules/postgres/src/test/scala/doobie/postgres/PostgresTypesSuite.scala +++ b/modules/postgres/src/test/scala/doobie/postgres/PostgresTypesSuite.scala @@ -134,14 +134,18 @@ object PostgresTypesSuite extends PostgresDatabaseSpec { skip("json"), // 8.15 Arrays - skip("bit[]", "Requires a cast"), - skip("smallint[]", "always comes back as Array[Int]"), + suiteGetPut[List[Boolean]]("bit[]", Gen.listOfBounded(0, 10)(Gen.boolean)), + suiteGetPut[List[Short]]("smallint[]", Gen.listOfBounded(0, 10)(Gen.short)), suiteGetPut[List[Int]]("integer[]", Gen.listOfBounded(0, 10)(Gen.int)), suiteGetPut[List[Long]]("bigint[]", Gen.listOfBounded(0, 10)(Gen.long)), suiteGetPut[List[Float]]("real[]", Gen.listOfBounded(0, 10)(Gen.float)), suiteGetPut[List[Double]]("double precision[]", Gen.listOfBounded(0, 10)(Gen.double)), suiteGetPut[List[String]]("varchar[]", Gen.listOfBounded(0, 10)(genString)), suiteGetPut[List[UUID]]("uuid[]", Gen.listOfBounded(0, 10)(Gen.uuid)), + suiteGetPut[List[java.sql.Date]]("date[]", Gen.listOfBounded(0, 10)(genSQLDateArray)), + suiteGetPut[List[java.time.LocalDate]]("date[]", Gen.listOfBounded(0, 10)(genLocalDateArray)), + suiteGetPut[List[java.sql.Timestamp]]("timestamp[]", Gen.listOfBounded(0, 10)(genSQLTimestampArray)), + suiteGetPut[List[java.time.Instant]]("timestamptz[]", Gen.listOfBounded(0, 10)(genInstantArray)), suiteGetPut("numeric[]", Gen.const(List[JBigDecimal](BigDecimal("3.14").bigDecimal, BigDecimal("42.0").bigDecimal))), suiteGetPut("numeric[]", Gen.const(List[BigDecimal](BigDecimal("3.14"), BigDecimal("42.0")))), diff --git a/modules/postgres/src/test/scala/doobie/postgres/util/generators/SQLGenerators.scala b/modules/postgres/src/test/scala/doobie/postgres/util/generators/SQLGenerators.scala index e0efe9130..4b9de67c7 100644 --- a/modules/postgres/src/test/scala/doobie/postgres/util/generators/SQLGenerators.scala +++ b/modules/postgres/src/test/scala/doobie/postgres/util/generators/SQLGenerators.scala @@ -14,6 +14,8 @@ object SQLGenerators { val genSQLTime: Gen[Any, Time] = TimeGenerators.genLocalTime.map(Time.valueOf(_)) val genSQLDate: Gen[Any, Date] = TimeGenerators.genLocalDate.map(Date.valueOf(_)) + val genSQLDateArray: Gen[Any, Date] = TimeGenerators.genLocalDateArray.map(Date.valueOf(_)) val genSQLTimestamp: Gen[Any, Timestamp] = TimeGenerators.genLocalDateTime.map(Timestamp.valueOf(_)) + val genSQLTimestampArray: Gen[Any, Timestamp] = TimeGenerators.genLocalDateTimeArray.map(Timestamp.valueOf(_)) } diff --git a/modules/postgres/src/test/scala/doobie/postgres/util/generators/TimeGenerators.scala b/modules/postgres/src/test/scala/doobie/postgres/util/generators/TimeGenerators.scala index c4039c8e4..5fd2e0025 100644 --- a/modules/postgres/src/test/scala/doobie/postgres/util/generators/TimeGenerators.scala +++ b/modules/postgres/src/test/scala/doobie/postgres/util/generators/TimeGenerators.scala @@ -28,6 +28,8 @@ object TimeGenerators { // 4713 BC to 5874897 AD val genLocalDate: Gen[Any, LocalDate] = Gen.localDate(MinDate, MaxDate) + val genLocalDateArray: Gen[Any, LocalDate] = Gen.localDate(LocalDate.of(1, 1, 1), LocalDate.of(9999, 12, 31)) + // 00:00:00.000000 to 23:59:59.999999 val genLocalTime: Gen[Any, LocalTime] = { val min = micros(LocalTime.MIN.toNanoOfDay) @@ -40,10 +42,19 @@ object TimeGenerators { time <- genLocalTime } yield LocalDateTime.of(date, time) + val genLocalDateTimeArray: Gen[Any, LocalDateTime] = for { + date <- genLocalDateArray + time <- genLocalTime + } yield LocalDateTime.of(date, time) + val genInstant: Gen[Any, Instant] = { genLocalDateTime.map(_.toInstant(ZoneOffset.UTC)) } + val genInstantArray: Gen[Any, Instant] = { + genLocalDateTimeArray.map(_.toInstant(ZoneOffset.UTC)) + } + val genZoneOffset: Gen[Any, ZoneOffset] = { Gen.int(MinOffset.getTotalSeconds, MaxOffset.getTotalSeconds).map(ZoneOffset.ofTotalSeconds(_)) } diff --git a/project/build.properties b/project/build.properties index 489e0a72d..5e6884d37 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.11.4 +sbt.version=1.11.6 From b84dcd623f814218ce977ddc1543cc95bbb566dd Mon Sep 17 00:00:00 2001 From: Sam Guymer Date: Sat, 11 Oct 2025 11:41:12 +1000 Subject: [PATCH 2/2] a --- .../src/main/scala/doobie/bench/README.md | 18 +++---- .../src/main/scala/doobie/bench/insert.scala | 21 ++++---- .../doobie/postgres/instances/array.scala | 50 ------------------- .../doobie/postgres/PostgresTypesSuite.scala | 1 - .../util/generators/TimeGenerators.scala | 4 -- project/build.properties | 2 +- 6 files changed, 21 insertions(+), 75 deletions(-) diff --git a/modules/bench/src/main/scala/doobie/bench/README.md b/modules/bench/src/main/scala/doobie/bench/README.md index 564f0981d..0cde272ab 100644 --- a/modules/bench/src/main/scala/doobie/bench/README.md +++ b/modules/bench/src/main/scala/doobie/bench/README.md @@ -25,13 +25,13 @@ d.b.t.copy_stream avgt 2 382.481 ns/op # JMH version: 1.37 # VM version: JDK 21.0.8, OpenJDK 64-Bit Server VM Benchmark Mode Cnt Score Error Units -d.b.i.array_512 ss 1500 2108.146 ± 29.363 ns/op -d.b.i.array_1024 ss 1500 1282.125 ± 17.193 ns/op -d.b.i.array_2048 ss 1500 901.204 ± 27.426 ns/op -d.b.i.batch_512 ss 1500 3610.360 ± 49.533 ns/op -d.b.i.batch_1024 ss 1500 2824.224 ± 158.971 ns/op -d.b.i.batch_2048 ss 1500 2413.790 ± 15.661 ns/op -d.b.i.values_512 ss 1500 2661.017 ± 70.472 ns/op -d.b.i.values_1024 ss 1500 1930.406 ± 25.466 ns/op -d.b.i.values_2048 ss 1500 1759.005 ± 13.514 ns/op +d.b.i.array_512 ss 1500 2637.671 ± 74.508 ns/op +d.b.i.array_1024 ss 1500 1536.033 ± 17.970 ns/op +d.b.i.array_2048 ss 1500 1115.185 ± 20.325 ns/op +d.b.i.batch_512 ss 1500 3894.445 ± 42.689 ns/op +d.b.i.batch_1024 ss 1500 3040.211 ± 21.192 ns/op +d.b.i.batch_2048 ss 1500 2648.115 ± 19.796 ns/op +d.b.i.values_512 ss 1500 3057.419 ± 34.115 ns/op +d.b.i.values_1024 ss 1500 2360.481 ± 41.843 ns/op +d.b.i.values_2048 ss 1500 2343.947 ± 116.611 ns/op ``` diff --git a/modules/bench/src/main/scala/doobie/bench/insert.scala b/modules/bench/src/main/scala/doobie/bench/insert.scala index b60783123..414845b37 100644 --- a/modules/bench/src/main/scala/doobie/bench/insert.scala +++ b/modules/bench/src/main/scala/doobie/bench/insert.scala @@ -9,7 +9,7 @@ import doobie.util.Write import doobie.util.update.Update import org.openjdk.jmh.annotations.* -import java.time.Instant +import java.time.LocalDate import java.util.concurrent.TimeUnit @BenchmarkMode(Array(Mode.SingleShotTime)) @@ -71,13 +71,13 @@ class insert { object insert { import doobie.postgres.instances.array.* - final case class Widget(name: String, extensions: Int, produced: Instant) + final case class Widget(name: String, extensions: Int, produced: LocalDate) object Widget { implicit val write: Write[Widget] = Write.derived - val now = Instant.now() + private val now = LocalDate.now() - def generate(n: Int) = List.fill(n)(Widget("widget", n, now.plusMillis(n.toLong))) + def generate(n: Int) = Vector.fill(n)(Widget("widget", n, now.plusDays(n.toLong))) } private val insertBatch = { @@ -90,7 +90,7 @@ object insert { Update[Widget](sql) } - private def insertValues(widgets: List[Widget]) = { + private def insertValues(widgets: Vector[Widget]) = { val sql = fr""" INSERT INTO bench_widget (name, extensions, produced) @@ -100,13 +100,14 @@ object insert { sql.update } - private def insertArrayValues(widgets: List[Widget]) = { + private def insertArrayValues(widgets: Vector[Widget]) = { val n = widgets.length val names = Array.ofDim[String](n) val extensions = Array.ofDim[Int](n) - val produced = Array.ofDim[Instant](n) + val produced = Array.ofDim[LocalDate](n) var i = 0 - widgets.foreach { w => + while (i < n) { + val w = widgets(i) names(i) = w.name extensions(i) = w.extensions produced(i) = w.produced @@ -119,7 +120,7 @@ object insert { SELECT * FROM unnest( $names::text[], $extensions::int4[], - $produced::timestamptz[] + $produced::date[] ) """ sql.update @@ -130,7 +131,7 @@ object insert { sql"""create table bench_widget ( name text not null, extensions integer not null, - produced timestamptz not null + produced date not null )""".update.run.void @State(Scope.Thread) diff --git a/modules/postgres/src/main/scala/doobie/postgres/instances/array.scala b/modules/postgres/src/main/scala/doobie/postgres/instances/array.scala index d1ef192df..23a920d97 100644 --- a/modules/postgres/src/main/scala/doobie/postgres/instances/array.scala +++ b/modules/postgres/src/main/scala/doobie/postgres/instances/array.scala @@ -4,17 +4,10 @@ package doobie.postgres.instances -import cats.data.NonEmptyList -import doobie.enumerated.JdbcType -import doobie.util.Get -import doobie.util.Put import doobie.util.invariant.* import doobie.util.meta.Meta -import org.postgresql.jdbc.PgArray -import java.time.Instant import java.time.LocalDate -import java.time.ZoneOffset import scala.reflect.ClassTag @SuppressWarnings(Array("org.wartremover.warts.AutoUnboxing")) @@ -111,49 +104,6 @@ object array { implicit val unliftedTimestampArrayType: Meta[Array[java.sql.Timestamp]] = boxedPairTimestamp._1 implicit val liftedTimestampArrayType: Meta[Array[Option[java.sql.Timestamp]]] = boxedPairTimestamp._2 - private class InstantValidToString(val i: Instant) { - override def toString = i.toString.replace('T', ' ').stripSuffix("Z") - } - - @SuppressWarnings(Array("org.wartremover.warts.AsInstanceOf", "org.wartremover.warts.Null")) - private val boxedPairInstant = { - val schema = NonEmptyList.of("_timestamptz") - - val fieldStringField = classOf[PgArray].getDeclaredField("fieldString") - fieldStringField.setAccessible(true) - - val get = Get.Advanced.one( - JdbcType.Array, - schema, - (r, n) => { - val a = r.getArray(n) - if (a == null) { - null - } else { - a match { - case a: PgArray => - // the array string looks like "0001-01-01 20:03:28.658293+10:12:08" for some reason... - val fieldString = fieldStringField.get(a).asInstanceOf[String] - val newFieldString = fieldString.replaceAll("\\+[0-9]{2}(:[0-9]{2})?(:[0-9]{2})?", "") - fieldStringField.set(a, newFieldString) - case _ => () - } - // cant override, a timestamptz array always has java.sql.Timestamp elements - a.getArray.asInstanceOf[Array[java.sql.Timestamp]] - .map(_.toLocalDateTime.toInstant(ZoneOffset.UTC)) - } - }, - ) - - // postgres driver calls .toString() ... - val put = Put.Advanced.array[InstantValidToString](schema, "timestamptz") - .contramap[Array[Instant]](_.map(new InstantValidToString(_))) - - boxedPairMeta(new Meta(get, put)) - } - implicit val unliftedInstantArrayType: Meta[Array[Instant]] = boxedPairInstant._1 - implicit val liftedInstantArrayType: Meta[Array[Option[Instant]]] = boxedPairInstant._2 - // Unboxed equivalents (actually identical in the lifted case). We require that B is the unboxed // equivalent of A, otherwise this will fail in spectacular fashion, and we're using a cast in the // lifted case because the representation is identical, assuming no nulls. In the long run this diff --git a/modules/postgres/src/test/scala/doobie/postgres/PostgresTypesSuite.scala b/modules/postgres/src/test/scala/doobie/postgres/PostgresTypesSuite.scala index 21768bb5f..384589557 100644 --- a/modules/postgres/src/test/scala/doobie/postgres/PostgresTypesSuite.scala +++ b/modules/postgres/src/test/scala/doobie/postgres/PostgresTypesSuite.scala @@ -145,7 +145,6 @@ object PostgresTypesSuite extends PostgresDatabaseSpec { suiteGetPut[List[java.sql.Date]]("date[]", Gen.listOfBounded(0, 10)(genSQLDateArray)), suiteGetPut[List[java.time.LocalDate]]("date[]", Gen.listOfBounded(0, 10)(genLocalDateArray)), suiteGetPut[List[java.sql.Timestamp]]("timestamp[]", Gen.listOfBounded(0, 10)(genSQLTimestampArray)), - suiteGetPut[List[java.time.Instant]]("timestamptz[]", Gen.listOfBounded(0, 10)(genInstantArray)), suiteGetPut("numeric[]", Gen.const(List[JBigDecimal](BigDecimal("3.14").bigDecimal, BigDecimal("42.0").bigDecimal))), suiteGetPut("numeric[]", Gen.const(List[BigDecimal](BigDecimal("3.14"), BigDecimal("42.0")))), diff --git a/modules/postgres/src/test/scala/doobie/postgres/util/generators/TimeGenerators.scala b/modules/postgres/src/test/scala/doobie/postgres/util/generators/TimeGenerators.scala index 5fd2e0025..7355d5980 100644 --- a/modules/postgres/src/test/scala/doobie/postgres/util/generators/TimeGenerators.scala +++ b/modules/postgres/src/test/scala/doobie/postgres/util/generators/TimeGenerators.scala @@ -51,10 +51,6 @@ object TimeGenerators { genLocalDateTime.map(_.toInstant(ZoneOffset.UTC)) } - val genInstantArray: Gen[Any, Instant] = { - genLocalDateTimeArray.map(_.toInstant(ZoneOffset.UTC)) - } - val genZoneOffset: Gen[Any, ZoneOffset] = { Gen.int(MinOffset.getTotalSeconds, MaxOffset.getTotalSeconds).map(ZoneOffset.ofTotalSeconds(_)) } diff --git a/project/build.properties b/project/build.properties index 5e6884d37..01a16ed14 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.11.6 +sbt.version=1.11.7