From 7a2f1ccd8124a5e6fbad816bbdf0b5ec6a6d86e4 Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Thu, 8 Jan 2026 02:39:52 +0100 Subject: [PATCH 1/5] Make geometry handling more generic, suitable for use by libraries other than `geo` This is accomplished by: * Add `MyXQL.Protocol.Encoder`, a protocol for encoding structs * Providing encoders for Geo structs if the `geo` library is available at compile-time * Fetch the wkb decoder from configuration, defaulting to the `geo` library, via `Application.get_env(:myxql, :wkb_decoder)` This preserves the existing functionality, while allowing other libraries to provide WKB encoding (via the protocol) and decoding (via the wkb decoder configuration). --- README.md | 15 +++-- lib/myxql/protocol/encoder.ex | 5 ++ lib/myxql/protocol/encoder/geo.ex | 45 +++++++++++++++ lib/myxql/protocol/values.ex | 92 +++++++++++++++++++++---------- 4 files changed, 122 insertions(+), 35 deletions(-) create mode 100644 lib/myxql/protocol/encoder.ex create mode 100644 lib/myxql/protocol/encoder/geo.ex diff --git a/README.md b/README.md index bb147be..3506a04 100644 --- a/README.md +++ b/README.md @@ -135,19 +135,22 @@ config :myxql, :json_library, SomeJSONModule ## Geometry support -MyXQL comes with Geometry types support via the [Geo](https://github.com/bryanjos/geo) package. +MyXQL supports data stored in `geometry` columns with the help of external geometry libraries such as `geo`. -To use it, add `:geo` to your dependencies: +After adding one of these libraries as a dependency in `mix.exs`, you can register that library's parser +function in a config file (e.g. `runtime.exs`): ```elixir -{:geo, "~> 3.3"} +config :myxql, wkb_decoder: {Geo.WKB, :decode!} ``` -Note, some structs like `%Geo.PointZ{}` does not have equivalent on the MySQL server side and thus -shouldn't be used. +If using the `geo` library, this is the default and no explicit configuration is required. + +Note: some geometry types available in these libraries, such as `PointZ`, are not supported by MySQL and thus +should not be used. If you're using MyXQL geometry types with Ecto and need to for example accept a WKT format as user -input, consider implementing an [custom Ecto type](https://hexdocs.pm/ecto/Ecto.Type.html). +input, consider implementing a [custom Ecto type](https://hexdocs.pm/ecto/Ecto.Type.html). ## UTC required diff --git a/lib/myxql/protocol/encoder.ex b/lib/myxql/protocol/encoder.ex new file mode 100644 index 0000000..a42ef90 --- /dev/null +++ b/lib/myxql/protocol/encoder.ex @@ -0,0 +1,5 @@ +defprotocol MyXQL.Protocol.Encoder do + @spec encode(struct()) :: {MyXQL.Protocol.Values.storage_type(), binary()} + def encode(struct) +end + diff --git a/lib/myxql/protocol/encoder/geo.ex b/lib/myxql/protocol/encoder/geo.ex new file mode 100644 index 0000000..acde8ae --- /dev/null +++ b/lib/myxql/protocol/encoder/geo.ex @@ -0,0 +1,45 @@ + +if Code.ensure_loaded?(Geo) do + defmodule GeoEncoderHelper do + @moduledoc false + import MyXQL.Protocol.Types, only: [uint4: 0] + + def encode_geometry(geo) do + srid = geo.srid || 0 + binary = %{geo | srid: nil} |> Geo.WKB.encode_to_iodata(:ndr) |> IO.iodata_to_binary() + + { + :mysql_type_var_string, + MyXQL.Protocol.Types.encode_string_lenenc(<>) + } + end + end + + defimpl MyXQL.Protocol.Encoder, for: Geo.Point do + def encode(geo), do: GeoEncoderHelper.encode_geometry(geo) + end + + defimpl MyXQL.Protocol.Encoder, for: Geo.MultiPoint do + def encode(geo), do: GeoEncoderHelper.encode_geometry(geo) + end + + defimpl MyXQL.Protocol.Encoder, for: Geo.LineString do + def encode(geo), do: GeoEncoderHelper.encode_geometry(geo) + end + + defimpl MyXQL.Protocol.Encoder, for: Geo.MultiLineString do + def encode(geo), do: GeoEncoderHelper.encode_geometry(geo) + end + + defimpl MyXQL.Protocol.Encoder, for: Geo.Polygon do + def encode(geo), do: GeoEncoderHelper.encode_geometry(geo) + end + + defimpl MyXQL.Protocol.Encoder, for: Geo.MultiPolygon do + def encode(geo), do: GeoEncoderHelper.encode_geometry(geo) + end + + defimpl MyXQL.Protocol.Encoder, for: Geo.GeometryCollection do + def encode(geo), do: GeoEncoderHelper.encode_geometry(geo) + end +end diff --git a/lib/myxql/protocol/values.ex b/lib/myxql/protocol/values.ex index 3baf5ae..7652151 100644 --- a/lib/myxql/protocol/values.ex +++ b/lib/myxql/protocol/values.ex @@ -21,6 +21,38 @@ defmodule MyXQL.Protocol.Values do # MySQL TIMESTAMP is equal to Postgres TIMESTAMP WITH TIME ZONE # MySQL DATETIME is equal to Postgres TIMESTAMP [WITHOUT TIME ZONE] + @type storage_type :: + :mysql_type_tiny + | :mysql_type_short + | :mysql_type_long + | :mysql_type_float + | :mysql_type_double + | :mysql_type_null + | :mysql_type_timestamp + | :mysql_type_longlong + | :mysql_type_int24 + | :mysql_type_date + | :mysql_type_time + | :mysql_type_datetime + | :mysql_type_year + | :mysql_type_varchar + | :mysql_type_bit + | :mysql_type_json + | :mysql_type_newdecimal + | :mysql_type_enum + | :mysql_type_set + | :mysql_type_tiny_blob + | :mysql_type_medium_blob + | :mysql_type_long_blob + | :mysql_type_blob + | :mysql_type_var_string + | :mysql_type_string + | :mysql_type_geometry + | :mysql_type_newdate + | :mysql_type_timestamp2 + | :mysql_type_datetime2 + | :mysql_type_date2 + types = [ mysql_type_tiny: 0x01, mysql_type_short: 0x02, @@ -250,14 +282,13 @@ defmodule MyXQL.Protocol.Values do {:mysql_type_tiny, <<0>>} end - if Code.ensure_loaded?(Geo) do - def encode_binary_value(%Geo.Point{} = geo), do: encode_geometry(geo) - def encode_binary_value(%Geo.MultiPoint{} = geo), do: encode_geometry(geo) - def encode_binary_value(%Geo.LineString{} = geo), do: encode_geometry(geo) - def encode_binary_value(%Geo.MultiLineString{} = geo), do: encode_geometry(geo) - def encode_binary_value(%Geo.Polygon{} = geo), do: encode_geometry(geo) - def encode_binary_value(%Geo.MultiPolygon{} = geo), do: encode_geometry(geo) - def encode_binary_value(%Geo.GeometryCollection{} = geo), do: encode_geometry(geo) + def encode_binary_value(struct) when is_struct(struct) do + try do + MyXQL.Protocol.Encoder.encode(struct) + rescue + Protocol.UndefinedError -> + raise ArgumentError, "query has invalid parameter #{inspect(struct)}" + end end def encode_binary_value(term) when is_list(term) or is_map(term) do @@ -269,14 +300,6 @@ defmodule MyXQL.Protocol.Values do raise ArgumentError, "query has invalid parameter #{inspect(other)}" end - if Code.ensure_loaded?(Geo) do - defp encode_geometry(geo) do - srid = geo.srid || 0 - binary = %{geo | srid: nil} |> Geo.WKB.encode_to_iodata(:ndr) |> IO.iodata_to_binary() - {:mysql_type_var_string, encode_string_lenenc(<>)} - end - end - ## Time/DateTime # MySQL supports negative time and days, we don't. @@ -423,21 +446,32 @@ defmodule MyXQL.Protocol.Values do Enum.reverse(acc) end - if Code.ensure_loaded?(Geo) do - # https://dev.mysql.com/doc/refman/8.0/en/gis-data-formats.html#gis-internal-format - defp decode_geometry(<>) do - srid = if srid == 0, do: nil, else: srid - r |> Geo.WKB.decode!() |> Map.put(:srid, srid) - end - else - defp decode_geometry(_) do - raise """ - encoding/decoding geometry types requires :geo package, add: + defp decode_geometry(<>) do + # Assumptions made in this function: + # + # * :srid is a top-level field in the struct returned by the wkb decoder + # * the wkb decoder can decode any value that can be stored in a geometry column - {:geo, "~> 3.4"} + try do + {mod, fun} = Application.get_env(:myxql, :wkb_decoder, {Geo.WKB, :decode!}) - to your mix.exs and run `mix deps.compile --force myxql`. - """ + srid = if srid == 0, do: nil, else: srid + decoded = apply(mod, fun, [data]) + + if srid != 0 do + Map.put(decoded, :srid, srid) + else + decoded + end + rescue + _ -> + raise """ + Encoding/decoding geometry types requires WKB decoder that returns a map. + Libraries such as `geo` and `geometry` provide WKB decoders, which can be + registered in the application configuration such as `runtime.exs`: + + config :myxql, wkb_decoder: {Geometry, :from_wkb!} + """ end end From 97edefc364db3a4efe35c3dbbb4b1e48f695ea49 Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Fri, 9 Jan 2026 12:22:29 +0100 Subject: [PATCH 2/5] If there is no protocol for the struct, attempt to json encode it --- lib/myxql/protocol/values.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/myxql/protocol/values.ex b/lib/myxql/protocol/values.ex index 7652151..dd6abb5 100644 --- a/lib/myxql/protocol/values.ex +++ b/lib/myxql/protocol/values.ex @@ -287,7 +287,9 @@ defmodule MyXQL.Protocol.Values do MyXQL.Protocol.Encoder.encode(struct) rescue Protocol.UndefinedError -> - raise ArgumentError, "query has invalid parameter #{inspect(struct)}" + # No protocol defined for this struct, so try to json encode it + string = json_library().encode!(struct) + {:mysql_type_var_string, encode_string_lenenc(string)} end end From c3e6e41f231855ac49a579a709bacb7b2ea5df57 Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Fri, 9 Jan 2026 15:34:56 +0100 Subject: [PATCH 3/5] Provide a GeometryCodec behaviour to en/decode geometry data Can be configured by setting the `:myxql, :geometry_codec` configuration key to the name of a module that implements the GeometryCodec behaviour default to geo, if available. --- README.md | 8 +-- lib/myxql/protocol/encoder.ex | 5 -- lib/myxql/protocol/encoder/geo.ex | 45 ---------------- lib/myxql/protocol/geometry_codec.ex | 39 ++++++++++++++ lib/myxql/protocol/geometry_codec/geo.ex | 26 ++++++++++ lib/myxql/protocol/values.ex | 66 ++++-------------------- 6 files changed, 80 insertions(+), 109 deletions(-) delete mode 100644 lib/myxql/protocol/encoder.ex delete mode 100644 lib/myxql/protocol/encoder/geo.ex create mode 100644 lib/myxql/protocol/geometry_codec.ex create mode 100644 lib/myxql/protocol/geometry_codec/geo.ex diff --git a/README.md b/README.md index 3506a04..7c97f34 100644 --- a/README.md +++ b/README.md @@ -137,14 +137,14 @@ config :myxql, :json_library, SomeJSONModule MyXQL supports data stored in `geometry` columns with the help of external geometry libraries such as `geo`. -After adding one of these libraries as a dependency in `mix.exs`, you can register that library's parser -function in a config file (e.g. `runtime.exs`): +If using a library other than `geo`, `myxql` can be configured to use a different codec for encoding +and decoding in the application config, e.g.: ```elixir -config :myxql, wkb_decoder: {Geo.WKB, :decode!} +config :myxql, geometry_coded: GeoSQL.MySQL.Codec ``` -If using the `geo` library, this is the default and no explicit configuration is required. +If using the `geo` library, no explicit configuration is required. Note: some geometry types available in these libraries, such as `PointZ`, are not supported by MySQL and thus should not be used. diff --git a/lib/myxql/protocol/encoder.ex b/lib/myxql/protocol/encoder.ex deleted file mode 100644 index a42ef90..0000000 --- a/lib/myxql/protocol/encoder.ex +++ /dev/null @@ -1,5 +0,0 @@ -defprotocol MyXQL.Protocol.Encoder do - @spec encode(struct()) :: {MyXQL.Protocol.Values.storage_type(), binary()} - def encode(struct) -end - diff --git a/lib/myxql/protocol/encoder/geo.ex b/lib/myxql/protocol/encoder/geo.ex deleted file mode 100644 index acde8ae..0000000 --- a/lib/myxql/protocol/encoder/geo.ex +++ /dev/null @@ -1,45 +0,0 @@ - -if Code.ensure_loaded?(Geo) do - defmodule GeoEncoderHelper do - @moduledoc false - import MyXQL.Protocol.Types, only: [uint4: 0] - - def encode_geometry(geo) do - srid = geo.srid || 0 - binary = %{geo | srid: nil} |> Geo.WKB.encode_to_iodata(:ndr) |> IO.iodata_to_binary() - - { - :mysql_type_var_string, - MyXQL.Protocol.Types.encode_string_lenenc(<>) - } - end - end - - defimpl MyXQL.Protocol.Encoder, for: Geo.Point do - def encode(geo), do: GeoEncoderHelper.encode_geometry(geo) - end - - defimpl MyXQL.Protocol.Encoder, for: Geo.MultiPoint do - def encode(geo), do: GeoEncoderHelper.encode_geometry(geo) - end - - defimpl MyXQL.Protocol.Encoder, for: Geo.LineString do - def encode(geo), do: GeoEncoderHelper.encode_geometry(geo) - end - - defimpl MyXQL.Protocol.Encoder, for: Geo.MultiLineString do - def encode(geo), do: GeoEncoderHelper.encode_geometry(geo) - end - - defimpl MyXQL.Protocol.Encoder, for: Geo.Polygon do - def encode(geo), do: GeoEncoderHelper.encode_geometry(geo) - end - - defimpl MyXQL.Protocol.Encoder, for: Geo.MultiPolygon do - def encode(geo), do: GeoEncoderHelper.encode_geometry(geo) - end - - defimpl MyXQL.Protocol.Encoder, for: Geo.GeometryCollection do - def encode(geo), do: GeoEncoderHelper.encode_geometry(geo) - end -end diff --git a/lib/myxql/protocol/geometry_codec.ex b/lib/myxql/protocol/geometry_codec.ex new file mode 100644 index 0000000..3f14352 --- /dev/null +++ b/lib/myxql/protocol/geometry_codec.ex @@ -0,0 +1,39 @@ +defmodule MyXQL.Protocol.GeometryCodec do + @type no_srid :: 0 + @type some_srid :: pos_integer + + @callback encode(struct) :: {srid :: integer, wkb :: binary()} | :unknown + @callback decode(srid :: no_srid | some_srid, wkb :: binary) :: struct() | :unknown + + import MyXQL.Protocol.Types + + @doc false + def do_encode(struct) do + with codec when not is_nil(codec) <- geometry_codec(), + {srid, wkb} <- codec.encode(struct) do + { + :mysql_type_var_string, + MyXQL.Protocol.Types.encode_string_lenenc(<>) + } + else + _ -> :unknown + end + end + + @doc false + def do_decode(srid, wkb) do + codec = geometry_codec() + + if codec != nil do + codec.decode(srid, wkb) + else + :unknown + end + end + + default_codec = if Code.ensure_loaded?(Geo), do: MyXQL.Protocol.GeometryCodec.Geo, else: nil + + defp(geometry_codec) do + Application.get_env(:myxql, :geometry_codec, unquote(default_codec)) + end +end diff --git a/lib/myxql/protocol/geometry_codec/geo.ex b/lib/myxql/protocol/geometry_codec/geo.ex new file mode 100644 index 0000000..dcdd346 --- /dev/null +++ b/lib/myxql/protocol/geometry_codec/geo.ex @@ -0,0 +1,26 @@ +if Code.ensure_loaded?(Geo) do + defmodule MyXQL.Protocol.GeometryCodec.Geo do + @behaviour MyXQL.Protocol.GeometryCodec + + supported_structs = [ + Geo.Point, + Geo.GeometryCollection, + Geo.LineString, + Geo.MultiPoint, + Geo.MultiLineString, + Geo.MultiPolygon, + Geo.Polygon + ] + + def encode(%x{} = geo) when x in unquote(supported_structs) do + srid = geo.srid || 0 + wkb = %{geo | srid: nil} |> Geo.WKB.encode_to_iodata(:ndr) |> IO.iodata_to_binary() + {srid, wkb} + end + + def encode(_), do: :unknown + + def decode(0, wkb), do: Geo.WKB.decode!(wkb) + def decode(srid, wkb), do: Geo.WKB.decode!(wkb) |> Map.put(:srid, srid) + end +end diff --git a/lib/myxql/protocol/values.ex b/lib/myxql/protocol/values.ex index dd6abb5..cb00f84 100644 --- a/lib/myxql/protocol/values.ex +++ b/lib/myxql/protocol/values.ex @@ -21,38 +21,6 @@ defmodule MyXQL.Protocol.Values do # MySQL TIMESTAMP is equal to Postgres TIMESTAMP WITH TIME ZONE # MySQL DATETIME is equal to Postgres TIMESTAMP [WITHOUT TIME ZONE] - @type storage_type :: - :mysql_type_tiny - | :mysql_type_short - | :mysql_type_long - | :mysql_type_float - | :mysql_type_double - | :mysql_type_null - | :mysql_type_timestamp - | :mysql_type_longlong - | :mysql_type_int24 - | :mysql_type_date - | :mysql_type_time - | :mysql_type_datetime - | :mysql_type_year - | :mysql_type_varchar - | :mysql_type_bit - | :mysql_type_json - | :mysql_type_newdecimal - | :mysql_type_enum - | :mysql_type_set - | :mysql_type_tiny_blob - | :mysql_type_medium_blob - | :mysql_type_long_blob - | :mysql_type_blob - | :mysql_type_var_string - | :mysql_type_string - | :mysql_type_geometry - | :mysql_type_newdate - | :mysql_type_timestamp2 - | :mysql_type_datetime2 - | :mysql_type_date2 - types = [ mysql_type_tiny: 0x01, mysql_type_short: 0x02, @@ -283,13 +251,14 @@ defmodule MyXQL.Protocol.Values do end def encode_binary_value(struct) when is_struct(struct) do - try do - MyXQL.Protocol.Encoder.encode(struct) - rescue - Protocol.UndefinedError -> - # No protocol defined for this struct, so try to json encode it + # see if it is a geometry struct + case MyXQL.Protocol.GeometryCodec.do_encode(struct) do + :unknown -> string = json_library().encode!(struct) {:mysql_type_var_string, encode_string_lenenc(string)} + + encoded -> + encoded end end @@ -449,24 +418,8 @@ defmodule MyXQL.Protocol.Values do end defp decode_geometry(<>) do - # Assumptions made in this function: - # - # * :srid is a top-level field in the struct returned by the wkb decoder - # * the wkb decoder can decode any value that can be stored in a geometry column - - try do - {mod, fun} = Application.get_env(:myxql, :wkb_decoder, {Geo.WKB, :decode!}) - - srid = if srid == 0, do: nil, else: srid - decoded = apply(mod, fun, [data]) - - if srid != 0 do - Map.put(decoded, :srid, srid) - else - decoded - end - rescue - _ -> + case MyXQL.Protocol.GeometryCodec.do_decode(srid, data) do + :unknown -> raise """ Encoding/decoding geometry types requires WKB decoder that returns a map. Libraries such as `geo` and `geometry` provide WKB decoders, which can be @@ -474,6 +427,9 @@ defmodule MyXQL.Protocol.Values do config :myxql, wkb_decoder: {Geometry, :from_wkb!} """ + + decoded -> + decoded end end From 808dcb6ad91bf6b180fdf11ba1e11d66fd091c51 Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Fri, 9 Jan 2026 17:16:18 +0100 Subject: [PATCH 4/5] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Valim --- README.md | 2 +- lib/myxql/protocol/values.ex | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7c97f34..1196092 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ If using a library other than `geo`, `myxql` can be configured to use a differen and decoding in the application config, e.g.: ```elixir -config :myxql, geometry_coded: GeoSQL.MySQL.Codec +config :myxql, geometry_codec: GeoSQL.MySQL.Codec ``` If using the `geo` library, no explicit configuration is required. diff --git a/lib/myxql/protocol/values.ex b/lib/myxql/protocol/values.ex index cb00f84..514fb2c 100644 --- a/lib/myxql/protocol/values.ex +++ b/lib/myxql/protocol/values.ex @@ -421,9 +421,8 @@ defmodule MyXQL.Protocol.Values do case MyXQL.Protocol.GeometryCodec.do_decode(srid, data) do :unknown -> raise """ - Encoding/decoding geometry types requires WKB decoder that returns a map. - Libraries such as `geo` and `geometry` provide WKB decoders, which can be - registered in the application configuration such as `runtime.exs`: + Decoding geometry types requires a geometry library with a MySQL codec. Add a library such + as `geo` or `geometry` as a dependency, and register its codec in the application configuration: config :myxql, wkb_decoder: {Geometry, :from_wkb!} """ From 0cf16d04642ce1c482b8b921666f2d2b59436a19 Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Tue, 13 Jan 2026 13:41:54 +0100 Subject: [PATCH 5/5] Apply suggestions from code review Co-authored-by: Wojtek Mach --- lib/myxql/protocol/geometry_codec.ex | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/myxql/protocol/geometry_codec.ex b/lib/myxql/protocol/geometry_codec.ex index 3f14352..2b3a6f7 100644 --- a/lib/myxql/protocol/geometry_codec.ex +++ b/lib/myxql/protocol/geometry_codec.ex @@ -1,9 +1,9 @@ defmodule MyXQL.Protocol.GeometryCodec do - @type no_srid :: 0 - @type some_srid :: pos_integer + @type no_srid() :: 0 + @type some_srid() :: pos_integer() - @callback encode(struct) :: {srid :: integer, wkb :: binary()} | :unknown - @callback decode(srid :: no_srid | some_srid, wkb :: binary) :: struct() | :unknown + @callback encode(struct()) :: {srid :: integer(), wkb :: binary()} | :unknown + @callback decode(srid :: no_srid() | some_srid(), wkb :: binary()) :: struct() | :unknown import MyXQL.Protocol.Types @@ -33,7 +33,7 @@ defmodule MyXQL.Protocol.GeometryCodec do default_codec = if Code.ensure_loaded?(Geo), do: MyXQL.Protocol.GeometryCodec.Geo, else: nil - defp(geometry_codec) do + defp geometry_codec do Application.get_env(:myxql, :geometry_codec, unquote(default_codec)) end end