From 7223b5d6f36224ffa878df871a69074b34c1aa5b Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Sat, 7 Mar 2026 10:26:10 +0000 Subject: [PATCH 01/15] checkpint --- go.mod | 19 +- go.sum | 62 ++- internal/cmd/run.go | 24 +- internal/engine/cache_fs.go | 105 +++++ internal/engine/cache_sql.go | 316 +++++++++++++ internal/engine/cache_sql_test.go | 81 ++++ internal/engine/codegen.go | 4 +- internal/engine/engine.go | 13 +- internal/engine/handler_result.go | 26 -- internal/engine/handler_resulter.go | 26 +- internal/engine/link.go | 4 + internal/engine/local_cache.go | 424 ++++++++++++------ internal/engine/remote_cache.go | 67 ++- .../engine/{caches.go => remote_caches.go} | 0 internal/engine/schedule.go | 144 +++--- internal/enginee2e/deps_cache_test.go | 6 +- .../pluginscyclicprovider/provider.go | 8 +- .../pluginsmartprovidertest/provider.go | 3 +- internal/enginee2e/sanity_remotecache_test.go | 6 +- internal/hartifact/manifest.go | 81 ++-- internal/hartifact/reader.go | 81 +++- internal/hartifact/unpack.go | 23 +- internal/hartifact/well_known.go | 12 +- internal/remotecache/exec.go | 5 +- internal/remotecache/gcs.go | 17 +- lib/pluginsdk/artifact.go | 49 ++ lib/pluginsdk/init.go | 24 +- lib/pluginsdk/plugin_cache.go | 2 +- lib/pluginsdk/plugin_driver.go | 12 +- .../pluginsdkconnect/plugin_driver.go | 33 +- plugin/pluginbin/plugin.go | 2 +- plugin/pluginexec/plugin_test.go | 91 ++-- plugin/pluginexec/run.go | 11 +- plugin/pluginexec/sandbox.go | 88 ++-- plugin/pluginfs/driver.go | 2 +- plugin/plugingo/get_std.go | 3 +- plugin/plugingo/pkg_analysis.go | 22 +- plugin/plugingroup/plugin.go | 2 +- plugin/pluginnix/driver_test.go | 21 +- plugin/plugintextfile/plugin.go | 2 +- plugin/proto/heph/core/v1/result.proto | 9 +- 41 files changed, 1406 insertions(+), 524 deletions(-) create mode 100644 internal/engine/cache_fs.go create mode 100644 internal/engine/cache_sql.go create mode 100644 internal/engine/cache_sql_test.go delete mode 100644 internal/engine/handler_result.go rename internal/engine/{caches.go => remote_caches.go} (100%) create mode 100644 lib/pluginsdk/artifact.go diff --git a/go.mod b/go.mod index e270d536..1e6a2135 100644 --- a/go.mod +++ b/go.mod @@ -58,10 +58,11 @@ require ( go.opentelemetry.io/otel/trace v1.35.0 go.starlark.net v0.0.0-20241226192728-8dfa5b98479f go.uber.org/mock v0.5.2 - golang.org/x/net v0.39.0 + golang.org/x/net v0.46.0 golang.org/x/sync v0.19.0 golang.org/x/sys v0.41.0 google.golang.org/protobuf v1.36.7-0.20250625222701-8e8926ef675d + modernc.org/sqlite v1.46.1 ) require ( @@ -87,6 +88,7 @@ require ( github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect github.com/colega/zeropool v0.0.0-20230505084239-6fb4a4f75381 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect @@ -105,9 +107,11 @@ require ( github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect github.com/muesli/cancelreader v0.2.2 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect @@ -120,17 +124,20 @@ require ( go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect go.opentelemetry.io/proto/otlp v1.5.0 // indirect - golang.org/x/crypto v0.37.0 // indirect - golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 // indirect - golang.org/x/mod v0.23.0 // indirect + golang.org/x/crypto v0.43.0 // indirect + golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect + golang.org/x/mod v0.29.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/text v0.24.0 // indirect + golang.org/x/text v0.30.0 // indirect golang.org/x/time v0.11.0 // indirect - golang.org/x/tools v0.30.0 // indirect + golang.org/x/tools v0.38.0 // indirect google.golang.org/api v0.232.0 // indirect google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250505200425-f936aa4a68b2 // indirect google.golang.org/grpc v1.72.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + modernc.org/libc v1.67.6 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect ) diff --git a/go.sum b/go.sum index aaf3b7a5..1b953c3c 100644 --- a/go.sum +++ b/go.sum @@ -84,6 +84,8 @@ github.com/creack/pty v1.1.20/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= @@ -123,6 +125,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -136,6 +140,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 h1:e9Rjr40Z98/clHv5Yg79Is0Ntos github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1/go.mod h1:tIxuGz/9mpox++sgp9fJjHO0+q1X9/UOWd798aAm22M= github.com/hanwen/go-fuse/v2 v2.7.2 h1:SbJP1sUP+n1UF8NXBA14BuojmTez+mDgOk0bC057HQw= github.com/hanwen/go-fuse/v2 v2.7.2/go.mod h1:ugNaD/iv5JYyS1Rcvi57Wz7/vrLQJo10mmketmoef48= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/heimdalr/dag v1.5.0 h1:hqVtijvY776P5OKP3QbdVBRt3Xxq6BYopz3XgklsGvo= github.com/heimdalr/dag v1.5.0/go.mod h1:lthekrHl01dddmzqyBQ1YZbi7XcVGGzjFo0jIky5knc= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -164,6 +170,8 @@ github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vyg github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= github.com/pkg/xattr v0.4.12 h1:rRTkSyFNTRElv6pkA3zpjHpQ90p/OdHQC1GmGh1aTjM= @@ -174,6 +182,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= @@ -260,20 +270,20 @@ go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= -golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 h1:1UoZQm6f0P/ZO0w1Ri+f+ifG/gXhegadRdwBIXEFWDo= -golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= +golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= +golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY= +golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM= -golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= -golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= +golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -303,16 +313,16 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= +golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY= -golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.232.0 h1:qGnmaIMf7KcuwHOlF3mERVzChloDYwRfOJOrHt8YC3I= google.golang.org/api v0.232.0/go.mod h1:p9QCfBWZk1IJETUdbTKloR5ToFdKbYh2fkjsUL6vNoY= @@ -332,3 +342,31 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis= +modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.30.1 h1:4r4U1J6Fhj98NKfSjnPUN7Ze2c6MnAdL0hWw6+LrJpc= +modernc.org/ccgo/v4 v4.30.1/go.mod h1:bIOeI1JL54Utlxn+LwrFyjCx2n2RDiYEaJVSrgdrRfM= +modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA= +modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.1 h1:k8T3gkXWY9sEiytKhcgyiZ2L0DTyCQ/nvX+LoCljoRE= +modernc.org/gc/v3 v3.1.1/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.67.6 h1:eVOQvpModVLKOdT+LvBPjdQqfrZq+pC39BygcT+E7OI= +modernc.org/libc v1.67.6/go.mod h1:JAhxUVlolfYDErnwiqaLvUqc8nfb2r6S6slAgZOnaiE= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.46.1 h1:eFJ2ShBLIEnUWlLy12raN0Z1plqmFX9Qe3rjQTKt6sU= +modernc.org/sqlite v1.46.1/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/cmd/run.go b/internal/cmd/run.go index 9cb9a6be..6b10c5e8 100644 --- a/internal/cmd/run.go +++ b/internal/cmd/run.go @@ -207,18 +207,18 @@ func init() { for _, output := range re.Artifacts { fmt.Println(output.GetName()) fmt.Println(" group: ", output.GetGroup()) - switch output.WhichContent() { - case pluginv1.Artifact_File_case: - fmt.Println(" content:", output.GetFile()) - case pluginv1.Artifact_Raw_case: - fmt.Println(" content:", output.GetRaw()) - case pluginv1.Artifact_TarPath_case: - fmt.Println(" content:", output.GetTarPath()) - case pluginv1.Artifact_TargzPath_case: - fmt.Println(" content:", output.GetTargzPath()) - case pluginv1.Artifact_Content_not_set_case: - fmt.Println(" content: ") - } + //switch output.WhichContent() { + //case pluginv1.Artifact_File_case: + // fmt.Println(" content:", output.GetFile()) + //case pluginv1.Artifact_Raw_case: + // fmt.Println(" content:", output.GetRaw()) + //case pluginv1.Artifact_TarPath_case: + // fmt.Println(" content:", output.GetTarPath()) + //case pluginv1.Artifact_TargzPath_case: + // fmt.Println(" content:", output.GetTargzPath()) + //case pluginv1.Artifact_Content_not_set_case: + // fmt.Println(" content: ") + //} fmt.Println(" type: ", output.GetType().String()) } } diff --git a/internal/engine/cache_fs.go b/internal/engine/cache_fs.go new file mode 100644 index 00000000..745a8da2 --- /dev/null +++ b/internal/engine/cache_fs.go @@ -0,0 +1,105 @@ +package engine + +import ( + "context" + "encoding/hex" + "errors" + "io" + "iter" + + "github.com/hephbuild/heph/internal/hfs" + "github.com/hephbuild/heph/internal/hproto/hashpb" + "github.com/hephbuild/heph/lib/tref" + pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" + "github.com/zeebo/xxh3" +) + +type FSCache struct { + root hfs.OS // absolute cache root +} + +func (c FSCache) Exists(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (bool, error) { + _, err := c.path(ref, hashin, name).Lstat() + if err != nil { + if errors.Is(err, hfs.ErrNotExist) { + return false, nil + } + + return false, err + } + + return true, nil +} + +var _ LocalCache = (*FSCache)(nil) + +func NewFSCache(root hfs.OS) *FSCache { + return &FSCache{root: root} +} + +func (c FSCache) targetDirName(ref *pluginv1.TargetRef) string { + if len(ref.GetArgs()) == 0 { + return "__" + ref.GetName() + } + + h := xxh3.New() + hashpb.Hash(h, ref, tref.OmitHashPb) + + return "__" + ref.GetName() + "_" + hex.EncodeToString(h.Sum(nil)) +} + +func (c FSCache) path(ref *pluginv1.TargetRef, hashin, name string) hfs.Node { + return c.root.At(ref.GetPackage(), c.targetDirName(ref), hashin) +} + +func (c FSCache) Reader(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.ReadCloser, error) { + return hfs.Open(c.path(ref, hashin, name)) +} + +func (c FSCache) Writer(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.WriteCloser, error) { + return hfs.Create(c.path(ref, hashin, name)) +} + +func (c FSCache) Delete(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) error { + return c.path(ref, hashin, name).RemoveAll() +} + +func (c FSCache) ListArtifacts(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) iter.Seq2[string, error] { + return func(yield func(string, error) bool) { + entries, err := c.path(ref, hashin, "").ReadDir() + if err != nil { + yield("", err) + return + } + + for _, entry := range entries { + if entry.IsDir() { + continue + } + + if !yield(entry.Name(), nil) { + return + } + } + } +} + +func (c FSCache) ListVersions(ctx context.Context, ref *pluginv1.TargetRef, name string) iter.Seq2[string, error] { + return func(yield func(string, error) bool) { + entries, err := c.path(ref, "", "").ReadDir() + if err != nil { + yield("", err) + return + } + + for _, entry := range entries { + if !entry.IsDir() { + continue + } + + if !yield(entry.Name(), nil) { + return + } + } + } +} diff --git a/internal/engine/cache_sql.go b/internal/engine/cache_sql.go new file mode 100644 index 00000000..56c6b4e9 --- /dev/null +++ b/internal/engine/cache_sql.go @@ -0,0 +1,316 @@ +package engine + +import ( + "bytes" + "context" + "database/sql" + "encoding/hex" + "errors" + "fmt" + "io" + "iter" + "os" + "path/filepath" + "time" + + "github.com/hephbuild/heph/internal/hfs" + "github.com/hephbuild/heph/internal/hproto/hashpb" + "github.com/hephbuild/heph/lib/tref" + pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" + "github.com/zeebo/xxh3" + _ "modernc.org/sqlite" +) + +type SQLCache struct { + db *sql.DB + root hfs.OS +} + +func (c *SQLCache) Exists(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (bool, error) { + targetAddr := c.targetKey(ref) + + var count int + err := c.db.QueryRowContext( + ctx, + `SELECT COUNT(*) FROM cache_blobs WHERE target_addr = ? AND hashin = ? AND artifact_name = ? LIMIT 1`, + targetAddr, hashin, name, + ).Scan(&count) + if err != nil { + return false, fmt.Errorf("exists: %w", err) + } + + return count > 0, nil +} + +func (c *SQLCache) Delete(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) error { + targetAddr := c.targetKey(ref) + + var err error + if name == "" { + _, err = c.db.ExecContext( + ctx, + `DELETE FROM cache_blobs WHERE target_addr = ? AND hashin = ?`, + targetAddr, hashin, + ) + } else { + _, err = c.db.ExecContext( + ctx, + `DELETE FROM cache_blobs WHERE target_addr = ? AND hashin = ? AND artifact_name = ?`, + targetAddr, hashin, name, + ) + } + if err != nil { + return fmt.Errorf("delete: %w", err) + } + + return nil +} + +var _ LocalCache = (*SQLCache)(nil) + +func migrateSQLCacheDB(db *sql.DB) error { + _, err := db.Exec(` + CREATE TABLE IF NOT EXISTS cache_blobs ( + target_addr TEXT NOT NULL, + hashin TEXT NOT NULL, + artifact_name TEXT NOT NULL, + data BLOB, + created_at INTEGER NOT NULL, + PRIMARY KEY (target_addr, hashin, artifact_name) + ); + + CREATE INDEX IF NOT EXISTS cache_blobs_target_hashin + ON cache_blobs (target_addr, hashin); + `) + return err +} + +func OpenSQLCacheDB(path string) (*sql.DB, error) { + if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil { + return nil, fmt.Errorf("OpenSQLCacheDB mkdir: %w", err) + } + + dsn := path + + "?_pragma=journal_mode(WAL)" + + "&_pragma=busy_timeout(10000)" + + "&_pragma=synchronous(NORMAL)" + + "&_pragma=foreign_keys(ON)" + + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, fmt.Errorf("OpenSQLCacheDB open: %w", err) + } + + if err := migrateSQLCacheDB(db); err != nil { + _ = db.Close() + return nil, fmt.Errorf("OpenSQLCacheDB migrate: %w", err) + } + + return db, nil +} + +func NewSQLCache(db *sql.DB) *SQLCache { + return &SQLCache{ + db: db, + } +} + +// targetAddr computes a stable filesystem-safe address for a TargetRef. +// When the ref has no args, this is just "__". When args are present, +// a hash of the full ref is appended to disambiguate. +func (c *SQLCache) targetAddr(ref *pluginv1.TargetRef) string { + if len(ref.GetArgs()) == 0 { + return "__" + ref.GetName() + } + + h := xxh3.New() + hashpb.Hash(h, ref, tref.OmitHashPb) + + return "__" + ref.GetName() + "_" + hex.EncodeToString(h.Sum(nil)) +} + +// targetKey returns the compound target address used as target_addr in the DB. +func (c *SQLCache) targetKey(ref *pluginv1.TargetRef) string { + return ref.GetPackage() + "/" + c.targetAddr(ref) +} + +func (c *SQLCache) Reader(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.ReadCloser, error) { + targetAddr := c.targetKey(ref) + + var dataBytes sql.NullString + + err := c.db.QueryRowContext( + ctx, + ` + SELECT data + FROM cache_blobs + WHERE target_addr = ? AND hashin = ? AND artifact_name = ? + `, + targetAddr, hashin, name, + ).Scan(&dataBytes) + + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, LocalCacheNotFoundError + } + + return nil, fmt.Errorf("reader scan: %w", err) + } + + if !dataBytes.Valid { + return io.NopCloser(bytes.NewReader(nil)), nil + } + + scanned := []byte(dataBytes.String) + return io.NopCloser(bytes.NewReader(scanned)), nil +} + +type sqlWriter struct { + pw *io.PipeWriter + errC chan error + closed bool +} + +func (w *sqlWriter) Write(p []byte) (n int, err error) { + return w.pw.Write(p) +} + +func (w *sqlWriter) Close() error { + if w.closed { + return nil + } + w.closed = true + + err := w.pw.Close() + writeErr := <-w.errC + if writeErr != nil { + return writeErr + } + return err +} + +func (c *SQLCache) writeEntry(ctx context.Context, targetAddr, hashin, name string, data io.Reader) error { + _, err := c.db.ExecContext( + ctx, + ` + INSERT INTO cache_blobs (target_addr, hashin, artifact_name, data, created_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(target_addr, hashin, artifact_name) DO UPDATE SET + data = excluded.data, + created_at = excluded.created_at + `, + targetAddr, hashin, name, []byte{}, time.Now().UnixNano(), + ) + if err != nil { + return fmt.Errorf("writeEntry upsert %w", err) + } + + buf := make([]byte, 32*1024) // 32KB chunks + for { + n, err := data.Read(buf) + if n > 0 { + _, errAppend := c.db.ExecContext(ctx, ` + UPDATE cache_blobs + SET data = data || ? + WHERE target_addr = ? AND hashin = ? AND artifact_name = ? + `, buf[:n], targetAddr, hashin, name) + if errAppend != nil { + return fmt.Errorf("writeEntry append chunk %w", errAppend) + } + } + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("writeEntry read %w", err) + } + } + + return nil +} + +func (c *SQLCache) Writer(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.WriteCloser, error) { + targetAddr := c.targetKey(ref) + + pr, pw := io.Pipe() + + errC := make(chan error, 1) + + go func() { + defer pr.Close() + err := c.writeEntry(ctx, targetAddr, hashin, name, pr) + if err != nil { + err = fmt.Errorf("writer write: %q %q %q %w", tref.Format(ref), hashin, name, err) + } + errC <- err + }() + + return &sqlWriter{ + pw: pw, + errC: errC, + }, nil +} + +func (c *SQLCache) ListArtifacts(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) iter.Seq2[string, error] { + return func(yield func(string, error) bool) { + targetAddr := c.targetKey(ref) + + rows, err := c.db.QueryContext(ctx, + "SELECT artifact_name FROM cache_blobs WHERE target_addr = ? AND hashin = ?", + targetAddr, hashin, + ) + if err != nil { + yield("", err) + return + } + defer rows.Close() + + for rows.Next() { + var artifactName string + if err := rows.Scan(&artifactName); err != nil { + if !yield("", err) { + return + } + continue + } + if !yield(artifactName, nil) { + return + } + } + if err := rows.Err(); err != nil { + yield("", err) + } + } +} + +func (c *SQLCache) ListVersions(ctx context.Context, ref *pluginv1.TargetRef, name string) iter.Seq2[string, error] { + return func(yield func(string, error) bool) { + targetAddr := c.targetKey(ref) + + rows, err := c.db.QueryContext(ctx, + "SELECT DISTINCT hashin FROM cache_blobs WHERE target_addr = ?", + targetAddr, + ) + if err != nil { + yield("", err) + return + } + defer rows.Close() + + for rows.Next() { + var hashin string + if err := rows.Scan(&hashin); err != nil { + if !yield("", err) { + return + } + continue + } + if !yield(hashin, nil) { + return + } + } + if err := rows.Err(); err != nil { + yield("", err) + } + } +} diff --git a/internal/engine/cache_sql_test.go b/internal/engine/cache_sql_test.go new file mode 100644 index 00000000..0ff2baf1 --- /dev/null +++ b/internal/engine/cache_sql_test.go @@ -0,0 +1,81 @@ +package engine_test + +import ( + "context" + "io" + "path/filepath" + "testing" + + "github.com/hephbuild/heph/internal/engine" + pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" + "github.com/stretchr/testify/require" +) + +func TestSQLCache(t *testing.T) { + tempDir := t.TempDir() + dbPath := filepath.Join(tempDir, "cache.db") + db, err := engine.OpenSQLCacheDB(dbPath) + require.NoError(t, err) + defer db.Close() + + cache := engine.NewSQLCache(db) + + ctx := context.Background() + pkg := "pkg" + name := "target" + ref := (&pluginv1.TargetRef_builder{ + Package: &pkg, + Name: &name, + }).Build() + hashin := "hash123" + + // Test Write & Read + w, err := cache.Writer(ctx, ref, hashin, "art1") + require.NoError(t, err) + _, err = io.WriteString(w, "hello world") + require.NoError(t, err) + require.NoError(t, w.Close()) + + exists, err := cache.Exists(ctx, ref, hashin, "art1") + require.NoError(t, err) + require.True(t, exists) + r, err := cache.Reader(ctx, ref, hashin, "art1") + require.NoError(t, err) + b, err := io.ReadAll(r) + require.NoError(t, err) + require.NoError(t, r.Close()) + require.Equal(t, "hello world", string(b)) + + // Test Read Not Exist + exists, err = cache.Exists(ctx, ref, hashin, "art2") + require.NoError(t, err) + require.False(t, exists) + _, err = cache.Reader(ctx, ref, hashin, "art2") + require.ErrorIs(t, err, engine.LocalCacheNotFoundError) + + // Test ListArtifacts + w, err = cache.Writer(ctx, ref, hashin, "art2") + require.NoError(t, err) + require.NoError(t, w.Close()) + + artSeq := cache.ListArtifacts(ctx, ref, hashin, "") + var artifacts []string + for a, e := range artSeq { + require.NoError(t, e) + artifacts = append(artifacts, a) + } + require.ElementsMatch(t, []string{"art1", "art2"}, artifacts) + + // Test ListVersions + w, err = cache.Writer(ctx, ref, "hash456", "art1") + require.NoError(t, err) + require.NoError(t, w.Close()) + + verSeq := cache.ListVersions(ctx, ref, "") + var versions []string + for v, e := range verSeq { + require.NoError(t, e) + versions = append(versions, v) + } + require.ElementsMatch(t, []string{"hash123", "hash456"}, versions) +} diff --git a/internal/engine/codegen.go b/internal/engine/codegen.go index 8012df75..41b3c78d 100644 --- a/internal/engine/codegen.go +++ b/internal/engine/codegen.go @@ -15,7 +15,7 @@ import ( "github.com/hephbuild/heph/plugin/pluginfs" ) -func (e *Engine) codegenTree(ctx context.Context, def *LightLinkedTarget, outputs []ExecuteResultArtifact) error { +func (e *Engine) codegenTree(ctx context.Context, def *LightLinkedTarget, outputs []*ResultArtifact) error { step, ctx := hstep.New(ctx, "Copying to tree...") defer step.Done() @@ -32,7 +32,7 @@ func (e *Engine) codegenTree(ctx context.Context, def *LightLinkedTarget, output return nil } -func (e *Engine) codegenTreeCopy(ctx context.Context, def *LightLinkedTarget, outputs []ExecuteResultArtifact, mode pluginv1.TargetDef_Path_CodegenMode) error { +func (e *Engine) codegenTreeCopy(ctx context.Context, def *LightLinkedTarget, outputs []*ResultArtifact, mode pluginv1.TargetDef_Path_CodegenMode) error { codegenPaths := make([]string, 0) for _, output := range def.GetOutputs() { for _, path := range output.GetPaths() { diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 66c2339d..efc6976b 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -90,6 +90,9 @@ type Engine struct { Sandbox hfs.OS RootSpan trace.Span + CacheSmall LocalCache + CacheLarge LocalCache + WellKnownPackages []string // For testing, for now... CoreHandle EngineHandle @@ -131,6 +134,13 @@ func New(ctx context.Context, root string, cfg Config) (*Engine, error) { FSLock: cfg.LockDriver == "fs", } + db, err := OpenSQLCacheDB(filepath.Join(cachefs.Path(), "cache.db")) + if err != nil { + return nil, fmt.Errorf("open cache db: %w", err) + } + e.CacheSmall = NewSQLCache(db) + e.CacheLarge = NewFSCache(cachefs) + for _, s := range cfg.Packages.Exclude { e.PackagesExclude = append(e.PackagesExclude, filepath.Join(root, s)) } @@ -173,7 +183,6 @@ func New(ctx context.Context, root string, cfg Config) (*Engine, error) { srvh.Mux.Handle(corev1connect.NewLogServiceHandler(hlog.NewLoggerHandler(hlog.From(ctx)))) srvh.Mux.Handle(corev1connect.NewStepServiceHandler(hstep.NewHandler(hstep.HandlerFromContext(ctx)), handlerOpts...)) - srvh.Mux.Handle(corev1connect.NewResultServiceHandler(e.ResultHandler(), handlerOpts...)) e.CoreHandle = EngineHandle{ ServerHandle: srvh, @@ -181,7 +190,7 @@ func New(ctx context.Context, root string, cfg Config) (*Engine, error) { return pluginsdk.Engine{ LogClient: corev1connect.NewLogServiceClient(srvh.HTTPClient(), srvh.GetBaseURL()), StepClient: corev1connect.NewStepServiceClient(srvh.HTTPClient(), srvh.GetBaseURL(), clientOpts...), - ResultClient: e.Resulter(), + ResultClient: pluginsdk.Resulter{Engine: e.Resulter()}, } }, } diff --git a/internal/engine/handler_result.go b/internal/engine/handler_result.go deleted file mode 100644 index 51768409..00000000 --- a/internal/engine/handler_result.go +++ /dev/null @@ -1,26 +0,0 @@ -package engine - -import ( - "context" - - "connectrpc.com/connect" - corev1 "github.com/hephbuild/heph/plugin/gen/heph/core/v1" - "github.com/hephbuild/heph/plugin/gen/heph/core/v1/corev1connect" -) - -func (e *Engine) ResultHandler() corev1connect.ResultServiceHandler { - return &resultServiceHandler{Engine: e} -} - -type resultServiceHandler struct { - *Engine -} - -func (r resultServiceHandler) Get(ctx context.Context, req *connect.Request[corev1.ResultRequest]) (*connect.Response[corev1.ResultResponse], error) { - res, err := r.Resulter().Get(ctx, req.Msg) - if err != nil { - return nil, err - } - - return connect.NewResponse(res), nil -} diff --git a/internal/engine/handler_resulter.go b/internal/engine/handler_resulter.go index a60db858..6778fa00 100644 --- a/internal/engine/handler_resulter.go +++ b/internal/engine/handler_resulter.go @@ -5,23 +5,26 @@ import ( "errors" "fmt" + "github.com/google/uuid" "github.com/hephbuild/heph/lib/pluginsdk" corev1 "github.com/hephbuild/heph/plugin/gen/heph/core/v1" - pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" + sync_map "github.com/zolstein/sync-map" "go.opentelemetry.io/otel" ) -func (e *Engine) Resulter() pluginsdk.Resulter { +func (e *Engine) Resulter() pluginsdk.EngineResulter { return &resulterHandler{Engine: e} } type resulterHandler struct { *Engine + + m sync_map.Map[string, *ExecuteResultLocks] } var tracer = otel.Tracer("heph/engine") -func (r resulterHandler) Get(ctx context.Context, req *corev1.ResultRequest) (*corev1.ResultResponse, error) { +func (r *resulterHandler) Get(ctx context.Context, req *corev1.ResultRequest) (*pluginsdk.GetResult, error) { rs, err := r.GetRequestState(req.GetRequestId()) if err != nil { return nil, err @@ -52,16 +55,21 @@ func (r resulterHandler) Get(ctx context.Context, req *corev1.ResultRequest) (*c default: return nil, fmt.Errorf("unexpected message type: %v", kind) } - // TODO: this is in the wrong place, the caller should be responsible for releasing the locks - defer res.Unlock(ctx) - artifacts := make([]*pluginv1.Artifact, 0, len(res.Artifacts)) + id := uuid.New().String() + r.m.Store(id, res) + + artifacts := make([]pluginsdk.Artifact, 0, len(res.Artifacts)) for _, artifact := range res.Artifacts { - artifacts = append(artifacts, artifact.Artifact) + artifacts = append(artifacts, artifact) } - return corev1.ResultResponse_builder{ + return &pluginsdk.GetResult{ + Release: func() { + res.Unlock(ctx) + }, Artifacts: artifacts, Def: res.Def.TargetDef.TargetDef, - }.Build(), nil + }, nil + } diff --git a/internal/engine/link.go b/internal/engine/link.go index 4d0c3d09..c8544203 100644 --- a/internal/engine/link.go +++ b/internal/engine/link.go @@ -193,6 +193,10 @@ func (e *Engine) resolveSpecQuery(ctx context.Context, rs *RequestState, ref *pl deps = append(deps, structpb.NewStringValue(tref.Format(qref))) } + slices.SortFunc(deps, func(a, b *structpb.Value) int { + return strings.Compare(a.GetStringValue(), b.GetStringValue()) + }) + return pluginv1.TargetSpec_builder{ Ref: ref, Driver: htypes.Ptr(plugingroup.Name), diff --git a/internal/engine/local_cache.go b/internal/engine/local_cache.go index 73c79be3..7f38ac0d 100644 --- a/internal/engine/local_cache.go +++ b/internal/engine/local_cache.go @@ -5,18 +5,22 @@ import ( "bytes" "context" "encoding/hex" + "encoding/json" "errors" "fmt" "hash" "io" + "iter" "os" "slices" "strconv" "strings" + "sync" "time" "github.com/hephbuild/heph/internal/hcore/hstep" "github.com/hephbuild/heph/internal/hproto/hashpb" + "github.com/hephbuild/heph/lib/pluginsdk" "github.com/hephbuild/heph/lib/tref" @@ -28,7 +32,18 @@ import ( "github.com/zeebo/xxh3" ) -func (e *Engine) hashout(ctx context.Context, ref *pluginv1.TargetRef, artifact *pluginv1.Artifact) (string, error) { +var LocalCacheNotFoundError = errors.New("artifact not found") + +type LocalCache interface { + Reader(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.ReadCloser, error) + Exists(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (bool, error) + Writer(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.WriteCloser, error) + Delete(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) error + ListArtifacts(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) iter.Seq2[string, error] + ListVersions(ctx context.Context, ref *pluginv1.TargetRef, name string) iter.Seq2[string, error] +} + +func (e *Engine) hashout(ctx context.Context, ref *pluginv1.TargetRef, artifact pluginsdk.Artifact) (string, error) { var h interface { hash.Hash io.StringWriter @@ -76,156 +91,199 @@ func (e *Engine) CacheLocally( ctx context.Context, def *LightLinkedTarget, hashin string, - sandboxArtifacts []ExecuteResultArtifact, -) ([]ExecuteResultArtifact, *hartifact.Manifest, error) { + sandboxArtifacts []*ResultArtifact, +) ([]*ResultArtifact, *hartifact.Manifest, error) { step, ctx := hstep.New(ctx, "Caching...") defer step.Done() - cachedir := hfs.At(e.Cache, def.GetRef().GetPackage(), e.targetDirName(def.GetRef()), hashin) - - cacheArtifacts := make([]ExecuteResultArtifact, 0, len(sandboxArtifacts)) + cacheArtifacts := make([]*ResultArtifact, 0, len(sandboxArtifacts)) for _, artifact := range sandboxArtifacts { if artifact.GetType() == pluginv1.Artifact_TYPE_MANIFEST_V1 { continue } - if artifact.HasFile() { - content := artifact.GetFile() + src, err := e.contentReaderNormalizer(artifact) + if err != nil { + return nil, nil, err + } + defer src.Close() + + cacheArtifact, err := e.cacheLocally(ctx, def.GetRef(), hashin, CacheLocallyArtifact{ + Reader: src, + Size: 0, + Type: artifact.GetType(), + Group: artifact.GetGroup(), + Name: artifact.GetName(), + }, artifact.Hashout) + if err != nil { + return nil, nil, err + } - artifact.SetName(artifact.GetName() + ".tar") - tarf, err := hfs.Create(cachedir.At(artifact.GetName())) - if err != nil { - return nil, nil, err - } - defer tarf.Close() - p := htar.NewPacker(tarf) + cacheArtifacts = append(cacheArtifacts, cacheArtifact) + } - sourcefs := hfs.NewOS(content.GetSourcePath()) + m := hartifact.Manifest{ + Version: "v1", + Target: tref.Format(def.GetRef()), + CreatedAt: time.Now(), + Hashin: hashin, + } + for _, artifact := range cacheArtifacts { + martifact, err := hartifact.ProtoArtifactToManifest(artifact.Hashout, artifact.Artifact) + if err != nil { + return nil, nil, err + } - f, err := hfs.Open(sourcefs) - if err != nil { - return nil, nil, err - } - defer f.Close() + m.Artifacts = append(m.Artifacts, martifact) + } - err = p.WriteFile(f, content.GetOutPath()) - if err != nil { - return nil, nil, err - } + cacheArtifacts = append(cacheArtifacts, &ResultArtifact{ + Artifact: &manifestPluginArtifact{manifest: m}, + }) - _ = f.Close() - _ = tarf.Close() + return cacheArtifacts, &m, nil +} - artifact.SetTarPath(tarf.Name()) - } - if artifact.HasRaw() { - content := artifact.GetRaw() +type cachePluginArtifact struct { + group, name string + type_ pluginv1.Artifact_Type + cache LocalCache +} - artifact.SetName(artifact.GetName() + ".tar") - tarf, err := hfs.Create(cachedir.At(artifact.GetName())) - if err != nil { - return nil, nil, err - } - defer tarf.Close() - p := htar.NewPacker(tarf) +func (e cachePluginArtifact) GetGroup() string { + return e.group +} +func (e cachePluginArtifact) GetName() string { + return e.name +} +func (e cachePluginArtifact) GetType() pluginv1.Artifact_Type { + return e.type_ +} - mode := int64(os.ModePerm) - if content.GetX() { - mode |= 0111 // executable - } +func (e cachePluginArtifact) GetProto() *pluginv1.Artifact { + panic("TO REMOVE") +} - err = p.Write(bytes.NewReader(content.GetData()), &tar.Header{ - Typeflag: tar.TypeReg, - Name: content.GetPath(), - Size: int64(len(content.GetData())), - Mode: mode, - }) - if err != nil { - return nil, nil, err - } +func (e cachePluginArtifact) GetContentReader() (io.ReadCloser, error) { + return hartifact.Reader(e) +} - _ = tarf.Close() +func (e cachePluginArtifact) GetContentSize() (int64, error) { + return hartifact.Size(e) +} - artifact.SetTarPath(tarf.Name()) - } +type manifestPluginArtifact struct { + manifest hartifact.Manifest + + manifestBytes []byte + manifestBytesOnce sync.Once +} + +func (e *manifestPluginArtifact) GetGroup() string { + return "" +} +func (e *manifestPluginArtifact) GetName() string { + return hartifact.ManifestName +} +func (e *manifestPluginArtifact) GetType() pluginv1.Artifact_Type { + return pluginv1.Artifact_TYPE_MANIFEST_V1 +} + +func (e *manifestPluginArtifact) GetProto() *pluginv1.Artifact { + panic("TO REMOVE") +} - fromPath, err := hartifact.Path(artifact.Artifact) +func (e *manifestPluginArtifact) marshal() { + e.manifestBytesOnce.Do(func() { + b, err := json.Marshal(e.manifest) if err != nil { - return nil, nil, err + panic(err) } - if fromPath == "" { - return nil, nil, fmt.Errorf("artifact %s has no path", artifact.GetName()) - } + e.manifestBytes = b + }) +} - var prefix string - switch artifact.GetType() { - case pluginv1.Artifact_TYPE_OUTPUT: - prefix = "out_" - case pluginv1.Artifact_TYPE_SUPPORT_FILE: - prefix = "support_" - case pluginv1.Artifact_TYPE_LOG: - prefix = "log_" - case pluginv1.Artifact_TYPE_OUTPUT_LIST_V1, pluginv1.Artifact_TYPE_MANIFEST_V1, pluginv1.Artifact_TYPE_UNSPECIFIED: - fallthrough - default: - return nil, nil, fmt.Errorf("invalid artifact type: %s", artifact.GetType()) - } +func (e *manifestPluginArtifact) GetContentReader() (io.ReadCloser, error) { + e.marshal() + + return io.NopCloser(bytes.NewReader(e.manifestBytes)), nil +} + +func (e *manifestPluginArtifact) GetContentSize() (int64, error) { + e.marshal() + + return int64(len(e.manifestBytes)), nil +} - artifact.SetName(prefix + artifact.GetName()) - fromfs := hfs.NewOS(fromPath) - tofs := hfs.At(cachedir, artifact.GetName()) +func (e *Engine) contentReaderNormalizer(artifact pluginsdk.Artifact) (io.ReadCloser, error) { + partifact := artifact.GetProto() - if false && strings.HasPrefix(fromfs.Path(), e.Home.Path()) { - // TODO: there was a bug here where when a target was running in tree, it would move things out from tree - err = hfs.Move(fromfs, tofs) + if partifact.HasFile() { + content := partifact.GetFile() + + pr, pw := io.Pipe() + + p := htar.NewPacker(pw) + + sourcefs := hfs.NewOS(content.GetSourcePath()) + + go func() { + f, err := hfs.Open(sourcefs) if err != nil { - return nil, nil, fmt.Errorf("move: %w", err) + _ = pr.CloseWithError(err) + + return } - } else { - err = hfs.Copy(fromfs, tofs) + defer f.Close() + + err = p.WriteFile(f, content.GetOutPath()) if err != nil { - return nil, nil, fmt.Errorf("copy: %w", err) + _ = pr.CloseWithError(err) + + return } - } - cachedArtifact, err := hartifact.Relocated(artifact.Artifact, tofs.Path()) - if err != nil { - return nil, nil, fmt.Errorf("relocated: %w", err) - } + _ = f.Close() + _ = pw.Close() + }() - cacheArtifacts = append(cacheArtifacts, ExecuteResultArtifact{ - Hashout: artifact.Hashout, - Artifact: cachedArtifact, - }) + return pr, nil } - m := hartifact.Manifest{ - Version: "v1", - Target: tref.Format(def.GetRef()), - CreatedAt: time.Now(), - Hashin: hashin, - } - for _, artifact := range cacheArtifacts { - martifact, err := hartifact.ProtoArtifactToManifest(artifact.Hashout, artifact.Artifact) - if err != nil { - return nil, nil, err + if partifact.HasRaw() { + content := partifact.GetRaw() + + pr, pw := io.Pipe() + + p := htar.NewPacker(pw) + + mode := int64(os.ModePerm) + if content.GetX() { + mode |= 0111 // executable } - m.Artifacts = append(m.Artifacts, martifact) - } + go func() { + err := p.Write(bytes.NewReader(content.GetData()), &tar.Header{ + Typeflag: tar.TypeReg, + Name: content.GetPath(), + Size: int64(len(content.GetData())), + Mode: mode, + }) + if err != nil { + _ = pr.CloseWithError(err) - manifestArtifact, err := hartifact.WriteManifest(cachedir, m) - if err != nil { - return nil, nil, err - } + return + } + }() - cacheArtifacts = append(cacheArtifacts, ExecuteResultArtifact{ - Artifact: manifestArtifact, - }) + _ = pw.Close() - return cacheArtifacts, &m, nil + return pr, nil + } + + return artifact.GetContentReader() } func (e *Engine) ClearCacheLocally( @@ -236,9 +294,15 @@ func (e *Engine) ClearCacheLocally( step, ctx := hstep.New(ctx, "Clearing...") defer step.Done() - cachedir := hfs.At(e.Cache, ref.GetPackage(), e.targetDirName(ref), hashin) + if err := e.CacheLarge.Delete(ctx, ref, hashin, ""); err != nil { + return fmt.Errorf("fs: %w", err) + } - return cachedir.RemoveAll() + if err := e.CacheSmall.Delete(ctx, ref, hashin, ""); err != nil { + return fmt.Errorf("db: %w", err) + } + + return nil } func keyRefOutputs(ref *pluginv1.TargetRef, outputs []string) string { @@ -261,56 +325,154 @@ func (e *Engine) ResultFromLocalCache(ctx context.Context, def *LightLinkedTarge res, ok, err := e.resultFromLocalCacheInner(ctx, def, outputs, hashin) if err != nil { - // if the file doesnt exist, thats not an error, just means the cache doesnt exist locally if errors.Is(err, hfs.ErrNotExist) { return nil, false, nil } - return nil, false, err } return res, ok, nil } +func (e *Engine) readAnyCache(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.ReadCloser, error) { + for _, c := range []LocalCache{e.CacheSmall, e.CacheLarge} { + r, err := c.Reader(ctx, ref, hashin, name) + if err != nil { + if errors.Is(err, LocalCacheNotFoundError) { + continue + } + + return nil, err + } + + return r, nil + } + + return nil, LocalCacheNotFoundError +} + +func (e *Engine) existsAnyCache(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (bool, LocalCache, error) { + for _, c := range [...]LocalCache{e.CacheSmall, e.CacheLarge} { + exists, err := c.Exists(ctx, ref, hashin, name) + if err != nil { + return false, c, err + } + + if exists { + return true, c, nil + } + } + + return false, nil, nil +} + func (e *Engine) resultFromLocalCacheInner(ctx context.Context, def *LightLinkedTarget, outputs []string, hashin string) (*ExecuteResult, bool, error) { - dirfs := hfs.At(e.Cache, def.GetRef().GetPackage(), e.targetDirName(def.GetRef()), hashin) + r, err := e.readAnyCache(ctx, def.GetRef(), hashin, hartifact.ManifestName) + if err != nil { + if errors.Is(err, LocalCacheNotFoundError) { + return nil, false, nil + } + + return nil, false, err + } + defer r.Close() - manifest, manifestArtifact, err := hartifact.ManifestFromFS(dirfs) + m, err := hartifact.DecodeManifest(r) if err != nil { - return nil, false, fmt.Errorf("ManifestFromFS: %w", err) + return nil, false, err } artifacts := make([]hartifact.ManifestArtifact, 0, len(outputs)) for _, output := range outputs { - outputArtifacts := manifest.GetArtifacts(output) + outputArtifacts := m.GetArtifacts(output) artifacts = append(artifacts, outputArtifacts...) } - execArtifacts := make([]ExecuteResultArtifact, 0, len(artifacts)) + execArtifacts := make([]*ResultArtifact, 0, len(artifacts)) for _, artifact := range artifacts { - if !hfs.Exists(dirfs.At(artifact.Name)) { - return nil, false, nil + exists, cache, err := e.existsAnyCache(ctx, def.GetRef(), hashin, artifact.Name) + if err != nil { + return nil, false, err } - partifact, err := hartifact.ManifestArtifactToProto(artifact, dirfs.At(artifact.Name).Path()) - if err != nil { - return nil, false, fmt.Errorf("ManifestArtifactToProto: %w", err) + if !exists { + return nil, false, nil } - execArtifacts = append(execArtifacts, ExecuteResultArtifact{ - Hashout: artifact.Hashout, - Artifact: partifact, + execArtifacts = append(execArtifacts, &ResultArtifact{ + Hashout: artifact.Hashout, + Artifact: cachePluginArtifact{ + group: artifact.Group, + name: artifact.Name, + type_: pluginv1.Artifact_Type(artifact.Type), + cache: cache, + }, }) } - execArtifacts = append(execArtifacts, ExecuteResultArtifact{ - Artifact: manifestArtifact, + execArtifacts = append(execArtifacts, &ResultArtifact{ + Artifact: &manifestPluginArtifact{manifest: m}, }) return ExecuteResult{ Def: def, - Hashin: manifest.Hashin, + Hashin: m.Hashin, Artifacts: execArtifacts, }.Sorted(), true, nil } + +type CacheLocallyArtifact struct { + Reader io.Reader + Size int64 + Type pluginv1.Artifact_Type + Group string + Name string +} + +func (e *Engine) cacheLocally(ctx context.Context, ref *pluginv1.TargetRef, hashin string, art CacheLocallyArtifact, hashout string) (*ResultArtifact, error) { + cache := e.CacheSmall + if art.Size > 100_000 { + cache = e.CacheLarge + } + + var prefix string + switch art.Type { + case pluginv1.Artifact_TYPE_OUTPUT: + prefix = "out_" + case pluginv1.Artifact_TYPE_SUPPORT_FILE: + prefix = "support_" + case pluginv1.Artifact_TYPE_LOG: + prefix = "log_" + case pluginv1.Artifact_TYPE_OUTPUT_LIST_V1, pluginv1.Artifact_TYPE_MANIFEST_V1, pluginv1.Artifact_TYPE_UNSPECIFIED: + fallthrough + default: + return nil, fmt.Errorf("invalid artifact type: %s", art.Type) + } + + dst, err := cache.Writer(ctx, ref, hashin, prefix+art.Name) + if err != nil { + return nil, err + } + defer dst.Close() + + _, err = io.Copy(dst, art.Reader) + if err != nil { + return nil, err + } + + err = dst.Close() + if err != nil { + return nil, err + } + + return &ResultArtifact{ + Hashout: hashout, + Artifact: cachePluginArtifact{ + group: art.Group, + name: prefix + art.Name, + type_: art.Type, + cache: cache, + }, + }, nil +} diff --git a/internal/engine/remote_cache.go b/internal/engine/remote_cache.go index 129fc59b..00002342 100644 --- a/internal/engine/remote_cache.go +++ b/internal/engine/remote_cache.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "io" "log/slog" "os" "path" @@ -23,10 +22,9 @@ import ( pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - "golang.org/x/sync/errgroup" ) -func (e *Engine) CacheRemotely(ctx context.Context, def *LightLinkedTarget, hashin string, manifest *hartifact.Manifest, artifacts []ExecuteResultArtifact) { +func (e *Engine) CacheRemotely(ctx context.Context, def *LightLinkedTarget, hashin string, manifest *hartifact.Manifest, artifacts []*ResultArtifact) { if def.GetDisableRemoteCache() { return } @@ -71,7 +69,7 @@ func (e *Engine) cacheRemotelyInner(ctx context.Context, ref *pluginv1.TargetRef, hashin string, manifest *hartifact.Manifest, - artifacts []ExecuteResultArtifact, + artifacts []*ResultArtifact, cache CacheHandle, ) error { step, ctx := hstep.New(ctx, fmt.Sprintf("Caching %q...", cache.Name)) @@ -80,7 +78,7 @@ func (e *Engine) cacheRemotelyInner(ctx context.Context, // TODO: remote lock ? for _, artifact := range artifacts { - r, err := hartifact.Reader(ctx, artifact.Artifact) + r, err := hartifact.Reader(artifact.Artifact) if err != nil { return err } @@ -155,7 +153,7 @@ func (e *Engine) ResultFromRemoteCache(ctx context.Context, rs *RequestState, de } if ok { - localArtifacts := make([]ExecuteResultArtifact, 0, len(artifacts)) + localArtifacts := make([]*ResultArtifact, 0, len(artifacts)) for _, artifact := range artifacts { to := cacheDir.At(artifact.GetName()) @@ -164,14 +162,14 @@ func (e *Engine) ResultFromRemoteCache(ctx context.Context, rs *RequestState, de return nil, false, err } - rartifact, err := hartifact.Relocated(artifact.Artifact, to.Path()) + rartifact, err := hartifact.Relocated(artifact.GetProto(), to.Path()) if err != nil { return nil, false, err } - localArtifacts = append(localArtifacts, ExecuteResultArtifact{ + localArtifacts = append(localArtifacts, &ResultArtifact{ Hashout: artifact.Hashout, - Artifact: rartifact, + Artifact: protoPluginArtifact{Artifact: rartifact}, }) } @@ -192,7 +190,7 @@ func (e *Engine) ResultFromRemoteCache(ctx context.Context, rs *RequestState, de func (e *Engine) manifestFromRemoteCache(ctx context.Context, ref *pluginv1.TargetRef, hashin string, cache CacheHandle) (hartifact.Manifest, bool, error) { manifestKey := e.remoteCacheKey(ref, hashin, hartifact.ManifestName) - r, err := cache.Client.Get(ctx, manifestKey) + r, _, err := cache.Client.Get(ctx, manifestKey) if err != nil { if errors.Is(err, pluginsdk.ErrCacheNotFound) { return hartifact.Manifest{}, false, nil @@ -227,7 +225,7 @@ func (e *Engine) resultFromRemoteCacheInner( hashin string, cache CacheHandle, cachedir hfs.OS, -) ([]ExecuteResultArtifact, bool, error) { +) ([]*ResultArtifact, bool, error) { ctx, span := tracer.Start(ctx, "ResultFromLocalCacheInner", trace.WithAttributes(attribute.String("cache", cache.Name))) defer span.End() @@ -256,17 +254,15 @@ func (e *Engine) resultFromRemoteCacheInner( } var localArtifactsm sync.Mutex - localArtifacts := make([]ExecuteResultArtifact, 0, len(outputs)) + localArtifacts := make([]*ResultArtifact, 0, len(outputs)) - g, ctx := errgroup.WithContext(ctx) + g := herrgroup.NewContext(ctx, true) for _, artifact := range remoteArtifacts { key := e.remoteCacheKey(ref, hashin, artifact.Name) - tofs := cachedir.At(artifact.Name) - - g.Go(func() error { - r, err := cache.Client.Get(ctx, key) + g.Go(func(ctx context.Context) error { + r, size, err := cache.Client.Get(ctx, key) if err != nil { if errors.Is(err, pluginsdk.ErrCacheNotFound) { return pluginsdk.ErrCacheNotFound @@ -274,29 +270,21 @@ func (e *Engine) resultFromRemoteCacheInner( return err } - - f, err := hfs.Create(tofs) - if err != nil { - return err - } - defer f.Close() - - _, err = io.Copy(f, r) - if err != nil { - return err - } - _ = f.Close() - - lartifact, err := hartifact.ManifestArtifactToProto(artifact, tofs.Path()) + defer r.Close() + + localArtifact, err := e.cacheLocally(ctx, ref, hashin, CacheLocallyArtifact{ + Reader: r, + Size: size, + Type: pluginv1.Artifact_Type(artifact.Type), + Group: artifact.Group, + Name: artifact.Name, + }, artifact.Hashout) if err != nil { return err } localArtifactsm.Lock() - localArtifacts = append(localArtifacts, ExecuteResultArtifact{ - Hashout: artifact.Hashout, - Artifact: lartifact, - }) + localArtifacts = append(localArtifacts, localArtifact) localArtifactsm.Unlock() return nil @@ -311,13 +299,8 @@ func (e *Engine) resultFromRemoteCacheInner( return nil, false, err } - manifestArtifact, err := hartifact.WriteManifest(cachedir, manifest) - if err != nil { - return nil, false, err - } - - localArtifacts = append(localArtifacts, ExecuteResultArtifact{ - Artifact: manifestArtifact, + localArtifacts = append(localArtifacts, &ResultArtifact{ + Artifact: &manifestPluginArtifact{manifest: manifest}, }) return localArtifacts, true, nil diff --git a/internal/engine/caches.go b/internal/engine/remote_caches.go similarity index 100% rename from internal/engine/caches.go rename to internal/engine/remote_caches.go diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index 7209d241..e7a7ef78 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -23,7 +23,6 @@ import ( "github.com/hephbuild/heph/internal/herrgroup" "github.com/hephbuild/heph/internal/hinstance" "github.com/hephbuild/heph/internal/hpanic" - "github.com/hephbuild/heph/internal/hproto" "github.com/hephbuild/heph/internal/hproto/hashpb" "github.com/hephbuild/heph/internal/htypes" "github.com/hephbuild/heph/internal/tmatch" @@ -278,11 +277,11 @@ func (e *Engine) result(ctx context.Context, rs *RequestState, c DefContainer, o } if onlyManifest { - res.Artifacts = slices.DeleteFunc(res.Artifacts, func(artifact ExecuteResultArtifact) bool { + res.Artifacts = slices.DeleteFunc(res.Artifacts, func(artifact *ResultArtifact) bool { return artifact.GetType() != pluginv1.Artifact_TYPE_MANIFEST_V1 }) } else { - res.Artifacts = slices.DeleteFunc(res.Artifacts, func(artifact ExecuteResultArtifact) bool { + res.Artifacts = slices.DeleteFunc(res.Artifacts, func(artifact *ResultArtifact) bool { if artifact.GetType() == pluginv1.Artifact_TYPE_MANIFEST_V1 { return true } @@ -341,7 +340,7 @@ func (e *Engine) depsResults(ctx context.Context, rs *RequestState, t *LightLink return err } - res.Artifacts = slices.DeleteFunc(res.Artifacts, func(output ExecuteResultArtifact) bool { + res.Artifacts = slices.DeleteFunc(res.Artifacts, func(output *ResultArtifact) bool { return output.GetType() != pluginv1.Artifact_TYPE_OUTPUT && output.GetType() != pluginv1.Artifact_TYPE_SUPPORT_FILE }) @@ -456,7 +455,7 @@ func (e *Engine) ResultMetaFromDef(ctx context.Context, rs *RequestState, def *T return ResultMeta{}, errors.New("no manifest") } - manifest, err := hartifact.ManifestFromArtifact(ctx, manifestArtifact.Artifact) + manifest, err := hartifact.ManifestFromArtifact(ctx, manifestArtifact) if err != nil { return ResultMeta{}, fmt.Errorf("manifest from artifact: %w", err) } @@ -656,19 +655,21 @@ func (e *Engine) innerResult(ctx context.Context, rs *RequestState, def *LightLi Hashin: hashin, } - var artifacts []ExecuteResultArtifact + var artifacts []*ResultArtifact var locks CacheLocks for _, result := range results { for _, artifact := range result.Artifacts { - gartifact := hproto.Clone(artifact.Artifact) + gartifact := artifact.GetProto() gartifact.ClearGroup() // TODO support output group - artifacts = append(artifacts, ExecuteResultArtifact{ + partifact := protoPluginArtifact{Artifact: gartifact} + + artifacts = append(artifacts, &ResultArtifact{ Hashout: artifact.Hashout, - Artifact: gartifact, + Artifact: partifact, }) - martifact, err := hartifact.ProtoArtifactToManifest(artifact.Hashout, gartifact) + martifact, err := hartifact.ProtoArtifactToManifest(artifact.Hashout, partifact) if err != nil { return nil, fmt.Errorf("proto artifact to manifest: %w", err) } @@ -684,8 +685,8 @@ func (e *Engine) innerResult(ctx context.Context, rs *RequestState, def *LightLi return nil, fmt.Errorf("new manifest artifact: %w", err) } - artifacts = append(artifacts, ExecuteResultArtifact{ - Artifact: martifact, + artifacts = append(artifacts, &ResultArtifact{ + Artifact: protoPluginArtifact{Artifact: martifact}, }) err = locks.RLock(ctx) @@ -739,7 +740,7 @@ func (e *Engine) innerResult(ctx context.Context, rs *RequestState, def *LightLi Hashin: hashin, } for _, artifact := range res.Artifacts { - martifact, err := hartifact.ProtoArtifactToManifest(artifact.Hashout, artifact.Artifact) + martifact, err := hartifact.ProtoArtifactToManifest(artifact.Hashout, artifact) if err != nil { err = errors.Join(err, locks.Unlock()) @@ -756,8 +757,8 @@ func (e *Engine) innerResult(ctx context.Context, rs *RequestState, def *LightLi return nil, err } - res.Artifacts = append(res.Artifacts, ExecuteResultArtifact{ - Artifact: manifestArtifact, + res.Artifacts = append(res.Artifacts, &ResultArtifact{ + Artifact: protoPluginArtifact{Artifact: manifestArtifact}, }) } @@ -881,19 +882,35 @@ func (e *Engine) hashin(ctx context.Context, def *LightLinkedTarget, results []* return e.hashin2(ctx, def, metas, "res") } -type ExecuteResultArtifact struct { - Hashout string +type protoPluginArtifact struct { *pluginv1.Artifact } +func (e protoPluginArtifact) GetProto() *pluginv1.Artifact { + return e.Artifact +} + +func (e protoPluginArtifact) GetContentReader() (io.ReadCloser, error) { + return hartifact.Reader(e) +} + +func (e protoPluginArtifact) GetContentSize() (int64, error) { + return hartifact.Size(e) +} + type ExecuteResult struct { Def *LightLinkedTarget Executed bool Hashin string - Artifacts []ExecuteResultArtifact + Artifacts []*ResultArtifact +} + +type ResultArtifact struct { + Hashout string + pluginsdk.Artifact } -func (r ExecuteResult) FindManifest() (ExecuteResultArtifact, bool) { +func (r ExecuteResult) FindManifest() (*ResultArtifact, bool) { for _, artifact := range r.Artifacts { if artifact.GetType() != pluginv1.Artifact_TYPE_MANIFEST_V1 { continue @@ -902,11 +919,11 @@ func (r ExecuteResult) FindManifest() (ExecuteResultArtifact, bool) { return artifact, true } - return ExecuteResultArtifact{}, false + return nil, false } -func (r ExecuteResult) FindOutputs(group string) []ExecuteResultArtifact { - res := make([]ExecuteResultArtifact, 0, len(r.Artifacts)) +func (r ExecuteResult) FindOutputs(group string) []*ResultArtifact { + res := make([]*ResultArtifact, 0, len(r.Artifacts)) for _, artifact := range r.Artifacts { if artifact.GetType() != pluginv1.Artifact_TYPE_OUTPUT { continue @@ -921,8 +938,8 @@ func (r ExecuteResult) FindOutputs(group string) []ExecuteResultArtifact { return res } -func (r ExecuteResult) FindSupport() []ExecuteResultArtifact { - res := make([]ExecuteResultArtifact, 0, len(r.Artifacts)) +func (r ExecuteResult) FindSupport() []*ResultArtifact { + res := make([]*ResultArtifact, 0, len(r.Artifacts)) for _, artifact := range r.Artifacts { if artifact.GetType() != pluginv1.Artifact_TYPE_SUPPORT_FILE { continue @@ -965,7 +982,7 @@ func (r *ExecuteResultLocks) Unlock(ctx context.Context) { } func (r ExecuteResult) Sorted() *ExecuteResult { - slices.SortFunc(r.Artifacts, func(a, b ExecuteResultArtifact) int { + slices.SortFunc(r.Artifacts, func(a, b *ResultArtifact) int { return strings.Compare(a.Hashout, b.Hashout) }) @@ -1256,12 +1273,22 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked for _, result := range results { for _, artifact := range result.Artifacts { inputs = append(inputs, pluginv1.ArtifactWithOrigin_builder{ - Artifact: artifact.Artifact, + Artifact: artifact.GetProto(), Origin: result.InputOrigin, }.Build()) } } + inputsSdk := make([]*pluginsdk.ArtifactWithOrigin, 0, len(inputs)) + for _, input := range inputs { + inputsSdk = append(inputsSdk, &pluginsdk.ArtifactWithOrigin{ + Artifact: protoPluginArtifact{ + Artifact: input.GetArtifact(), + }, + Origin: input.GetOrigin(), + }) + } + var runRes *pluginv1.RunResponse var runErr error err = execWrapper(ctx, InteractiveExecOptions{ @@ -1288,15 +1315,18 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked }() runRes, runErr = hpanic.RecoverV(func() (*pluginv1.RunResponse, error) { - return driver.Run(ctx, pluginv1.RunRequest_builder{ - RequestId: htypes.Ptr(rs.ID), - Target: def.TargetDef.TargetDef, - SandboxPath: htypes.Ptr(sandboxfs.Path()), - TreeRootPath: htypes.Ptr(e.Root.Path()), - Inputs: inputs, - Pipes: pipes, - Hashin: htypes.Ptr(hashin), - }.Build()) + return driver.Run(ctx, &pluginsdk.RunRequest{ + RunRequest: pluginv1.RunRequest_builder{ + RequestId: htypes.Ptr(rs.ID), + Target: def.TargetDef.TargetDef, + SandboxPath: htypes.Ptr(sandboxfs.Path()), + TreeRootPath: htypes.Ptr(e.Root.Path()), + Inputs: inputs, + Pipes: pipes, + Hashin: htypes.Ptr(hashin), + }.Build(), + Inputs: inputsSdk, + }) }) }, Pty: def.GetPty(), @@ -1328,7 +1358,7 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked } cachefs := hfs.At(e.Cache, def.GetRef().GetPackage(), e.targetDirName(def.GetRef()), hashin) - execArtifacts := make([]ExecuteResultArtifact, 0, len(def.OutputNames())) + execArtifacts := make([]*ResultArtifact, 0, len(def.OutputNames())) for _, output := range def.GetOutputs() { shouldCollect := false @@ -1394,19 +1424,21 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked return nil, err } - execArtifact := pluginv1.Artifact_builder{ - Group: htypes.Ptr(output.GetGroup()), - Name: htypes.Ptr(tarname), - Type: htypes.Ptr(pluginv1.Artifact_TYPE_OUTPUT), - TarPath: proto.String(tarf.Name()), - }.Build() + execArtifact := protoPluginArtifact{ + Artifact: pluginv1.Artifact_builder{ + Group: htypes.Ptr(output.GetGroup()), + Name: htypes.Ptr(tarname), + Type: htypes.Ptr(pluginv1.Artifact_TYPE_OUTPUT), + TarPath: proto.String(tarf.Name()), + }.Build(), + } hashout, err := e.hashout(ctx, def.GetRef(), execArtifact) if err != nil { return nil, err } - execArtifacts = append(execArtifacts, ExecuteResultArtifact{ + execArtifacts = append(execArtifacts, &ResultArtifact{ Hashout: hashout, Artifact: execArtifact, }) @@ -1468,35 +1500,41 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked return nil, err } - execArtifact := pluginv1.Artifact_builder{ - Name: htypes.Ptr(tarname), - Type: htypes.Ptr(pluginv1.Artifact_TYPE_SUPPORT_FILE), - TarPath: proto.String(tarf.Name()), - }.Build() + execArtifact := protoPluginArtifact{ + Artifact: pluginv1.Artifact_builder{ + Name: htypes.Ptr(tarname), + Type: htypes.Ptr(pluginv1.Artifact_TYPE_SUPPORT_FILE), + TarPath: proto.String(tarf.Name()), + }.Build(), + } hashout, err := e.hashout(ctx, def.GetRef(), execArtifact) if err != nil { return nil, err } - execArtifacts = append(execArtifacts, ExecuteResultArtifact{ + execArtifacts = append(execArtifacts, &ResultArtifact{ Hashout: hashout, Artifact: execArtifact, }) } } - for _, artifact := range runRes.GetArtifacts() { - if artifact.GetType() != pluginv1.Artifact_TYPE_OUTPUT { + for _, partifact := range runRes.GetArtifacts() { + if partifact.GetType() != pluginv1.Artifact_TYPE_OUTPUT { continue } + artifact := protoPluginArtifact{ + Artifact: partifact, + } + hashout, err := e.hashout(ctx, def.GetRef(), artifact) if err != nil { return nil, fmt.Errorf("hashout: %w", err) } - execArtifacts = append(execArtifacts, ExecuteResultArtifact{ + execArtifacts = append(execArtifacts, &ResultArtifact{ Hashout: hashout, Artifact: artifact, }) @@ -1534,7 +1572,7 @@ func (e *Engine) ExecuteAndCache(ctx context.Context, rs *RequestState, def *Lig return nil, fmt.Errorf("execute: %w", err) } - var cachedArtifacts []ExecuteResultArtifact + var cachedArtifacts []*ResultArtifact if res.Executed { artifacts, manifest, err := e.CacheLocally(ctx, def, res.Hashin, res.Artifacts) if err != nil { diff --git a/internal/enginee2e/deps_cache_test.go b/internal/enginee2e/deps_cache_test.go index 05da0345..15b094bb 100644 --- a/internal/enginee2e/deps_cache_test.go +++ b/internal/enginee2e/deps_cache_test.go @@ -170,7 +170,7 @@ func TestDepsCache2(t *testing.T) { driver.EXPECT(). Run(gomock.Any(), gomock.Any()). - DoAndReturn(func(ctx context.Context, request *pluginv1.RunRequest) (*pluginv1.RunResponse, error) { + DoAndReturn(func(ctx context.Context, request *pluginsdk.RunRequest) (*pluginv1.RunResponse, error) { return pluginv1.RunResponse_builder{ Artifacts: []*pluginv1.Artifact{ pluginv1.Artifact_builder{ @@ -193,7 +193,7 @@ func TestDepsCache2(t *testing.T) { cache.EXPECT(). Get(gomock.Any(), "__child/e5d50f4b478a3687/manifest.v1.json"). - Return(nil, pluginsdk.ErrNotFound).Times(1) + Return(nil, 0, pluginsdk.ErrNotFound).Times(1) for _, key := range []string{"__child/e5d50f4b478a3687/manifest.v1.json", "__child/e5d50f4b478a3687/out_out.tar"} { cache.EXPECT(). @@ -206,7 +206,7 @@ func TestDepsCache2(t *testing.T) { cache.EXPECT(). Get(gomock.Any(), key). - Return(io.NopCloser(bytes.NewReader(b)), nil).Times(1) + Return(io.NopCloser(bytes.NewReader(b)), int64(len(b)), nil).AnyTimes() return nil }).Times(1) diff --git a/internal/enginee2e/pluginscyclicprovider/provider.go b/internal/enginee2e/pluginscyclicprovider/provider.go index 46b996e7..b3e2e4da 100644 --- a/internal/enginee2e/pluginscyclicprovider/provider.go +++ b/internal/enginee2e/pluginscyclicprovider/provider.go @@ -82,7 +82,7 @@ func (p *Provider) List(ctx context.Context, req *pluginv1.ListRequest) (plugins } if p.resultOnList { - _, err := p.resultClient.ResultClient.Get(ctx, corev1.ResultRequest_builder{ + res, err := p.resultClient.ResultClient.Get(ctx, corev1.ResultRequest_builder{ RequestId: htypes.Ptr(req.GetRequestId()), Spec: pluginv1.TargetSpec_builder{ Ref: tref.New(hephPackage, "think", nil), @@ -100,6 +100,8 @@ func (p *Provider) List(ctx context.Context, req *pluginv1.ListRequest) (plugins if err != nil { return err } + + defer res.Release() } for _, spec := range p.targets() { @@ -125,7 +127,7 @@ func (p *Provider) Get(ctx context.Context, req *pluginv1.GetRequest) (*pluginv1 } if p.resultOnGet { - _, err := p.resultClient.ResultClient.Get(ctx, corev1.ResultRequest_builder{ + res, err := p.resultClient.ResultClient.Get(ctx, corev1.ResultRequest_builder{ RequestId: htypes.Ptr(req.GetRequestId()), Spec: pluginv1.TargetSpec_builder{ Ref: tref.New(hephPackage, "think", nil), @@ -143,6 +145,8 @@ func (p *Provider) Get(ctx context.Context, req *pluginv1.GetRequest) (*pluginv1 if err != nil { return nil, err } + + defer res.Release() } for _, spec := range p.targets() { diff --git a/internal/enginee2e/pluginsmartprovidertest/provider.go b/internal/enginee2e/pluginsmartprovidertest/provider.go index 9371aee6..b3875ce8 100644 --- a/internal/enginee2e/pluginsmartprovidertest/provider.go +++ b/internal/enginee2e/pluginsmartprovidertest/provider.go @@ -63,8 +63,9 @@ func (p *Provider) Get(ctx context.Context, req *pluginv1.GetRequest) (*pluginv1 if err != nil { return nil, err } + defer res.Release() - artifacts := hartifact.FindOutputs(res.GetArtifacts(), "") + artifacts := hartifact.FindOutputs(res.Artifacts, "") r, err := hartifact.FileReader(ctx, artifacts[0]) if err != nil { diff --git a/internal/enginee2e/sanity_remotecache_test.go b/internal/enginee2e/sanity_remotecache_test.go index 1df3cdeb..3b8e2a9b 100644 --- a/internal/enginee2e/sanity_remotecache_test.go +++ b/internal/enginee2e/sanity_remotecache_test.go @@ -49,13 +49,13 @@ func (m *mockCache) Store(ctx context.Context, key string, r io.Reader) error { return nil } -func (m *mockCache) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (m *mockCache) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) { b, ok := m.store[key] if !ok { - return nil, pluginsdk.ErrCacheNotFound + return nil, 0, pluginsdk.ErrCacheNotFound } - return io.NopCloser(bytes.NewReader(b)), nil + return io.NopCloser(bytes.NewReader(b)), int64(len(b)), nil } func TestSanityRemoteCache(t *testing.T) { diff --git a/internal/hartifact/manifest.go b/internal/hartifact/manifest.go index a9929d1f..c1ed0c77 100644 --- a/internal/hartifact/manifest.go +++ b/internal/hartifact/manifest.go @@ -9,8 +9,7 @@ import ( "time" "github.com/hephbuild/heph/internal/htypes" - - "github.com/hephbuild/heph/internal/hfs" + "github.com/hephbuild/heph/lib/pluginsdk" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" ) @@ -58,6 +57,7 @@ type ManifestArtifact struct { Group string Name string + Size int64 Type ManifestArtifactType ContentType ManifestArtifactContentType OutPath string `json:",omitempty"` // set for file & raw @@ -85,18 +85,13 @@ func (m Manifest) GetArtifacts(output string) []ManifestArtifact { return a } -func WriteManifest(node hfs.Node, m Manifest) (*pluginv1.Artifact, error) { - b, err := json.Marshal(m) //nolint:musttag - if err != nil { - return nil, err - } - - err = hfs.WriteFile(node.At(ManifestName), b) +func EncodeManifest(w io.Writer, m Manifest) error { + err := json.NewEncoder(w).Encode(m) //nolint:musttag if err != nil { - return nil, err + return err } - return newManifestArtifact(node), nil + return nil } func NewManifestArtifact(m Manifest) (*pluginv1.Artifact, error) { @@ -115,18 +110,7 @@ func NewManifestArtifact(m Manifest) (*pluginv1.Artifact, error) { }.Build(), nil } -func newManifestArtifact(node hfs.Node) *pluginv1.Artifact { - return pluginv1.Artifact_builder{ - Name: htypes.Ptr(ManifestName), - Type: htypes.Ptr(pluginv1.Artifact_TYPE_MANIFEST_V1), - File: pluginv1.Artifact_ContentFile_builder{ - SourcePath: htypes.Ptr(node.At(ManifestName).Path()), - OutPath: htypes.Ptr(ManifestName), - }.Build(), - }.Build() -} - -func ManifestFromArtifact(ctx context.Context, a *pluginv1.Artifact) (Manifest, error) { +func ManifestFromArtifact(ctx context.Context, a pluginsdk.Artifact) (Manifest, error) { r, err := FileReader(ctx, a) if err != nil { return Manifest{}, err @@ -136,21 +120,6 @@ func ManifestFromArtifact(ctx context.Context, a *pluginv1.Artifact) (Manifest, return DecodeManifest(r) } -func ManifestFromFS(node hfs.Node) (Manifest, *pluginv1.Artifact, error) { - f, err := hfs.Open(node.At(ManifestName)) - if err != nil { - return Manifest{}, nil, err - } - defer f.Close() - - m, err := DecodeManifest(f) - if err != nil { - return Manifest{}, nil, err - } - - return m, newManifestArtifact(node), nil -} - func DecodeManifest(r io.Reader) (Manifest, error) { var manifest Manifest err := json.NewDecoder(r).Decode(&manifest) //nolint:musttag @@ -161,8 +130,10 @@ func DecodeManifest(r io.Reader) (Manifest, error) { return manifest, nil } -func ManifestContentType(a *pluginv1.Artifact) (ManifestArtifactContentType, error) { - switch a.WhichContent() { +func ManifestContentType(a pluginsdk.Artifact) (ManifestArtifactContentType, error) { + ap := a.GetProto() + + switch ap.WhichContent() { case pluginv1.Artifact_TargzPath_case: return ManifestArtifactContentTypeTarGz, nil case pluginv1.Artifact_TarPath_case: @@ -174,31 +145,39 @@ func ManifestContentType(a *pluginv1.Artifact) (ManifestArtifactContentType, err default: } - return "", fmt.Errorf("unsupported content %v", a.WhichContent().String()) + return "", fmt.Errorf("unsupported content %v", ap.WhichContent().String()) } -func ProtoArtifactToManifest(hashout string, artifact *pluginv1.Artifact) (ManifestArtifact, error) { - contentType, err := ManifestContentType(artifact) +func ProtoArtifactToManifest(hashout string, a pluginsdk.Artifact) (ManifestArtifact, error) { + contentType, err := ManifestContentType(a) if err != nil { return ManifestArtifact{}, err } + ap := a.GetProto() + var outPath string var x bool - switch artifact.WhichContent() { //nolint:exhaustive + switch ap.WhichContent() { //nolint:exhaustive case pluginv1.Artifact_File_case: - outPath = artifact.GetFile().GetOutPath() - x = artifact.GetFile().GetX() + outPath = ap.GetFile().GetOutPath() + x = ap.GetFile().GetX() case pluginv1.Artifact_Raw_case: - outPath = artifact.GetRaw().GetPath() - x = artifact.GetRaw().GetX() + outPath = ap.GetRaw().GetPath() + x = ap.GetRaw().GetX() + } + + size, err := a.GetContentSize() + if err != nil { + return ManifestArtifact{}, err } return ManifestArtifact{ Hashout: hashout, - Group: artifact.GetGroup(), - Name: artifact.GetName(), - Type: ManifestArtifactType(artifact.GetType()), + Group: ap.GetGroup(), + Name: ap.GetName(), + Size: size, + Type: ManifestArtifactType(ap.GetType()), ContentType: contentType, OutPath: outPath, X: x, diff --git a/internal/hartifact/reader.go b/internal/hartifact/reader.go index 21add7c1..3f20b781 100644 --- a/internal/hartifact/reader.go +++ b/internal/hartifact/reader.go @@ -11,35 +11,66 @@ import ( "github.com/hephbuild/heph/internal/hio" "github.com/hephbuild/heph/internal/htar" + "github.com/hephbuild/heph/lib/pluginsdk" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" ) // Reader gives a raw io.Reader of an artifact, useful for things like hashing. -func Reader(ctx context.Context, a *pluginv1.Artifact) (io.ReadCloser, error) { - switch a.WhichContent() { +func Reader(a pluginsdk.Artifact) (io.ReadCloser, error) { + ap := a.GetProto() + + switch ap.WhichContent() { case pluginv1.Artifact_File_case: - return os.Open(a.GetFile().GetSourcePath()) + return os.Open(ap.GetFile().GetSourcePath()) case pluginv1.Artifact_Raw_case: - return io.NopCloser(bytes.NewReader(a.GetRaw().GetData())), nil + return io.NopCloser(bytes.NewReader(ap.GetRaw().GetData())), nil case pluginv1.Artifact_TargzPath_case: - return os.Open(a.GetTargzPath()) + return os.Open(ap.GetTargzPath()) case pluginv1.Artifact_TarPath_case: - return os.Open(a.GetTarPath()) + return os.Open(ap.GetTarPath()) default: - return nil, fmt.Errorf("unsupported encoding %v", a.WhichContent()) + return nil, fmt.Errorf("unsupported encoding %v", ap.WhichContent()) + } +} + +func fileSize(p string) (int64, error) { + info, err := os.Stat(p) + if err != nil { + return 0, err + } + + return info.Size(), nil +} + +func Size(a pluginsdk.Artifact) (int64, error) { + ap := a.GetProto() + + switch ap.WhichContent() { + case pluginv1.Artifact_File_case: + return fileSize(ap.GetFile().GetSourcePath()) + case pluginv1.Artifact_Raw_case: + return int64(len(ap.GetRaw().GetData())), nil + case pluginv1.Artifact_TargzPath_case: + return fileSize(ap.GetTargzPath()) + case pluginv1.Artifact_TarPath_case: + return fileSize(ap.GetTarPath()) + default: + return -1, fmt.Errorf("unsupported encoding %v", ap.WhichContent()) } } // FileReader Assumes the output has a single file, and provides a reader for it (no matter the packaging). -func FileReader(ctx context.Context, a *pluginv1.Artifact) (io.ReadCloser, error) { - switch a.WhichContent() { +func FileReader(ctx context.Context, a pluginsdk.Artifact) (io.ReadCloser, error) { + ap := a.GetProto() + + switch ap.WhichContent() { case pluginv1.Artifact_File_case: - return os.Open(a.GetFile().GetSourcePath()) + return os.Open(ap.GetFile().GetSourcePath()) case pluginv1.Artifact_Raw_case: - return io.NopCloser(bytes.NewReader(a.GetRaw().GetData())), nil + return io.NopCloser(bytes.NewReader(ap.GetRaw().GetData())), nil case pluginv1.Artifact_TarPath_case: - r, err := Reader(ctx, a) + r, err := Reader(a) if err != nil { return nil, err } @@ -52,14 +83,14 @@ func FileReader(ctx context.Context, a *pluginv1.Artifact) (io.ReadCloser, error } return hio.NewReadCloser(tr, r), nil - // case *pluginv1.Artifact_TargzPath: + // case pluginsdk.Artifact_TargzPath: default: - return nil, fmt.Errorf("unsupported encoding %v", a.WhichContent()) + return nil, fmt.Errorf("unsupported encoding %v", ap.WhichContent()) } } // FileReader Assumes the output has a single file, and provides the bytes for it (no matter the packaging). -func FileReadAll(ctx context.Context, a *pluginv1.Artifact) ([]byte, error) { +func FileReadAll(ctx context.Context, a pluginsdk.Artifact) ([]byte, error) { f, err := FileReader(ctx, a) if err != nil { return nil, err @@ -75,11 +106,13 @@ type File struct { } // FilesReader provides a reader for each file it (no matter the packaging). -func FilesReader(ctx context.Context, a *pluginv1.Artifact) iter.Seq2[*File, error] { +func FilesReader(ctx context.Context, a pluginsdk.Artifact) iter.Seq2[*File, error] { + ap := a.GetProto() + return func(yield func(*File, error) bool) { - switch a.WhichContent() { + switch ap.WhichContent() { case pluginv1.Artifact_File_case: - f, err := os.Open(a.GetFile().GetSourcePath()) + f, err := os.Open(ap.GetFile().GetSourcePath()) if err != nil { if !yield(nil, err) { return @@ -89,21 +122,21 @@ func FilesReader(ctx context.Context, a *pluginv1.Artifact) iter.Seq2[*File, err if !yield(&File{ ReadCloser: f, - Path: a.GetFile().GetOutPath(), + Path: ap.GetFile().GetOutPath(), }, nil) { return } case pluginv1.Artifact_Raw_case: - f := io.NopCloser(bytes.NewReader(a.GetRaw().GetData())) + f := io.NopCloser(bytes.NewReader(ap.GetRaw().GetData())) if !yield(&File{ ReadCloser: f, - Path: a.GetRaw().GetPath(), + Path: ap.GetRaw().GetPath(), }, nil) { return } case pluginv1.Artifact_TarPath_case: - r, err := Reader(ctx, a) + r, err := Reader(a) if err != nil { if !yield(nil, err) { return @@ -134,9 +167,9 @@ func FilesReader(ctx context.Context, a *pluginv1.Artifact) iter.Seq2[*File, err } return } - // case *pluginv1.Artifact_TargzPath: + // case pluginsdk.Artifact_TargzPath: default: - if !yield(nil, fmt.Errorf("unsupported encoding %v", a.WhichContent())) { + if !yield(nil, fmt.Errorf("unsupported encoding %v", ap.WhichContent())) { return } } diff --git a/internal/hartifact/unpack.go b/internal/hartifact/unpack.go index 7e3cc913..e8a85133 100644 --- a/internal/hartifact/unpack.go +++ b/internal/hartifact/unpack.go @@ -6,6 +6,7 @@ import ( "io" "github.com/hephbuild/heph/internal/htar" + "github.com/hephbuild/heph/lib/pluginsdk" "github.com/hephbuild/heph/internal/hfs" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" @@ -30,7 +31,7 @@ func WithFilter(filter func(from string) bool) UnpackOption { } } -func Unpack(ctx context.Context, artifact *pluginv1.Artifact, node hfs.Node, options ...UnpackOption) error { +func Unpack(ctx context.Context, artifact pluginsdk.Artifact, node hfs.Node, options ...UnpackOption) error { var cfg unpackConfig for _, option := range options { option(&cfg) @@ -44,24 +45,26 @@ func Unpack(ctx context.Context, artifact *pluginv1.Artifact, node hfs.Node, opt } } - r, err := Reader(ctx, artifact) + r, err := Reader(artifact) if err != nil { return err } defer r.Close() - switch artifact.WhichContent() { + artifactp := artifact.GetProto() + + switch artifactp.WhichContent() { case pluginv1.Artifact_File_case: - if !cfg.filter(artifact.GetFile().GetOutPath()) { + if !cfg.filter(artifactp.GetFile().GetOutPath()) { return nil } create := hfs.Create - if artifact.GetFile().GetX() { + if artifactp.GetFile().GetX() { create = hfs.CreateExec } - f, err := create(node.At(artifact.GetFile().GetOutPath())) + f, err := create(node.At(artifactp.GetFile().GetOutPath())) if err != nil { return fmt.Errorf("file: create: %w", err) } @@ -78,16 +81,16 @@ func Unpack(ctx context.Context, artifact *pluginv1.Artifact, node hfs.Node, opt return err } case pluginv1.Artifact_Raw_case: - if !cfg.filter(artifact.GetRaw().GetPath()) { + if !cfg.filter(artifactp.GetRaw().GetPath()) { return nil } create := hfs.Create - if artifact.GetRaw().GetX() { + if artifactp.GetRaw().GetX() { create = hfs.CreateExec } - f, err := create(node.At(artifact.GetRaw().GetPath())) + f, err := create(node.At(artifactp.GetRaw().GetPath())) if err != nil { return fmt.Errorf("raw: create: %w", err) } @@ -110,7 +113,7 @@ func Unpack(ctx context.Context, artifact *pluginv1.Artifact, node hfs.Node, opt } // case *pluginv1.Artifact_TargzPath: default: - return fmt.Errorf("unsupported encoding %v", artifact.WhichContent()) + return fmt.Errorf("unsupported encoding %v", artifactp.WhichContent()) } return nil diff --git a/internal/hartifact/well_known.go b/internal/hartifact/well_known.go index cd4ca386..d5870897 100644 --- a/internal/hartifact/well_known.go +++ b/internal/hartifact/well_known.go @@ -1,18 +1,12 @@ package hartifact import ( - "iter" - "slices" - + "github.com/hephbuild/heph/lib/pluginsdk" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" ) -func FindOutputs(artifacts []*pluginv1.Artifact, group string) []*pluginv1.Artifact { - return findOutputs(slices.All(artifacts), group) -} - -func findOutputs(artifacts iter.Seq2[int, *pluginv1.Artifact], group string) []*pluginv1.Artifact { - out := make([]*pluginv1.Artifact, 0, 1) +func FindOutputs(artifacts []pluginsdk.Artifact, group string) []pluginsdk.Artifact { + out := make([]pluginsdk.Artifact, 0, 1) for _, artifact := range artifacts { if artifact.GetType() != pluginv1.Artifact_TYPE_OUTPUT { continue diff --git a/internal/remotecache/exec.go b/internal/remotecache/exec.go index c4b21215..180696cb 100644 --- a/internal/remotecache/exec.go +++ b/internal/remotecache/exec.go @@ -4,6 +4,7 @@ import ( "context" "errors" "io" + "math" "os/exec" "strings" "sync" @@ -102,7 +103,7 @@ func (e *execReader) Close() error { } // Get should return engine.ErrCacheNotFound if the key cannot be found, engine.ErrCacheNotFound can also be returned from Close(). -func (e Exec) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (e Exec) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) { cmd := exec.CommandContext(ctx, e.args[0], e.args[1:]...) //nolint:gosec cmd.Env = append(cmd.Environ(), "CACHE_KEY="+key) @@ -110,5 +111,5 @@ func (e Exec) Get(ctx context.Context, key string) (io.ReadCloser, error) { runCh: make(chan struct{}), notFoundSentinels: e.notFoundSentinels, cmd: cmd, - }, nil + }, math.MaxInt64, nil } diff --git a/internal/remotecache/gcs.go b/internal/remotecache/gcs.go index d832bb8f..690649c5 100644 --- a/internal/remotecache/gcs.go +++ b/internal/remotecache/gcs.go @@ -57,17 +57,26 @@ func (g GCS) Store(ctx context.Context, key string, r io.Reader) error { return w.Close() } -func (g GCS) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (g GCS) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) { obj := g.bucket.Object(key) + attr, err := obj.Attrs(ctx) + if err != nil { + if errors.Is(err, storage.ErrObjectNotExist) { + return nil, 0, pluginsdk.ErrCacheNotFound + } + + return nil, 0, err + } + r, err := obj.NewReader(ctx) if err != nil { if errors.Is(err, storage.ErrObjectNotExist) { - return nil, pluginsdk.ErrCacheNotFound + return nil, 0, pluginsdk.ErrCacheNotFound } - return nil, err + return nil, 0, err } - return r, nil + return r, attr.Size, nil } diff --git a/lib/pluginsdk/artifact.go b/lib/pluginsdk/artifact.go new file mode 100644 index 00000000..98abcd10 --- /dev/null +++ b/lib/pluginsdk/artifact.go @@ -0,0 +1,49 @@ +package pluginsdk + +import ( + "io" + + pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" +) + +type Artifact interface { + GetGroup() string + GetName() string + GetType() pluginv1.Artifact_Type + GetContentReader() (io.ReadCloser, error) + GetContentSize() (int64, error) + + GetProto() *pluginv1.Artifact +} + +var _ Artifact = (*ProtoArtifact)(nil) + +type ProtoArtifact struct { + *pluginv1.Artifact + ContentReaderFunc func(e ProtoArtifact) (io.ReadCloser, error) + ContentSizeFunc func(e ProtoArtifact) (int64, error) +} + +func (e ProtoArtifact) GetProto() *pluginv1.Artifact { + return e.Artifact +} + +func (e ProtoArtifact) GetContentReader() (io.ReadCloser, error) { + return e.ContentReaderFunc(e) +} +func (e ProtoArtifact) GetContentSize() (int64, error) { + return e.ContentSizeFunc(e) +} + +type ArtifactWithOrigin struct { + Artifact + Origin *pluginv1.TargetDef_InputOrigin +} + +func (a ArtifactWithOrigin) GetArtifact() Artifact { + return a.Artifact +} + +func (a ArtifactWithOrigin) GetOrigin() *pluginv1.TargetDef_InputOrigin { + return a.Origin +} diff --git a/lib/pluginsdk/init.go b/lib/pluginsdk/init.go index f7b525e9..79f9d437 100644 --- a/lib/pluginsdk/init.go +++ b/lib/pluginsdk/init.go @@ -5,10 +5,30 @@ import ( corev1 "github.com/hephbuild/heph/plugin/gen/heph/core/v1" "github.com/hephbuild/heph/plugin/gen/heph/core/v1/corev1connect" + pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" ) -type Resulter interface { - Get(context.Context, *corev1.ResultRequest) (*corev1.ResultResponse, error) +type GetResult struct { + Release func() + Artifacts []Artifact + Def *pluginv1.TargetDef +} + +type EngineResulter interface { + Get(context.Context, *corev1.ResultRequest) (*GetResult, error) +} + +type Resulter struct { + Engine EngineResulter +} + +func (r *Resulter) Get(ctx context.Context, req *corev1.ResultRequest) (*GetResult, error) { + res, err := r.Engine.Get(ctx, req) + if err != nil { + return nil, err + } + + return res, nil } type InitPayload struct { diff --git a/lib/pluginsdk/plugin_cache.go b/lib/pluginsdk/plugin_cache.go index 6b392567..b20ff1fa 100644 --- a/lib/pluginsdk/plugin_cache.go +++ b/lib/pluginsdk/plugin_cache.go @@ -10,7 +10,7 @@ var ErrCacheNotFound = errors.New("not found") type Cache interface { Store(ctx context.Context, key string, r io.Reader) error - Get(ctx context.Context, key string) (io.ReadCloser, error) + Get(ctx context.Context, key string) (io.ReadCloser, int64, error) } type CacheHas interface { diff --git a/lib/pluginsdk/plugin_driver.go b/lib/pluginsdk/plugin_driver.go index b198d955..33d99fa3 100644 --- a/lib/pluginsdk/plugin_driver.go +++ b/lib/pluginsdk/plugin_driver.go @@ -6,10 +6,20 @@ import ( pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" ) +type RunRequest struct { + *pluginv1.RunRequest + + Inputs []*ArtifactWithOrigin +} + +func (r *RunRequest) GetInputs() []*ArtifactWithOrigin { + return r.Inputs +} + type Driver interface { Config(context.Context, *pluginv1.ConfigRequest) (*pluginv1.ConfigResponse, error) Parse(context.Context, *pluginv1.ParseRequest) (*pluginv1.ParseResponse, error) ApplyTransitive(context.Context, *pluginv1.ApplyTransitiveRequest) (*pluginv1.ApplyTransitiveResponse, error) - Run(context.Context, *pluginv1.RunRequest) (*pluginv1.RunResponse, error) + Run(context.Context, *RunRequest) (*pluginv1.RunResponse, error) Pipe(context.Context, *pluginv1.PipeRequest) (*pluginv1.PipeResponse, error) } diff --git a/lib/pluginsdk/pluginsdkconnect/plugin_driver.go b/lib/pluginsdk/pluginsdkconnect/plugin_driver.go index dabe9d5a..fd96f22c 100644 --- a/lib/pluginsdk/pluginsdkconnect/plugin_driver.go +++ b/lib/pluginsdk/pluginsdkconnect/plugin_driver.go @@ -3,8 +3,10 @@ package pluginsdkconnect import ( "context" "errors" + "io" "connectrpc.com/connect" + "github.com/hephbuild/heph/internal/hartifact" "github.com/hephbuild/heph/internal/hcore/hlog" "github.com/hephbuild/heph/lib/pluginsdk" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" @@ -49,7 +51,7 @@ func (p driverConnectClient) Parse(ctx context.Context, req *pluginv1.ParseReque return res.Msg, nil } -func (p driverConnectClient) Run(ctx context.Context, req *pluginv1.RunRequest) (*pluginv1.RunResponse, error) { +func (p driverConnectClient) Run(ctx context.Context, req *pluginsdk.RunRequest) (*pluginv1.RunResponse, error) { hardCtx, cancelHardCtx := context.WithCancel(context.WithoutCancel(ctx)) defer cancelHardCtx() @@ -69,7 +71,7 @@ func (p driverConnectClient) Run(ctx context.Context, req *pluginv1.RunRequest) } }() - err := strm.Send(pluginv1.RunContainer_builder{Start: proto.ValueOrDefault(req)}.Build()) + err := strm.Send(pluginv1.RunContainer_builder{Start: req.RunRequest}.Build()) if err != nil { return nil, p.handleErr(ctx, err) } @@ -137,11 +139,34 @@ func (p driverConnectHandler) Parse(ctx context.Context, req *connect.Request[pl return connect.NewResponse(res), nil } +func (p driverConnectHandler) runRequest(rr *pluginv1.RunRequest) *pluginsdk.RunRequest { + inputs := make([]*pluginsdk.ArtifactWithOrigin, 0, len(rr.GetInputs())) + for _, input := range rr.GetInputs() { + inputs = append(inputs, &pluginsdk.ArtifactWithOrigin{ + Artifact: pluginsdk.ProtoArtifact{ + Artifact: input.GetArtifact(), + ContentReaderFunc: func(e pluginsdk.ProtoArtifact) (io.ReadCloser, error) { + return hartifact.Reader(e) + }, + ContentSizeFunc: func(e pluginsdk.ProtoArtifact) (int64, error) { + return hartifact.Size(e) + }, + }, + Origin: input.GetOrigin(), + }) + } + + return &pluginsdk.RunRequest{ + RunRequest: rr, + Inputs: inputs, + } +} + func (p driverConnectHandler) Run(ctx context.Context, strm *connect.BidiStream[pluginv1.RunContainer, pluginv1.RunResponse]) error { ctx, cancel := context.WithCancel(ctx) defer cancel() - startCh := make(chan *pluginv1.RunRequest, 1) + startCh := make(chan *pluginsdk.RunRequest, 1) errCh := make(chan error, 1) go func() { @@ -156,7 +181,7 @@ func (p driverConnectHandler) Run(ctx context.Context, strm *connect.BidiStream[ switch msg.WhichMsg() { case pluginv1.RunContainer_Start_case: - startCh <- msg.GetStart() + startCh <- p.runRequest(msg.GetStart()) close(startCh) case pluginv1.RunContainer_Cancel_case: cancel() diff --git a/plugin/pluginbin/plugin.go b/plugin/pluginbin/plugin.go index 18ac5e10..92affa51 100644 --- a/plugin/pluginbin/plugin.go +++ b/plugin/pluginbin/plugin.go @@ -71,7 +71,7 @@ const wrapperScript = ` exec "$@" ` -func (p Plugin) Run(ctx context.Context, req *pluginv1.RunRequest) (*pluginv1.RunResponse, error) { +func (p Plugin) Run(ctx context.Context, req *pluginsdk.RunRequest) (*pluginv1.RunResponse, error) { t := &binv1.Target{} err := req.GetTarget().GetDef().UnmarshalTo(t) if err != nil { diff --git a/plugin/pluginexec/plugin_test.go b/plugin/pluginexec/plugin_test.go index 27836eef..aff7e238 100644 --- a/plugin/pluginexec/plugin_test.go +++ b/plugin/pluginexec/plugin_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/hephbuild/heph/internal/htypes" + "github.com/hephbuild/heph/lib/pluginsdk" "github.com/hephbuild/heph/internal/herrgroup" "github.com/hephbuild/heph/lib/hpipe" @@ -67,10 +68,12 @@ func TestSanity(t *testing.T) { } { - res, err := p.Run(ctx, pluginv1.RunRequest_builder{ - Target: def, - SandboxPath: htypes.Ptr(sandboxPath), - }.Build()) + res, err := p.Run(ctx, &pluginsdk.RunRequest{ + RunRequest: pluginv1.RunRequest_builder{ + Target: def, + SandboxPath: htypes.Ptr(sandboxPath), + }.Build(), + }) require.NoError(t, err) assert.Len(t, res.GetArtifacts(), 1) @@ -110,17 +113,19 @@ func TestPipeStdout(t *testing.T) { require.NoError(t, err) go func() { - _, err = p.Run(ctx, pluginv1.RunRequest_builder{ - Target: pluginv1.TargetDef_builder{ - Ref: pluginv1.TargetRef_builder{ - Package: htypes.Ptr("some/pkg"), - Name: htypes.Ptr("target"), + _, err = p.Run(ctx, &pluginsdk.RunRequest{ + RunRequest: pluginv1.RunRequest_builder{ + Target: pluginv1.TargetDef_builder{ + Ref: pluginv1.TargetRef_builder{ + Package: htypes.Ptr("some/pkg"), + Name: htypes.Ptr("target"), + }.Build(), + Def: def, }.Build(), - Def: def, + SandboxPath: htypes.Ptr(sandboxPath), + Pipes: []string{"", res.Msg.GetId(), ""}, }.Build(), - SandboxPath: htypes.Ptr(sandboxPath), - Pipes: []string{"", res.Msg.GetId(), ""}, - }.Build()) + }) if err != nil { panic(err) } @@ -182,17 +187,19 @@ func TestPipeStdin(t *testing.T) { require.NoError(t, err) go func() { - _, err = p.Run(ctx, pluginv1.RunRequest_builder{ - Target: pluginv1.TargetDef_builder{ - Ref: pluginv1.TargetRef_builder{ - Package: htypes.Ptr("some/pkg"), - Name: htypes.Ptr("target"), + _, err = p.Run(ctx, &pluginsdk.RunRequest{ + RunRequest: pluginv1.RunRequest_builder{ + Target: pluginv1.TargetDef_builder{ + Ref: pluginv1.TargetRef_builder{ + Package: htypes.Ptr("some/pkg"), + Name: htypes.Ptr("target"), + }.Build(), + Def: def, }.Build(), - Def: def, + SandboxPath: htypes.Ptr(sandboxPath), + Pipes: []string{pipeIn.Msg.GetId(), pipeOut.Msg.GetId(), ""}, }.Build(), - SandboxPath: htypes.Ptr(sandboxPath), - Pipes: []string{pipeIn.Msg.GetId(), pipeOut.Msg.GetId(), ""}, - }.Build()) + }) if err != nil { panic(err) } @@ -266,17 +273,19 @@ func TestPipeStdinLargeAndSlow(t *testing.T) { require.NoError(t, err) go func() { - _, err = p.Run(ctx, pluginv1.RunRequest_builder{ - Target: pluginv1.TargetDef_builder{ - Ref: pluginv1.TargetRef_builder{ - Package: htypes.Ptr("some/pkg"), - Name: htypes.Ptr("target"), + _, err = p.Run(ctx, &pluginsdk.RunRequest{ + RunRequest: pluginv1.RunRequest_builder{ + Target: pluginv1.TargetDef_builder{ + Ref: pluginv1.TargetRef_builder{ + Package: htypes.Ptr("some/pkg"), + Name: htypes.Ptr("target"), + }.Build(), + Def: def, }.Build(), - Def: def, + SandboxPath: htypes.Ptr(sandboxPath), + Pipes: []string{pipeIn.Msg.GetId(), pipeOut.Msg.GetId(), ""}, }.Build(), - SandboxPath: htypes.Ptr(sandboxPath), - Pipes: []string{pipeIn.Msg.GetId(), pipeOut.Msg.GetId(), ""}, - }.Build()) + }) if err != nil { panic(err) } @@ -336,17 +345,19 @@ func TestPipe404(t *testing.T) { return err }) - _, err = p.Run(ctx, pluginv1.RunRequest_builder{ - Target: pluginv1.TargetDef_builder{ - Ref: pluginv1.TargetRef_builder{ - Package: htypes.Ptr("some/pkg"), - Name: htypes.Ptr("target"), + _, err = p.Run(ctx, &pluginsdk.RunRequest{ + RunRequest: pluginv1.RunRequest_builder{ + Target: pluginv1.TargetDef_builder{ + Ref: pluginv1.TargetRef_builder{ + Package: htypes.Ptr("some/pkg"), + Name: htypes.Ptr("target"), + }.Build(), + Def: def, }.Build(), - Def: def, + SandboxPath: htypes.Ptr(sandboxPath), + Pipes: []string{"", res.Msg.GetId(), ""}, }.Build(), - SandboxPath: htypes.Ptr(sandboxPath), - Pipes: []string{"", res.Msg.GetId(), ""}, - }.Build()) + }) require.NoError(t, err) err = eg.Wait() diff --git a/plugin/pluginexec/run.go b/plugin/pluginexec/run.go index 49491805..9cb41baf 100644 --- a/plugin/pluginexec/run.go +++ b/plugin/pluginexec/run.go @@ -17,6 +17,7 @@ import ( "time" "github.com/hephbuild/heph/internal/htypes" + "github.com/hephbuild/heph/lib/pluginsdk" "github.com/hephbuild/heph/internal/hdebug" "github.com/hephbuild/heph/lib/tref" @@ -40,7 +41,7 @@ var tracer = otel.Tracer("heph/pluginexec") var sem = semaphore.NewWeighted(int64(runtime.GOMAXPROCS(-1))) -func (p *Plugin[S]) Run(ctx context.Context, req *pluginv1.RunRequest) (*pluginv1.RunResponse, error) { +func (p *Plugin[S]) Run(ctx context.Context, req *pluginsdk.RunRequest) (*pluginv1.RunResponse, error) { ctx, cleanLabels := hdebug.SetLabels(ctx, func() []string { return []string{ "where", fmt.Sprintf("hephpluginexec %v: %v", p.name, tref.Format(req.GetTarget().GetRef())), @@ -355,8 +356,8 @@ func getEnvEntryWithName(name, value string) string { return name + "=" + value } -func (p *Plugin[S]) inputEnv(ctx context.Context, inputs []*pluginv1.ArtifactWithOrigin, t *execv1.Target, listfs hfs.Node, basePath string) ([]string, error) { - m := map[string][]*pluginv1.ArtifactWithOrigin{} +func (p *Plugin[S]) inputEnv(ctx context.Context, inputs []*pluginsdk.ArtifactWithOrigin, t *execv1.Target, listfs hfs.Node, basePath string) ([]string, error) { + m := map[string][]*pluginsdk.ArtifactWithOrigin{} for _, dep := range t.GetDeps() { id := dep.GetId() @@ -402,7 +403,7 @@ func (p *Plugin[S]) inputEnv(ctx context.Context, inputs []*pluginv1.ArtifactWit seenFiles := map[string]struct{}{} envSb.Reset() - slices.SortFunc(artifacts, func(a, b *pluginv1.ArtifactWithOrigin) int { + slices.SortFunc(artifacts, func(a, b *pluginsdk.ArtifactWithOrigin) int { if v := strings.Compare(a.GetOrigin().GetId(), b.GetOrigin().GetId()); v != 0 { return v } @@ -415,7 +416,7 @@ func (p *Plugin[S]) inputEnv(ctx context.Context, inputs []*pluginv1.ArtifactWit return v } - if v := hproto.Compare(a, b, nil); v != 0 { + if v := hproto.Compare(a.GetProto(), b.GetProto(), nil); v != 0 { return v } diff --git a/plugin/pluginexec/sandbox.go b/plugin/pluginexec/sandbox.go index 69741f69..247d5461 100644 --- a/plugin/pluginexec/sandbox.go +++ b/plugin/pluginexec/sandbox.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "fmt" + "io" "iter" "os" "path/filepath" @@ -11,6 +12,7 @@ import ( "github.com/hephbuild/heph/internal/hproto/hashpb" "github.com/hephbuild/heph/internal/htypes" + "github.com/hephbuild/heph/lib/pluginsdk" "github.com/hephbuild/heph/lib/tref" @@ -24,14 +26,14 @@ import ( ) type SetupSandboxResult struct { - ListArtifacts []*pluginv1.ArtifactWithOrigin + ListArtifacts []*pluginsdk.ArtifactWithOrigin Sourcemap map[string]string } func SetupSandbox( ctx context.Context, t *execv1.Target, - results []*pluginv1.ArtifactWithOrigin, + results []*pluginsdk.ArtifactWithOrigin, workfs, binfs, cwdfs, @@ -53,7 +55,7 @@ func SetupSandbox( sourcemap := map[string]string{} - var listArtifacts []*pluginv1.ArtifactWithOrigin + var listArtifacts []*pluginsdk.ArtifactWithOrigin if setupWd { err = cwdfs.MkdirAll(os.ModePerm) if err != nil { @@ -62,58 +64,58 @@ func SetupSandbox( for _, target := range t.GetDeps() { for artifact := range ArtifactsForId(results, target.GetId(), pluginv1.Artifact_TYPE_OUTPUT) { - listArtifact, err := SetupSandboxArtifact(ctx, artifact.GetArtifact(), target, workfs, target.GetRef().GetFilters(), sourcemap) + listArtifact, err := SetupSandboxArtifact(ctx, artifact, target, workfs, target.GetRef().GetFilters(), sourcemap) if err != nil { return nil, fmt.Errorf("setup artifact: %v: %w", target.GetId(), err) } - listArtifacts = append(listArtifacts, pluginv1.ArtifactWithOrigin_builder{ + listArtifacts = append(listArtifacts, &pluginsdk.ArtifactWithOrigin{ Artifact: listArtifact, Origin: pluginv1.TargetDef_InputOrigin_builder{ Id: htypes.Ptr(target.GetId()), }.Build(), - }.Build()) + }) } for artifact := range ArtifactsForId(results, target.GetId(), pluginv1.Artifact_TYPE_SUPPORT_FILE) { - listArtifact, err := SetupSandboxArtifact(ctx, artifact.GetArtifact(), target, workfs, nil, nil) + listArtifact, err := SetupSandboxArtifact(ctx, artifact, target, workfs, nil, nil) if err != nil { return nil, fmt.Errorf("setup support file artifact: %v: %w", target.GetId(), err) } - listArtifacts = append(listArtifacts, pluginv1.ArtifactWithOrigin_builder{ + listArtifacts = append(listArtifacts, &pluginsdk.ArtifactWithOrigin{ Artifact: listArtifact, Origin: pluginv1.TargetDef_InputOrigin_builder{ Id: htypes.Ptr(target.GetId()), }.Build(), - }.Build()) + }) } } } for _, tool := range t.GetTools() { for artifact := range ArtifactsForId(results, tool.GetId(), pluginv1.Artifact_TYPE_OUTPUT) { - listArtifact, err := SetupSandboxBinArtifact(ctx, artifact.GetArtifact(), binfs) + listArtifact, err := SetupSandboxBinArtifact(ctx, artifact, binfs) if err != nil { return nil, fmt.Errorf("%v: %w", tref.FormatOut(tool.GetRef()), err) } - listArtifacts = append(listArtifacts, pluginv1.ArtifactWithOrigin_builder{ + listArtifacts = append(listArtifacts, &pluginsdk.ArtifactWithOrigin{ Artifact: listArtifact, Origin: pluginv1.TargetDef_InputOrigin_builder{ Id: htypes.Ptr(tool.GetId()), }.Build(), - }.Build()) + }) } for artifact := range ArtifactsForId(results, tool.GetId(), pluginv1.Artifact_TYPE_SUPPORT_FILE) { - listArtifact, err := SetupSandboxArtifact(ctx, artifact.GetArtifact(), nil, workfs, nil, nil) + listArtifact, err := SetupSandboxArtifact(ctx, artifact, nil, workfs, nil, nil) if err != nil { return nil, fmt.Errorf("setup support file artifact: %v: %w", tref.FormatOut(tool.GetRef()), err) } - listArtifacts = append(listArtifacts, pluginv1.ArtifactWithOrigin_builder{ + listArtifacts = append(listArtifacts, &pluginsdk.ArtifactWithOrigin{ Artifact: listArtifact, Origin: pluginv1.TargetDef_InputOrigin_builder{ Id: htypes.Ptr(tool.GetId()), }.Build(), - }.Build()) + }) } } @@ -132,8 +134,8 @@ func SetupSandbox( }, nil } -func ArtifactsForId(inputs []*pluginv1.ArtifactWithOrigin, id string, typ pluginv1.Artifact_Type) iter.Seq[*pluginv1.ArtifactWithOrigin] { - return func(yield func(origin *pluginv1.ArtifactWithOrigin) bool) { +func ArtifactsForId(inputs []*pluginsdk.ArtifactWithOrigin, id string, typ pluginv1.Artifact_Type) iter.Seq[*pluginsdk.ArtifactWithOrigin] { + return func(yield func(origin *pluginsdk.ArtifactWithOrigin) bool) { for _, input := range inputs { if input.GetArtifact().GetType() != typ { continue @@ -150,12 +152,12 @@ func ArtifactsForId(inputs []*pluginv1.ArtifactWithOrigin, id string, typ plugin } } -func SetupSandboxArtifact(ctx context.Context, artifact *pluginv1.Artifact, source *execv1.Target_Dep, node hfs.Node, filters []string, sourcemap map[string]string) (*pluginv1.Artifact, error) { +func SetupSandboxArtifact(ctx context.Context, artifact pluginsdk.Artifact, source *execv1.Target_Dep, node hfs.Node, filters []string, sourcemap map[string]string) (pluginsdk.Artifact, error) { ctx, span := tracer.Start(ctx, "SetupSandboxArtifact") defer span.End() h := xxh3.New() - hashpb.Hash(h, artifact, tref.OmitHashPb) + hashpb.Hash(h, artifact.GetProto(), tref.OmitHashPb) for _, f := range filters { _, _ = h.WriteString(f) } @@ -205,17 +207,25 @@ func SetupSandboxArtifact(ctx context.Context, artifact *pluginv1.Artifact, sour listType = pluginv1.Artifact_TYPE_SUPPORT_FILE_LIST_V1 } - return pluginv1.Artifact_builder{ - Group: htypes.Ptr(artifact.GetGroup()), - Name: htypes.Ptr(artifact.GetName() + ".list"), - Type: htypes.Ptr(listType), - File: pluginv1.Artifact_ContentFile_builder{ - SourcePath: htypes.Ptr(listf.Name()), + return &pluginsdk.ProtoArtifact{ + Artifact: pluginv1.Artifact_builder{ + Group: htypes.Ptr(artifact.GetGroup()), + Name: htypes.Ptr(artifact.GetName() + ".list"), + Type: htypes.Ptr(listType), + File: pluginv1.Artifact_ContentFile_builder{ + SourcePath: htypes.Ptr(listf.Name()), + }.Build(), }.Build(), - }.Build(), nil + ContentReaderFunc: func(e pluginsdk.ProtoArtifact) (io.ReadCloser, error) { + return hartifact.Reader(e) + }, + ContentSizeFunc: func(e pluginsdk.ProtoArtifact) (int64, error) { + return hartifact.Size(e) + }, + }, nil } -func SetupSandboxBinArtifact(ctx context.Context, artifact *pluginv1.Artifact, node hfs.Node) (*pluginv1.Artifact, error) { +func SetupSandboxBinArtifact(ctx context.Context, artifact pluginsdk.Artifact, node hfs.Node) (pluginsdk.Artifact, error) { ctx, span := tracer.Start(ctx, "SetupSandboxBinArtifact") defer span.End() @@ -251,13 +261,21 @@ func SetupSandboxBinArtifact(ctx context.Context, artifact *pluginv1.Artifact, n return nil, err } - return pluginv1.Artifact_builder{ - Group: htypes.Ptr(artifact.GetGroup()), - Name: htypes.Ptr(artifact.GetName() + ".list"), - Type: htypes.Ptr(pluginv1.Artifact_TYPE_OUTPUT_LIST_V1), - Raw: pluginv1.Artifact_ContentRaw_builder{ - Data: []byte(dest.Path()), - Path: htypes.Ptr(artifact.GetName() + ".list"), + return &pluginsdk.ProtoArtifact{ + Artifact: pluginv1.Artifact_builder{ + Group: htypes.Ptr(artifact.GetGroup()), + Name: htypes.Ptr(artifact.GetName() + ".list"), + Type: htypes.Ptr(pluginv1.Artifact_TYPE_OUTPUT_LIST_V1), + Raw: pluginv1.Artifact_ContentRaw_builder{ + Data: []byte(dest.Path()), + Path: htypes.Ptr(artifact.GetName() + ".list"), + }.Build(), }.Build(), - }.Build(), nil + ContentReaderFunc: func(e pluginsdk.ProtoArtifact) (io.ReadCloser, error) { + return hartifact.Reader(e) + }, + ContentSizeFunc: func(e pluginsdk.ProtoArtifact) (int64, error) { + return hartifact.Size(e) + }, + }, nil } diff --git a/plugin/pluginfs/driver.go b/plugin/pluginfs/driver.go index 75ae4e62..0d02990b 100644 --- a/plugin/pluginfs/driver.go +++ b/plugin/pluginfs/driver.go @@ -103,7 +103,7 @@ func (p *Driver) Parse(ctx context.Context, req *pluginv1.ParseRequest) (*plugin }.Build(), nil } -func (p *Driver) Run(ctx context.Context, req *pluginv1.RunRequest) (*pluginv1.RunResponse, error) { +func (p *Driver) Run(ctx context.Context, req *pluginsdk.RunRequest) (*pluginv1.RunResponse, error) { t := &fsv1.Target{} err := req.GetTarget().GetDef().UnmarshalTo(t) if err != nil { diff --git a/plugin/plugingo/get_std.go b/plugin/plugingo/get_std.go index 7145f137..193448ee 100644 --- a/plugin/plugingo/get_std.go +++ b/plugin/plugingo/get_std.go @@ -36,8 +36,9 @@ func (p *Plugin) resultStdListInner(ctx context.Context, factors Factors, reques if err != nil { return nil, err } + defer res.Release() - outputs := hartifact.FindOutputs(res.GetArtifacts(), "list") + outputs := hartifact.FindOutputs(res.Artifacts, "list") if len(outputs) == 0 { return nil, errors.New("no install artifact found") diff --git a/plugin/plugingo/pkg_analysis.go b/plugin/plugingo/pkg_analysis.go index a207ac94..70898b3b 100644 --- a/plugin/plugingo/pkg_analysis.go +++ b/plugin/plugingo/pkg_analysis.go @@ -33,25 +33,19 @@ import ( var errNoGoFiles = errors.New("no Go files in package") -func (p *Plugin) goListPkg(ctx context.Context, pkg string, factors Factors, imp, requestId string) ([]*pluginv1.Artifact, *pluginv1.TargetRef, error) { +func (p *Plugin) goListPkgResult(ctx context.Context, basePkg, runPkg, imp string, factors Factors, requestId string) (Package, error) { res, err := p.resultClient.ResultClient.Get(ctx, corev1.ResultRequest_builder{ RequestId: htypes.Ptr(requestId), - Ref: tref.New(pkg, "_golist", hmaps.Concat(factors.Args(), map[string]string{ + Ref: tref.New(runPkg, "_golist", hmaps.Concat(factors.Args(), map[string]string{ "imp": imp, })), }.Build()) if err != nil { - return nil, nil, fmt.Errorf("golist: %v (in %v): %w", imp, pkg, err) + return Package{}, fmt.Errorf("golist: %v (in %v): %w", imp, runPkg, err) } + defer res.Release() - return res.GetArtifacts(), res.GetDef().GetRef(), nil -} - -func (p *Plugin) goListPkgResult(ctx context.Context, basePkg, runPkg, imp string, factors Factors, requestId string) (Package, error) { - artifacts, _, err := p.goListPkg(ctx, runPkg, factors, imp, requestId) - if err != nil { - return Package{}, fmt.Errorf("go list: %w", err) - } + artifacts := res.Artifacts jsonArtifacts := hartifact.FindOutputs(artifacts, "json") rootArtifacts := hartifact.FindOutputs(artifacts, "root") @@ -699,8 +693,10 @@ func (p *Plugin) goModules(ctx context.Context, pkg, requestId string) ([]Module return nil, fmt.Errorf("gomod: %w", err) } - jsonArtifacts := hartifact.FindOutputs(res.GetArtifacts(), "json") - rootArtifacts := hartifact.FindOutputs(res.GetArtifacts(), "root") + defer res.Release() + + jsonArtifacts := hartifact.FindOutputs(res.Artifacts, "json") + rootArtifacts := hartifact.FindOutputs(res.Artifacts, "root") if len(jsonArtifacts) == 0 || len(rootArtifacts) == 0 { return nil, connect.NewError(connect.CodeInternal, errors.New("gomodules: no output found")) diff --git a/plugin/plugingroup/plugin.go b/plugin/plugingroup/plugin.go index cec8eaac..0fff7455 100644 --- a/plugin/plugingroup/plugin.go +++ b/plugin/plugingroup/plugin.go @@ -75,7 +75,7 @@ func (p Plugin) Parse(ctx context.Context, req *pluginv1.ParseRequest) (*pluginv }.Build(), nil } -func (p Plugin) Run(ctx context.Context, request *pluginv1.RunRequest) (*pluginv1.RunResponse, error) { +func (p Plugin) Run(ctx context.Context, request *pluginsdk.RunRequest) (*pluginv1.RunResponse, error) { panic("group: run should not be called") // custom handling in engine } diff --git a/plugin/pluginnix/driver_test.go b/plugin/pluginnix/driver_test.go index d7d72a16..a57306f2 100644 --- a/plugin/pluginnix/driver_test.go +++ b/plugin/pluginnix/driver_test.go @@ -7,6 +7,7 @@ import ( "github.com/hephbuild/heph/internal/hproto/hstructpb" "github.com/hephbuild/heph/internal/htypes" + "github.com/hephbuild/heph/lib/pluginsdk" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" "github.com/stretchr/testify/assert" @@ -64,10 +65,12 @@ func TestSanity(t *testing.T) { } { - res, err := p.Run(ctx, pluginv1.RunRequest_builder{ - Target: def, - SandboxPath: htypes.Ptr(sandboxPath), - }.Build()) + res, err := p.Run(ctx, &pluginsdk.RunRequest{ + RunRequest: pluginv1.RunRequest_builder{ + Target: def, + SandboxPath: htypes.Ptr(sandboxPath), + }.Build(), + }) require.NoError(t, err) assert.Len(t, res.GetArtifacts(), 1) @@ -108,10 +111,12 @@ func TestToolSanity(t *testing.T) { } { - res, err := p.Run(ctx, pluginv1.RunRequest_builder{ - Target: def, - SandboxPath: htypes.Ptr(sandboxPath), - }.Build()) + res, err := p.Run(ctx, &pluginsdk.RunRequest{ + RunRequest: pluginv1.RunRequest_builder{ + Target: def, + SandboxPath: htypes.Ptr(sandboxPath), + }.Build(), + }) require.NoError(t, err) assert.Len(t, res.GetArtifacts(), 1) diff --git a/plugin/plugintextfile/plugin.go b/plugin/plugintextfile/plugin.go index 0482b7e2..6626f6fa 100644 --- a/plugin/plugintextfile/plugin.go +++ b/plugin/plugintextfile/plugin.go @@ -75,7 +75,7 @@ func (p Plugin) Parse(ctx context.Context, req *pluginv1.ParseRequest) (*pluginv }.Build(), nil } -func (p Plugin) Run(ctx context.Context, req *pluginv1.RunRequest) (*pluginv1.RunResponse, error) { +func (p Plugin) Run(ctx context.Context, req *pluginsdk.RunRequest) (*pluginv1.RunResponse, error) { t := &textfilev1.Target{} err := req.GetTarget().GetDef().UnmarshalTo(t) if err != nil { diff --git a/plugin/proto/heph/core/v1/result.proto b/plugin/proto/heph/core/v1/result.proto index 88af3304..bdb1bdda 100644 --- a/plugin/proto/heph/core/v1/result.proto +++ b/plugin/proto/heph/core/v1/result.proto @@ -18,10 +18,7 @@ message ResultRequest { } message ResultResponse { - repeated heph.plugin.v1.Artifact artifacts = 1; - heph.plugin.v1.TargetDef def = 2; -} - -service ResultService { - rpc Get(ResultRequest) returns (ResultResponse) {} + string id = 1; + repeated heph.plugin.v1.Artifact artifacts = 2; + heph.plugin.v1.TargetDef def = 3; } From 57eb0a2416853f69186546a5841960ffc4766b28 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Sat, 7 Mar 2026 13:55:58 +0000 Subject: [PATCH 02/15] checkpint --- internal/engine/local_cache.go | 175 +++++----- internal/engine/remote_cache.go | 163 ++++----- internal/engine/schedule.go | 316 ++++++++---------- internal/enginee2e/sanity_remotecache_test.go | 6 +- internal/hartifact/manifest.go | 68 ++-- internal/hartifact/reader.go | 36 +- internal/remotecache/exec.go | 5 +- internal/remotecache/gcs.go | 8 +- lib/pluginsdk/artifact.go | 20 ++ lib/pluginsdk/plugin_cache.go | 2 +- 10 files changed, 353 insertions(+), 446 deletions(-) diff --git a/internal/engine/local_cache.go b/internal/engine/local_cache.go index 7f38ac0d..6a31660e 100644 --- a/internal/engine/local_cache.go +++ b/internal/engine/local_cache.go @@ -5,7 +5,6 @@ import ( "bytes" "context" "encoding/hex" - "encoding/json" "errors" "fmt" "hash" @@ -15,7 +14,6 @@ import ( "slices" "strconv" "strings" - "sync" "time" "github.com/hephbuild/heph/internal/hcore/hstep" @@ -87,11 +85,11 @@ func (e *Engine) targetDirName(ref *pluginv1.TargetRef) string { return "__" + ref.GetName() + "_" + hex.EncodeToString(h.Sum(nil)) } -func (e *Engine) CacheLocally( +func (e *Engine) cacheLocally( ctx context.Context, def *LightLinkedTarget, hashin string, - sandboxArtifacts []*ResultArtifact, + sandboxArtifacts []*ExecuteArtifact, ) ([]*ResultArtifact, *hartifact.Manifest, error) { step, ctx := hstep.New(ctx, "Caching...") defer step.Done() @@ -99,22 +97,19 @@ func (e *Engine) CacheLocally( cacheArtifacts := make([]*ResultArtifact, 0, len(sandboxArtifacts)) for _, artifact := range sandboxArtifacts { - if artifact.GetType() == pluginv1.Artifact_TYPE_MANIFEST_V1 { - continue - } - src, err := e.contentReaderNormalizer(artifact) if err != nil { return nil, nil, err } defer src.Close() - cacheArtifact, err := e.cacheLocally(ctx, def.GetRef(), hashin, CacheLocallyArtifact{ - Reader: src, - Size: 0, - Type: artifact.GetType(), - Group: artifact.GetGroup(), - Name: artifact.GetName(), + cacheArtifact, err := e.cacheArtifactLocally(ctx, def.GetRef(), hashin, CacheLocallyArtifact{ + Reader: src, + Size: 0, + Type: artifact.GetType(), + Group: artifact.GetGroup(), + Name: artifact.GetName(), + ContentType: artifact.GetContentType(), }, artifact.Hashout) if err != nil { return nil, nil, err @@ -123,42 +118,62 @@ func (e *Engine) CacheLocally( cacheArtifacts = append(cacheArtifacts, cacheArtifact) } - m := hartifact.Manifest{ + manifest, err := e.createLocalCacheManifest(ctx, def.GetRef(), hashin, cacheArtifacts) + if err != nil { + return nil, nil, err + } + + return cacheArtifacts, manifest, nil +} + +func (e *Engine) createLocalCacheManifest(ctx context.Context, ref *pluginv1.TargetRef, hashin string, artifacts []*ResultArtifact) (*hartifact.Manifest, error) { + m := &hartifact.Manifest{ Version: "v1", - Target: tref.Format(def.GetRef()), + Target: tref.Format(ref), CreatedAt: time.Now(), Hashin: hashin, } - for _, artifact := range cacheArtifacts { + for _, artifact := range artifacts { martifact, err := hartifact.ProtoArtifactToManifest(artifact.Hashout, artifact.Artifact) if err != nil { - return nil, nil, err + return nil, err } m.Artifacts = append(m.Artifacts, martifact) } - cacheArtifacts = append(cacheArtifacts, &ResultArtifact{ - Artifact: &manifestPluginArtifact{manifest: m}, - }) + w, err := e.CacheSmall.Writer(ctx, ref, hashin, hartifact.ManifestName) + if err != nil { + return nil, err + } + defer w.Close() + + err = hartifact.EncodeManifest(w, m) + if err != nil { + return nil, err + } - return cacheArtifacts, &m, nil + err = w.Close() + if err != nil { + return nil, err + } + + return m, nil } type cachePluginArtifact struct { - group, name string - type_ pluginv1.Artifact_Type - cache LocalCache + artifact hartifact.ManifestArtifact + cache LocalCache } func (e cachePluginArtifact) GetGroup() string { - return e.group + return e.artifact.Group } func (e cachePluginArtifact) GetName() string { - return e.name + return e.artifact.Name } func (e cachePluginArtifact) GetType() pluginv1.Artifact_Type { - return e.type_ + return pluginv1.Artifact_Type(e.artifact.Type) } func (e cachePluginArtifact) GetProto() *pluginv1.Artifact { @@ -173,48 +188,19 @@ func (e cachePluginArtifact) GetContentSize() (int64, error) { return hartifact.Size(e) } -type manifestPluginArtifact struct { - manifest hartifact.Manifest - - manifestBytes []byte - manifestBytesOnce sync.Once -} - -func (e *manifestPluginArtifact) GetGroup() string { - return "" -} -func (e *manifestPluginArtifact) GetName() string { - return hartifact.ManifestName -} -func (e *manifestPluginArtifact) GetType() pluginv1.Artifact_Type { - return pluginv1.Artifact_TYPE_MANIFEST_V1 -} - -func (e *manifestPluginArtifact) GetProto() *pluginv1.Artifact { - panic("TO REMOVE") -} - -func (e *manifestPluginArtifact) marshal() { - e.manifestBytesOnce.Do(func() { - b, err := json.Marshal(e.manifest) - if err != nil { - panic(err) - } - - e.manifestBytes = b - }) -} - -func (e *manifestPluginArtifact) GetContentReader() (io.ReadCloser, error) { - e.marshal() - - return io.NopCloser(bytes.NewReader(e.manifestBytes)), nil -} - -func (e *manifestPluginArtifact) GetContentSize() (int64, error) { - e.marshal() +func (e cachePluginArtifact) GetContentType() (pluginsdk.ArtifactContentType, error) { + switch e.artifact.ContentType { + case hartifact.ManifestArtifactContentTypeTar: + return pluginsdk.ArtifactContentTypeTar, nil + case hartifact.ManifestArtifactContentTypeTarGz: + return pluginsdk.ArtifactContentTypeTarGz, nil + case hartifact.ManifestArtifactContentTypeFile: + return pluginsdk.ArtifactContentTypeFile, nil + case hartifact.ManifestArtifactContentTypeRaw: + return pluginsdk.ArtifactContentTypeRaw, nil + } - return int64(len(e.manifestBytes)), nil + return "", fmt.Errorf("invalid artifact content type: %q", e.artifact.ContentType) } func (e *Engine) contentReaderNormalizer(artifact pluginsdk.Artifact) (io.ReadCloser, error) { @@ -319,7 +305,7 @@ func keyRefOutputs(ref *pluginv1.TargetRef, outputs []string) string { return refKey(ref) + fmt.Sprintf("%#v", outputs) } -func (e *Engine) ResultFromLocalCache(ctx context.Context, def *LightLinkedTarget, outputs []string, hashin string) (*ExecuteResult, bool, error) { +func (e *Engine) ResultFromLocalCache(ctx context.Context, def *LightLinkedTarget, outputs []string, hashin string) (*Result, bool, error) { ctx, span := tracer.Start(ctx, "ResultFromLocalCache") defer span.End() @@ -335,7 +321,7 @@ func (e *Engine) ResultFromLocalCache(ctx context.Context, def *LightLinkedTarge } func (e *Engine) readAnyCache(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.ReadCloser, error) { - for _, c := range []LocalCache{e.CacheSmall, e.CacheLarge} { + for _, c := range [...]LocalCache{e.CacheSmall, e.CacheLarge} { r, err := c.Reader(ctx, ref, hashin, name) if err != nil { if errors.Is(err, LocalCacheNotFoundError) { @@ -366,7 +352,7 @@ func (e *Engine) existsAnyCache(ctx context.Context, ref *pluginv1.TargetRef, ha return false, nil, nil } -func (e *Engine) resultFromLocalCacheInner(ctx context.Context, def *LightLinkedTarget, outputs []string, hashin string) (*ExecuteResult, bool, error) { +func (e *Engine) resultFromLocalCacheInner(ctx context.Context, def *LightLinkedTarget, outputs []string, hashin string) (*Result, bool, error) { r, err := e.readAnyCache(ctx, def.GetRef(), hashin, hartifact.ManifestName) if err != nil { if errors.Is(err, LocalCacheNotFoundError) { @@ -403,34 +389,31 @@ func (e *Engine) resultFromLocalCacheInner(ctx context.Context, def *LightLinked execArtifacts = append(execArtifacts, &ResultArtifact{ Hashout: artifact.Hashout, Artifact: cachePluginArtifact{ - group: artifact.Group, - name: artifact.Name, - type_: pluginv1.Artifact_Type(artifact.Type), - cache: cache, + artifact: artifact, + cache: cache, }, + Manifest: artifact, }) } - execArtifacts = append(execArtifacts, &ResultArtifact{ - Artifact: &manifestPluginArtifact{manifest: m}, - }) - - return ExecuteResult{ + return Result{ Def: def, Hashin: m.Hashin, Artifacts: execArtifacts, + Manifest: m, }.Sorted(), true, nil } type CacheLocallyArtifact struct { - Reader io.Reader - Size int64 - Type pluginv1.Artifact_Type - Group string - Name string + Reader io.Reader + Size int64 + Type pluginv1.Artifact_Type + Group string + Name string + ContentType hartifact.ManifestArtifactContentType } -func (e *Engine) cacheLocally(ctx context.Context, ref *pluginv1.TargetRef, hashin string, art CacheLocallyArtifact, hashout string) (*ResultArtifact, error) { +func (e *Engine) cacheArtifactLocally(ctx context.Context, ref *pluginv1.TargetRef, hashin string, art CacheLocallyArtifact, hashout string) (*ResultArtifact, error) { cache := e.CacheSmall if art.Size > 100_000 { cache = e.CacheLarge @@ -444,7 +427,7 @@ func (e *Engine) cacheLocally(ctx context.Context, ref *pluginv1.TargetRef, hash prefix = "support_" case pluginv1.Artifact_TYPE_LOG: prefix = "log_" - case pluginv1.Artifact_TYPE_OUTPUT_LIST_V1, pluginv1.Artifact_TYPE_MANIFEST_V1, pluginv1.Artifact_TYPE_UNSPECIFIED: + case pluginv1.Artifact_TYPE_OUTPUT_LIST_V1, pluginv1.Artifact_TYPE_UNSPECIFIED: fallthrough default: return nil, fmt.Errorf("invalid artifact type: %s", art.Type) @@ -466,13 +449,21 @@ func (e *Engine) cacheLocally(ctx context.Context, ref *pluginv1.TargetRef, hash return nil, err } + manifestArtifact := hartifact.ManifestArtifact{ + Hashout: hashout, + Group: art.Group, + Name: art.Name, + Size: art.Size, + Type: hartifact.ManifestArtifactType(art.Type), + ContentType: art.ContentType, + } + return &ResultArtifact{ Hashout: hashout, Artifact: cachePluginArtifact{ - group: art.Group, - name: prefix + art.Name, - type_: art.Type, - cache: cache, + artifact: manifestArtifact, + cache: cache, }, + Manifest: manifestArtifact, }, nil } diff --git a/internal/engine/remote_cache.go b/internal/engine/remote_cache.go index 00002342..4687b4e6 100644 --- a/internal/engine/remote_cache.go +++ b/internal/engine/remote_cache.go @@ -4,19 +4,15 @@ import ( "context" "errors" "fmt" + "io" "log/slog" - "os" "path" - "sync" "github.com/hephbuild/heph/internal/herrgroup" "github.com/hephbuild/heph/internal/hartifact" "github.com/hephbuild/heph/internal/hcore/hlog" "github.com/hephbuild/heph/internal/hcore/hstep" - "github.com/hephbuild/heph/internal/hfs" - "github.com/hephbuild/heph/internal/hinstance" - "github.com/hephbuild/heph/internal/hrand" "github.com/hephbuild/heph/internal/hslices" "github.com/hephbuild/heph/lib/pluginsdk" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" @@ -77,32 +73,50 @@ func (e *Engine) cacheRemotelyInner(ctx context.Context, // TODO: remote lock ? + g := herrgroup.NewContext(ctx, true) + for _, artifact := range artifacts { - r, err := hartifact.Reader(artifact.Artifact) - if err != nil { - return err - } - defer r.Close() + g.Go(func(ctx context.Context) error { + r, err := hartifact.Reader(artifact.Artifact) + if err != nil { + return err + } + defer r.Close() - artifactName := artifact.GetName() - if artifact.GetType() == pluginv1.Artifact_TYPE_MANIFEST_V1 { - artifactName = hartifact.ManifestName - } + key := e.remoteCacheKey(ref, hashin, artifact.GetName()) - key := e.remoteCacheKey(ref, hashin, artifactName) + err = cache.Client.Store(ctx, key, r) + if err != nil { + return err + } - err = cache.Client.Store(ctx, key, r) + return nil + }) + } + + key := e.remoteCacheKey(ref, hashin, hartifact.ManifestName) + + pr, pw := io.Pipe() + + go func() { + defer pw.Close() + err := hartifact.EncodeManifest(pw, manifest) if err != nil { - return err + pr.CloseWithError(err) + + return } + }() - _ = r.Close() + err := cache.Client.Store(ctx, key, pr) + if err != nil { + return err } return nil } -func (e *Engine) ResultFromRemoteCache(ctx context.Context, rs *RequestState, def *LightLinkedTarget, outputs []string, hashin string) (*ExecuteResult, bool, error) { +func (e *Engine) ResultFromRemoteCache(ctx context.Context, rs *RequestState, def *LightLinkedTarget, outputs []string, hashin string) (*Result, bool, error) { ref := def.GetRef() if def.GetDisableRemoteCache() { @@ -132,81 +146,50 @@ func (e *Engine) ResultFromRemoteCache(ctx context.Context, rs *RequestState, de continue } - tmpCacheDir := hfs.At( - e.Cache, - def.GetRef().GetPackage(), - e.targetDirName(def.GetRef())+"_remote_tmp_"+hinstance.UID+"_"+hrand.Str(7)+"_"+hashin, - ) - err := tmpCacheDir.MkdirAll(os.ModePerm) - if err != nil { - return nil, false, err - } - - defer tmpCacheDir.RemoveAll() - - cacheDir := hfs.At(e.Cache, def.GetRef().GetPackage(), e.targetDirName(def.GetRef()), hashin) - - artifacts, ok, err := e.resultFromRemoteCacheInner(ctx, ref, outputs, hashin, cache, tmpCacheDir) + ok, err := e.resultFromRemoteCacheInner(ctx, ref, outputs, hashin, cache) if err != nil { hlog.From(ctx).With(slog.String("cache", cache.Name), slog.String("err", err.Error())).Error("failed to get from cache") continue } + if !ok { + continue + } - if ok { - localArtifacts := make([]*ResultArtifact, 0, len(artifacts)) - for _, artifact := range artifacts { - to := cacheDir.At(artifact.GetName()) - - err = hfs.Move(tmpCacheDir.At(artifact.GetName()), to) - if err != nil { - return nil, false, err - } - - rartifact, err := hartifact.Relocated(artifact.GetProto(), to.Path()) - if err != nil { - return nil, false, err - } - - localArtifacts = append(localArtifacts, &ResultArtifact{ - Hashout: artifact.Hashout, - Artifact: protoPluginArtifact{Artifact: rartifact}, - }) - } + res, ok, err := e.ResultFromLocalCache(ctx, def, outputs, hashin) + if err != nil { + // this is really not supposed to happen... - return ExecuteResult{ - Def: def, - Executed: false, - Hashin: hashin, - Artifacts: localArtifacts, - }.Sorted(), true, nil + return nil, false, fmt.Errorf("malformed local cache from remote cache: %w", err) } - _ = tmpCacheDir.RemoveAll() + if ok { + return res, true, nil + } } return nil, false, nil } -func (e *Engine) manifestFromRemoteCache(ctx context.Context, ref *pluginv1.TargetRef, hashin string, cache CacheHandle) (hartifact.Manifest, bool, error) { +func (e *Engine) manifestFromRemoteCache(ctx context.Context, ref *pluginv1.TargetRef, hashin string, cache CacheHandle) (*hartifact.Manifest, bool, error) { manifestKey := e.remoteCacheKey(ref, hashin, hartifact.ManifestName) - r, _, err := cache.Client.Get(ctx, manifestKey) + r, err := cache.Client.Get(ctx, manifestKey) if err != nil { if errors.Is(err, pluginsdk.ErrCacheNotFound) { - return hartifact.Manifest{}, false, nil + return nil, false, nil } - return hartifact.Manifest{}, false, nil + return nil, false, nil } defer r.Close() m, err := hartifact.DecodeManifest(r) if err != nil { if errors.Is(err, pluginsdk.ErrCacheNotFound) { - return hartifact.Manifest{}, false, nil + return nil, false, nil } - return hartifact.Manifest{}, false, err + return nil, false, err } return m, true, nil @@ -224,17 +207,16 @@ func (e *Engine) resultFromRemoteCacheInner( outputs []string, hashin string, cache CacheHandle, - cachedir hfs.OS, -) ([]*ResultArtifact, bool, error) { +) (bool, error) { ctx, span := tracer.Start(ctx, "ResultFromLocalCacheInner", trace.WithAttributes(attribute.String("cache", cache.Name))) defer span.End() manifest, ok, err := e.manifestFromRemoteCache(ctx, ref, hashin, cache) if err != nil { - return nil, false, err + return false, err } if !ok { - return nil, false, nil + return false, nil } remoteArtifacts := make([]hartifact.ManifestArtifact, 0, len(outputs)) @@ -247,22 +229,21 @@ func (e *Engine) resultFromRemoteCacheInner( return artifact.Group == output }) if !ok { - return nil, false, nil + return false, nil } remoteArtifacts = append(remoteArtifacts, artifact) } - var localArtifactsm sync.Mutex - localArtifacts := make([]*ResultArtifact, 0, len(outputs)) + localArtifacts := make([]*ResultArtifact, len(outputs)) g := herrgroup.NewContext(ctx, true) - for _, artifact := range remoteArtifacts { + for i, artifact := range remoteArtifacts { key := e.remoteCacheKey(ref, hashin, artifact.Name) g.Go(func(ctx context.Context) error { - r, size, err := cache.Client.Get(ctx, key) + r, err := cache.Client.Get(ctx, key) if err != nil { if errors.Is(err, pluginsdk.ErrCacheNotFound) { return pluginsdk.ErrCacheNotFound @@ -272,20 +253,19 @@ func (e *Engine) resultFromRemoteCacheInner( } defer r.Close() - localArtifact, err := e.cacheLocally(ctx, ref, hashin, CacheLocallyArtifact{ - Reader: r, - Size: size, - Type: pluginv1.Artifact_Type(artifact.Type), - Group: artifact.Group, - Name: artifact.Name, + localArtifact, err := e.cacheArtifactLocally(ctx, ref, hashin, CacheLocallyArtifact{ + Reader: r, + Size: artifact.Size, + Type: pluginv1.Artifact_Type(artifact.Type), + Group: artifact.Group, + Name: artifact.Name, + ContentType: artifact.ContentType, }, artifact.Hashout) if err != nil { return err } - localArtifactsm.Lock() - localArtifacts = append(localArtifacts, localArtifact) - localArtifactsm.Unlock() + localArtifacts[i] = localArtifact return nil }) @@ -293,15 +273,16 @@ func (e *Engine) resultFromRemoteCacheInner( err = g.Wait() if err != nil { if errors.Is(err, pluginsdk.ErrCacheNotFound) { - return nil, false, nil + return false, nil } - return nil, false, err + return false, err } - localArtifacts = append(localArtifacts, &ResultArtifact{ - Artifact: &manifestPluginArtifact{manifest: manifest}, - }) + _, err = e.createLocalCacheManifest(ctx, ref, hashin, localArtifacts) + if err != nil { + return false, err + } - return localArtifacts, true, nil + return true, nil } diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index e7a7ef78..19e8c847 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -56,7 +56,7 @@ type ExecuteOptions struct { shell bool force bool interactive bool - hashin string + metaHashin string } type InteractiveExecOptions struct { @@ -450,15 +450,7 @@ func (e *Engine) ResultMetaFromDef(ctx context.Context, rs *RequestState, def *T } defer res.Unlock(ctx) - manifestArtifact, ok := res.FindManifest() - if !ok { - return ResultMeta{}, errors.New("no manifest") - } - - manifest, err := hartifact.ManifestFromArtifact(ctx, manifestArtifact) - if err != nil { - return ResultMeta{}, fmt.Errorf("manifest from artifact: %w", err) - } + manifest := res.Manifest m := ResultMeta{ Hashin: manifest.Hashin, @@ -562,7 +554,7 @@ func (e *Engine) depsResultMetas(ctx context.Context, rs *RequestState, def *Lig const AllOutputs = "__all_outputs__" -func (e *Engine) resultFromCache(ctx context.Context, rs *RequestState, def *LightLinkedTarget, outputs []string, hashin string) (*ExecuteResult, bool, error) { +func (e *Engine) resultFromCache(ctx context.Context, rs *RequestState, def *LightLinkedTarget, outputs []string, hashin string) (*Result, bool, error) { res, ok, err := e.ResultFromLocalCache(ctx, def, outputs, hashin) if err != nil { return nil, false, fmt.Errorf("result from local cache: %w", err) @@ -629,8 +621,8 @@ func (e *Engine) innerResult(ctx context.Context, rs *RequestState, def *LightLi if ok { return &ExecuteResultLocks{ - ExecuteResult: res, - Locks: locks, + Result: res, + Locks: locks, }, nil } @@ -648,7 +640,7 @@ func (e *Engine) innerResult(ctx context.Context, rs *RequestState, def *LightLi } defer results.Unlock(ctx) - manifest := hartifact.Manifest{ + manifest := &hartifact.Manifest{ Version: "v1", Target: tref.Format(def.GetRef()), CreatedAt: time.Now(), @@ -680,26 +672,17 @@ func (e *Engine) innerResult(ctx context.Context, rs *RequestState, def *LightLi } } - martifact, err := hartifact.NewManifestArtifact(manifest) - if err != nil { - return nil, fmt.Errorf("new manifest artifact: %w", err) - } - - artifacts = append(artifacts, &ResultArtifact{ - Artifact: protoPluginArtifact{Artifact: martifact}, - }) - err = locks.RLock(ctx) if err != nil { return nil, err } return &ExecuteResultLocks{ - ExecuteResult: ExecuteResult{ + Result: Result{ Def: def, - Executed: true, Hashin: hashin, Artifacts: artifacts, + Manifest: manifest, }.Sorted(), Locks: &locks, }, nil @@ -714,10 +697,10 @@ func (e *Engine) innerResult(ctx context.Context, rs *RequestState, def *LightLi shell: shouldShell, force: shouldForce, interactive: tref.Equal(rs.Shell, def.GetRef()) || tref.Equal(rs.Interactive, def.GetRef()), - hashin: hashin, + metaHashin: hashin, } - var res *ExecuteResult + var res *Result if storeCache { res, err = e.ExecuteAndCache(ctx, rs, def, execOptions) if err != nil { @@ -726,40 +709,12 @@ func (e *Engine) innerResult(ctx context.Context, rs *RequestState, def *LightLi return nil, err } } else { - res, err = e.Execute(ctx, rs, def, execOptions) - if err != nil { - err = errors.Join(err, locks.Unlock()) - - return nil, err - } - - m := hartifact.Manifest{ - Version: "v1", - Target: tref.Format(def.GetRef()), - CreatedAt: time.Now(), - Hashin: hashin, - } - for _, artifact := range res.Artifacts { - martifact, err := hartifact.ProtoArtifactToManifest(artifact.Hashout, artifact) - if err != nil { - err = errors.Join(err, locks.Unlock()) - - return nil, err - } - - m.Artifacts = append(m.Artifacts, martifact) - } - - manifestArtifact, err := hartifact.NewManifestArtifact(m) + res, err = e.ExecuteNoCache(ctx, rs, def, execOptions) if err != nil { err = errors.Join(err, locks.Unlock()) return nil, err } - - res.Artifacts = append(res.Artifacts, &ResultArtifact{ - Artifact: protoPluginArtifact{Artifact: manifestArtifact}, - }) } err = locks.Lock2RLock(ctx) @@ -770,8 +725,8 @@ func (e *Engine) innerResult(ctx context.Context, rs *RequestState, def *LightLi } return &ExecuteResultLocks{ - ExecuteResult: res, - Locks: locks, + Result: res, + Locks: locks, }, nil }) if err != nil { @@ -882,44 +837,47 @@ func (e *Engine) hashin(ctx context.Context, def *LightLinkedTarget, results []* return e.hashin2(ctx, def, metas, "res") } -type protoPluginArtifact struct { - *pluginv1.Artifact +type ExecuteResult struct { + Def *LightLinkedTarget + Hashin string + Artifacts []*ExecuteArtifact + AfterCache func() } -func (e protoPluginArtifact) GetProto() *pluginv1.Artifact { - return e.Artifact +type Result struct { + Def *LightLinkedTarget + Hashin string + Artifacts []*ResultArtifact + Manifest *hartifact.Manifest } -func (e protoPluginArtifact) GetContentReader() (io.ReadCloser, error) { - return hartifact.Reader(e) -} +func (r Result) Sorted() *Result { + slices.SortFunc(r.Artifacts, func(a, b *ResultArtifact) int { + return strings.Compare(a.Hashout, b.Hashout) + }) -func (e protoPluginArtifact) GetContentSize() (int64, error) { - return hartifact.Size(e) + return &r } -type ExecuteResult struct { - Def *LightLinkedTarget - Executed bool - Hashin string - Artifacts []*ResultArtifact +func (r Result) Clone() *Result { + return &Result{ + Def: r.Def.Clone(), + Hashin: r.Hashin, + Artifacts: slices.Clone(r.Artifacts), + Manifest: r.Manifest, + } } -type ResultArtifact struct { +type ExecuteArtifact struct { Hashout string pluginsdk.Artifact } -func (r ExecuteResult) FindManifest() (*ResultArtifact, bool) { - for _, artifact := range r.Artifacts { - if artifact.GetType() != pluginv1.Artifact_TYPE_MANIFEST_V1 { - continue - } - - return artifact, true - } +type ResultArtifact struct { + Hashout string + Manifest hartifact.ManifestArtifact - return nil, false + pluginsdk.Artifact } func (r ExecuteResult) FindOutputs(group string) []*ResultArtifact { @@ -952,21 +910,21 @@ func (r ExecuteResult) FindSupport() []*ResultArtifact { } type ExecuteResultLocks struct { - *ExecuteResult + *Result Locks *CacheLocks } func (r *ExecuteResultLocks) Clone() *ExecuteResultLocks { return &ExecuteResultLocks{ - ExecuteResult: r.ExecuteResult.Clone(), - Locks: r.Locks.Clone(), + Result: r.Result.Clone(), + Locks: r.Locks.Clone(), } } func (r *ExecuteResultLocks) CloneWithoutLocks() *ExecuteResultLocks { return &ExecuteResultLocks{ - ExecuteResult: r.ExecuteResult.Clone(), - Locks: r.Locks, + Result: r.Result.Clone(), + Locks: r.Locks, } } @@ -982,7 +940,7 @@ func (r *ExecuteResultLocks) Unlock(ctx context.Context) { } func (r ExecuteResult) Sorted() *ExecuteResult { - slices.SortFunc(r.Artifacts, func(a, b *ResultArtifact) int { + slices.SortFunc(r.Artifacts, func(a, b *ExecuteArtifact) int { return strings.Compare(a.Hashout, b.Hashout) }) @@ -991,10 +949,10 @@ func (r ExecuteResult) Sorted() *ExecuteResult { func (r ExecuteResult) Clone() *ExecuteResult { return &ExecuteResult{ - Def: r.Def.Clone(), - Executed: r.Executed, - Hashin: r.Hashin, - Artifacts: slices.Clone(r.Artifacts), + Def: r.Def.Clone(), + Hashin: r.Hashin, + Artifacts: slices.Clone(r.Artifacts), + AfterCache: r.AfterCache, } } @@ -1174,7 +1132,9 @@ func (e *Engine) pickShellDriver(ctx context.Context, def *LightLinkedTarget) (p var sem = semaphore.NewWeighted(1000 * int64(runtime.GOMAXPROCS(-1))) -func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinkedTarget, options ExecuteOptions) (*ExecuteResult, error) { +func (e *Engine) execute(ctx context.Context, rs *RequestState, def *LightLinkedTarget, results DepsResults, options ExecuteOptions) (*ExecuteResult, error) { + hashin := options.metaHashin + ctx, span := tracer.Start(ctx, "Execute") defer span.End() @@ -1185,18 +1145,13 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked }) defer cleanLabels() - results, err := e.depsResults(ctx, rs, def) - if err != nil { - return nil, fmt.Errorf("deps results: %w", err) - } - defer results.Unlock(ctx) - driver, ok := e.DriversByName[def.GetDriver()] if !ok { return nil, fmt.Errorf("driver not found: %v", def.GetDriver()) } if options.shell { + var err error driver, err = e.pickShellDriver(ctx, def) if err != nil { return nil, err @@ -1211,30 +1166,7 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked targetfolder = fmt.Sprintf("__%v__%v", e.targetDirName(def.GetRef()), time.Now().UnixNano()) } - hashin, err := e.hashin(ctx, def, results) - if err != nil { - return nil, fmt.Errorf("hashin1: %w", err) - } - - if hashin != options.hashin { - return nil, fmt.Errorf("results hashin (%v) != meta hashin (%v)", hashin, options.hashin) - } - - if def.GetCache() && !options.force && !options.shell { - res, ok, err := e.ResultFromLocalCache(ctx, def, def.OutputNames(), hashin) - if err != nil { - hlog.From(ctx).With(slog.String("target", tref.Format(def.GetRef())), slog.String("err", err.Error())).Warn("failed to get result from local cache") - } - - if ok { - step := hstep.From(ctx) - step.SetText(fmt.Sprintf("%v: cached", tref.Format(def.GetRef()))) - - return res, nil - } - } - - err = sem.Acquire(ctx, 1) + err := sem.Acquire(ctx, 1) if err != nil { return nil, err } @@ -1269,24 +1201,25 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked } } - var inputs []*pluginv1.ArtifactWithOrigin + var inputLen int + for _, result := range results { + inputLen += len(result.Artifacts) + } + + inputs := make([]*pluginv1.ArtifactWithOrigin, 0, inputLen) + inputsSdk := make([]*pluginsdk.ArtifactWithOrigin, 0, inputLen) for _, result := range results { for _, artifact := range result.Artifacts { inputs = append(inputs, pluginv1.ArtifactWithOrigin_builder{ Artifact: artifact.GetProto(), Origin: result.InputOrigin, }.Build()) - } - } - inputsSdk := make([]*pluginsdk.ArtifactWithOrigin, 0, len(inputs)) - for _, input := range inputs { - inputsSdk = append(inputsSdk, &pluginsdk.ArtifactWithOrigin{ - Artifact: protoPluginArtifact{ - Artifact: input.GetArtifact(), - }, - Origin: input.GetOrigin(), - }) + inputsSdk = append(inputsSdk, &pluginsdk.ArtifactWithOrigin{ + Artifact: artifact.Artifact, + Origin: result.InputOrigin, + }) + } } var runRes *pluginv1.RunResponse @@ -1351,14 +1284,18 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked if options.shell { return &ExecuteResult{ - Def: def, - Hashin: hashin, - Executed: true, + Def: def, + Hashin: hashin, + AfterCache: func() { + err := sandboxfs.RemoveAll() + if err != nil { + hlog.From(ctx).Error(fmt.Sprintf("failed to remove sandboxfs: %v", err)) + } + }, }, nil } - cachefs := hfs.At(e.Cache, def.GetRef().GetPackage(), e.targetDirName(def.GetRef()), hashin) - execArtifacts := make([]*ResultArtifact, 0, len(def.OutputNames())) + execArtifacts := make([]*ExecuteArtifact, 0, len(def.OutputNames())) for _, output := range def.GetOutputs() { shouldCollect := false @@ -1374,7 +1311,7 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked } tarname := output.GetGroup() + ".tar" - tarf, err := hfs.Create(cachefs.At(tarname)) + tarf, err := hfs.Create(sandboxfs.At("collect", tarname)) if err != nil { return nil, err } @@ -1438,7 +1375,7 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked return nil, err } - execArtifacts = append(execArtifacts, &ResultArtifact{ + execArtifacts = append(execArtifacts, &ExecuteArtifact{ Hashout: hashout, Artifact: execArtifact, }) @@ -1455,7 +1392,7 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked if shouldCollect { tarname := "support.tar" - tarf, err := hfs.Create(cachefs.At(tarname)) + tarf, err := hfs.Create(sandboxfs.At("collect", tarname)) if err != nil { return nil, err } @@ -1513,7 +1450,7 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked return nil, err } - execArtifacts = append(execArtifacts, &ResultArtifact{ + execArtifacts = append(execArtifacts, &ExecuteArtifact{ Hashout: hashout, Artifact: execArtifact, }) @@ -1534,62 +1471,91 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked return nil, fmt.Errorf("hashout: %w", err) } - execArtifacts = append(execArtifacts, &ResultArtifact{ + execArtifacts = append(execArtifacts, &ExecuteArtifact{ Hashout: hashout, Artifact: artifact, }) - - // panic("copy to cache not implemented yet") - - // TODO: copy to cache - // hfs.Copy() - // - // artifact.Uri - // - // execOutputs = append(execOutputs, ExecuteResultOutput{ - // Name: artifact.Group, - // Hashout: "", - // TarPath: "", - // }) - } - - err = sandboxfs.RemoveAll() - if err != nil { - return nil, err } return ExecuteResult{ Hashin: hashin, Def: def, - Executed: true, Artifacts: execArtifacts, + AfterCache: func() { + err := sandboxfs.RemoveAll() + if err != nil { + hlog.From(ctx).Error(fmt.Sprintf("failed to remove sandboxfs: %v", err)) + } + }, }.Sorted(), nil } -func (e *Engine) ExecuteAndCache(ctx context.Context, rs *RequestState, def *LightLinkedTarget, options ExecuteOptions) (*ExecuteResult, error) { - res, err := e.Execute(ctx, rs, def, options) +func (e *Engine) executeAndCacheInner(ctx context.Context, rs *RequestState, def *LightLinkedTarget, options ExecuteOptions, persistentCache bool) (*Result, error) { + results, err := e.depsResults(ctx, rs, def) if err != nil { - return nil, fmt.Errorf("execute: %w", err) + return nil, fmt.Errorf("deps results: %w", err) + } + defer results.Unlock(ctx) + + hashin, err := e.hashin(ctx, def, results) + if err != nil { + return nil, fmt.Errorf("results hashin: %w", err) } - var cachedArtifacts []*ResultArtifact - if res.Executed { - artifacts, manifest, err := e.CacheLocally(ctx, def, res.Hashin, res.Artifacts) + if hashin != options.metaHashin { + return nil, fmt.Errorf("results hashin (%v) != meta hashin (%v)", hashin, options.metaHashin) + } + + cacheHashin := hashin + if !persistentCache { + cacheHashin = hinstance.UID + "_" + hashin + } + + if def.GetCache() && !options.force && !options.shell { + // One last cache check after the deps have completed + res, ok, err := e.ResultFromLocalCache(ctx, def, def.OutputNames(), cacheHashin) if err != nil { - return nil, fmt.Errorf("cache locally: %w", err) + hlog.From(ctx).With(slog.String("target", tref.Format(def.GetRef())), slog.String("err", err.Error())).Warn("failed to get result from local cache") } - cachedArtifacts = artifacts + if ok { + return res, nil + } + } + + res, err := e.execute(ctx, rs, def, results, options) + if err != nil { + return nil, fmt.Errorf("execute: %w", err) + } - // TODO: move this to a background execution model , so that local build can proceed, while this is uploading in the background + cachedArtifacts, manifest, err := e.cacheLocally(ctx, def, cacheHashin, res.Artifacts) + if err != nil { + return nil, fmt.Errorf("cache locally: %w", err) + } + + if res.AfterCache != nil { + res.AfterCache() + } + + if persistentCache { + // TODO: move this to a background execution so that local build can proceed, while this is uploading in the background e.CacheRemotely(ctx, def, res.Hashin, manifest, cachedArtifacts) - } else { - cachedArtifacts = res.Artifacts } - return ExecuteResult{ + return Result{ Def: def, Hashin: res.Hashin, Artifacts: cachedArtifacts, + Manifest: manifest, }.Sorted(), nil } + +func (e *Engine) ExecuteAndCache(ctx context.Context, rs *RequestState, def *LightLinkedTarget, options ExecuteOptions) (*Result, error) { + return e.executeAndCacheInner(ctx, rs, def, options, true) +} + +func (e *Engine) ExecuteNoCache(ctx context.Context, rs *RequestState, def *LightLinkedTarget, options ExecuteOptions) (*Result, error) { + // cached and uncached targets should go through the same mechanism, the only difference is that uncached targets should be removed upon session completion + + return e.executeAndCacheInner(ctx, rs, def, options, false) +} diff --git a/internal/enginee2e/sanity_remotecache_test.go b/internal/enginee2e/sanity_remotecache_test.go index 3b8e2a9b..1df3cdeb 100644 --- a/internal/enginee2e/sanity_remotecache_test.go +++ b/internal/enginee2e/sanity_remotecache_test.go @@ -49,13 +49,13 @@ func (m *mockCache) Store(ctx context.Context, key string, r io.Reader) error { return nil } -func (m *mockCache) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) { +func (m *mockCache) Get(ctx context.Context, key string) (io.ReadCloser, error) { b, ok := m.store[key] if !ok { - return nil, 0, pluginsdk.ErrCacheNotFound + return nil, pluginsdk.ErrCacheNotFound } - return io.NopCloser(bytes.NewReader(b)), int64(len(b)), nil + return io.NopCloser(bytes.NewReader(b)), nil } func TestSanityRemoteCache(t *testing.T) { diff --git a/internal/hartifact/manifest.go b/internal/hartifact/manifest.go index c1ed0c77..500b96ae 100644 --- a/internal/hartifact/manifest.go +++ b/internal/hartifact/manifest.go @@ -60,8 +60,6 @@ type ManifestArtifact struct { Size int64 Type ManifestArtifactType ContentType ManifestArtifactContentType - OutPath string `json:",omitempty"` // set for file & raw - X bool `json:",omitempty"` // set for file & raw } type Manifest struct { @@ -85,7 +83,7 @@ func (m Manifest) GetArtifacts(output string) []ManifestArtifact { return a } -func EncodeManifest(w io.Writer, m Manifest) error { +func EncodeManifest(w io.Writer, m *Manifest) error { err := json.NewEncoder(w).Encode(m) //nolint:musttag if err != nil { return err @@ -94,58 +92,45 @@ func EncodeManifest(w io.Writer, m Manifest) error { return nil } -func NewManifestArtifact(m Manifest) (*pluginv1.Artifact, error) { - b, err := json.Marshal(m) //nolint:musttag - if err != nil { - return nil, err - } - - return pluginv1.Artifact_builder{ - Name: htypes.Ptr(ManifestName), - Type: htypes.Ptr(pluginv1.Artifact_TYPE_MANIFEST_V1), - Raw: pluginv1.Artifact_ContentRaw_builder{ - Data: b, - Path: htypes.Ptr(ManifestName), - }.Build(), - }.Build(), nil -} - -func ManifestFromArtifact(ctx context.Context, a pluginsdk.Artifact) (Manifest, error) { +func ManifestFromArtifact(ctx context.Context, a pluginsdk.Artifact) (*Manifest, error) { r, err := FileReader(ctx, a) if err != nil { - return Manifest{}, err + return nil, err } defer r.Close() return DecodeManifest(r) } -func DecodeManifest(r io.Reader) (Manifest, error) { +func DecodeManifest(r io.Reader) (*Manifest, error) { var manifest Manifest err := json.NewDecoder(r).Decode(&manifest) //nolint:musttag if err != nil { - return Manifest{}, err + return nil, err } - return manifest, nil + return &manifest, nil } func ManifestContentType(a pluginsdk.Artifact) (ManifestArtifactContentType, error) { - ap := a.GetProto() + contentType, err := a.GetContentType() + if err != nil { + return "", err + } - switch ap.WhichContent() { - case pluginv1.Artifact_TargzPath_case: + switch contentType { + case pluginsdk.ArtifactContentTypeTarGz: return ManifestArtifactContentTypeTarGz, nil - case pluginv1.Artifact_TarPath_case: + case pluginsdk.ArtifactContentTypeTar: return ManifestArtifactContentTypeTar, nil - case pluginv1.Artifact_File_case: + case pluginsdk.ArtifactContentTypeFile: return ManifestArtifactContentTypeFile, nil - case pluginv1.Artifact_Raw_case: + case pluginsdk.ArtifactContentTypeRaw: return ManifestArtifactContentTypeRaw, nil default: } - return "", fmt.Errorf("unsupported content %v", ap.WhichContent().String()) + return "", fmt.Errorf("unsupported content %v", contentType) } func ProtoArtifactToManifest(hashout string, a pluginsdk.Artifact) (ManifestArtifact, error) { @@ -154,19 +139,6 @@ func ProtoArtifactToManifest(hashout string, a pluginsdk.Artifact) (ManifestArti return ManifestArtifact{}, err } - ap := a.GetProto() - - var outPath string - var x bool - switch ap.WhichContent() { //nolint:exhaustive - case pluginv1.Artifact_File_case: - outPath = ap.GetFile().GetOutPath() - x = ap.GetFile().GetX() - case pluginv1.Artifact_Raw_case: - outPath = ap.GetRaw().GetPath() - x = ap.GetRaw().GetX() - } - size, err := a.GetContentSize() if err != nil { return ManifestArtifact{}, err @@ -174,13 +146,11 @@ func ProtoArtifactToManifest(hashout string, a pluginsdk.Artifact) (ManifestArti return ManifestArtifact{ Hashout: hashout, - Group: ap.GetGroup(), - Name: ap.GetName(), + Group: a.GetGroup(), + Name: a.GetName(), Size: size, - Type: ManifestArtifactType(ap.GetType()), + Type: ManifestArtifactType(a.GetType()), ContentType: contentType, - OutPath: outPath, - X: x, }, nil } diff --git a/internal/hartifact/reader.go b/internal/hartifact/reader.go index 3f20b781..b97b5b9a 100644 --- a/internal/hartifact/reader.go +++ b/internal/hartifact/reader.go @@ -107,35 +107,15 @@ type File struct { // FilesReader provides a reader for each file it (no matter the packaging). func FilesReader(ctx context.Context, a pluginsdk.Artifact) iter.Seq2[*File, error] { - ap := a.GetProto() - return func(yield func(*File, error) bool) { - switch ap.WhichContent() { - case pluginv1.Artifact_File_case: - f, err := os.Open(ap.GetFile().GetSourcePath()) - if err != nil { - if !yield(nil, err) { - return - } - return - } - - if !yield(&File{ - ReadCloser: f, - Path: ap.GetFile().GetOutPath(), - }, nil) { - return - } - case pluginv1.Artifact_Raw_case: - f := io.NopCloser(bytes.NewReader(ap.GetRaw().GetData())) + contentType, err := a.GetContentType() + if err != nil { + yield(nil, fmt.Errorf("get content type: %w", err)) + return + } - if !yield(&File{ - ReadCloser: f, - Path: ap.GetRaw().GetPath(), - }, nil) { - return - } - case pluginv1.Artifact_TarPath_case: + switch contentType { + case pluginsdk.ArtifactContentTypeTar: r, err := Reader(a) if err != nil { if !yield(nil, err) { @@ -169,7 +149,7 @@ func FilesReader(ctx context.Context, a pluginsdk.Artifact) iter.Seq2[*File, err } // case pluginsdk.Artifact_TargzPath: default: - if !yield(nil, fmt.Errorf("unsupported encoding %v", ap.WhichContent())) { + if !yield(nil, fmt.Errorf("unsupported encoding %v", contentType)) { return } } diff --git a/internal/remotecache/exec.go b/internal/remotecache/exec.go index 180696cb..c4b21215 100644 --- a/internal/remotecache/exec.go +++ b/internal/remotecache/exec.go @@ -4,7 +4,6 @@ import ( "context" "errors" "io" - "math" "os/exec" "strings" "sync" @@ -103,7 +102,7 @@ func (e *execReader) Close() error { } // Get should return engine.ErrCacheNotFound if the key cannot be found, engine.ErrCacheNotFound can also be returned from Close(). -func (e Exec) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) { +func (e Exec) Get(ctx context.Context, key string) (io.ReadCloser, error) { cmd := exec.CommandContext(ctx, e.args[0], e.args[1:]...) //nolint:gosec cmd.Env = append(cmd.Environ(), "CACHE_KEY="+key) @@ -111,5 +110,5 @@ func (e Exec) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) runCh: make(chan struct{}), notFoundSentinels: e.notFoundSentinels, cmd: cmd, - }, math.MaxInt64, nil + }, nil } diff --git a/internal/remotecache/gcs.go b/internal/remotecache/gcs.go index 690649c5..ce0e9bfa 100644 --- a/internal/remotecache/gcs.go +++ b/internal/remotecache/gcs.go @@ -57,7 +57,7 @@ func (g GCS) Store(ctx context.Context, key string, r io.Reader) error { return w.Close() } -func (g GCS) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) { +func (g GCS) Get(ctx context.Context, key string) (io.ReadCloser, error) { obj := g.bucket.Object(key) attr, err := obj.Attrs(ctx) @@ -66,7 +66,7 @@ func (g GCS) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) return nil, 0, pluginsdk.ErrCacheNotFound } - return nil, 0, err + return nil, err } r, err := obj.NewReader(ctx) @@ -75,8 +75,8 @@ func (g GCS) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) return nil, 0, pluginsdk.ErrCacheNotFound } - return nil, 0, err + return nil, err } - return r, attr.Size, nil + return r, nil } diff --git a/lib/pluginsdk/artifact.go b/lib/pluginsdk/artifact.go index 98abcd10..64aa767c 100644 --- a/lib/pluginsdk/artifact.go +++ b/lib/pluginsdk/artifact.go @@ -1,17 +1,26 @@ package pluginsdk import ( + "fmt" "io" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" ) +type ArtifactContentType string + +const ( + ArtifactContentTypeTar ArtifactContentType = "application/x-tar" + ArtifactContentTypeTarGz ArtifactContentType = "application/x-gtar" +) + type Artifact interface { GetGroup() string GetName() string GetType() pluginv1.Artifact_Type GetContentReader() (io.ReadCloser, error) GetContentSize() (int64, error) + GetContentType() (ArtifactContentType, error) GetProto() *pluginv1.Artifact } @@ -34,6 +43,17 @@ func (e ProtoArtifact) GetContentReader() (io.ReadCloser, error) { func (e ProtoArtifact) GetContentSize() (int64, error) { return e.ContentSizeFunc(e) } +func (e ProtoArtifact) GetContentType() (ArtifactContentType, error) { + switch e.Artifact.WhichContent() { + case pluginv1.Artifact_TargzPath_case: + return ArtifactContentTypeTarGz, nil + case pluginv1.Artifact_TarPath_case: + return ArtifactContentTypeTar, nil + default: + } + + return "", fmt.Errorf("unsupported content %v", e.WhichContent().String()) +} type ArtifactWithOrigin struct { Artifact diff --git a/lib/pluginsdk/plugin_cache.go b/lib/pluginsdk/plugin_cache.go index b20ff1fa..6b392567 100644 --- a/lib/pluginsdk/plugin_cache.go +++ b/lib/pluginsdk/plugin_cache.go @@ -10,7 +10,7 @@ var ErrCacheNotFound = errors.New("not found") type Cache interface { Store(ctx context.Context, key string, r io.Reader) error - Get(ctx context.Context, key string) (io.ReadCloser, int64, error) + Get(ctx context.Context, key string) (io.ReadCloser, error) } type CacheHas interface { From 57f490f5f37ebf11d4af3b360ff0d11d1cd345f0 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Sat, 7 Mar 2026 15:34:53 +0000 Subject: [PATCH 03/15] mostly working --- internal/engine/artifact_group.go | 12 ++ internal/engine/cache_fs.go | 6 +- internal/engine/cache_sql.go | 8 +- internal/engine/local_cache.go | 116 +++----------- internal/engine/remote_cache.go | 2 +- internal/engine/schedule.go | 60 ++++--- internal/enginee2e/deps_cache_test.go | 88 +++++++++- internal/enginee2e/hash_deps_test.go | 2 +- internal/hartifact/manifest.go | 55 ------- internal/hartifact/reader.go | 84 ++-------- internal/hartifact/unpack.go | 71 ++------ internal/remotecache/gcs.go | 11 +- lib/pluginsdk/artifact.go | 33 ---- lib/pluginsdk/artifact_proto.go | 151 ++++++++++++++++++ .../pluginsdkconnect/artifact_proto.go | 27 ++++ .../pluginsdkconnect/plugin_driver.go | 12 +- plugin/pluginbin/plugin.go | 2 +- plugin/pluginexec/run.go | 4 - plugin/pluginexec/sandbox.go | 22 +-- plugin/proto/heph/plugin/v1/driver.proto | 20 ++- 20 files changed, 385 insertions(+), 401 deletions(-) create mode 100644 internal/engine/artifact_group.go create mode 100644 lib/pluginsdk/artifact_proto.go create mode 100644 lib/pluginsdk/pluginsdkconnect/artifact_proto.go diff --git a/internal/engine/artifact_group.go b/internal/engine/artifact_group.go new file mode 100644 index 00000000..6298736b --- /dev/null +++ b/internal/engine/artifact_group.go @@ -0,0 +1,12 @@ +package engine + +import "github.com/hephbuild/heph/lib/pluginsdk" + +type artifactGroupMap struct { + pluginsdk.Artifact + group string +} + +func (a artifactGroupMap) GetGroup() string { + return a.group +} diff --git a/internal/engine/cache_fs.go b/internal/engine/cache_fs.go index 745a8da2..812515cd 100644 --- a/internal/engine/cache_fs.go +++ b/internal/engine/cache_fs.go @@ -49,7 +49,7 @@ func (c FSCache) targetDirName(ref *pluginv1.TargetRef) string { } func (c FSCache) path(ref *pluginv1.TargetRef, hashin, name string) hfs.Node { - return c.root.At(ref.GetPackage(), c.targetDirName(ref), hashin) + return c.root.At(ref.GetPackage(), c.targetDirName(ref), hashin, name) } func (c FSCache) Reader(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.ReadCloser, error) { @@ -64,7 +64,7 @@ func (c FSCache) Delete(ctx context.Context, ref *pluginv1.TargetRef, hashin, na return c.path(ref, hashin, name).RemoveAll() } -func (c FSCache) ListArtifacts(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) iter.Seq2[string, error] { +func (c FSCache) ListArtifacts(ctx context.Context, ref *pluginv1.TargetRef, hashin string) iter.Seq2[string, error] { return func(yield func(string, error) bool) { entries, err := c.path(ref, hashin, "").ReadDir() if err != nil { @@ -84,7 +84,7 @@ func (c FSCache) ListArtifacts(ctx context.Context, ref *pluginv1.TargetRef, has } } -func (c FSCache) ListVersions(ctx context.Context, ref *pluginv1.TargetRef, name string) iter.Seq2[string, error] { +func (c FSCache) ListVersions(ctx context.Context, ref *pluginv1.TargetRef) iter.Seq2[string, error] { return func(yield func(string, error) bool) { entries, err := c.path(ref, "", "").ReadDir() if err != nil { diff --git a/internal/engine/cache_sql.go b/internal/engine/cache_sql.go index 56c6b4e9..eaf04e8b 100644 --- a/internal/engine/cache_sql.go +++ b/internal/engine/cache_sql.go @@ -13,7 +13,6 @@ import ( "path/filepath" "time" - "github.com/hephbuild/heph/internal/hfs" "github.com/hephbuild/heph/internal/hproto/hashpb" "github.com/hephbuild/heph/lib/tref" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" @@ -22,8 +21,7 @@ import ( ) type SQLCache struct { - db *sql.DB - root hfs.OS + db *sql.DB } func (c *SQLCache) Exists(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (bool, error) { @@ -251,7 +249,7 @@ func (c *SQLCache) Writer(ctx context.Context, ref *pluginv1.TargetRef, hashin, }, nil } -func (c *SQLCache) ListArtifacts(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) iter.Seq2[string, error] { +func (c *SQLCache) ListArtifacts(ctx context.Context, ref *pluginv1.TargetRef, hashin string) iter.Seq2[string, error] { return func(yield func(string, error) bool) { targetAddr := c.targetKey(ref) @@ -283,7 +281,7 @@ func (c *SQLCache) ListArtifacts(ctx context.Context, ref *pluginv1.TargetRef, h } } -func (c *SQLCache) ListVersions(ctx context.Context, ref *pluginv1.TargetRef, name string) iter.Seq2[string, error] { +func (c *SQLCache) ListVersions(ctx context.Context, ref *pluginv1.TargetRef) iter.Seq2[string, error] { return func(yield func(string, error) bool) { targetAddr := c.targetKey(ref) diff --git a/internal/engine/local_cache.go b/internal/engine/local_cache.go index 6a31660e..94b6ee73 100644 --- a/internal/engine/local_cache.go +++ b/internal/engine/local_cache.go @@ -1,8 +1,6 @@ package engine import ( - "archive/tar" - "bytes" "context" "encoding/hex" "errors" @@ -10,7 +8,6 @@ import ( "hash" "io" "iter" - "os" "slices" "strconv" "strings" @@ -22,8 +19,6 @@ import ( "github.com/hephbuild/heph/lib/tref" - "github.com/hephbuild/heph/internal/htar" - "github.com/hephbuild/heph/internal/hartifact" "github.com/hephbuild/heph/internal/hfs" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" @@ -37,8 +32,8 @@ type LocalCache interface { Exists(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (bool, error) Writer(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.WriteCloser, error) Delete(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) error - ListArtifacts(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) iter.Seq2[string, error] - ListVersions(ctx context.Context, ref *pluginv1.TargetRef, name string) iter.Seq2[string, error] + ListArtifacts(ctx context.Context, ref *pluginv1.TargetRef, hashin string) iter.Seq2[string, error] + ListVersions(ctx context.Context, ref *pluginv1.TargetRef) iter.Seq2[string, error] } func (e *Engine) hashout(ctx context.Context, ref *pluginv1.TargetRef, artifact pluginsdk.Artifact) (string, error) { @@ -97,7 +92,17 @@ func (e *Engine) cacheLocally( cacheArtifacts := make([]*ResultArtifact, 0, len(sandboxArtifacts)) for _, artifact := range sandboxArtifacts { - src, err := e.contentReaderNormalizer(artifact) + contentType, err := artifact.GetContentType() + if err != nil { + return nil, nil, err + } + + contentSize, err := artifact.GetContentSize() + if err != nil { + return nil, nil, err + } + + src, err := artifact.GetContentReader() if err != nil { return nil, nil, err } @@ -105,14 +110,14 @@ func (e *Engine) cacheLocally( cacheArtifact, err := e.cacheArtifactLocally(ctx, def.GetRef(), hashin, CacheLocallyArtifact{ Reader: src, - Size: 0, + Size: contentSize, Type: artifact.GetType(), Group: artifact.GetGroup(), Name: artifact.GetName(), - ContentType: artifact.GetContentType(), + ContentType: hartifact.ManifestArtifactContentType(contentType), }, artifact.Hashout) if err != nil { - return nil, nil, err + return nil, nil, fmt.Errorf("%q/%q: %w", artifact.GetGroup(), artifact.GetName(), err) } cacheArtifacts = append(cacheArtifacts, cacheArtifact) @@ -163,6 +168,8 @@ func (e *Engine) createLocalCacheManifest(ctx context.Context, ref *pluginv1.Tar type cachePluginArtifact struct { artifact hartifact.ManifestArtifact + ref *pluginv1.TargetRef + hashin string cache LocalCache } @@ -176,16 +183,12 @@ func (e cachePluginArtifact) GetType() pluginv1.Artifact_Type { return pluginv1.Artifact_Type(e.artifact.Type) } -func (e cachePluginArtifact) GetProto() *pluginv1.Artifact { - panic("TO REMOVE") -} - func (e cachePluginArtifact) GetContentReader() (io.ReadCloser, error) { - return hartifact.Reader(e) + return e.cache.Reader(context.TODO(), e.ref, e.hashin, e.artifact.Name) } func (e cachePluginArtifact) GetContentSize() (int64, error) { - return hartifact.Size(e) + return e.artifact.Size, nil } func (e cachePluginArtifact) GetContentType() (pluginsdk.ArtifactContentType, error) { @@ -194,84 +197,11 @@ func (e cachePluginArtifact) GetContentType() (pluginsdk.ArtifactContentType, er return pluginsdk.ArtifactContentTypeTar, nil case hartifact.ManifestArtifactContentTypeTarGz: return pluginsdk.ArtifactContentTypeTarGz, nil - case hartifact.ManifestArtifactContentTypeFile: - return pluginsdk.ArtifactContentTypeFile, nil - case hartifact.ManifestArtifactContentTypeRaw: - return pluginsdk.ArtifactContentTypeRaw, nil } return "", fmt.Errorf("invalid artifact content type: %q", e.artifact.ContentType) } -func (e *Engine) contentReaderNormalizer(artifact pluginsdk.Artifact) (io.ReadCloser, error) { - partifact := artifact.GetProto() - - if partifact.HasFile() { - content := partifact.GetFile() - - pr, pw := io.Pipe() - - p := htar.NewPacker(pw) - - sourcefs := hfs.NewOS(content.GetSourcePath()) - - go func() { - f, err := hfs.Open(sourcefs) - if err != nil { - _ = pr.CloseWithError(err) - - return - } - defer f.Close() - - err = p.WriteFile(f, content.GetOutPath()) - if err != nil { - _ = pr.CloseWithError(err) - - return - } - - _ = f.Close() - _ = pw.Close() - }() - - return pr, nil - } - - if partifact.HasRaw() { - content := partifact.GetRaw() - - pr, pw := io.Pipe() - - p := htar.NewPacker(pw) - - mode := int64(os.ModePerm) - if content.GetX() { - mode |= 0111 // executable - } - - go func() { - err := p.Write(bytes.NewReader(content.GetData()), &tar.Header{ - Typeflag: tar.TypeReg, - Name: content.GetPath(), - Size: int64(len(content.GetData())), - Mode: mode, - }) - if err != nil { - _ = pr.CloseWithError(err) - - return - } - }() - - _ = pw.Close() - - return pr, nil - } - - return artifact.GetContentReader() -} - func (e *Engine) ClearCacheLocally( ctx context.Context, ref *pluginv1.TargetRef, @@ -390,6 +320,8 @@ func (e *Engine) resultFromLocalCacheInner(ctx context.Context, def *LightLinked Hashout: artifact.Hashout, Artifact: cachePluginArtifact{ artifact: artifact, + ref: def.GetRef(), + hashin: hashin, cache: cache, }, Manifest: artifact, @@ -452,7 +384,7 @@ func (e *Engine) cacheArtifactLocally(ctx context.Context, ref *pluginv1.TargetR manifestArtifact := hartifact.ManifestArtifact{ Hashout: hashout, Group: art.Group, - Name: art.Name, + Name: prefix + art.Name, Size: art.Size, Type: hartifact.ManifestArtifactType(art.Type), ContentType: art.ContentType, @@ -462,6 +394,8 @@ func (e *Engine) cacheArtifactLocally(ctx context.Context, ref *pluginv1.TargetR Hashout: hashout, Artifact: cachePluginArtifact{ artifact: manifestArtifact, + ref: ref, + hashin: hashin, cache: cache, }, Manifest: manifestArtifact, diff --git a/internal/engine/remote_cache.go b/internal/engine/remote_cache.go index 4687b4e6..4f4cf868 100644 --- a/internal/engine/remote_cache.go +++ b/internal/engine/remote_cache.go @@ -77,7 +77,7 @@ func (e *Engine) cacheRemotelyInner(ctx context.Context, for _, artifact := range artifacts { g.Go(func(ctx context.Context) error { - r, err := hartifact.Reader(artifact.Artifact) + r, err := artifact.GetContentReader() if err != nil { return err } diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index 19e8c847..62d9e6c8 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -277,15 +277,9 @@ func (e *Engine) result(ctx context.Context, rs *RequestState, c DefContainer, o } if onlyManifest { - res.Artifacts = slices.DeleteFunc(res.Artifacts, func(artifact *ResultArtifact) bool { - return artifact.GetType() != pluginv1.Artifact_TYPE_MANIFEST_V1 - }) + res.Artifacts = nil } else { res.Artifacts = slices.DeleteFunc(res.Artifacts, func(artifact *ResultArtifact) bool { - if artifact.GetType() == pluginv1.Artifact_TYPE_MANIFEST_V1 { - return true - } - if artifact.GetType() == pluginv1.Artifact_TYPE_SUPPORT_FILE { return false } @@ -651,10 +645,7 @@ func (e *Engine) innerResult(ctx context.Context, rs *RequestState, def *LightLi var locks CacheLocks for _, result := range results { for _, artifact := range result.Artifacts { - gartifact := artifact.GetProto() - gartifact.ClearGroup() // TODO support output group - - partifact := protoPluginArtifact{Artifact: gartifact} + partifact := artifactGroupMap{Artifact: artifact, group: ""} // TODO support output group artifacts = append(artifacts, &ResultArtifact{ Hashout: artifact.Hashout, @@ -868,19 +859,7 @@ func (r Result) Clone() *Result { } } -type ExecuteArtifact struct { - Hashout string - pluginsdk.Artifact -} - -type ResultArtifact struct { - Hashout string - Manifest hartifact.ManifestArtifact - - pluginsdk.Artifact -} - -func (r ExecuteResult) FindOutputs(group string) []*ResultArtifact { +func (r Result) FindOutputs(group string) []*ResultArtifact { res := make([]*ResultArtifact, 0, len(r.Artifacts)) for _, artifact := range r.Artifacts { if artifact.GetType() != pluginv1.Artifact_TYPE_OUTPUT { @@ -896,7 +875,7 @@ func (r ExecuteResult) FindOutputs(group string) []*ResultArtifact { return res } -func (r ExecuteResult) FindSupport() []*ResultArtifact { +func (r Result) FindSupport() []*ResultArtifact { res := make([]*ResultArtifact, 0, len(r.Artifacts)) for _, artifact := range r.Artifacts { if artifact.GetType() != pluginv1.Artifact_TYPE_SUPPORT_FILE { @@ -909,6 +888,18 @@ func (r ExecuteResult) FindSupport() []*ResultArtifact { return res } +type ExecuteArtifact struct { + Hashout string + pluginsdk.Artifact +} + +type ResultArtifact struct { + Hashout string + Manifest hartifact.ManifestArtifact + + pluginsdk.Artifact +} + type ExecuteResultLocks struct { *Result Locks *CacheLocks @@ -1206,12 +1197,19 @@ func (e *Engine) execute(ctx context.Context, rs *RequestState, def *LightLinked inputLen += len(result.Artifacts) } - inputs := make([]*pluginv1.ArtifactWithOrigin, 0, inputLen) + inputs := make([]*pluginv1.RunRequest_Input, 0, inputLen) inputsSdk := make([]*pluginsdk.ArtifactWithOrigin, 0, inputLen) for _, result := range results { for _, artifact := range result.Artifacts { - inputs = append(inputs, pluginv1.ArtifactWithOrigin_builder{ - Artifact: artifact.GetProto(), + partifact := pluginv1.RunRequest_Input_Artifact_builder{ + Group: htypes.Ptr(artifact.GetGroup()), + Name: htypes.Ptr(artifact.GetName()), + Type: htypes.Ptr(artifact.GetType()), + Id: htypes.Ptr("TODO"), // to be implemented along with the pluginsdkconnect.ProtoArtifact + }.Build() + + inputs = append(inputs, pluginv1.RunRequest_Input_builder{ + Artifact: partifact, Origin: result.InputOrigin, }.Build()) @@ -1361,7 +1359,7 @@ func (e *Engine) execute(ctx context.Context, rs *RequestState, def *LightLinked return nil, err } - execArtifact := protoPluginArtifact{ + execArtifact := pluginsdk.ProtoArtifact{ Artifact: pluginv1.Artifact_builder{ Group: htypes.Ptr(output.GetGroup()), Name: htypes.Ptr(tarname), @@ -1437,7 +1435,7 @@ func (e *Engine) execute(ctx context.Context, rs *RequestState, def *LightLinked return nil, err } - execArtifact := protoPluginArtifact{ + execArtifact := pluginsdk.ProtoArtifact{ Artifact: pluginv1.Artifact_builder{ Name: htypes.Ptr(tarname), Type: htypes.Ptr(pluginv1.Artifact_TYPE_SUPPORT_FILE), @@ -1462,7 +1460,7 @@ func (e *Engine) execute(ctx context.Context, rs *RequestState, def *LightLinked continue } - artifact := protoPluginArtifact{ + artifact := pluginsdk.ProtoArtifact{ Artifact: partifact, } diff --git a/internal/enginee2e/deps_cache_test.go b/internal/enginee2e/deps_cache_test.go index 15b094bb..3ab5f546 100644 --- a/internal/enginee2e/deps_cache_test.go +++ b/internal/enginee2e/deps_cache_test.go @@ -111,6 +111,90 @@ func TestDepsCache(t *testing.T) { } } +func TestDepsCacheLarge(t *testing.T) { + ctx := t.Context() + + dir := t.TempDir() + + e, err := engine.New(ctx, dir, engine.Config{}) + require.NoError(t, err) + + staticprovider := pluginstaticprovider.New([]pluginstaticprovider.Target{ + { + Spec: pluginv1.TargetSpec_builder{ + Ref: pluginv1.TargetRef_builder{ + Package: htypes.Ptr(""), + Name: htypes.Ptr("child"), + }.Build(), + Driver: htypes.Ptr("bash"), + Config: map[string]*structpb.Value{ + "run": hstructpb.NewStringsValue([]string{`dd if=/dev/zero of=$OUT bs=1k count=200`}), + "out": hstructpb.NewStringsValue([]string{"out_child"}), + }, + }.Build(), + }, + { + Spec: pluginv1.TargetSpec_builder{ + Ref: pluginv1.TargetRef_builder{ + Package: htypes.Ptr(""), + Name: htypes.Ptr("parent"), + }.Build(), + Driver: htypes.Ptr("bash"), + Config: map[string]*structpb.Value{ + "run": hstructpb.NewStringsValue([]string{`wc -c < "${SRC}" | tr -d ' ' > $OUT`}), + "deps": hstructpb.NewStringsValue([]string{"//:child"}), + "out": hstructpb.NewStringsValue([]string{"out_parent"}), + }, + }.Build(), + }, + }) + + _, err = e.RegisterProvider(ctx, staticprovider, engine.RegisterProviderConfig{}) + require.NoError(t, err) + + _, err = e.RegisterDriver(ctx, pluginexec.NewExec(), nil) + require.NoError(t, err) + _, err = e.RegisterDriver(ctx, pluginexec.NewSh(), nil) + require.NoError(t, err) + _, err = e.RegisterDriver(ctx, pluginexec.NewBash(), nil) + require.NoError(t, err) + + assertOut := func(res *engine.ExecuteResultLocks) { + fs := hfstest.New(t) + err = hartifact.Unpack(ctx, res.FindOutputs("")[0].Artifact, fs) + require.NoError(t, err) + + b, err := hfs.ReadFile(fs.At("out_parent")) + require.NoError(t, err) + + assert.Equal(t, "204800\n", string(b)) + } + + { // This will run all + rs, clean := e.NewRequestState() + defer clean() + + res, err := e.Result(ctx, rs, "", "parent", []string{engine.AllOutputs}) + require.NoError(t, err) + defer res.Unlock(ctx) + + assertOut(res) + res.Unlock(ctx) + } + + { // this should reuse cache from deps + rs, clean := e.NewRequestState() + defer clean() + + res, err := e.Result(ctx, rs, "", "parent", []string{engine.AllOutputs}) + require.NoError(t, err) + defer res.Unlock(ctx) + + assertOut(res) + res.Unlock(ctx) + } +} + func TestDepsCache2(t *testing.T) { ctx := t.Context() c := gomock.NewController(t) @@ -193,7 +277,7 @@ func TestDepsCache2(t *testing.T) { cache.EXPECT(). Get(gomock.Any(), "__child/e5d50f4b478a3687/manifest.v1.json"). - Return(nil, 0, pluginsdk.ErrNotFound).Times(1) + Return(nil, pluginsdk.ErrNotFound).Times(1) for _, key := range []string{"__child/e5d50f4b478a3687/manifest.v1.json", "__child/e5d50f4b478a3687/out_out.tar"} { cache.EXPECT(). @@ -206,7 +290,7 @@ func TestDepsCache2(t *testing.T) { cache.EXPECT(). Get(gomock.Any(), key). - Return(io.NopCloser(bytes.NewReader(b)), int64(len(b)), nil).AnyTimes() + Return(io.NopCloser(bytes.NewReader(b)), nil).AnyTimes() return nil }).Times(1) diff --git a/internal/enginee2e/hash_deps_test.go b/internal/enginee2e/hash_deps_test.go index 285e0185..a490cc0d 100644 --- a/internal/enginee2e/hash_deps_test.go +++ b/internal/enginee2e/hash_deps_test.go @@ -67,7 +67,7 @@ func TestHashDeps(t *testing.T) { m, err := e.ResultMetaFromRef(ctx, rs, tref.New("some/package", "sometarget", nil), nil) require.NoError(t, err) - at = m.CreatedAt + at = m.CreatedAt.UTC() res.Unlock(ctx) } diff --git a/internal/hartifact/manifest.go b/internal/hartifact/manifest.go index 500b96ae..976f0712 100644 --- a/internal/hartifact/manifest.go +++ b/internal/hartifact/manifest.go @@ -1,14 +1,11 @@ package hartifact import ( - "context" - "encoding/base64" "encoding/json" "fmt" "io" "time" - "github.com/hephbuild/heph/internal/htypes" "github.com/hephbuild/heph/lib/pluginsdk" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" @@ -48,8 +45,6 @@ type ManifestArtifactContentType string const ( ManifestArtifactContentTypeTar ManifestArtifactContentType = "application/x-tar" ManifestArtifactContentTypeTarGz ManifestArtifactContentType = "application/x-gtar" - ManifestArtifactContentTypeFile ManifestArtifactContentType = "file" - ManifestArtifactContentTypeRaw ManifestArtifactContentType = "raw" ) type ManifestArtifact struct { @@ -92,16 +87,6 @@ func EncodeManifest(w io.Writer, m *Manifest) error { return nil } -func ManifestFromArtifact(ctx context.Context, a pluginsdk.Artifact) (*Manifest, error) { - r, err := FileReader(ctx, a) - if err != nil { - return nil, err - } - defer r.Close() - - return DecodeManifest(r) -} - func DecodeManifest(r io.Reader) (*Manifest, error) { var manifest Manifest err := json.NewDecoder(r).Decode(&manifest) //nolint:musttag @@ -123,10 +108,6 @@ func ManifestContentType(a pluginsdk.Artifact) (ManifestArtifactContentType, err return ManifestArtifactContentTypeTarGz, nil case pluginsdk.ArtifactContentTypeTar: return ManifestArtifactContentTypeTar, nil - case pluginsdk.ArtifactContentTypeFile: - return ManifestArtifactContentTypeFile, nil - case pluginsdk.ArtifactContentTypeRaw: - return ManifestArtifactContentTypeRaw, nil default: } @@ -153,39 +134,3 @@ func ProtoArtifactToManifest(hashout string, a pluginsdk.Artifact) (ManifestArti ContentType: contentType, }, nil } - -func ManifestArtifactToProto(artifact ManifestArtifact, path string) (*pluginv1.Artifact, error) { - partifact := pluginv1.Artifact_builder{ - Group: htypes.Ptr(artifact.Group), - Name: htypes.Ptr(artifact.Name), - Type: htypes.Ptr(pluginv1.Artifact_Type(artifact.Type)), - }.Build() - - switch artifact.ContentType { - case ManifestArtifactContentTypeTar: - partifact.SetTarPath(path) - case ManifestArtifactContentTypeTarGz: - partifact.SetTargzPath(path) - case ManifestArtifactContentTypeFile: - partifact.SetFile(pluginv1.Artifact_ContentFile_builder{ - SourcePath: &path, - OutPath: &artifact.OutPath, - X: &artifact.X, - }.Build()) - case ManifestArtifactContentTypeRaw: - b, err := base64.StdEncoding.DecodeString(path) - if err != nil { - return nil, err - } - - partifact.SetRaw(pluginv1.Artifact_ContentRaw_builder{ - Data: b, - Path: &artifact.OutPath, - X: &artifact.X, - }.Build()) - default: - return nil, fmt.Errorf("unsupported content type %q", artifact.ContentType) - } - - return partifact, nil -} diff --git a/internal/hartifact/reader.go b/internal/hartifact/reader.go index b97b5b9a..6a6b8f3e 100644 --- a/internal/hartifact/reader.go +++ b/internal/hartifact/reader.go @@ -2,75 +2,26 @@ package hartifact import ( "archive/tar" - "bytes" "context" "fmt" "io" "iter" - "os" "github.com/hephbuild/heph/internal/hio" "github.com/hephbuild/heph/internal/htar" "github.com/hephbuild/heph/lib/pluginsdk" - - pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" ) -// Reader gives a raw io.Reader of an artifact, useful for things like hashing. -func Reader(a pluginsdk.Artifact) (io.ReadCloser, error) { - ap := a.GetProto() - - switch ap.WhichContent() { - case pluginv1.Artifact_File_case: - return os.Open(ap.GetFile().GetSourcePath()) - case pluginv1.Artifact_Raw_case: - return io.NopCloser(bytes.NewReader(ap.GetRaw().GetData())), nil - case pluginv1.Artifact_TargzPath_case: - return os.Open(ap.GetTargzPath()) - case pluginv1.Artifact_TarPath_case: - return os.Open(ap.GetTarPath()) - default: - return nil, fmt.Errorf("unsupported encoding %v", ap.WhichContent()) - } -} - -func fileSize(p string) (int64, error) { - info, err := os.Stat(p) +// FileReader Assumes the output has a single file, and provides a reader for it (no matter the packaging). +func FileReader(ctx context.Context, a pluginsdk.Artifact) (io.ReadCloser, error) { + contentType, err := a.GetContentType() if err != nil { - return 0, err - } - - return info.Size(), nil -} - -func Size(a pluginsdk.Artifact) (int64, error) { - ap := a.GetProto() - - switch ap.WhichContent() { - case pluginv1.Artifact_File_case: - return fileSize(ap.GetFile().GetSourcePath()) - case pluginv1.Artifact_Raw_case: - return int64(len(ap.GetRaw().GetData())), nil - case pluginv1.Artifact_TargzPath_case: - return fileSize(ap.GetTargzPath()) - case pluginv1.Artifact_TarPath_case: - return fileSize(ap.GetTarPath()) - default: - return -1, fmt.Errorf("unsupported encoding %v", ap.WhichContent()) + return nil, fmt.Errorf("get content type: %w", err) } -} -// FileReader Assumes the output has a single file, and provides a reader for it (no matter the packaging). -func FileReader(ctx context.Context, a pluginsdk.Artifact) (io.ReadCloser, error) { - ap := a.GetProto() - - switch ap.WhichContent() { - case pluginv1.Artifact_File_case: - return os.Open(ap.GetFile().GetSourcePath()) - case pluginv1.Artifact_Raw_case: - return io.NopCloser(bytes.NewReader(ap.GetRaw().GetData())), nil - case pluginv1.Artifact_TarPath_case: - r, err := Reader(a) + switch contentType { + case pluginsdk.ArtifactContentTypeTar: + r, err := a.GetContentReader() if err != nil { return nil, err } @@ -83,9 +34,9 @@ func FileReader(ctx context.Context, a pluginsdk.Artifact) (io.ReadCloser, error } return hio.NewReadCloser(tr, r), nil - // case pluginsdk.Artifact_TargzPath: + //case pluginsdk.ArtifactContentTypeTarGz: default: - return nil, fmt.Errorf("unsupported encoding %v", ap.WhichContent()) + return nil, fmt.Errorf("unsupported encoding %v", contentType) } } @@ -116,11 +67,9 @@ func FilesReader(ctx context.Context, a pluginsdk.Artifact) iter.Seq2[*File, err switch contentType { case pluginsdk.ArtifactContentTypeTar: - r, err := Reader(a) + r, err := a.GetContentReader() if err != nil { - if !yield(nil, err) { - return - } + yield(nil, err) return } defer r.Close() @@ -142,16 +91,15 @@ func FilesReader(ctx context.Context, a pluginsdk.Artifact) iter.Seq2[*File, err return nil }) if err != nil { - if !yield(nil, err) { - return - } + yield(nil, err) return } + + return // case pluginsdk.Artifact_TargzPath: default: - if !yield(nil, fmt.Errorf("unsupported encoding %v", contentType)) { - return - } + yield(nil, fmt.Errorf("unsupported encoding %v", contentType)) + return } } } diff --git a/internal/hartifact/unpack.go b/internal/hartifact/unpack.go index e8a85133..4fe13390 100644 --- a/internal/hartifact/unpack.go +++ b/internal/hartifact/unpack.go @@ -3,13 +3,11 @@ package hartifact import ( "context" "fmt" - "io" "github.com/hephbuild/heph/internal/htar" "github.com/hephbuild/heph/lib/pluginsdk" "github.com/hephbuild/heph/internal/hfs" - pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" ) type unpackConfig struct { @@ -45,76 +43,27 @@ func Unpack(ctx context.Context, artifact pluginsdk.Artifact, node hfs.Node, opt } } - r, err := Reader(artifact) + r, err := artifact.GetContentReader() if err != nil { return err } defer r.Close() - artifactp := artifact.GetProto() - - switch artifactp.WhichContent() { - case pluginv1.Artifact_File_case: - if !cfg.filter(artifactp.GetFile().GetOutPath()) { - return nil - } - - create := hfs.Create - if artifactp.GetFile().GetX() { - create = hfs.CreateExec - } - - f, err := create(node.At(artifactp.GetFile().GetOutPath())) - if err != nil { - return fmt.Errorf("file: create: %w", err) - } - defer cfg.onFile(f.Name()) - defer f.Close() - - _, err = io.Copy(f, r) - if err != nil { - return err - } - - err = hfs.CloseEnsureROFD(f) - if err != nil { - return err - } - case pluginv1.Artifact_Raw_case: - if !cfg.filter(artifactp.GetRaw().GetPath()) { - return nil - } - - create := hfs.Create - if artifactp.GetRaw().GetX() { - create = hfs.CreateExec - } - - f, err := create(node.At(artifactp.GetRaw().GetPath())) - if err != nil { - return fmt.Errorf("raw: create: %w", err) - } - defer cfg.onFile(f.Name()) - defer f.Close() - - _, err = io.Copy(f, r) - if err != nil { - return err - } + contentType, err := artifact.GetContentType() + if err != nil { + return err + } - err = hfs.CloseEnsureROFD(f) - if err != nil { - return err - } - case pluginv1.Artifact_TarPath_case: + switch contentType { + case pluginsdk.ArtifactContentTypeTar: err = htar.Unpack(ctx, r, node, htar.WithOnFile(cfg.onFile), htar.WithFilter(cfg.filter)) if err != nil { return fmt.Errorf("tar: %w", err) } + + return nil // case *pluginv1.Artifact_TargzPath: default: - return fmt.Errorf("unsupported encoding %v", artifactp.WhichContent()) + return fmt.Errorf("unsupported encoding %v", contentType) } - - return nil } diff --git a/internal/remotecache/gcs.go b/internal/remotecache/gcs.go index ce0e9bfa..d832bb8f 100644 --- a/internal/remotecache/gcs.go +++ b/internal/remotecache/gcs.go @@ -60,19 +60,10 @@ func (g GCS) Store(ctx context.Context, key string, r io.Reader) error { func (g GCS) Get(ctx context.Context, key string) (io.ReadCloser, error) { obj := g.bucket.Object(key) - attr, err := obj.Attrs(ctx) - if err != nil { - if errors.Is(err, storage.ErrObjectNotExist) { - return nil, 0, pluginsdk.ErrCacheNotFound - } - - return nil, err - } - r, err := obj.NewReader(ctx) if err != nil { if errors.Is(err, storage.ErrObjectNotExist) { - return nil, 0, pluginsdk.ErrCacheNotFound + return nil, pluginsdk.ErrCacheNotFound } return nil, err diff --git a/lib/pluginsdk/artifact.go b/lib/pluginsdk/artifact.go index 64aa767c..71a6522f 100644 --- a/lib/pluginsdk/artifact.go +++ b/lib/pluginsdk/artifact.go @@ -1,7 +1,6 @@ package pluginsdk import ( - "fmt" "io" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" @@ -21,38 +20,6 @@ type Artifact interface { GetContentReader() (io.ReadCloser, error) GetContentSize() (int64, error) GetContentType() (ArtifactContentType, error) - - GetProto() *pluginv1.Artifact -} - -var _ Artifact = (*ProtoArtifact)(nil) - -type ProtoArtifact struct { - *pluginv1.Artifact - ContentReaderFunc func(e ProtoArtifact) (io.ReadCloser, error) - ContentSizeFunc func(e ProtoArtifact) (int64, error) -} - -func (e ProtoArtifact) GetProto() *pluginv1.Artifact { - return e.Artifact -} - -func (e ProtoArtifact) GetContentReader() (io.ReadCloser, error) { - return e.ContentReaderFunc(e) -} -func (e ProtoArtifact) GetContentSize() (int64, error) { - return e.ContentSizeFunc(e) -} -func (e ProtoArtifact) GetContentType() (ArtifactContentType, error) { - switch e.Artifact.WhichContent() { - case pluginv1.Artifact_TargzPath_case: - return ArtifactContentTypeTarGz, nil - case pluginv1.Artifact_TarPath_case: - return ArtifactContentTypeTar, nil - default: - } - - return "", fmt.Errorf("unsupported content %v", e.WhichContent().String()) } type ArtifactWithOrigin struct { diff --git a/lib/pluginsdk/artifact_proto.go b/lib/pluginsdk/artifact_proto.go new file mode 100644 index 00000000..aa07f5c4 --- /dev/null +++ b/lib/pluginsdk/artifact_proto.go @@ -0,0 +1,151 @@ +package pluginsdk + +import ( + "archive/tar" + "bytes" + "fmt" + "io" + "os" + + "github.com/hephbuild/heph/internal/hfs" + "github.com/hephbuild/heph/internal/htar" + pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" +) + +type ProtoArtifact struct { + *pluginv1.Artifact +} + +var _ Artifact = (*ProtoArtifact)(nil) + +func (a ProtoArtifact) GetContentReader() (io.ReadCloser, error) { + partifact := a.Artifact + + switch partifact.WhichContent() { + case pluginv1.Artifact_File_case: + content := partifact.GetFile() + + pr, pw := io.Pipe() + + sourcefs := hfs.NewOS(content.GetSourcePath()) + + go func() { + defer pw.Close() + + tarPacker := htar.NewPacker(pw) + + f, err := hfs.Open(sourcefs) + if err != nil { + _ = pr.CloseWithError(err) + + return + } + defer f.Close() + + err = tarPacker.WriteFile(f, content.GetOutPath()) + if err != nil { + _ = pr.CloseWithError(err) + + return + } + + _ = f.Close() + }() + + return pr, nil + case pluginv1.Artifact_Raw_case: + content := partifact.GetRaw() + + pr, pw := io.Pipe() + + go func() { + defer pw.Close() + + mode := int64(os.ModePerm) + if content.GetX() { + mode |= 0111 // executable + } + + tarPacker := htar.NewPacker(pw) + + err := tarPacker.Write(bytes.NewReader(content.GetData()), &tar.Header{ + Typeflag: tar.TypeReg, + Name: content.GetPath(), + Size: int64(len(content.GetData())), + Mode: mode, + }) + if err != nil { + _ = pr.CloseWithError(err) + + return + } + }() + + return pr, nil + case pluginv1.Artifact_TargzPath_case: + return os.Open(partifact.GetTargzPath()) + case pluginv1.Artifact_TarPath_case: + return os.Open(partifact.GetTarPath()) + default: + return nil, fmt.Errorf("unsupported encoding %v", partifact.WhichContent()) + } +} + +func (a ProtoArtifact) GetContentSize() (int64, error) { + partifact := a.Artifact + + switch partifact.WhichContent() { + case pluginv1.Artifact_File_case: + content := partifact.GetFile() + + sourcefs := hfs.NewOS(content.GetSourcePath()) + + info, err := sourcefs.Lstat() + if err != nil { + return 0, err + } + + return info.Size(), nil + case pluginv1.Artifact_Raw_case: + content := partifact.GetRaw() + + return int64(len(content.GetData())), nil + case pluginv1.Artifact_TargzPath_case: + sourcefs := hfs.NewOS(partifact.GetTargzPath()) + + info, err := sourcefs.Lstat() + if err != nil { + return 0, err + } + + return info.Size(), nil + case pluginv1.Artifact_TarPath_case: + sourcefs := hfs.NewOS(partifact.GetTarPath()) + + info, err := sourcefs.Lstat() + if err != nil { + return 0, err + } + + return info.Size(), nil + default: + return -1, fmt.Errorf("unsupported encoding %v", partifact.WhichContent()) + } +} + +func (a ProtoArtifact) GetContentType() (ArtifactContentType, error) { + partifact := a.Artifact + + switch partifact.WhichContent() { + case pluginv1.Artifact_File_case: + return ArtifactContentTypeTar, nil + case pluginv1.Artifact_Raw_case: + return ArtifactContentTypeTar, nil + case pluginv1.Artifact_TargzPath_case: + return ArtifactContentTypeTarGz, nil + case pluginv1.Artifact_TarPath_case: + return ArtifactContentTypeTar, nil + default: + return "", fmt.Errorf("unsupported encoding %v", partifact.WhichContent()) + } +} diff --git a/lib/pluginsdk/pluginsdkconnect/artifact_proto.go b/lib/pluginsdk/pluginsdkconnect/artifact_proto.go new file mode 100644 index 00000000..5db2b904 --- /dev/null +++ b/lib/pluginsdk/pluginsdkconnect/artifact_proto.go @@ -0,0 +1,27 @@ +package pluginsdkconnect + +import ( + "io" + "math" + + "github.com/hephbuild/heph/lib/pluginsdk" + pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" +) + +type ProtoArtifact struct { + *pluginv1.RunRequest_Input_Artifact +} + +var _ pluginsdk.Artifact = (*ProtoArtifact)(nil) + +func (a ProtoArtifact) GetContentReader() (io.ReadCloser, error) { + panic("TODO: implement call from plugin to core to get the content") +} + +func (a ProtoArtifact) GetContentSize() (int64, error) { + return math.MaxInt64, nil // will not be used +} + +func (a ProtoArtifact) GetContentType() (pluginsdk.ArtifactContentType, error) { + panic("TODO: implement call from plugin to core to get the content") +} diff --git a/lib/pluginsdk/pluginsdkconnect/plugin_driver.go b/lib/pluginsdk/pluginsdkconnect/plugin_driver.go index fd96f22c..d14d4e87 100644 --- a/lib/pluginsdk/pluginsdkconnect/plugin_driver.go +++ b/lib/pluginsdk/pluginsdkconnect/plugin_driver.go @@ -3,10 +3,8 @@ package pluginsdkconnect import ( "context" "errors" - "io" "connectrpc.com/connect" - "github.com/hephbuild/heph/internal/hartifact" "github.com/hephbuild/heph/internal/hcore/hlog" "github.com/hephbuild/heph/lib/pluginsdk" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" @@ -143,14 +141,8 @@ func (p driverConnectHandler) runRequest(rr *pluginv1.RunRequest) *pluginsdk.Run inputs := make([]*pluginsdk.ArtifactWithOrigin, 0, len(rr.GetInputs())) for _, input := range rr.GetInputs() { inputs = append(inputs, &pluginsdk.ArtifactWithOrigin{ - Artifact: pluginsdk.ProtoArtifact{ - Artifact: input.GetArtifact(), - ContentReaderFunc: func(e pluginsdk.ProtoArtifact) (io.ReadCloser, error) { - return hartifact.Reader(e) - }, - ContentSizeFunc: func(e pluginsdk.ProtoArtifact) (int64, error) { - return hartifact.Size(e) - }, + Artifact: ProtoArtifact{ + RunRequest_Input_Artifact: input.GetArtifact(), }, Origin: input.GetOrigin(), }) diff --git a/plugin/pluginbin/plugin.go b/plugin/pluginbin/plugin.go index 92affa51..518a518b 100644 --- a/plugin/pluginbin/plugin.go +++ b/plugin/pluginbin/plugin.go @@ -22,7 +22,7 @@ type Plugin struct { const Name = "bin" func (p Plugin) Pipe(ctx context.Context, request *pluginv1.PipeRequest) (*pluginv1.PipeResponse, error) { - return nil, connect.NewError(connect.CodeUnimplemented, errors.New("not implemented")) + return nil, pluginsdk.ErrNotImplemented } func (p Plugin) Config(ctx context.Context, request *pluginv1.ConfigRequest) (*pluginv1.ConfigResponse, error) { diff --git a/plugin/pluginexec/run.go b/plugin/pluginexec/run.go index 9cb41baf..f96546f4 100644 --- a/plugin/pluginexec/run.go +++ b/plugin/pluginexec/run.go @@ -416,10 +416,6 @@ func (p *Plugin[S]) inputEnv(ctx context.Context, inputs []*pluginsdk.ArtifactWi return v } - if v := hproto.Compare(a.GetProto(), b.GetProto(), nil); v != 0 { - return v - } - return 0 }) diff --git a/plugin/pluginexec/sandbox.go b/plugin/pluginexec/sandbox.go index 247d5461..2fda55e9 100644 --- a/plugin/pluginexec/sandbox.go +++ b/plugin/pluginexec/sandbox.go @@ -4,13 +4,12 @@ import ( "context" "encoding/hex" "fmt" - "io" "iter" "os" "path/filepath" "slices" + "sync/atomic" - "github.com/hephbuild/heph/internal/hproto/hashpb" "github.com/hephbuild/heph/internal/htypes" "github.com/hephbuild/heph/lib/pluginsdk" @@ -152,15 +151,14 @@ func ArtifactsForId(inputs []*pluginsdk.ArtifactWithOrigin, id string, typ plugi } } +var artifactId atomic.Int64 + func SetupSandboxArtifact(ctx context.Context, artifact pluginsdk.Artifact, source *execv1.Target_Dep, node hfs.Node, filters []string, sourcemap map[string]string) (pluginsdk.Artifact, error) { ctx, span := tracer.Start(ctx, "SetupSandboxArtifact") defer span.End() h := xxh3.New() - hashpb.Hash(h, artifact.GetProto(), tref.OmitHashPb) - for _, f := range filters { - _, _ = h.WriteString(f) - } + _, _ = fmt.Fprintf(h, "%d", artifactId.Add(1)) listf, err := hfs.Create(node.At(hex.EncodeToString(h.Sum(nil)) + ".list")) if err != nil { @@ -216,12 +214,6 @@ func SetupSandboxArtifact(ctx context.Context, artifact pluginsdk.Artifact, sour SourcePath: htypes.Ptr(listf.Name()), }.Build(), }.Build(), - ContentReaderFunc: func(e pluginsdk.ProtoArtifact) (io.ReadCloser, error) { - return hartifact.Reader(e) - }, - ContentSizeFunc: func(e pluginsdk.ProtoArtifact) (int64, error) { - return hartifact.Size(e) - }, }, nil } @@ -271,11 +263,5 @@ func SetupSandboxBinArtifact(ctx context.Context, artifact pluginsdk.Artifact, n Path: htypes.Ptr(artifact.GetName() + ".list"), }.Build(), }.Build(), - ContentReaderFunc: func(e pluginsdk.ProtoArtifact) (io.ReadCloser, error) { - return hartifact.Reader(e) - }, - ContentSizeFunc: func(e pluginsdk.ProtoArtifact) (int64, error) { - return hartifact.Size(e) - }, }, nil } diff --git a/plugin/proto/heph/plugin/v1/driver.proto b/plugin/proto/heph/plugin/v1/driver.proto index 5af59c7e..365dcb66 100644 --- a/plugin/proto/heph/plugin/v1/driver.proto +++ b/plugin/proto/heph/plugin/v1/driver.proto @@ -25,17 +25,24 @@ message ApplyTransitiveResponse { TargetDef target = 1; } -message ArtifactWithOrigin { - Artifact artifact = 1; - TargetDef.InputOrigin origin = 2; -} - message RunRequest { + message Input { + message Artifact { + string group = 1; + string name = 2; + heph.plugin.v1.Artifact.Type type = 3; + string id = 4; + } + + Artifact artifact = 1; + TargetDef.InputOrigin origin = 2; + } + string request_id = 1; TargetDef target = 2; string sandbox_path = 3; string tree_root_path = 4; - repeated ArtifactWithOrigin inputs = 5; + repeated Input inputs = 5; repeated string pipes = 6; string hashin = 7; } @@ -53,7 +60,6 @@ message Artifact { TYPE_OUTPUT = 1; TYPE_OUTPUT_LIST_V1 = 2; TYPE_LOG = 3; - TYPE_MANIFEST_V1 = 4; TYPE_SUPPORT_FILE = 5; TYPE_SUPPORT_FILE_LIST_V1 = 6; } From c9f7fc1096f08fa82457515c9a4d3bac700b340a Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Sun, 8 Mar 2026 01:17:44 +0000 Subject: [PATCH 04/15] working! --- internal/cmd/inspect_cache.go | 132 ++++++++++++++++++++++++++ internal/cmd/run.go | 6 +- internal/engine/cache_sql.go | 85 +++++++++-------- internal/engine/cache_sql_test.go | 6 +- internal/engine/local_cache.go | 6 +- internal/engine/query.go | 2 +- internal/engine/remote_cache.go | 6 +- internal/engine/schedule.go | 9 +- internal/enginee2e/cache_test.go | 90 ++++++++++++++++++ internal/enginee2e/deps_cache_test.go | 2 +- internal/hartifact/reader.go | 2 +- internal/hiter/group.go | 4 +- internal/htar/pack.go | 22 ++--- internal/htar/unpack.go | 73 ++++++++------ lib/pluginsdk/artifact_proto.go | 13 +-- plugin/pluginexec/run.go | 3 + plugin/pluginexec/sandbox.go | 8 +- plugin/pluginfs/driver.go | 7 +- plugin/plugingo/pkg_analysis.go | 4 +- 19 files changed, 360 insertions(+), 120 deletions(-) create mode 100644 internal/cmd/inspect_cache.go create mode 100644 internal/enginee2e/cache_test.go diff --git a/internal/cmd/inspect_cache.go b/internal/cmd/inspect_cache.go new file mode 100644 index 00000000..9e93036c --- /dev/null +++ b/internal/cmd/inspect_cache.go @@ -0,0 +1,132 @@ +package cmd + +import ( + "errors" + "fmt" + + "github.com/hephbuild/heph/internal/engine" + "github.com/spf13/cobra" +) + +var cacheCmd *cobra.Command + +func init() { + cacheCmd = &cobra.Command{ + Use: "cache", + Short: "Cache utilities", + } + + inspectCmd.AddCommand(cacheCmd) +} + +func init() { + cmdArgs := parseRefArgs{cmdName: "list-versions"} + + cmd := &cobra.Command{ + Use: cmdArgs.Use(), + Short: "List versions for hashin", + Args: cmdArgs.Args(), + ValidArgsFunction: cmdArgs.ValidArgsFunction(), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + ctx, stop := newSignalNotifyContext(ctx) + defer stop() + + cwd, err := engine.Cwd() + if err != nil { + return err + } + + root, err := engine.Root() + if err != nil { + return err + } + + ref, err := cmdArgs.Parse(args[0], cwd, root) + if err != nil { + return err + } + + e, err := newEngine(ctx, root) + if err != nil { + return err + } + + for hashin, err := range e.CacheSmall.ListVersions(ctx, ref) { + if err != nil { + return err + } + + fmt.Println(hashin) + } + + return nil + }, + } + + cacheCmd.AddCommand(cmd) +} + +func init() { + cmdArgs := parseRefArgs{cmdName: "list-artifacts"} + + cmd := &cobra.Command{ + Use: cmdArgs.Use(), + Short: "List artifacts at revision", + Args: func(cmd *cobra.Command, args []string) error { + switch len(args) { + case 2: + return nil + default: + return errors.New("must be `list-artifacts `") + } + }, + ValidArgsFunction: cmdArgs.ValidArgsFunction(), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + ctx, stop := newSignalNotifyContext(ctx) + defer stop() + + cwd, err := engine.Cwd() + if err != nil { + return err + } + + root, err := engine.Root() + if err != nil { + return err + } + + ref, err := cmdArgs.Parse(args[0], cwd, root) + if err != nil { + return err + } + + hashin := args[1] + + e, err := newEngine(ctx, root) + if err != nil { + return err + } + + for i, cache := range [...]engine.LocalCache{e.CacheSmall, e.CacheLarge} { + fmt.Println("======", i) + + for name, err := range cache.ListArtifacts(ctx, ref, hashin) { + if err != nil { + return err + } + + fmt.Println(name) + } + + } + + return nil + }, + } + + cacheCmd.AddCommand(cmd) +} diff --git a/internal/cmd/run.go b/internal/cmd/run.go index 6b10c5e8..d2fc2229 100644 --- a/internal/cmd/run.go +++ b/internal/cmd/run.go @@ -200,6 +200,10 @@ func init() { _ = f.Close() } + + if stdout.Written && stdout.LastByte != '\n' { + _, _ = io.WriteString(stdout, "\n") + } } } case listArtifacts: @@ -262,7 +266,7 @@ func init() { hcobra.AddLocalFlagSet(cmd, runFlagGroup) outFlagGroup := hcobra.NewFlagSet("Output Flags") - outFlagGroup.BoolVarP(&listArtifacts, "list-artifacts", "", false, "List output artifacts") + //outFlagGroup.BoolVarP(&listArtifacts, "list-artifacts", "", false, "List output artifacts") outFlagGroup.BoolVarP(&listOut, "list-out", "", false, "List output paths") outFlagGroup.StringVarP(©Out, "copy-out", "", "", "Copy output to path") outFlagGroup.BoolVarP(&catOut, "cat-out", "", false, "Print outputs to stdout") diff --git a/internal/engine/cache_sql.go b/internal/engine/cache_sql.go index eaf04e8b..66d9e55f 100644 --- a/internal/engine/cache_sql.go +++ b/internal/engine/cache_sql.go @@ -14,6 +14,7 @@ import ( "time" "github.com/hephbuild/heph/internal/hproto/hashpb" + "github.com/hephbuild/heph/internal/hsync" "github.com/hephbuild/heph/lib/tref" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" "github.com/zeebo/xxh3" @@ -22,6 +23,8 @@ import ( type SQLCache struct { db *sql.DB + + pool hsync.Pool[[]byte] } func (c *SQLCache) Exists(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (bool, error) { @@ -99,6 +102,11 @@ func OpenSQLCacheDB(path string) (*sql.DB, error) { return nil, fmt.Errorf("OpenSQLCacheDB open: %w", err) } + // SQLite supports only one concurrent writer. Capping the pool to a single + // connection serialises all access at the Go level and avoids SQLITE_BUSY + // races that busy_timeout alone cannot prevent across multiple pool conns. + db.SetMaxOpenConns(1) + if err := migrateSQLCacheDB(db); err != nil { _ = db.Close() return nil, fmt.Errorf("OpenSQLCacheDB migrate: %w", err) @@ -110,6 +118,9 @@ func OpenSQLCacheDB(path string) (*sql.DB, error) { func NewSQLCache(db *sql.DB) *SQLCache { return &SQLCache{ db: db, + pool: hsync.Pool[[]byte]{New: func() []byte { + return make([]byte, 100_000) + }}, } } @@ -118,18 +129,18 @@ func NewSQLCache(db *sql.DB) *SQLCache { // a hash of the full ref is appended to disambiguate. func (c *SQLCache) targetAddr(ref *pluginv1.TargetRef) string { if len(ref.GetArgs()) == 0 { - return "__" + ref.GetName() + return ref.GetName() } h := xxh3.New() hashpb.Hash(h, ref, tref.OmitHashPb) - return "__" + ref.GetName() + "_" + hex.EncodeToString(h.Sum(nil)) + return ref.GetName() + "@" + hex.EncodeToString(h.Sum(nil)) } // targetKey returns the compound target address used as target_addr in the DB. func (c *SQLCache) targetKey(ref *pluginv1.TargetRef) string { - return ref.GetPackage() + "/" + c.targetAddr(ref) + return ref.GetPackage() + ":" + c.targetAddr(ref) } func (c *SQLCache) Reader(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.ReadCloser, error) { @@ -163,30 +174,6 @@ func (c *SQLCache) Reader(ctx context.Context, ref *pluginv1.TargetRef, hashin, return io.NopCloser(bytes.NewReader(scanned)), nil } -type sqlWriter struct { - pw *io.PipeWriter - errC chan error - closed bool -} - -func (w *sqlWriter) Write(p []byte) (n int, err error) { - return w.pw.Write(p) -} - -func (w *sqlWriter) Close() error { - if w.closed { - return nil - } - w.closed = true - - err := w.pw.Close() - writeErr := <-w.errC - if writeErr != nil { - return writeErr - } - return err -} - func (c *SQLCache) writeEntry(ctx context.Context, targetAddr, hashin, name string, data io.Reader) error { _, err := c.db.ExecContext( ctx, @@ -200,10 +187,12 @@ func (c *SQLCache) writeEntry(ctx context.Context, targetAddr, hashin, name stri targetAddr, hashin, name, []byte{}, time.Now().UnixNano(), ) if err != nil { - return fmt.Errorf("writeEntry upsert %w", err) + return fmt.Errorf("writeEntry upsert: %w", err) } - buf := make([]byte, 32*1024) // 32KB chunks + buf := c.pool.Get() + defer c.pool.Put(buf) + for { n, err := data.Read(buf) if n > 0 { @@ -213,40 +202,58 @@ func (c *SQLCache) writeEntry(ctx context.Context, targetAddr, hashin, name stri WHERE target_addr = ? AND hashin = ? AND artifact_name = ? `, buf[:n], targetAddr, hashin, name) if errAppend != nil { - return fmt.Errorf("writeEntry append chunk %w", errAppend) + return fmt.Errorf("writeEntry append chunk: %w", errAppend) } } if err == io.EOF { break } if err != nil { - return fmt.Errorf("writeEntry read %w", err) + return fmt.Errorf("writeEntry read: %w", err) } } return nil } +type sqlCacheWriter struct { + pw *io.PipeWriter + done <-chan error +} + +func (w *sqlCacheWriter) Write(p []byte) (int, error) { + return w.pw.Write(p) +} + +func (w *sqlCacheWriter) Close() error { + // Close the write end; this unblocks the goroutine's reader. + if err := w.pw.Close(); err != nil { + return err + } + // Wait for the goroutine to finish and return any write error. + return <-w.done +} + func (c *SQLCache) Writer(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.WriteCloser, error) { targetAddr := c.targetKey(ref) pr, pw := io.Pipe() - - errC := make(chan error, 1) + done := make(chan error, 1) go func() { defer pr.Close() err := c.writeEntry(ctx, targetAddr, hashin, name, pr) if err != nil { - err = fmt.Errorf("writer write: %q %q %q %w", tref.Format(ref), hashin, name, err) + wrappedErr := fmt.Errorf("writer write: %q %q %q %w", tref.Format(ref), hashin, name, err) + _ = pr.CloseWithError(wrappedErr) + done <- wrappedErr + } else { + done <- nil } - errC <- err + close(done) }() - return &sqlWriter{ - pw: pw, - errC: errC, - }, nil + return &sqlCacheWriter{pw: pw, done: done}, nil } func (c *SQLCache) ListArtifacts(ctx context.Context, ref *pluginv1.TargetRef, hashin string) iter.Seq2[string, error] { diff --git a/internal/engine/cache_sql_test.go b/internal/engine/cache_sql_test.go index 0ff2baf1..7dbcaf2d 100644 --- a/internal/engine/cache_sql_test.go +++ b/internal/engine/cache_sql_test.go @@ -58,9 +58,8 @@ func TestSQLCache(t *testing.T) { require.NoError(t, err) require.NoError(t, w.Close()) - artSeq := cache.ListArtifacts(ctx, ref, hashin, "") var artifacts []string - for a, e := range artSeq { + for a, e := range cache.ListArtifacts(ctx, ref, hashin) { require.NoError(t, e) artifacts = append(artifacts, a) } @@ -71,9 +70,8 @@ func TestSQLCache(t *testing.T) { require.NoError(t, err) require.NoError(t, w.Close()) - verSeq := cache.ListVersions(ctx, ref, "") var versions []string - for v, e := range verSeq { + for v, e := range cache.ListVersions(ctx, ref) { require.NoError(t, e) versions = append(versions, v) } diff --git a/internal/engine/local_cache.go b/internal/engine/local_cache.go index 94b6ee73..930cec46 100644 --- a/internal/engine/local_cache.go +++ b/internal/engine/local_cache.go @@ -121,6 +121,8 @@ func (e *Engine) cacheLocally( } cacheArtifacts = append(cacheArtifacts, cacheArtifact) + + _ = src.Close() } manifest, err := e.createLocalCacheManifest(ctx, def.GetRef(), hashin, cacheArtifacts) @@ -347,7 +349,7 @@ type CacheLocallyArtifact struct { func (e *Engine) cacheArtifactLocally(ctx context.Context, ref *pluginv1.TargetRef, hashin string, art CacheLocallyArtifact, hashout string) (*ResultArtifact, error) { cache := e.CacheSmall - if art.Size > 100_000 { + if art.Size > 100_000 { // 100kb cache = e.CacheLarge } @@ -373,7 +375,7 @@ func (e *Engine) cacheArtifactLocally(ctx context.Context, ref *pluginv1.TargetR _, err = io.Copy(dst, art.Reader) if err != nil { - return nil, err + return nil, fmt.Errorf("copy to local: %w", err) } err = dst.Close() diff --git a/internal/engine/query.go b/internal/engine/query.go index 90bf1f6f..cc9443f2 100644 --- a/internal/engine/query.go +++ b/internal/engine/query.go @@ -150,7 +150,7 @@ func (e *Engine) query(ctx context.Context, rs *RequestState, matcher *pluginv1. wg := hiter.NewGroup(ctx, yield) for pkg, err := range e.Packages(ctx, matcher) { if err != nil { - yield(nil, err) + wg.Yield(nil, err) return } diff --git a/internal/engine/remote_cache.go b/internal/engine/remote_cache.go index 4f4cf868..b099970a 100644 --- a/internal/engine/remote_cache.go +++ b/internal/engine/remote_cache.go @@ -102,7 +102,7 @@ func (e *Engine) cacheRemotelyInner(ctx context.Context, defer pw.Close() err := hartifact.EncodeManifest(pw, manifest) if err != nil { - pr.CloseWithError(err) + _ = pw.CloseWithError(err) return } @@ -245,10 +245,6 @@ func (e *Engine) resultFromRemoteCacheInner( g.Go(func(ctx context.Context) error { r, err := cache.Client.Get(ctx, key) if err != nil { - if errors.Is(err, pluginsdk.ErrCacheNotFound) { - return pluginsdk.ErrCacheNotFound - } - return err } defer r.Close() diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index 62d9e6c8..e3b56973 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -852,7 +852,7 @@ func (r Result) Sorted() *Result { func (r Result) Clone() *Result { return &Result{ - Def: r.Def.Clone(), + Def: r.Def, Hashin: r.Hashin, Artifacts: slices.Clone(r.Artifacts), Manifest: r.Manifest, @@ -1305,7 +1305,7 @@ func (e *Engine) execute(ctx context.Context, rs *RequestState, def *LightLinked } if !shouldCollect { - break + continue } tarname := output.GetGroup() + ".tar" @@ -1359,6 +1359,11 @@ func (e *Engine) execute(ctx context.Context, rs *RequestState, def *LightLinked return nil, err } + err = tarf.Close() + if err != nil { + return nil, err + } + execArtifact := pluginsdk.ProtoArtifact{ Artifact: pluginv1.Artifact_builder{ Group: htypes.Ptr(output.GetGroup()), diff --git a/internal/enginee2e/cache_test.go b/internal/enginee2e/cache_test.go new file mode 100644 index 00000000..74c40f54 --- /dev/null +++ b/internal/enginee2e/cache_test.go @@ -0,0 +1,90 @@ +package enginee2e + +import ( + "testing" + + "github.com/hephbuild/heph/internal/htypes" + + "github.com/hephbuild/heph/internal/hproto/hstructpb" + + "github.com/hephbuild/heph/internal/engine" + "github.com/hephbuild/heph/internal/hartifact" + "github.com/hephbuild/heph/internal/hfs" + "github.com/hephbuild/heph/internal/hfs/hfstest" + pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" + "github.com/hephbuild/heph/plugin/pluginexec" + "github.com/hephbuild/heph/plugin/pluginstaticprovider" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +func TestCache(t *testing.T) { + ctx := t.Context() + + dir := t.TempDir() + + e, err := engine.New(ctx, dir, engine.Config{}) + require.NoError(t, err) + + staticprovider := pluginstaticprovider.New([]pluginstaticprovider.Target{ + { + Spec: pluginv1.TargetSpec_builder{ + Ref: pluginv1.TargetRef_builder{ + Package: htypes.Ptr(""), + Name: htypes.Ptr("child"), + }.Build(), + Driver: htypes.Ptr("bash"), + Config: map[string]*structpb.Value{ + "run": hstructpb.NewStringsValue([]string{`echo hello > $OUT`}), + "out": hstructpb.NewStringsValue([]string{"out_child"}), + }, + }.Build(), + }, + }) + + _, err = e.RegisterProvider(ctx, staticprovider, engine.RegisterProviderConfig{}) + require.NoError(t, err) + + _, err = e.RegisterDriver(ctx, pluginexec.NewExec(), nil) + require.NoError(t, err) + _, err = e.RegisterDriver(ctx, pluginexec.NewSh(), nil) + require.NoError(t, err) + _, err = e.RegisterDriver(ctx, pluginexec.NewBash(), nil) + require.NoError(t, err) + + assertOut := func(res *engine.ExecuteResultLocks) { + fs := hfstest.New(t) + err = hartifact.Unpack(ctx, res.FindOutputs("")[0].Artifact, fs) + require.NoError(t, err) + + b, err := hfs.ReadFile(fs.At("out_child")) + require.NoError(t, err) + + assert.Equal(t, "hello\n", string(b)) + } + + { // This will run all + rs, clean := e.NewRequestState() + defer clean() + + res, err := e.Result(ctx, rs, "", "child", []string{engine.AllOutputs}) + require.NoError(t, err) + defer res.Unlock(ctx) + + assertOut(res) + res.Unlock(ctx) + } + + { // this should reuse cache from deps + rs, clean := e.NewRequestState() + defer clean() + + res, err := e.Result(ctx, rs, "", "child", []string{engine.AllOutputs}) + require.NoError(t, err) + defer res.Unlock(ctx) + + assertOut(res) + res.Unlock(ctx) + } +} diff --git a/internal/enginee2e/deps_cache_test.go b/internal/enginee2e/deps_cache_test.go index 3ab5f546..bb777225 100644 --- a/internal/enginee2e/deps_cache_test.go +++ b/internal/enginee2e/deps_cache_test.go @@ -195,7 +195,7 @@ func TestDepsCacheLarge(t *testing.T) { } } -func TestDepsCache2(t *testing.T) { +func TestDepsCacheRemote(t *testing.T) { ctx := t.Context() c := gomock.NewController(t) diff --git a/internal/hartifact/reader.go b/internal/hartifact/reader.go index 6a6b8f3e..18a9ffac 100644 --- a/internal/hartifact/reader.go +++ b/internal/hartifact/reader.go @@ -76,7 +76,7 @@ func FilesReader(ctx context.Context, a pluginsdk.Artifact) iter.Seq2[*File, err tr := tar.NewReader(r) - err = htar.Walk(tr, func(header *tar.Header, reader *tar.Reader) error { + err = htar.Walk(tr, func(header *tar.Header, reader io.Reader) error { if header.Typeflag != tar.TypeReg { return nil } diff --git a/internal/hiter/group.go b/internal/hiter/group.go index b4bbf384..aa4caf92 100644 --- a/internal/hiter/group.go +++ b/internal/hiter/group.go @@ -34,7 +34,7 @@ func (g *Group[K, V]) Go(f func(ctx context.Context, yield func(K, V) bool)) { return errYieldExited } - f(ctx, g.gyield) + f(ctx, g.Yield) if g.yieldExited { return errYieldExited @@ -52,7 +52,7 @@ func (g *Group[K, V]) SetLimit(n int) { g.wg.SetLimit(n) } -func (g *Group[K, V]) gyield(k K, v V) bool { +func (g *Group[K, V]) Yield(k K, v V) bool { g.yieldmu.Lock() defer g.yieldmu.Unlock() diff --git a/internal/htar/pack.go b/internal/htar/pack.go index 1261f740..afe8a187 100644 --- a/internal/htar/pack.go +++ b/internal/htar/pack.go @@ -4,6 +4,7 @@ import ( "archive/tar" "fmt" "io" + "io/fs" "os" "path/filepath" @@ -11,7 +12,8 @@ import ( ) type Packer struct { - tw *tar.Writer + tw *tar.Writer + AllowAbsLink bool } func NewPacker(w io.Writer) *Packer { @@ -25,7 +27,7 @@ func (p *Packer) WriteFile(f hfs.File, path string) error { } var link string - if info.Mode().Type()&os.ModeSymlink != 0 { + if info.Mode().Type()&fs.ModeSymlink != 0 { l, err := os.Readlink(f.Name()) if err != nil { return err @@ -33,7 +35,7 @@ func (p *Packer) WriteFile(f hfs.File, path string) error { link = l - if filepath.IsAbs(link) { + if !p.AllowAbsLink && filepath.IsAbs(link) { return fmt.Errorf("absolute link not allowed: %v -> %v", f.Name(), link) } } @@ -45,19 +47,7 @@ func (p *Packer) WriteFile(f hfs.File, path string) error { hdr.Name = path - if err := p.tw.WriteHeader(hdr); err != nil { - return err - } - - if !info.Mode().IsRegular() { // nothing more to do for non-regular - return nil - } - - if _, err := io.Copy(p.tw, io.LimitReader(f, info.Size())); err != nil { - return err - } - - return nil + return p.Write(f, hdr) } func (p *Packer) Write(r io.Reader, hdr *tar.Header) error { diff --git a/internal/htar/unpack.go b/internal/htar/unpack.go index c66b3cc2..1dd32af8 100644 --- a/internal/htar/unpack.go +++ b/internal/htar/unpack.go @@ -43,31 +43,44 @@ type config struct { type Matcher = func(hdr *tar.Header) bool func FileReader(ctx context.Context, r io.Reader, match Matcher) (io.Reader, error) { - tr := tar.NewReader(r) + pr, pw := io.Pipe() - var fileReader io.Reader - err := Walk(tr, func(hdr *tar.Header, r *tar.Reader) error { - if !match(hdr) { - return nil - } + go func() { + defer pw.Close() - switch hdr.Typeflag { - case tar.TypeReg: - fileReader = r - return ErrStopWalk - default: - return fmt.Errorf("is not a file, is %v: %s", hdr.Typeflag, hdr.Name) + tr := tar.NewReader(r) + + var matched bool + err := Walk(tr, func(hdr *tar.Header, r io.Reader) error { + if !match(hdr) { + return nil + } + + switch hdr.Typeflag { + case tar.TypeReg: + matched = true + _, err := io.Copy(pw, r) + if err != nil { + return err + } + + return ErrStopWalk + default: + return fmt.Errorf("is not a file, is %v: %s", hdr.Typeflag, hdr.Name) + } + }) + if err != nil { + _ = pw.CloseWithError(err) + return } - }) - if err != nil { - return nil, err - } - if fileReader == nil { - return nil, errors.New("file not found") - } + if !matched { + _ = pw.CloseWithError(errors.New("tar is empty")) + return + } + }() - return fileReader, nil + return pr, nil } func Unpack(ctx context.Context, r io.Reader, to hfs.Node, options ...Option) error { @@ -86,7 +99,7 @@ func Unpack(ctx context.Context, r io.Reader, to hfs.Node, options ...Option) er tr := tar.NewReader(r) - return Walk(tr, func(hdr *tar.Header, r *tar.Reader) error { + return Walk(tr, func(hdr *tar.Header, r io.Reader) error { if !cfg.filter(hdr.Name) { return nil } @@ -134,7 +147,7 @@ func Unpack(ctx context.Context, r io.Reader, to hfs.Node, options ...Option) er }) } -func unpackFile(hdr *tar.Header, tr *tar.Reader, to hfs.Node, ro bool, onFile func(to string)) error { +func unpackFile(hdr *tar.Header, tr io.Reader, to hfs.Node, ro bool, onFile func(to string)) error { fileNode := to.At(hdr.Name) info, err := fileNode.Lstat() @@ -196,7 +209,7 @@ func unpackFile(hdr *tar.Header, tr *tar.Reader, to hfs.Node, ro bool, onFile fu var ErrStopWalk = errors.New("stop walk") -func Walk(tr *tar.Reader, fs ...func(*tar.Header, *tar.Reader) error) error { +func Walk(tr *tar.Reader, f func(*tar.Header, io.Reader) error) error { for { hdr, err := tr.Next() if err != nil { @@ -207,15 +220,13 @@ func Walk(tr *tar.Reader, fs ...func(*tar.Header, *tar.Reader) error) error { return fmt.Errorf("walk: %w", err) } - for _, f := range fs { - err = f(hdr, tr) - if err != nil { - if errors.Is(err, ErrStopWalk) { - return nil - } - - return err + err = f(hdr, io.LimitReader(tr, hdr.Size)) + if err != nil { + if errors.Is(err, ErrStopWalk) { + break } + + return err } } diff --git a/lib/pluginsdk/artifact_proto.go b/lib/pluginsdk/artifact_proto.go index aa07f5c4..0551655b 100644 --- a/lib/pluginsdk/artifact_proto.go +++ b/lib/pluginsdk/artifact_proto.go @@ -27,16 +27,15 @@ func (a ProtoArtifact) GetContentReader() (io.ReadCloser, error) { pr, pw := io.Pipe() - sourcefs := hfs.NewOS(content.GetSourcePath()) - go func() { defer pw.Close() tarPacker := htar.NewPacker(pw) + tarPacker.AllowAbsLink = true // TODO: This is wrong., but let's ignore for now... - f, err := hfs.Open(sourcefs) + f, err := hfs.Open(hfs.NewOS(content.GetSourcePath())) if err != nil { - _ = pr.CloseWithError(err) + _ = pw.CloseWithError(err) return } @@ -44,12 +43,10 @@ func (a ProtoArtifact) GetContentReader() (io.ReadCloser, error) { err = tarPacker.WriteFile(f, content.GetOutPath()) if err != nil { - _ = pr.CloseWithError(err) + _ = pw.CloseWithError(err) return } - - _ = f.Close() }() return pr, nil @@ -75,7 +72,7 @@ func (a ProtoArtifact) GetContentReader() (io.ReadCloser, error) { Mode: mode, }) if err != nil { - _ = pr.CloseWithError(err) + _ = pw.CloseWithError(err) return } diff --git a/plugin/pluginexec/run.go b/plugin/pluginexec/run.go index f96546f4..69b8c9cc 100644 --- a/plugin/pluginexec/run.go +++ b/plugin/pluginexec/run.go @@ -436,6 +436,9 @@ func (p *Plugin[S]) inputEnv(ctx context.Context, inputs []*pluginsdk.ArtifactWi for sc.Scan() { line := sc.Text() + if line == "" { + continue + } if _, ok := seenFiles[line]; ok { continue diff --git a/plugin/pluginexec/sandbox.go b/plugin/pluginexec/sandbox.go index 2fda55e9..0e1e618a 100644 --- a/plugin/pluginexec/sandbox.go +++ b/plugin/pluginexec/sandbox.go @@ -153,14 +153,14 @@ func ArtifactsForId(inputs []*pluginsdk.ArtifactWithOrigin, id string, typ plugi var artifactId atomic.Int64 -func SetupSandboxArtifact(ctx context.Context, artifact pluginsdk.Artifact, source *execv1.Target_Dep, node hfs.Node, filters []string, sourcemap map[string]string) (pluginsdk.Artifact, error) { +func SetupSandboxArtifact(ctx context.Context, artifact pluginsdk.Artifact, source *execv1.Target_Dep, workfs hfs.Node, filters []string, sourcemap map[string]string) (pluginsdk.Artifact, error) { ctx, span := tracer.Start(ctx, "SetupSandboxArtifact") defer span.End() h := xxh3.New() _, _ = fmt.Fprintf(h, "%d", artifactId.Add(1)) - listf, err := hfs.Create(node.At(hex.EncodeToString(h.Sum(nil)) + ".list")) + listf, err := hfs.Create(workfs.At(hex.EncodeToString(h.Sum(nil)) + ".list")) if err != nil { return nil, fmt.Errorf("create list file: %w", err) } @@ -168,7 +168,7 @@ func SetupSandboxArtifact(ctx context.Context, artifact pluginsdk.Artifact, sour writeNl := false - err = hartifact.Unpack(ctx, artifact, node, hartifact.WithOnFile(func(to string) { + err = hartifact.Unpack(ctx, artifact, workfs, hartifact.WithOnFile(func(to string) { if writeNl { _, _ = listf.Write([]byte("\n")) } @@ -176,7 +176,7 @@ func SetupSandboxArtifact(ctx context.Context, artifact pluginsdk.Artifact, sour writeNl = true if sourcemap != nil { - rel, err := filepath.Rel(node.Path(), to) + rel, err := filepath.Rel(workfs.Path(), to) if err != nil { panic(err) } diff --git a/plugin/pluginfs/driver.go b/plugin/pluginfs/driver.go index 0d02990b..039e5540 100644 --- a/plugin/pluginfs/driver.go +++ b/plugin/pluginfs/driver.go @@ -153,7 +153,12 @@ func (p *Driver) Run(ctx context.Context, req *pluginsdk.RunRequest) (*pluginv1. }.Build(), nil } - exclude := []string{} // TODO: exclude home, cache, git etc... + // TODO: exclude home, cache, git etc... + exclude := []string{ + ".git/**/*", + ".heph/**/*", + ".heph2/**/*", + } exclude = append(exclude, t.GetExclude()...) var artifacts []*pluginv1.Artifact diff --git a/plugin/plugingo/pkg_analysis.go b/plugin/plugingo/pkg_analysis.go index 70898b3b..16deaa1b 100644 --- a/plugin/plugingo/pkg_analysis.go +++ b/plugin/plugingo/pkg_analysis.go @@ -89,7 +89,7 @@ func (p *Plugin) goListPkgResult(ctx context.Context, basePkg, runPkg, imp strin err = json.NewDecoder(f).Decode(&goPkg) if err != nil { - return Package{}, fmt.Errorf("gopkg decode: %w", err) + return Package{}, fmt.Errorf("gopkg decode %q: %w", tref.Format(res.Def.GetRef()), err) } } @@ -103,7 +103,7 @@ func (p *Plugin) goListPkgResult(ctx context.Context, basePkg, runPkg, imp strin err = json.NewDecoder(f).Decode(&sourcemap) if err != nil { - return Package{}, fmt.Errorf("sourcemap decode: %w", err) + return Package{}, fmt.Errorf("sourcemap decode: %q: %w", tref.Format(res.Def.GetRef()), err) } } From 25e0cca080035caf4c453915106330ea8e147854 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Sun, 8 Mar 2026 02:22:18 +0000 Subject: [PATCH 05/15] concurrent reads --- internal/engine/cache_sql.go | 88 ++++++++++++++++++++++++++---------- 1 file changed, 64 insertions(+), 24 deletions(-) diff --git a/internal/engine/cache_sql.go b/internal/engine/cache_sql.go index 66d9e55f..7917ec9c 100644 --- a/internal/engine/cache_sql.go +++ b/internal/engine/cache_sql.go @@ -21,8 +21,31 @@ import ( _ "modernc.org/sqlite" ) +// SQLCacheDB holds the two connection pools used by SQLCache. +// Use OpenSQLCacheDB to open it and Close to shut down both pools. +type SQLCacheDB struct { + // rdb is the read pool: uncapped, WAL allows concurrent readers. + rdb *sql.DB + // wdb is the write pool: MaxOpenConns(1) serialises writers. + wdb *sql.DB +} + +func (s *SQLCacheDB) Close() error { + errR := s.rdb.Close() + errW := s.wdb.Close() + if errR != nil { + return errR + } + return errW +} + type SQLCache struct { - db *sql.DB + // rdb is used for all read operations. WAL mode allows many concurrent + // readers, so this pool is uncapped. + rdb *sql.DB + // wdb is used for all write operations. SetMaxOpenConns(1) serialises + // writers at the Go level, preventing SQLITE_BUSY races. + wdb *sql.DB pool hsync.Pool[[]byte] } @@ -31,7 +54,7 @@ func (c *SQLCache) Exists(ctx context.Context, ref *pluginv1.TargetRef, hashin, targetAddr := c.targetKey(ref) var count int - err := c.db.QueryRowContext( + err := c.rdb.QueryRowContext( ctx, `SELECT COUNT(*) FROM cache_blobs WHERE target_addr = ? AND hashin = ? AND artifact_name = ? LIMIT 1`, targetAddr, hashin, name, @@ -48,13 +71,13 @@ func (c *SQLCache) Delete(ctx context.Context, ref *pluginv1.TargetRef, hashin, var err error if name == "" { - _, err = c.db.ExecContext( + _, err = c.wdb.ExecContext( ctx, `DELETE FROM cache_blobs WHERE target_addr = ? AND hashin = ?`, targetAddr, hashin, ) } else { - _, err = c.db.ExecContext( + _, err = c.wdb.ExecContext( ctx, `DELETE FROM cache_blobs WHERE target_addr = ? AND hashin = ? AND artifact_name = ?`, targetAddr, hashin, name, @@ -86,11 +109,8 @@ func migrateSQLCacheDB(db *sql.DB) error { return err } -func OpenSQLCacheDB(path string) (*sql.DB, error) { - if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil { - return nil, fmt.Errorf("OpenSQLCacheDB mkdir: %w", err) - } - +// openSQLiteDB opens a connection pool to a SQLite file with common pragmas. +func openSQLiteDB(path string) (*sql.DB, error) { dsn := path + "?_pragma=journal_mode(WAL)" + "&_pragma=busy_timeout(10000)" + @@ -99,25 +119,45 @@ func OpenSQLCacheDB(path string) (*sql.DB, error) { db, err := sql.Open("sqlite", dsn) if err != nil { - return nil, fmt.Errorf("OpenSQLCacheDB open: %w", err) + return nil, err } + return db, nil +} - // SQLite supports only one concurrent writer. Capping the pool to a single - // connection serialises all access at the Go level and avoids SQLITE_BUSY - // races that busy_timeout alone cannot prevent across multiple pool conns. - db.SetMaxOpenConns(1) +// OpenSQLCacheDB opens a SQLCacheDB with two connection pools to the same SQLite file: +// - wdb: a single-connection write pool (serialises writers, no SQLITE_BUSY) +// - rdb: an uncapped read pool (WAL allows fully concurrent readers) +func OpenSQLCacheDB(path string) (*SQLCacheDB, error) { + if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil { + return nil, fmt.Errorf("OpenSQLCacheDB mkdir: %w", err) + } + + wdb, err := openSQLiteDB(path) + if err != nil { + return nil, fmt.Errorf("OpenSQLCacheDB open wdb: %w", err) + } + // One writer at a time — prevents SQLITE_BUSY without busy-retry overhead. + wdb.SetMaxOpenConns(1) - if err := migrateSQLCacheDB(db); err != nil { - _ = db.Close() + if err := migrateSQLCacheDB(wdb); err != nil { + _ = wdb.Close() return nil, fmt.Errorf("OpenSQLCacheDB migrate: %w", err) } - return db, nil + rdb, err := openSQLiteDB(path) + if err != nil { + _ = wdb.Close() + return nil, fmt.Errorf("OpenSQLCacheDB open rdb: %w", err) + } + // No cap — WAL lets concurrent readers run in parallel. + + return &SQLCacheDB{rdb: rdb, wdb: wdb}, nil } -func NewSQLCache(db *sql.DB) *SQLCache { +func NewSQLCache(db *SQLCacheDB) *SQLCache { return &SQLCache{ - db: db, + rdb: db.rdb, + wdb: db.wdb, pool: hsync.Pool[[]byte]{New: func() []byte { return make([]byte, 100_000) }}, @@ -148,7 +188,7 @@ func (c *SQLCache) Reader(ctx context.Context, ref *pluginv1.TargetRef, hashin, var dataBytes sql.NullString - err := c.db.QueryRowContext( + err := c.rdb.QueryRowContext( ctx, ` SELECT data @@ -175,7 +215,7 @@ func (c *SQLCache) Reader(ctx context.Context, ref *pluginv1.TargetRef, hashin, } func (c *SQLCache) writeEntry(ctx context.Context, targetAddr, hashin, name string, data io.Reader) error { - _, err := c.db.ExecContext( + _, err := c.wdb.ExecContext( ctx, ` INSERT INTO cache_blobs (target_addr, hashin, artifact_name, data, created_at) @@ -196,7 +236,7 @@ func (c *SQLCache) writeEntry(ctx context.Context, targetAddr, hashin, name stri for { n, err := data.Read(buf) if n > 0 { - _, errAppend := c.db.ExecContext(ctx, ` + _, errAppend := c.wdb.ExecContext(ctx, ` UPDATE cache_blobs SET data = data || ? WHERE target_addr = ? AND hashin = ? AND artifact_name = ? @@ -260,7 +300,7 @@ func (c *SQLCache) ListArtifacts(ctx context.Context, ref *pluginv1.TargetRef, h return func(yield func(string, error) bool) { targetAddr := c.targetKey(ref) - rows, err := c.db.QueryContext(ctx, + rows, err := c.rdb.QueryContext(ctx, "SELECT artifact_name FROM cache_blobs WHERE target_addr = ? AND hashin = ?", targetAddr, hashin, ) @@ -292,7 +332,7 @@ func (c *SQLCache) ListVersions(ctx context.Context, ref *pluginv1.TargetRef) it return func(yield func(string, error) bool) { targetAddr := c.targetKey(ref) - rows, err := c.db.QueryContext(ctx, + rows, err := c.rdb.QueryContext(ctx, "SELECT DISTINCT hashin FROM cache_blobs WHERE target_addr = ?", targetAddr, ) From c698008026dac07e45df614f87d7f071ebfec07e Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Sun, 8 Mar 2026 11:00:38 +0000 Subject: [PATCH 06/15] sql rw and pool buf --- internal/engine/cache_sql.go | 217 +++++++++++++++++++++-------------- internal/hio/closer.go | 16 +++ 2 files changed, 149 insertions(+), 84 deletions(-) diff --git a/internal/engine/cache_sql.go b/internal/engine/cache_sql.go index 7917ec9c..1ae7e34b 100644 --- a/internal/engine/cache_sql.go +++ b/internal/engine/cache_sql.go @@ -11,8 +11,10 @@ import ( "iter" "os" "path/filepath" + "sync" "time" + "github.com/hephbuild/heph/internal/hio" "github.com/hephbuild/heph/internal/hproto/hashpb" "github.com/hephbuild/heph/internal/hsync" "github.com/hephbuild/heph/lib/tref" @@ -21,16 +23,56 @@ import ( _ "modernc.org/sqlite" ) -// SQLCacheDB holds the two connection pools used by SQLCache. -// Use OpenSQLCacheDB to open it and Close to shut down both pools. +// SQLCacheDB holds the path and lazily opens both connection pools on first use. +// Call Close only if the DB was actually used; it is safe to call regardless. type SQLCacheDB struct { - // rdb is the read pool: uncapped, WAL allows concurrent readers. - rdb *sql.DB - // wdb is the write pool: MaxOpenConns(1) serialises writers. - wdb *sql.DB + path string + once sync.Once + rdb *sql.DB + wdb *sql.DB + err error +} + +func (s *SQLCacheDB) pools() (*sql.DB, *sql.DB, error) { + s.once.Do(func() { + if err := os.MkdirAll(filepath.Dir(s.path), os.ModePerm); err != nil { + s.err = fmt.Errorf("OpenSQLCacheDB mkdir: %w", err) + return + } + + wdb, err := openSQLiteDB(s.path) + if err != nil { + s.err = fmt.Errorf("OpenSQLCacheDB open wdb: %w", err) + return + } + // One writer at a time — prevents SQLITE_BUSY. + wdb.SetMaxOpenConns(1) + + if err := initSQLCacheDB(wdb); err != nil { + _ = wdb.Close() + s.err = fmt.Errorf("OpenSQLCacheDB init: %w", err) + return + } + + rdb, err := openSQLiteDB(s.path) + if err != nil { + _ = wdb.Close() + s.err = fmt.Errorf("OpenSQLCacheDB open rdb: %w", err) + return + } + // No cap — WAL lets concurrent readers run in parallel. + + s.rdb = rdb + s.wdb = wdb + }) + return s.rdb, s.wdb, s.err } func (s *SQLCacheDB) Close() error { + // If pools() was never called, once.Do has never run and rdb/wdb are nil. + if s.rdb == nil { + return nil + } errR := s.rdb.Close() errW := s.wdb.Close() if errR != nil { @@ -40,21 +82,30 @@ func (s *SQLCacheDB) Close() error { } type SQLCache struct { - // rdb is used for all read operations. WAL mode allows many concurrent - // readers, so this pool is uncapped. - rdb *sql.DB - // wdb is used for all write operations. SetMaxOpenConns(1) serialises - // writers at the Go level, preventing SQLITE_BUSY races. - wdb *sql.DB - - pool hsync.Pool[[]byte] + db *SQLCacheDB + + rpool hsync.Pool[[]byte] + wpool hsync.Pool[[]byte] +} + +func (c *SQLCache) rwdb(ctx context.Context) (rdb, wdb *sql.DB, err error) { + rdb, wdb, err = c.db.pools() + if err != nil { + return nil, nil, fmt.Errorf("sqlcache open db: %w", err) + } + return rdb, wdb, nil } func (c *SQLCache) Exists(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (bool, error) { + rdb, _, err := c.rwdb(ctx) + if err != nil { + return false, err + } + targetAddr := c.targetKey(ref) var count int - err := c.rdb.QueryRowContext( + err = rdb.QueryRowContext( ctx, `SELECT COUNT(*) FROM cache_blobs WHERE target_addr = ? AND hashin = ? AND artifact_name = ? LIMIT 1`, targetAddr, hashin, name, @@ -67,17 +118,21 @@ func (c *SQLCache) Exists(ctx context.Context, ref *pluginv1.TargetRef, hashin, } func (c *SQLCache) Delete(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) error { + _, wdb, err := c.rwdb(ctx) + if err != nil { + return err + } + targetAddr := c.targetKey(ref) - var err error if name == "" { - _, err = c.wdb.ExecContext( + _, err = wdb.ExecContext( ctx, `DELETE FROM cache_blobs WHERE target_addr = ? AND hashin = ?`, targetAddr, hashin, ) } else { - _, err = c.wdb.ExecContext( + _, err = wdb.ExecContext( ctx, `DELETE FROM cache_blobs WHERE target_addr = ? AND hashin = ? AND artifact_name = ?`, targetAddr, hashin, name, @@ -92,7 +147,7 @@ func (c *SQLCache) Delete(ctx context.Context, ref *pluginv1.TargetRef, hashin, var _ LocalCache = (*SQLCache)(nil) -func migrateSQLCacheDB(db *sql.DB) error { +func initSQLCacheDB(db *sql.DB) error { _, err := db.Exec(` CREATE TABLE IF NOT EXISTS cache_blobs ( target_addr TEXT NOT NULL, @@ -124,41 +179,19 @@ func openSQLiteDB(path string) (*sql.DB, error) { return db, nil } -// OpenSQLCacheDB opens a SQLCacheDB with two connection pools to the same SQLite file: -// - wdb: a single-connection write pool (serialises writers, no SQLITE_BUSY) -// - rdb: an uncapped read pool (WAL allows fully concurrent readers) +// OpenSQLCacheDB returns a SQLCacheDB that opens its connection pools lazily +// on first use. No file I/O happens until the first cache operation. func OpenSQLCacheDB(path string) (*SQLCacheDB, error) { - if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil { - return nil, fmt.Errorf("OpenSQLCacheDB mkdir: %w", err) - } - - wdb, err := openSQLiteDB(path) - if err != nil { - return nil, fmt.Errorf("OpenSQLCacheDB open wdb: %w", err) - } - // One writer at a time — prevents SQLITE_BUSY without busy-retry overhead. - wdb.SetMaxOpenConns(1) - - if err := migrateSQLCacheDB(wdb); err != nil { - _ = wdb.Close() - return nil, fmt.Errorf("OpenSQLCacheDB migrate: %w", err) - } - - rdb, err := openSQLiteDB(path) - if err != nil { - _ = wdb.Close() - return nil, fmt.Errorf("OpenSQLCacheDB open rdb: %w", err) - } - // No cap — WAL lets concurrent readers run in parallel. - - return &SQLCacheDB{rdb: rdb, wdb: wdb}, nil + return &SQLCacheDB{path: path}, nil } func NewSQLCache(db *SQLCacheDB) *SQLCache { return &SQLCache{ - rdb: db.rdb, - wdb: db.wdb, - pool: hsync.Pool[[]byte]{New: func() []byte { + db: db, + wpool: hsync.Pool[[]byte]{New: func() []byte { + return make([]byte, 100_000) + }}, + rpool: hsync.Pool[[]byte]{New: func() []byte { return make([]byte, 100_000) }}, } @@ -184,11 +217,17 @@ func (c *SQLCache) targetKey(ref *pluginv1.TargetRef) string { } func (c *SQLCache) Reader(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.ReadCloser, error) { + rdb, _, err := c.rwdb(ctx) + if err != nil { + return nil, err + } + targetAddr := c.targetKey(ref) - var dataBytes sql.NullString + buf := c.rpool.Get() + buf = buf[0:] - err := c.rdb.QueryRowContext( + err = rdb.QueryRowContext( ctx, ` SELECT data @@ -196,9 +235,11 @@ func (c *SQLCache) Reader(ctx context.Context, ref *pluginv1.TargetRef, hashin, WHERE target_addr = ? AND hashin = ? AND artifact_name = ? `, targetAddr, hashin, name, - ).Scan(&dataBytes) + ).Scan(&buf) if err != nil { + c.rpool.Put(buf) + if errors.Is(err, sql.ErrNoRows) { return nil, LocalCacheNotFoundError } @@ -206,16 +247,35 @@ func (c *SQLCache) Reader(ctx context.Context, ref *pluginv1.TargetRef, hashin, return nil, fmt.Errorf("reader scan: %w", err) } - if !dataBytes.Valid { - return io.NopCloser(bytes.NewReader(nil)), nil - } + return hio.NewReadCloserFunc(bytes.NewReader(buf), func() error { + c.rpool.Put(buf) - scanned := []byte(dataBytes.String) - return io.NopCloser(bytes.NewReader(scanned)), nil + return nil + }), nil } func (c *SQLCache) writeEntry(ctx context.Context, targetAddr, hashin, name string, data io.Reader) error { - _, err := c.wdb.ExecContext( + _, wdb, err := c.rwdb(ctx) + if err != nil { + return err + } + + buf := c.wpool.Get() + defer c.wpool.Put(buf) + + // Read the full payload into the pool buffer without allocating. + // io.ReadFull returns: + // io.EOF — empty stream (zero bytes written) + // io.ErrUnexpectedEOF — stream shorter than buf (normal < 100KB case) + // nil — stream filled the buffer exactly + n, err := io.ReadFull(data, buf) + if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) && !errors.Is(err, io.EOF) { + return fmt.Errorf("writeEntry read: %w", err) + } + payload := buf[:n] + + // Single UPSERT — write lock held for exactly one statement. + _, err = wdb.ExecContext( ctx, ` INSERT INTO cache_blobs (target_addr, hashin, artifact_name, data, created_at) @@ -224,35 +284,12 @@ func (c *SQLCache) writeEntry(ctx context.Context, targetAddr, hashin, name stri data = excluded.data, created_at = excluded.created_at `, - targetAddr, hashin, name, []byte{}, time.Now().UnixNano(), + targetAddr, hashin, name, payload, time.Now().UnixNano(), ) if err != nil { return fmt.Errorf("writeEntry upsert: %w", err) } - buf := c.pool.Get() - defer c.pool.Put(buf) - - for { - n, err := data.Read(buf) - if n > 0 { - _, errAppend := c.wdb.ExecContext(ctx, ` - UPDATE cache_blobs - SET data = data || ? - WHERE target_addr = ? AND hashin = ? AND artifact_name = ? - `, buf[:n], targetAddr, hashin, name) - if errAppend != nil { - return fmt.Errorf("writeEntry append chunk: %w", errAppend) - } - } - if err == io.EOF { - break - } - if err != nil { - return fmt.Errorf("writeEntry read: %w", err) - } - } - return nil } @@ -298,9 +335,15 @@ func (c *SQLCache) Writer(ctx context.Context, ref *pluginv1.TargetRef, hashin, func (c *SQLCache) ListArtifacts(ctx context.Context, ref *pluginv1.TargetRef, hashin string) iter.Seq2[string, error] { return func(yield func(string, error) bool) { + rdb, _, err := c.rwdb(ctx) + if err != nil { + yield("", err) + return + } + targetAddr := c.targetKey(ref) - rows, err := c.rdb.QueryContext(ctx, + rows, err := rdb.QueryContext(ctx, "SELECT artifact_name FROM cache_blobs WHERE target_addr = ? AND hashin = ?", targetAddr, hashin, ) @@ -330,9 +373,15 @@ func (c *SQLCache) ListArtifacts(ctx context.Context, ref *pluginv1.TargetRef, h func (c *SQLCache) ListVersions(ctx context.Context, ref *pluginv1.TargetRef) iter.Seq2[string, error] { return func(yield func(string, error) bool) { + rdb, _, err := c.rwdb(ctx) + if err != nil { + yield("", err) + return + } + targetAddr := c.targetKey(ref) - rows, err := c.rdb.QueryContext(ctx, + rows, err := rdb.QueryContext(ctx, "SELECT DISTINCT hashin FROM cache_blobs WHERE target_addr = ?", targetAddr, ) diff --git a/internal/hio/closer.go b/internal/hio/closer.go index 358a65bc..9865c83d 100644 --- a/internal/hio/closer.go +++ b/internal/hio/closer.go @@ -13,3 +13,19 @@ func NewReadCloser(r io.Reader, c io.Closer) io.ReadCloser { Closer: c, } } + +type readCloserFunc struct { + io.Reader + CloserFunc func() error +} + +func (r readCloserFunc) Close() error { + return r.CloserFunc() +} + +func NewReadCloserFunc(r io.Reader, f func() error) io.ReadCloser { + return readCloserFunc{ + Reader: r, + CloserFunc: f, + } +} From 9898a6119fa589528985c58271fc98b82d84b7a4 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Sun, 8 Mar 2026 12:12:12 +0000 Subject: [PATCH 07/15] more efficient fs --- internal/cmd/run.go | 4 +- internal/engine/hash_utils.go | 2 +- internal/engine/local_cache.go | 21 +++---- internal/engine/schedule.go | 107 +++++++++++++++++++++----------- internal/hfs/atomic.go | 2 +- internal/hinstance/uid.go | 10 ++- lib/pluginsdk/artifact.go | 6 ++ lib/pluginsdk/artifact_proto.go | 17 +++++ 8 files changed, 114 insertions(+), 55 deletions(-) diff --git a/internal/cmd/run.go b/internal/cmd/run.go index d2fc2229..8e937f09 100644 --- a/internal/cmd/run.go +++ b/internal/cmd/run.go @@ -230,9 +230,9 @@ func init() { for _, re := range res { for _, output := range re.Artifacts { if output.GetGroup() == "" { - fmt.Println(output.Hashout) + fmt.Println(output.GetHashout()) } else { - fmt.Println(output.GetGroup(), output.Hashout) + fmt.Println(output.GetGroup(), output.GetHashout()) } } } diff --git a/internal/engine/hash_utils.go b/internal/engine/hash_utils.go index 25d717ec..8fea0841 100644 --- a/internal/engine/hash_utils.go +++ b/internal/engine/hash_utils.go @@ -25,7 +25,7 @@ func newHashWithDebug(w *xxh3.Hasher, name, hint string) hashWithDebug { c, _ := debugCounter.GetOrSet(name, &atomic.Int32{}) id := c.Add(1) - path := filepath.Join("/tmp/hashdebug", hinstance.UID, name, fmt.Sprintf("%d_%s.txt", id, hint)) + path := filepath.Join("/tmp/hashdebug", hinstance.LocalUID, name, fmt.Sprintf("%d_%s.txt", id, hint)) return hashWithDebug{Hasher: w, path: path} } diff --git a/internal/engine/local_cache.go b/internal/engine/local_cache.go index 930cec46..5db0f221 100644 --- a/internal/engine/local_cache.go +++ b/internal/engine/local_cache.go @@ -85,7 +85,7 @@ func (e *Engine) cacheLocally( def *LightLinkedTarget, hashin string, sandboxArtifacts []*ExecuteArtifact, -) ([]*ResultArtifact, *hartifact.Manifest, error) { +) ([]*ResultArtifact, error) { step, ctx := hstep.New(ctx, "Caching...") defer step.Done() @@ -94,17 +94,17 @@ func (e *Engine) cacheLocally( for _, artifact := range sandboxArtifacts { contentType, err := artifact.GetContentType() if err != nil { - return nil, nil, err + return nil, err } contentSize, err := artifact.GetContentSize() if err != nil { - return nil, nil, err + return nil, err } src, err := artifact.GetContentReader() if err != nil { - return nil, nil, err + return nil, err } defer src.Close() @@ -117,7 +117,7 @@ func (e *Engine) cacheLocally( ContentType: hartifact.ManifestArtifactContentType(contentType), }, artifact.Hashout) if err != nil { - return nil, nil, fmt.Errorf("%q/%q: %w", artifact.GetGroup(), artifact.GetName(), err) + return nil, fmt.Errorf("%q/%q: %w", artifact.GetGroup(), artifact.GetName(), err) } cacheArtifacts = append(cacheArtifacts, cacheArtifact) @@ -125,12 +125,7 @@ func (e *Engine) cacheLocally( _ = src.Close() } - manifest, err := e.createLocalCacheManifest(ctx, def.GetRef(), hashin, cacheArtifacts) - if err != nil { - return nil, nil, err - } - - return cacheArtifacts, manifest, nil + return cacheArtifacts, nil } func (e *Engine) createLocalCacheManifest(ctx context.Context, ref *pluginv1.TargetRef, hashin string, artifacts []*ResultArtifact) (*hartifact.Manifest, error) { @@ -141,7 +136,7 @@ func (e *Engine) createLocalCacheManifest(ctx context.Context, ref *pluginv1.Tar Hashin: hashin, } for _, artifact := range artifacts { - martifact, err := hartifact.ProtoArtifactToManifest(artifact.Hashout, artifact.Artifact) + martifact, err := hartifact.ProtoArtifactToManifest(artifact.GetHashout(), artifact.Artifact) if err != nil { return nil, err } @@ -319,7 +314,6 @@ func (e *Engine) resultFromLocalCacheInner(ctx context.Context, def *LightLinked } execArtifacts = append(execArtifacts, &ResultArtifact{ - Hashout: artifact.Hashout, Artifact: cachePluginArtifact{ artifact: artifact, ref: def.GetRef(), @@ -393,7 +387,6 @@ func (e *Engine) cacheArtifactLocally(ctx context.Context, ref *pluginv1.TargetR } return &ResultArtifact{ - Hashout: hashout, Artifact: cachePluginArtifact{ artifact: manifestArtifact, ref: ref, diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index e3b56973..62a3504b 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -57,6 +57,8 @@ type ExecuteOptions struct { force bool interactive bool metaHashin string + getCache bool + storeCache bool } type InteractiveExecOptions struct { @@ -339,7 +341,7 @@ func (e *Engine) depsResults(ctx context.Context, rs *RequestState, t *LightLink }) for _, artifact := range res.Artifacts { - if artifact.Hashout == "" { + if artifact.GetHashout() == "" { return fmt.Errorf("%v: output %q has empty hashout", tref.Format(dep.GetRef()), artifact.GetGroup()) } } @@ -647,16 +649,16 @@ func (e *Engine) innerResult(ctx context.Context, rs *RequestState, def *LightLi for _, artifact := range result.Artifacts { partifact := artifactGroupMap{Artifact: artifact, group: ""} // TODO support output group - artifacts = append(artifacts, &ResultArtifact{ - Hashout: artifact.Hashout, - Artifact: partifact, - }) - - martifact, err := hartifact.ProtoArtifactToManifest(artifact.Hashout, partifact) + martifact, err := hartifact.ProtoArtifactToManifest(artifact.GetHashout(), partifact) if err != nil { return nil, fmt.Errorf("proto artifact to manifest: %w", err) } + artifacts = append(artifacts, &ResultArtifact{ + Artifact: partifact, + Manifest: martifact, + }) + manifest.Artifacts = append(manifest.Artifacts, martifact) locks.AddFrom(result.Locks) @@ -689,23 +691,15 @@ func (e *Engine) innerResult(ctx context.Context, rs *RequestState, def *LightLi force: shouldForce, interactive: tref.Equal(rs.Shell, def.GetRef()) || tref.Equal(rs.Interactive, def.GetRef()), metaHashin: hashin, + getCache: getCache, + storeCache: storeCache, } - var res *Result - if storeCache { - res, err = e.ExecuteAndCache(ctx, rs, def, execOptions) - if err != nil { - err = errors.Join(err, locks.Unlock()) - - return nil, err - } - } else { - res, err = e.ExecuteNoCache(ctx, rs, def, execOptions) - if err != nil { - err = errors.Join(err, locks.Unlock()) + res, err := e.executeAndCacheInner(ctx, rs, def, execOptions) + if err != nil { + err = errors.Join(err, locks.Unlock()) - return nil, err - } + return nil, err } err = locks.Lock2RLock(ctx) @@ -810,7 +804,7 @@ func (e *Engine) hashin(ctx context.Context, def *LightLinkedTarget, results []* artifacts := make([]DepMetaArtifact, 0, len(result.Artifacts)) for _, artifact := range result.Artifacts { artifacts = append(artifacts, DepMetaArtifact{ - Hashout: artifact.Hashout, + Hashout: artifact.GetHashout(), }) } @@ -844,7 +838,7 @@ type Result struct { func (r Result) Sorted() *Result { slices.SortFunc(r.Artifacts, func(a, b *ResultArtifact) int { - return strings.Compare(a.Hashout, b.Hashout) + return strings.Compare(a.GetHashout(), b.GetHashout()) }) return &r @@ -894,12 +888,15 @@ type ExecuteArtifact struct { } type ResultArtifact struct { - Hashout string Manifest hartifact.ManifestArtifact pluginsdk.Artifact } +func (r ResultArtifact) GetHashout() string { + return r.Manifest.Hashout +} + type ExecuteResultLocks struct { *Result Locks *CacheLocks @@ -1493,7 +1490,7 @@ func (e *Engine) execute(ctx context.Context, rs *RequestState, def *LightLinked }.Sorted(), nil } -func (e *Engine) executeAndCacheInner(ctx context.Context, rs *RequestState, def *LightLinkedTarget, options ExecuteOptions, persistentCache bool) (*Result, error) { +func (e *Engine) executeAndCacheInner(ctx context.Context, rs *RequestState, def *LightLinkedTarget, options ExecuteOptions) (*Result, error) { results, err := e.depsResults(ctx, rs, def) if err != nil { return nil, fmt.Errorf("deps results: %w", err) @@ -1510,11 +1507,11 @@ func (e *Engine) executeAndCacheInner(ctx context.Context, rs *RequestState, def } cacheHashin := hashin - if !persistentCache { + if !options.storeCache { cacheHashin = hinstance.UID + "_" + hashin } - if def.GetCache() && !options.force && !options.shell { + if options.storeCache { // One last cache check after the deps have completed res, ok, err := e.ResultFromLocalCache(ctx, def, def.OutputNames(), cacheHashin) if err != nil { @@ -1531,16 +1528,48 @@ func (e *Engine) executeAndCacheInner(ctx context.Context, rs *RequestState, def return nil, fmt.Errorf("execute: %w", err) } - cachedArtifacts, manifest, err := e.cacheLocally(ctx, def, cacheHashin, res.Artifacts) + var artifactsToCache []*ExecuteArtifact + var artifactsToPassthrough []*ResultArtifact + if options.storeCache { + artifactsToCache = res.Artifacts + } else { + // this caters for the pluginfs case where it doesnt make sense to copy the tree into the cache, if it's + // an uncached target + artifactsToPassthrough = make([]*ResultArtifact, 0, len(res.Artifacts)) + for _, artifact := range res.Artifacts { + if e.isPassthroughArtifact(artifact) { + m, err := hartifact.ProtoArtifactToManifest(artifact.Hashout, artifact) + if err != nil { + return nil, fmt.Errorf("protoartifacttomanifest: %w", err) + } + + artifactsToPassthrough = append(artifactsToPassthrough, &ResultArtifact{ + Manifest: m, + Artifact: artifact, + }) + } else { + artifactsToCache = append(artifactsToCache, artifact) + } + } + } + + cachedArtifacts, err := e.cacheLocally(ctx, def, cacheHashin, artifactsToCache) if err != nil { return nil, fmt.Errorf("cache locally: %w", err) } + cachedArtifacts = append(cachedArtifacts, artifactsToPassthrough...) + + manifest, err := e.createLocalCacheManifest(ctx, def.GetRef(), hashin, cachedArtifacts) + if err != nil { + return nil, fmt.Errorf("create local cache manifest: %w", err) + } + if res.AfterCache != nil { res.AfterCache() } - if persistentCache { + if options.storeCache { // TODO: move this to a background execution so that local build can proceed, while this is uploading in the background e.CacheRemotely(ctx, def, res.Hashin, manifest, cachedArtifacts) } @@ -1553,12 +1582,20 @@ func (e *Engine) executeAndCacheInner(ctx context.Context, rs *RequestState, def }.Sorted(), nil } -func (e *Engine) ExecuteAndCache(ctx context.Context, rs *RequestState, def *LightLinkedTarget, options ExecuteOptions) (*Result, error) { - return e.executeAndCacheInner(ctx, rs, def, options, true) -} +func (e *Engine) isPassthroughArtifact(artifact *ExecuteArtifact) bool { + fsartifact, ok := artifact.Artifact.(pluginsdk.FSArtifact) + if !ok { + return false + } -func (e *Engine) ExecuteNoCache(ctx context.Context, rs *RequestState, def *LightLinkedTarget, options ExecuteOptions) (*Result, error) { - // cached and uncached targets should go through the same mechanism, the only difference is that uncached targets should be removed upon session completion + node := fsartifact.FSNode() + if node == nil { + return false + } + + if hfs.HasPathPrefix(node.Path(), e.Home.Path()) { + return false + } - return e.executeAndCacheInner(ctx, rs, def, options, false) + return true } diff --git a/internal/hfs/atomic.go b/internal/hfs/atomic.go index 6596d3e7..6a9e4bfb 100644 --- a/internal/hfs/atomic.go +++ b/internal/hfs/atomic.go @@ -6,7 +6,7 @@ import ( ) func processUniquePath(p string) string { - return p + "_tmp_" + hinstance.UID + "_" + hrand.Str(7) + return p + "_tmp_" + hinstance.LocalUID + "_" + hrand.Str(7) } type AtomicFile struct { diff --git a/internal/hinstance/uid.go b/internal/hinstance/uid.go index e4abc915..cf3f39ed 100644 --- a/internal/hinstance/uid.go +++ b/internal/hinstance/uid.go @@ -12,12 +12,18 @@ import ( func gen() string { host, _ := os.Hostname() - return fmt.Sprintf("%v_%v_%v", os.Getpid(), host, time.Now().UnixNano()) + return fmt.Sprintf("%v_%v_%v", host, os.Getpid(), time.Now().UnixNano()) } var UID = gen() -var Hash = sync.OnceValue(func() string { +func localgen() string { + return fmt.Sprintf("%v_%v_%v", os.Getpid(), time.Now().UnixNano()) +} + +var LocalUID = localgen() + +var HashExec = sync.OnceValue(func() string { p, err := os.Executable() if err != nil { panic(err) diff --git a/lib/pluginsdk/artifact.go b/lib/pluginsdk/artifact.go index 71a6522f..ca40b3f6 100644 --- a/lib/pluginsdk/artifact.go +++ b/lib/pluginsdk/artifact.go @@ -3,6 +3,7 @@ package pluginsdk import ( "io" + "github.com/hephbuild/heph/internal/hfs" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" ) @@ -22,6 +23,11 @@ type Artifact interface { GetContentType() (ArtifactContentType, error) } +type FSArtifact interface { + Artifact + FSNode() hfs.Node +} + type ArtifactWithOrigin struct { Artifact Origin *pluginv1.TargetDef_InputOrigin diff --git a/lib/pluginsdk/artifact_proto.go b/lib/pluginsdk/artifact_proto.go index 0551655b..741ac910 100644 --- a/lib/pluginsdk/artifact_proto.go +++ b/lib/pluginsdk/artifact_proto.go @@ -146,3 +146,20 @@ func (a ProtoArtifact) GetContentType() (ArtifactContentType, error) { return "", fmt.Errorf("unsupported encoding %v", partifact.WhichContent()) } } + +func (a ProtoArtifact) FSNode() hfs.Node { + partifact := a.Artifact + + switch partifact.WhichContent() { + case pluginv1.Artifact_File_case: + return hfs.NewOS(partifact.GetFile().GetSourcePath()) + case pluginv1.Artifact_Raw_case: + return nil + case pluginv1.Artifact_TargzPath_case: + return hfs.NewOS(partifact.GetTargzPath()) + case pluginv1.Artifact_TarPath_case: + return hfs.NewOS(partifact.GetTarPath()) + default: + return nil + } +} From 3c6e7ff45dcac28c164a936e59a1a05d99c8be23 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Sun, 8 Mar 2026 14:00:41 +0000 Subject: [PATCH 08/15] optimize sqlite --- internal/engine/cache_sql.go | 62 ++++++++++++++++++---------------- internal/termui/interactive.go | 8 +++-- 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/internal/engine/cache_sql.go b/internal/engine/cache_sql.go index 1ae7e34b..dabccc32 100644 --- a/internal/engine/cache_sql.go +++ b/internal/engine/cache_sql.go @@ -20,6 +20,7 @@ import ( "github.com/hephbuild/heph/lib/tref" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" "github.com/zeebo/xxh3" + "modernc.org/sqlite" _ "modernc.org/sqlite" ) @@ -35,6 +36,20 @@ type SQLCacheDB struct { func (s *SQLCacheDB) pools() (*sql.DB, *sql.DB, error) { s.once.Do(func() { + sqlite.RegisterConnectionHook(func(conn sqlite.ExecQuerierContext, _ string) error { + _, err := conn.ExecContext(context.Background(), ` + PRAGMA journal_mode = WAL; + PRAGMA busy_timeout = 10000; + PRAGMA synchronous = NORMAL; + PRAGMA foreign_keys = ON; + PRAGMA cache_size = -64000; + PRAGMA page_size = 8192; + PRAGMA mmap_size = 268435456; + PRAGMA temp_store = MEMORY; + `, nil) + return err + }) + if err := os.MkdirAll(filepath.Dir(s.path), os.ModePerm); err != nil { s.err = fmt.Errorf("OpenSQLCacheDB mkdir: %w", err) return @@ -85,7 +100,6 @@ type SQLCache struct { db *SQLCacheDB rpool hsync.Pool[[]byte] - wpool hsync.Pool[[]byte] } func (c *SQLCache) rwdb(ctx context.Context) (rdb, wdb *sql.DB, err error) { @@ -166,13 +180,7 @@ func initSQLCacheDB(db *sql.DB) error { // openSQLiteDB opens a connection pool to a SQLite file with common pragmas. func openSQLiteDB(path string) (*sql.DB, error) { - dsn := path + - "?_pragma=journal_mode(WAL)" + - "&_pragma=busy_timeout(10000)" + - "&_pragma=synchronous(NORMAL)" + - "&_pragma=foreign_keys(ON)" - - db, err := sql.Open("sqlite", dsn) + db, err := sql.Open("sqlite", path) if err != nil { return nil, err } @@ -188,9 +196,6 @@ func OpenSQLCacheDB(path string) (*SQLCacheDB, error) { func NewSQLCache(db *SQLCacheDB) *SQLCache { return &SQLCache{ db: db, - wpool: hsync.Pool[[]byte]{New: func() []byte { - return make([]byte, 100_000) - }}, rpool: hsync.Pool[[]byte]{New: func() []byte { return make([]byte, 100_000) }}, @@ -260,19 +265,10 @@ func (c *SQLCache) writeEntry(ctx context.Context, targetAddr, hashin, name stri return err } - buf := c.wpool.Get() - defer c.wpool.Put(buf) - - // Read the full payload into the pool buffer without allocating. - // io.ReadFull returns: - // io.EOF — empty stream (zero bytes written) - // io.ErrUnexpectedEOF — stream shorter than buf (normal < 100KB case) - // nil — stream filled the buffer exactly - n, err := io.ReadFull(data, buf) - if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) && !errors.Is(err, io.EOF) { + payload, err := io.ReadAll(data) + if err != nil { return fmt.Errorf("writeEntry read: %w", err) } - payload := buf[:n] // Single UPSERT — write lock held for exactly one statement. _, err = wdb.ExecContext( @@ -294,8 +290,10 @@ func (c *SQLCache) writeEntry(ctx context.Context, targetAddr, hashin, name stri } type sqlCacheWriter struct { - pw *io.PipeWriter - done <-chan error + pw *io.PipeWriter + done <-chan error + closeOnce sync.Once + closeErr error } func (w *sqlCacheWriter) Write(p []byte) (int, error) { @@ -303,12 +301,16 @@ func (w *sqlCacheWriter) Write(p []byte) (int, error) { } func (w *sqlCacheWriter) Close() error { - // Close the write end; this unblocks the goroutine's reader. - if err := w.pw.Close(); err != nil { - return err - } - // Wait for the goroutine to finish and return any write error. - return <-w.done + w.closeOnce.Do(func() { + // Close the write end; this unblocks the goroutine's reader. + if err := w.pw.Close(); err != nil { + w.closeErr = err + return + } + // Wait for the goroutine to finish and return any write error. + w.closeErr = <-w.done + }) + return w.closeErr } func (c *SQLCache) Writer(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.WriteCloser, error) { diff --git a/internal/termui/interactive.go b/internal/termui/interactive.go index 1a093c01..a46bc959 100644 --- a/internal/termui/interactive.go +++ b/internal/termui/interactive.go @@ -12,6 +12,7 @@ import ( "sync/atomic" "time" + "charm.land/lipgloss/v2" "github.com/hephbuild/heph/internal/hsoftcontext" "github.com/hephbuild/heph/internal/htime" @@ -125,12 +126,13 @@ func (m Model) View() tea.View { failed = fmt.Sprintf(" Failed jobs: %-6d", m.stepsState.failed) } - sb.WriteString(fmt.Sprintf( + _, _ = fmt.Fprintf( + &sb, "Total jobs: %-6d Completed jobs: %-6d%v Total time: %v\n", m.stepsState.total, m.stepsState.completed, failed, htime.FormatFixedWidthDuration(time.Since(m.startedAt)), - )) + ) if !m.finalRendering { sb.WriteString(strings.Repeat("-", m.width)) @@ -139,7 +141,7 @@ func (m Model) View() tea.View { buildStepsTree(m.stepsState.steps, m.stepsState.leafs, &sb, m.width, (m.height-2)/2) } - return tea.NewView(sb.String()) + return tea.NewView(lipgloss.NewStyle().MaxWidth(m.width).Render(sb.String())) } func NewStepsStore(ctx context.Context, p *tea.Program) (func(*corev1.Step), func()) { From cfcbcec6fda8e931b668610cdba453b29f0589ae Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Sun, 8 Mar 2026 14:10:48 +0000 Subject: [PATCH 09/15] optimize sql --- internal/engine/cache_sql.go | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/internal/engine/cache_sql.go b/internal/engine/cache_sql.go index dabccc32..2161a81b 100644 --- a/internal/engine/cache_sql.go +++ b/internal/engine/cache_sql.go @@ -118,17 +118,17 @@ func (c *SQLCache) Exists(ctx context.Context, ref *pluginv1.TargetRef, hashin, targetAddr := c.targetKey(ref) - var count int + var exists bool err = rdb.QueryRowContext( ctx, - `SELECT COUNT(*) FROM cache_blobs WHERE target_addr = ? AND hashin = ? AND artifact_name = ? LIMIT 1`, + `SELECT 1 FROM cache_blobs WHERE target_addr = ? AND hashin = ? AND artifact_name = ? LIMIT 1`, targetAddr, hashin, name, - ).Scan(&count) - if err != nil { + ).Scan(&exists) + if err != nil && !errors.Is(err, sql.ErrNoRows) { return false, fmt.Errorf("exists: %w", err) } - return count > 0, nil + return exists, nil } func (c *SQLCache) Delete(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) error { @@ -229,10 +229,7 @@ func (c *SQLCache) Reader(ctx context.Context, ref *pluginv1.TargetRef, hashin, targetAddr := c.targetKey(ref) - buf := c.rpool.Get() - buf = buf[0:] - - err = rdb.QueryRowContext( + rows, err := rdb.QueryContext( ctx, ` SELECT data @@ -240,18 +237,28 @@ func (c *SQLCache) Reader(ctx context.Context, ref *pluginv1.TargetRef, hashin, WHERE target_addr = ? AND hashin = ? AND artifact_name = ? `, targetAddr, hashin, name, - ).Scan(&buf) - + ) if err != nil { - c.rpool.Put(buf) + return nil, fmt.Errorf("reader query: %w", err) + } + defer rows.Close() - if errors.Is(err, sql.ErrNoRows) { - return nil, LocalCacheNotFoundError + if !rows.Next() { + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("reader next: %w", err) } + return nil, LocalCacheNotFoundError + } + var raw sql.RawBytes + err = rows.Scan(&raw) + if err != nil { return nil, fmt.Errorf("reader scan: %w", err) } + buf := c.rpool.Get() + buf = append(buf[:0], raw...) + return hio.NewReadCloserFunc(bytes.NewReader(buf), func() error { c.rpool.Put(buf) From b4ce8e7831df1f44464abdb8f261934d29b49bbe Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Sun, 8 Mar 2026 14:27:01 +0000 Subject: [PATCH 10/15] cache pkg list --- internal/engine/schedule.go | 4 +- plugin/plugingo/pkg_analysis.go | 155 +++++++++++++++++--------------- plugin/plugingo/plugin.go | 9 ++ 3 files changed, 96 insertions(+), 72 deletions(-) diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index 62a3504b..c4fac946 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -611,7 +611,7 @@ func (e *Engine) innerResult(ctx context.Context, rs *RequestState, def *LightLi } res, ok, err := e.resultFromCache(ctx, rs, def, outputs, hashin) - if err != nil { + if err != nil && !errors.Is(err, context.Canceled) { hlog.From(ctx).With(slog.String("target", tref.Format(def.GetRef())), slog.String("err", err.Error())).Warn("failed to get result from local cache") } @@ -1514,7 +1514,7 @@ func (e *Engine) executeAndCacheInner(ctx context.Context, rs *RequestState, def if options.storeCache { // One last cache check after the deps have completed res, ok, err := e.ResultFromLocalCache(ctx, def, def.OutputNames(), cacheHashin) - if err != nil { + if err != nil && !errors.Is(err, context.Canceled) { hlog.From(ctx).With(slog.String("target", tref.Format(def.GetRef())), slog.String("err", err.Error())).Warn("failed to get result from local cache") } diff --git a/plugin/plugingo/pkg_analysis.go b/plugin/plugingo/pkg_analysis.go index 16deaa1b..bfc492cc 100644 --- a/plugin/plugingo/pkg_analysis.go +++ b/plugin/plugingo/pkg_analysis.go @@ -34,101 +34,116 @@ import ( var errNoGoFiles = errors.New("no Go files in package") func (p *Plugin) goListPkgResult(ctx context.Context, basePkg, runPkg, imp string, factors Factors, requestId string) (Package, error) { - res, err := p.resultClient.ResultClient.Get(ctx, corev1.ResultRequest_builder{ - RequestId: htypes.Ptr(requestId), - Ref: tref.New(runPkg, "_golist", hmaps.Concat(factors.Args(), map[string]string{ - "imp": imp, - })), - }.Build()) - if err != nil { - return Package{}, fmt.Errorf("golist: %v (in %v): %w", imp, runPkg, err) + key := goListPkgResultKey{ + RequestId: requestId, + Factors: factors, + BasePkg: basePkg, + RunPkg: runPkg, + Imp: imp, } - defer res.Release() - artifacts := res.Artifacts + res, err, _ := p.goListPkgCache.Do(ctx, key, func(ctx context.Context) (Package, error) { + res, err := p.resultClient.ResultClient.Get(ctx, corev1.ResultRequest_builder{ + RequestId: htypes.Ptr(requestId), + Ref: tref.New(runPkg, "_golist", hmaps.Concat(factors.Args(), map[string]string{ + "imp": imp, + })), + }.Build()) + if err != nil { + return Package{}, fmt.Errorf("golist: %v (in %v): %w", imp, runPkg, err) + } + defer res.Release() - jsonArtifacts := hartifact.FindOutputs(artifacts, "json") - rootArtifacts := hartifact.FindOutputs(artifacts, "root") - smArtifacts := hartifact.FindOutputs(artifacts, "sm") - nogoArtifacts := hartifact.FindOutputs(artifacts, "nogo") + artifacts := res.Artifacts - if len(jsonArtifacts) == 0 || len(rootArtifacts) == 0 || len(smArtifacts) == 0 || len(nogoArtifacts) == 0 { - return Package{}, connect.NewError(connect.CodeInternal, errors.New("golist: no json found")) - } - - jsonArtifact := jsonArtifacts[0] - rootArtifact := rootArtifacts[0] - smArtifact := smArtifacts[0] - nogoArtifact := nogoArtifacts[0] + jsonArtifacts := hartifact.FindOutputs(artifacts, "json") + rootArtifacts := hartifact.FindOutputs(artifacts, "root") + smArtifacts := hartifact.FindOutputs(artifacts, "sm") + nogoArtifacts := hartifact.FindOutputs(artifacts, "nogo") - { - b, err := hartifact.FileReadAll(ctx, nogoArtifact) - if err != nil { - return Package{}, err + if len(jsonArtifacts) == 0 || len(rootArtifacts) == 0 || len(smArtifacts) == 0 || len(nogoArtifacts) == 0 { + return Package{}, connect.NewError(connect.CodeInternal, errors.New("golist: no json found")) } - if ok, _ := strconv.ParseBool(strings.TrimSpace(string(b))); ok { - return Package{}, errNoGoFiles - } - } + jsonArtifact := jsonArtifacts[0] + rootArtifact := rootArtifacts[0] + smArtifact := smArtifacts[0] + nogoArtifact := nogoArtifacts[0] - rootb, err := hartifact.FileReadAll(ctx, rootArtifact) - if err != nil { - return Package{}, fmt.Errorf("root artifact: %w", err) - } + { + b, err := hartifact.FileReadAll(ctx, nogoArtifact) + if err != nil { + return Package{}, err + } - root := strings.TrimSpace(string(rootb)) + if ok, _ := strconv.ParseBool(strings.TrimSpace(string(b))); ok { + return Package{}, errNoGoFiles + } + } - var goPkg Package - { - f, err := hartifact.FileReader(ctx, jsonArtifact) + rootb, err := hartifact.FileReadAll(ctx, rootArtifact) if err != nil { - return Package{}, err + return Package{}, fmt.Errorf("root artifact: %w", err) } - defer f.Close() - err = json.NewDecoder(f).Decode(&goPkg) - if err != nil { - return Package{}, fmt.Errorf("gopkg decode %q: %w", tref.Format(res.Def.GetRef()), err) + root := strings.TrimSpace(string(rootb)) + + var goPkg Package + { + f, err := hartifact.FileReader(ctx, jsonArtifact) + if err != nil { + return Package{}, err + } + defer f.Close() + + err = json.NewDecoder(f).Decode(&goPkg) + if err != nil { + return Package{}, fmt.Errorf("gopkg decode %q: %w", tref.Format(res.Def.GetRef()), err) + } } - } - var sourcemap map[string]string - { - f, err := hartifact.FileReader(ctx, smArtifact) - if err != nil { - return Package{}, err + var sourcemap map[string]string + { + f, err := hartifact.FileReader(ctx, smArtifact) + if err != nil { + return Package{}, err + } + defer f.Close() + + err = json.NewDecoder(f).Decode(&sourcemap) + if err != nil { + return Package{}, fmt.Errorf("sourcemap decode: %q: %w", tref.Format(res.Def.GetRef()), err) + } } - defer f.Close() - err = json.NewDecoder(f).Decode(&sourcemap) + goPkg.Sourcemap = sourcemap + + relPkg, err := tref.DirToPackage(goPkg.Dir, root) if err != nil { - return Package{}, fmt.Errorf("sourcemap decode: %q: %w", tref.Format(res.Def.GetRef()), err) - } - } + goPkg.Is3rdParty = true - goPkg.Sourcemap = sourcemap + if goPkg.Module == nil { + return Package{}, fmt.Errorf("%v: not in a module", imp) + } - relPkg, err := tref.DirToPackage(goPkg.Dir, root) - if err != nil { - goPkg.Is3rdParty = true + modPath := strings.ReplaceAll(goPkg.ImportPath, goPkg.Module.Path, "") + modPath = strings.TrimPrefix(modPath, "/") - if goPkg.Module == nil { - return Package{}, fmt.Errorf("%v: not in a module", imp) + goPkg.HephPackage = ThirdpartyBuildPackage(basePkg, goPkg.Module.Path, goPkg.Module.Version, modPath) + goPkg.HephBuildPackage = ThirdpartyBuildPackage(basePkg, goPkg.Module.Path, goPkg.Module.Version, modPath) + } else { + goPkg.Dir = filepath.Join(p.root, relPkg) + goPkg.HephPackage = relPkg } + goPkg.Factors = factors - modPath := strings.ReplaceAll(goPkg.ImportPath, goPkg.Module.Path, "") - modPath = strings.TrimPrefix(modPath, "/") - - goPkg.HephPackage = ThirdpartyBuildPackage(basePkg, goPkg.Module.Path, goPkg.Module.Version, modPath) - goPkg.HephBuildPackage = ThirdpartyBuildPackage(basePkg, goPkg.Module.Path, goPkg.Module.Version, modPath) - } else { - goPkg.Dir = filepath.Join(p.root, relPkg) - goPkg.HephPackage = relPkg + return goPkg, nil + }) + if err != nil { + return Package{}, err } - goPkg.Factors = factors - return goPkg, nil + return res, nil } type GetGoPackageCache struct { diff --git a/plugin/plugingo/plugin.go b/plugin/plugingo/plugin.go index 3fd37c30..2dc4b734 100644 --- a/plugin/plugingo/plugin.go +++ b/plugin/plugingo/plugin.go @@ -69,6 +69,14 @@ type moduleCacheKey struct { BasePkg string } +type goListPkgResultKey struct { + RequestId string + Factors Factors + BasePkg string + RunPkg string + Imp string +} + type Plugin struct { goTool string resultClient pluginsdk.Engine @@ -79,6 +87,7 @@ type Plugin struct { moduleCache hsingleflight.GroupMem[moduleCacheKey, []Module] stdCache hsingleflight.GroupMem[stdCacheKey, map[string]Package] goModGoWorkCache hsingleflight.GroupMemContext[string, goModRoot] + goListPkgCache hsingleflight.GroupMemContext[goListPkgResultKey, Package] } func (p *Plugin) getGoToolStructpb() *structpb.Value { From a9bacae98b0e564fe31d8abbd920e395838b90e5 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Sun, 8 Mar 2026 15:14:28 +0000 Subject: [PATCH 11/15] prevent proto copy --- internal/hmaps/sorted.go | 5 ++--- plugin/pluginexec/parse_config.go | 35 +++++++++++++++++++------------ 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/internal/hmaps/sorted.go b/internal/hmaps/sorted.go index afec6383..f0eef0fe 100644 --- a/internal/hmaps/sorted.go +++ b/internal/hmaps/sorted.go @@ -14,9 +14,8 @@ func Sorted[Map ~map[K]V, K cmp.Ordered, V any](m Map) iter.Seq2[K, V] { return case 1: for k, v := range m { - if !yield(k, v) { - return - } + yield(k, v) + break } return diff --git a/plugin/pluginexec/parse_config.go b/plugin/pluginexec/parse_config.go index 944c7ca5..2e23028e 100644 --- a/plugin/pluginexec/parse_config.go +++ b/plugin/pluginexec/parse_config.go @@ -9,7 +9,6 @@ import ( "github.com/hephbuild/heph/internal/hfs" "github.com/hephbuild/heph/internal/hmaps" - "github.com/hephbuild/heph/internal/hproto" "github.com/hephbuild/heph/internal/hproto/hashpb" "github.com/hephbuild/heph/internal/hproto/hstructpb" "github.com/hephbuild/heph/internal/hslices" @@ -190,41 +189,51 @@ func ToDef[S proto.Message](ref *pluginv1.TargetRef, target S, getTarget func(S) return def, nil } +var targetProtoName = string((&execv1.Target{}).ProtoReflect().Descriptor().Name()) + func hashTarget(target *execv1.Target) []byte { + omit := tref.OmitHashPb var cloned bool clonedOnce := func() { if !cloned { cloned = true - target = hproto.Clone(target) + omit = maps.Clone(omit) } } + h := xxh3.New() if hmaps.Has(target.GetEnv(), func(s string, env *execv1.Target_Env) bool { return !env.GetHash() }) { clonedOnce() + omit[targetProtoName+".env"] = struct{}{} - env := target.GetEnv() + for k, v := range hmaps.Sorted(target.GetEnv()) { + if !v.GetHash() { + continue + } - maps.DeleteFunc(env, func(s string, env *execv1.Target_Env) bool { - return !env.GetHash() - }) - target.SetEnv(env) + _, _ = h.WriteString(k) + hashpb.Hash(h, v, omit) + } } if hslices.Has(target.GetDeps(), func(dep *execv1.Target_Dep) bool { return !dep.GetHash() }) { clonedOnce() + omit[targetProtoName+".deps"] = struct{}{} - deps := slices.DeleteFunc(target.GetDeps(), func(dep *execv1.Target_Dep) bool { - return !dep.GetHash() - }) - target.SetDeps(deps) + for _, v := range target.GetDeps() { + if !v.GetHash() { + continue + } + + hashpb.Hash(h, v, omit) + } } - h := xxh3.New() - hashpb.Hash(h, target, tref.OmitHashPb) + hashpb.Hash(h, target, omit) return h.Sum(nil) } From 0fc7a201e485bbcd454117bad0639c515766c2d9 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Sun, 8 Mar 2026 18:12:29 +0000 Subject: [PATCH 12/15] lint --- internal/cmd/inspect_cache.go | 1 - internal/cmd/run.go | 18 ++----- internal/engine/cache_sql.go | 78 ++++++++++++++++++++--------- internal/engine/cache_sql_test.go | 2 +- internal/engine/handler_resulter.go | 1 - internal/engine/local_cache.go | 8 +-- internal/hartifact/reader.go | 2 +- internal/hinstance/uid.go | 2 +- 8 files changed, 65 insertions(+), 47 deletions(-) diff --git a/internal/cmd/inspect_cache.go b/internal/cmd/inspect_cache.go index 9e93036c..07bb33de 100644 --- a/internal/cmd/inspect_cache.go +++ b/internal/cmd/inspect_cache.go @@ -121,7 +121,6 @@ func init() { fmt.Println(name) } - } return nil diff --git a/internal/cmd/run.go b/internal/cmd/run.go index 8e937f09..e132f319 100644 --- a/internal/cmd/run.go +++ b/internal/cmd/run.go @@ -210,20 +210,8 @@ func init() { for _, re := range res { for _, output := range re.Artifacts { fmt.Println(output.GetName()) - fmt.Println(" group: ", output.GetGroup()) - //switch output.WhichContent() { - //case pluginv1.Artifact_File_case: - // fmt.Println(" content:", output.GetFile()) - //case pluginv1.Artifact_Raw_case: - // fmt.Println(" content:", output.GetRaw()) - //case pluginv1.Artifact_TarPath_case: - // fmt.Println(" content:", output.GetTarPath()) - //case pluginv1.Artifact_TargzPath_case: - // fmt.Println(" content:", output.GetTargzPath()) - //case pluginv1.Artifact_Content_not_set_case: - // fmt.Println(" content: ") - //} - fmt.Println(" type: ", output.GetType().String()) + fmt.Println(" group: ", output.GetGroup()) + fmt.Println(" type: ", output.GetType().String()) } } case hashOut: @@ -266,7 +254,7 @@ func init() { hcobra.AddLocalFlagSet(cmd, runFlagGroup) outFlagGroup := hcobra.NewFlagSet("Output Flags") - //outFlagGroup.BoolVarP(&listArtifacts, "list-artifacts", "", false, "List output artifacts") + outFlagGroup.BoolVarP(&listArtifacts, "list-artifacts", "", false, "List output artifacts") outFlagGroup.BoolVarP(&listOut, "list-out", "", false, "List output paths") outFlagGroup.StringVarP(©Out, "copy-out", "", "", "Copy output to path") outFlagGroup.BoolVarP(&catOut, "cat-out", "", false, "Print outputs to stdout") diff --git a/internal/engine/cache_sql.go b/internal/engine/cache_sql.go index 2161a81b..701d9a71 100644 --- a/internal/engine/cache_sql.go +++ b/internal/engine/cache_sql.go @@ -21,23 +21,26 @@ import ( pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" "github.com/zeebo/xxh3" "modernc.org/sqlite" - _ "modernc.org/sqlite" ) // SQLCacheDB holds the path and lazily opens both connection pools on first use. // Call Close only if the DB was actually used; it is safe to call regardless. type SQLCacheDB struct { - path string - once sync.Once - rdb *sql.DB - wdb *sql.DB - err error + path string + once sync.Once + rdb *sql.DB + wdb *sql.DB + readerStmt *sql.Stmt + existsStmt *sql.Stmt + err error } func (s *SQLCacheDB) pools() (*sql.DB, *sql.DB, error) { + ctx := context.Background() + s.once.Do(func() { sqlite.RegisterConnectionHook(func(conn sqlite.ExecQuerierContext, _ string) error { - _, err := conn.ExecContext(context.Background(), ` + _, err := conn.ExecContext(ctx, ` PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 10000; PRAGMA synchronous = NORMAL; @@ -63,7 +66,7 @@ func (s *SQLCacheDB) pools() (*sql.DB, *sql.DB, error) { // One writer at a time — prevents SQLITE_BUSY. wdb.SetMaxOpenConns(1) - if err := initSQLCacheDB(wdb); err != nil { + if err := initSQLCacheDB(ctx, wdb); err != nil { _ = wdb.Close() s.err = fmt.Errorf("OpenSQLCacheDB init: %w", err) return @@ -76,9 +79,34 @@ func (s *SQLCacheDB) pools() (*sql.DB, *sql.DB, error) { return } // No cap — WAL lets concurrent readers run in parallel. + rdb.SetMaxIdleConns(100) + rdb.SetMaxOpenConns(100) + + readerStmt, err := rdb.Prepare(` + SELECT data + FROM cache_blobs + WHERE target_addr = ? AND hashin = ? AND artifact_name = ? + `) + if err != nil { + _ = rdb.Close() + _ = wdb.Close() + s.err = fmt.Errorf("OpenSQLCacheDB prepare reader: %w", err) + return + } + + existsStmt, err := rdb.Prepare(`SELECT 1 FROM cache_blobs WHERE target_addr = ? AND hashin = ? AND artifact_name = ? LIMIT 1`) + if err != nil { + _ = readerStmt.Close() + _ = rdb.Close() + _ = wdb.Close() + s.err = fmt.Errorf("OpenSQLCacheDB prepare exists: %w", err) + return + } s.rdb = rdb s.wdb = wdb + s.readerStmt = readerStmt + s.existsStmt = existsStmt }) return s.rdb, s.wdb, s.err } @@ -90,9 +118,17 @@ func (s *SQLCacheDB) Close() error { } errR := s.rdb.Close() errW := s.wdb.Close() + errS := s.readerStmt.Close() + errE := s.existsStmt.Close() if errR != nil { return errR } + if errS != nil { + return errS + } + if errE != nil { + return errE + } return errW } @@ -102,16 +138,17 @@ type SQLCache struct { rpool hsync.Pool[[]byte] } -func (c *SQLCache) rwdb(ctx context.Context) (rdb, wdb *sql.DB, err error) { - rdb, wdb, err = c.db.pools() +func (c *SQLCache) rwdb(ctx context.Context) (*sql.DB, *sql.DB, error) { + rdb, wdb, err := c.db.pools() if err != nil { return nil, nil, fmt.Errorf("sqlcache open db: %w", err) } + return rdb, wdb, nil } func (c *SQLCache) Exists(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (bool, error) { - rdb, _, err := c.rwdb(ctx) + _, _, err := c.rwdb(ctx) if err != nil { return false, err } @@ -119,9 +156,8 @@ func (c *SQLCache) Exists(ctx context.Context, ref *pluginv1.TargetRef, hashin, targetAddr := c.targetKey(ref) var exists bool - err = rdb.QueryRowContext( + err = c.db.existsStmt.QueryRowContext( ctx, - `SELECT 1 FROM cache_blobs WHERE target_addr = ? AND hashin = ? AND artifact_name = ? LIMIT 1`, targetAddr, hashin, name, ).Scan(&exists) if err != nil && !errors.Is(err, sql.ErrNoRows) { @@ -161,8 +197,8 @@ func (c *SQLCache) Delete(ctx context.Context, ref *pluginv1.TargetRef, hashin, var _ LocalCache = (*SQLCache)(nil) -func initSQLCacheDB(db *sql.DB) error { - _, err := db.Exec(` +func initSQLCacheDB(ctx context.Context, db *sql.DB) error { + _, err := db.ExecContext(ctx, ` CREATE TABLE IF NOT EXISTS cache_blobs ( target_addr TEXT NOT NULL, hashin TEXT NOT NULL, @@ -175,6 +211,7 @@ func initSQLCacheDB(db *sql.DB) error { CREATE INDEX IF NOT EXISTS cache_blobs_target_hashin ON cache_blobs (target_addr, hashin); `) + return err } @@ -222,20 +259,15 @@ func (c *SQLCache) targetKey(ref *pluginv1.TargetRef) string { } func (c *SQLCache) Reader(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.ReadCloser, error) { - rdb, _, err := c.rwdb(ctx) + _, _, err := c.rwdb(ctx) if err != nil { return nil, err } targetAddr := c.targetKey(ref) - rows, err := rdb.QueryContext( + rows, err := c.db.readerStmt.QueryContext( ctx, - ` - SELECT data - FROM cache_blobs - WHERE target_addr = ? AND hashin = ? AND artifact_name = ? - `, targetAddr, hashin, name, ) if err != nil { @@ -247,7 +279,7 @@ func (c *SQLCache) Reader(ctx context.Context, ref *pluginv1.TargetRef, hashin, if err := rows.Err(); err != nil { return nil, fmt.Errorf("reader next: %w", err) } - return nil, LocalCacheNotFoundError + return nil, ErrLocalCacheNotFound } var raw sql.RawBytes diff --git a/internal/engine/cache_sql_test.go b/internal/engine/cache_sql_test.go index 7dbcaf2d..1b627cd8 100644 --- a/internal/engine/cache_sql_test.go +++ b/internal/engine/cache_sql_test.go @@ -51,7 +51,7 @@ func TestSQLCache(t *testing.T) { require.NoError(t, err) require.False(t, exists) _, err = cache.Reader(ctx, ref, hashin, "art2") - require.ErrorIs(t, err, engine.LocalCacheNotFoundError) + require.ErrorIs(t, err, engine.ErrLocalCacheNotFound) // Test ListArtifacts w, err = cache.Writer(ctx, ref, hashin, "art2") diff --git a/internal/engine/handler_resulter.go b/internal/engine/handler_resulter.go index 6778fa00..79f59149 100644 --- a/internal/engine/handler_resulter.go +++ b/internal/engine/handler_resulter.go @@ -71,5 +71,4 @@ func (r *resulterHandler) Get(ctx context.Context, req *corev1.ResultRequest) (* Artifacts: artifacts, Def: res.Def.TargetDef.TargetDef, }, nil - } diff --git a/internal/engine/local_cache.go b/internal/engine/local_cache.go index 5db0f221..1113eefc 100644 --- a/internal/engine/local_cache.go +++ b/internal/engine/local_cache.go @@ -25,7 +25,7 @@ import ( "github.com/zeebo/xxh3" ) -var LocalCacheNotFoundError = errors.New("artifact not found") +var ErrLocalCacheNotFound = errors.New("artifact not found") type LocalCache interface { Reader(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.ReadCloser, error) @@ -251,7 +251,7 @@ func (e *Engine) readAnyCache(ctx context.Context, ref *pluginv1.TargetRef, hash for _, c := range [...]LocalCache{e.CacheSmall, e.CacheLarge} { r, err := c.Reader(ctx, ref, hashin, name) if err != nil { - if errors.Is(err, LocalCacheNotFoundError) { + if errors.Is(err, ErrLocalCacheNotFound) { continue } @@ -261,7 +261,7 @@ func (e *Engine) readAnyCache(ctx context.Context, ref *pluginv1.TargetRef, hash return r, nil } - return nil, LocalCacheNotFoundError + return nil, ErrLocalCacheNotFound } func (e *Engine) existsAnyCache(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (bool, LocalCache, error) { @@ -282,7 +282,7 @@ func (e *Engine) existsAnyCache(ctx context.Context, ref *pluginv1.TargetRef, ha func (e *Engine) resultFromLocalCacheInner(ctx context.Context, def *LightLinkedTarget, outputs []string, hashin string) (*Result, bool, error) { r, err := e.readAnyCache(ctx, def.GetRef(), hashin, hartifact.ManifestName) if err != nil { - if errors.Is(err, LocalCacheNotFoundError) { + if errors.Is(err, ErrLocalCacheNotFound) { return nil, false, nil } diff --git a/internal/hartifact/reader.go b/internal/hartifact/reader.go index 18a9ffac..706f9518 100644 --- a/internal/hartifact/reader.go +++ b/internal/hartifact/reader.go @@ -34,7 +34,7 @@ func FileReader(ctx context.Context, a pluginsdk.Artifact) (io.ReadCloser, error } return hio.NewReadCloser(tr, r), nil - //case pluginsdk.ArtifactContentTypeTarGz: + // case pluginsdk.ArtifactContentTypeTarGz: default: return nil, fmt.Errorf("unsupported encoding %v", contentType) } diff --git a/internal/hinstance/uid.go b/internal/hinstance/uid.go index cf3f39ed..02b34b92 100644 --- a/internal/hinstance/uid.go +++ b/internal/hinstance/uid.go @@ -18,7 +18,7 @@ func gen() string { var UID = gen() func localgen() string { - return fmt.Sprintf("%v_%v_%v", os.Getpid(), time.Now().UnixNano()) + return fmt.Sprintf("%v_%v", os.Getpid(), time.Now().UnixNano()) } var LocalUID = localgen() From ea8ff026442aef7d9790f9cc1ffd95317201d8a3 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Sun, 8 Mar 2026 18:38:40 +0000 Subject: [PATCH 13/15] fix tests --- internal/engine/remote_cache.go | 7 +- internal/engine/schedule.go | 2 +- internal/enginee2e/deps_cache_test.go | 4 +- internal/enginee2e/hash_deps_test.go | 2 +- internal/enginee2e/sanity_remotecache_test.go | 103 +++++++++--------- lib/pluginsdk/artifact_proto.go | 11 ++ lib/tref/utils.go | 4 +- plugin/pluginexec/parse_config.go | 55 +++------- plugin/pluginnix/driver.go | 2 +- 9 files changed, 93 insertions(+), 97 deletions(-) diff --git a/internal/engine/remote_cache.go b/internal/engine/remote_cache.go index b099970a..694978a5 100644 --- a/internal/engine/remote_cache.go +++ b/internal/engine/remote_cache.go @@ -94,6 +94,11 @@ func (e *Engine) cacheRemotelyInner(ctx context.Context, }) } + err := g.Wait() + if err != nil { + return err + } + key := e.remoteCacheKey(ref, hashin, hartifact.ManifestName) pr, pw := io.Pipe() @@ -108,7 +113,7 @@ func (e *Engine) cacheRemotelyInner(ctx context.Context, } }() - err := cache.Client.Store(ctx, key, pr) + err = cache.Client.Store(ctx, key, pr) if err != nil { return err } diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index c4fac946..eb987b91 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -450,7 +450,7 @@ func (e *Engine) ResultMetaFromDef(ctx context.Context, rs *RequestState, def *T m := ResultMeta{ Hashin: manifest.Hashin, - CreatedAt: manifest.CreatedAt, + CreatedAt: manifest.CreatedAt.UTC(), } if len(outputs) == 1 && outputs[0] == AllOutputs { diff --git a/internal/enginee2e/deps_cache_test.go b/internal/enginee2e/deps_cache_test.go index bb777225..42aea1a3 100644 --- a/internal/enginee2e/deps_cache_test.go +++ b/internal/enginee2e/deps_cache_test.go @@ -276,10 +276,10 @@ func TestDepsCacheRemote(t *testing.T) { cache := pluginsdk.NewMockCache(c) cache.EXPECT(). - Get(gomock.Any(), "__child/e5d50f4b478a3687/manifest.v1.json"). + Get(gomock.Any(), "__child/16f2c56460420e9a/manifest.v1.json"). Return(nil, pluginsdk.ErrNotFound).Times(1) - for _, key := range []string{"__child/e5d50f4b478a3687/manifest.v1.json", "__child/e5d50f4b478a3687/out_out.tar"} { + for _, key := range []string{"__child/16f2c56460420e9a/manifest.v1.json", "__child/16f2c56460420e9a/out_out.tar"} { cache.EXPECT(). Store(gomock.Any(), key, gomock.Any()). DoAndReturn(func(ctx context.Context, key string, reader io.Reader) error { diff --git a/internal/enginee2e/hash_deps_test.go b/internal/enginee2e/hash_deps_test.go index a490cc0d..285e0185 100644 --- a/internal/enginee2e/hash_deps_test.go +++ b/internal/enginee2e/hash_deps_test.go @@ -67,7 +67,7 @@ func TestHashDeps(t *testing.T) { m, err := e.ResultMetaFromRef(ctx, rs, tref.New("some/package", "sometarget", nil), nil) require.NoError(t, err) - at = m.CreatedAt.UTC() + at = m.CreatedAt res.Unlock(ctx) } diff --git a/internal/enginee2e/sanity_remotecache_test.go b/internal/enginee2e/sanity_remotecache_test.go index 1df3cdeb..3947b7d7 100644 --- a/internal/enginee2e/sanity_remotecache_test.go +++ b/internal/enginee2e/sanity_remotecache_test.go @@ -3,6 +3,7 @@ package enginee2e import ( "bytes" "context" + "fmt" "io" "testing" @@ -112,76 +113,76 @@ func TestSanityRemoteCache(t *testing.T) { // Simulates 2 independent runs, with the same cache for i := 1; i <= 2; i++ { - t.Log("RUN", i) + t.Run(fmt.Sprintf("run %v", i), func(t *testing.T) { + dir := t.TempDir() - dir := t.TempDir() - - e, err := engine.New(ctx, dir, engine.Config{}) - require.NoError(t, err) + e, err := engine.New(ctx, dir, engine.Config{}) + require.NoError(t, err) - _, err = e.RegisterProvider(ctx, staticprovider, engine.RegisterProviderConfig{}) - require.NoError(t, err) + _, err = e.RegisterProvider(ctx, staticprovider, engine.RegisterProviderConfig{}) + require.NoError(t, err) - _, err = e.RegisterDriver(ctx, pluginexec.NewSh(), nil) - require.NoError(t, err) + _, err = e.RegisterDriver(ctx, pluginexec.NewSh(), nil) + require.NoError(t, err) - _, err = e.RegisterCache("test", cache, true, true) - require.NoError(t, err) + _, err = e.RegisterCache("test", cache, true, true) + require.NoError(t, err) - { - rs, clean := e.NewRequestState() - defer clean() + { + rs, clean := e.NewRequestState() + defer clean() - res, err := e.Result(ctx, rs, pkg, "t1", []string{""}) - require.NoError(t, err) - defer res.Unlock(ctx) + res, err := e.Result(ctx, rs, pkg, "t1", []string{""}) + require.NoError(t, err) + defer res.Unlock(ctx) - require.Len(t, res.Artifacts, 1) + require.Len(t, res.Artifacts, 1) - manifest, err := e.ResultMetaFromRef(ctx, rs, tref.New(pkg, "t1", nil), []string{""}) - require.NoError(t, err) + manifest, err := e.ResultMetaFromRef(ctx, rs, tref.New(pkg, "t1", nil), []string{""}) + require.NoError(t, err) - assert.Equal(t, "e52c3f7fe43c3c02", manifest.Hashin) - assert.Equal(t, "d4fd9c2c4c50146f", manifest.Artifacts[0].Hashout) - } + assert.Equal(t, "9d12ac88089ebc06", manifest.Hashin) + assert.Equal(t, "d4fd9c2c4c50146f", manifest.Artifacts[0].Hashout) + } - { - rs, clean := e.NewRequestState() - defer clean() + { + rs, clean := e.NewRequestState() + defer clean() - res, err := e.Result(ctx, rs, pkg, "t2", []string{""}) - require.NoError(t, err) - defer res.Unlock(ctx) + res, err := e.Result(ctx, rs, pkg, "t2", []string{""}) + require.NoError(t, err) + defer res.Unlock(ctx) - require.Len(t, res.Artifacts, 1) + require.Len(t, res.Artifacts, 1) - manifest, err := e.ResultMetaFromRef(ctx, rs, tref.New(pkg, "t2", nil), []string{""}) - require.NoError(t, err) + manifest, err := e.ResultMetaFromRef(ctx, rs, tref.New(pkg, "t2", nil), []string{""}) + require.NoError(t, err) - assert.Equal(t, "7103b83d26040e7a", manifest.Hashin) - assert.Equal(t, "3b0f519635c52211", manifest.Artifacts[0].Hashout) - } + assert.Equal(t, "c4bc4a188996ffc8", manifest.Hashin) + assert.Equal(t, "3b0f519635c52211", manifest.Artifacts[0].Hashout) + } - { - rs, clean := e.NewRequestState() - defer clean() + { + rs, clean := e.NewRequestState() + defer clean() - res, err := e.Result(ctx, rs, pkg, "t3", []string{""}) - require.NoError(t, err) - defer res.Unlock(ctx) + res, err := e.Result(ctx, rs, pkg, "t3", []string{""}) + require.NoError(t, err) + defer res.Unlock(ctx) - require.Len(t, res.Artifacts, 1) + require.Len(t, res.Artifacts, 1) - manifest, err := e.ResultMetaFromRef(ctx, rs, tref.New(pkg, "t3", nil), []string{""}) - require.NoError(t, err) + manifest, err := e.ResultMetaFromRef(ctx, rs, tref.New(pkg, "t3", nil), []string{""}) + require.NoError(t, err) - assert.Equal(t, "7a65fe455c8b6178", manifest.Hashin) - assert.Equal(t, "3b0f519635c52211", manifest.Artifacts[0].Hashout) - } + assert.Equal(t, "7e2b9b570c832965", manifest.Hashin) + assert.Equal(t, "3b0f519635c52211", manifest.Artifacts[0].Hashout) + } - assert.Len(t, cache.storeWrites, 6) - for k, c := range cache.storeWrites { - assert.Equalf(t, 1, c, "cache hit count for %v", k) - } + assert.Len(t, cache.storeWrites, 6) + for k, c := range cache.storeWrites { + assert.Equalf(t, 1, c, "cache hit count for %v", k) + } + }) } } diff --git a/lib/pluginsdk/artifact_proto.go b/lib/pluginsdk/artifact_proto.go index 741ac910..af668f21 100644 --- a/lib/pluginsdk/artifact_proto.go +++ b/lib/pluginsdk/artifact_proto.go @@ -147,6 +147,17 @@ func (a ProtoArtifact) GetContentType() (ArtifactContentType, error) { } } +func (a ProtoArtifact) GetName() string { + partifact := a.Artifact + + switch partifact.WhichContent() { + case pluginv1.Artifact_File_case, pluginv1.Artifact_Raw_case: + return a.Artifact.GetName() + ".tar" + default: + return a.Artifact.GetName() + } +} + func (a ProtoArtifact) FSNode() hfs.Node { partifact := a.Artifact diff --git a/lib/tref/utils.go b/lib/tref/utils.go index 940801eb..ff269127 100644 --- a/lib/tref/utils.go +++ b/lib/tref/utils.go @@ -115,6 +115,6 @@ func WithoutOut(ref *pluginv1.TargetRefWithOutput) *pluginv1.TargetRef { } var OmitHashPb = map[string]struct{}{ - string((&pluginv1.TargetRefWithOutput{}).ProtoReflect().Descriptor().Name()) + ".hash": {}, - string((&pluginv1.TargetRef{}).ProtoReflect().Descriptor().Name()) + ".hash": {}, + string((&pluginv1.TargetRefWithOutput{}).ProtoReflect().Descriptor().FullName()) + ".hash": {}, + string((&pluginv1.TargetRef{}).ProtoReflect().Descriptor().FullName()) + ".hash": {}, } diff --git a/plugin/pluginexec/parse_config.go b/plugin/pluginexec/parse_config.go index 2e23028e..066b61a5 100644 --- a/plugin/pluginexec/parse_config.go +++ b/plugin/pluginexec/parse_config.go @@ -3,7 +3,6 @@ package pluginexec import ( "context" "fmt" - "maps" "slices" "strings" @@ -11,7 +10,6 @@ import ( "github.com/hephbuild/heph/internal/hmaps" "github.com/hephbuild/heph/internal/hproto/hashpb" "github.com/hephbuild/heph/internal/hproto/hstructpb" - "github.com/hephbuild/heph/internal/hslices" "github.com/hephbuild/heph/internal/htypes" "github.com/hephbuild/heph/lib/tref" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" @@ -189,51 +187,32 @@ func ToDef[S proto.Message](ref *pluginv1.TargetRef, target S, getTarget func(S) return def, nil } -var targetProtoName = string((&execv1.Target{}).ProtoReflect().Descriptor().Name()) +var targetProtoName = string((&execv1.Target{}).ProtoReflect().Descriptor().FullName()) func hashTarget(target *execv1.Target) []byte { - omit := tref.OmitHashPb - var cloned bool - clonedOnce := func() { - if !cloned { - cloned = true - omit = maps.Clone(omit) - } - } + omit := hmaps.Concat(tref.OmitHashPb, map[string]struct{}{ + targetProtoName + ".env": {}, + targetProtoName + ".deps": {}, + }) h := xxh3.New() + hashpb.Hash(h, target, omit) - if hmaps.Has(target.GetEnv(), func(s string, env *execv1.Target_Env) bool { - return !env.GetHash() - }) { - clonedOnce() - omit[targetProtoName+".env"] = struct{}{} - - for k, v := range hmaps.Sorted(target.GetEnv()) { - if !v.GetHash() { - continue - } - - _, _ = h.WriteString(k) - hashpb.Hash(h, v, omit) + for k, v := range hmaps.Sorted(target.GetEnv()) { + if !v.GetHash() { + continue } - } - - if hslices.Has(target.GetDeps(), func(dep *execv1.Target_Dep) bool { - return !dep.GetHash() - }) { - clonedOnce() - omit[targetProtoName+".deps"] = struct{}{} - for _, v := range target.GetDeps() { - if !v.GetHash() { - continue - } + _, _ = h.WriteString(k) + hashpb.Hash(h, v, omit) + } - hashpb.Hash(h, v, omit) + for _, v := range target.GetDeps() { + if !v.GetHash() { + continue } - } - hashpb.Hash(h, target, omit) + hashpb.Hash(h, v, omit) + } return h.Sum(nil) } diff --git a/plugin/pluginnix/driver.go b/plugin/pluginnix/driver.go index 4e30daa6..3e766fc6 100644 --- a/plugin/pluginnix/driver.go +++ b/plugin/pluginnix/driver.go @@ -71,7 +71,7 @@ func parseConfig(ctx context.Context, ref *pluginv1.TargetRef, config map[string } var omitHashPb = hmaps.Concat(map[string]struct{}{ - string((&nixv1.Target{}).ProtoReflect().Descriptor().Name()) + ".target": {}, + string((&nixv1.Target{}).ProtoReflect().Descriptor().FullName()) + ".target": {}, }, tref.OmitHashPb) func hashTarget(nixTarget *nixv1.Target, execTargetHash []byte) []byte { From 063f21a6982682f73f39dcd9e50e7d65fa1da107 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Sun, 8 Mar 2026 18:41:53 +0000 Subject: [PATCH 14/15] lint --- internal/engine/cache_sql.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/engine/cache_sql.go b/internal/engine/cache_sql.go index 701d9a71..8525ffa8 100644 --- a/internal/engine/cache_sql.go +++ b/internal/engine/cache_sql.go @@ -82,7 +82,7 @@ func (s *SQLCacheDB) pools() (*sql.DB, *sql.DB, error) { rdb.SetMaxIdleConns(100) rdb.SetMaxOpenConns(100) - readerStmt, err := rdb.Prepare(` + readerStmt, err := rdb.PrepareContext(ctx, ` SELECT data FROM cache_blobs WHERE target_addr = ? AND hashin = ? AND artifact_name = ? @@ -94,9 +94,9 @@ func (s *SQLCacheDB) pools() (*sql.DB, *sql.DB, error) { return } - existsStmt, err := rdb.Prepare(`SELECT 1 FROM cache_blobs WHERE target_addr = ? AND hashin = ? AND artifact_name = ? LIMIT 1`) + existsStmt, err := rdb.PrepareContext(ctx, `SELECT 1 FROM cache_blobs WHERE target_addr = ? AND hashin = ? AND artifact_name = ? LIMIT 1`) if err != nil { - _ = readerStmt.Close() + _ = readerStmt.Close() //nolint:sqlclosecheck _ = rdb.Close() _ = wdb.Close() s.err = fmt.Errorf("OpenSQLCacheDB prepare exists: %w", err) From 0623f008e75aef706614115c00d4d29abc9f17e9 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Sun, 8 Mar 2026 18:57:06 +0000 Subject: [PATCH 15/15] very prepared --- internal/engine/cache_sql.go | 217 ++++++++++++++++++++--------------- 1 file changed, 125 insertions(+), 92 deletions(-) diff --git a/internal/engine/cache_sql.go b/internal/engine/cache_sql.go index 8525ffa8..79ffe8fd 100644 --- a/internal/engine/cache_sql.go +++ b/internal/engine/cache_sql.go @@ -26,21 +26,23 @@ import ( // SQLCacheDB holds the path and lazily opens both connection pools on first use. // Call Close only if the DB was actually used; it is safe to call regardless. type SQLCacheDB struct { - path string - once sync.Once - rdb *sql.DB - wdb *sql.DB - readerStmt *sql.Stmt - existsStmt *sql.Stmt - err error + path string + once sync.Once + rdb *sql.DB + wdb *sql.DB + readerStmt *sql.Stmt + existsStmt *sql.Stmt + listArtifactsStmt *sql.Stmt + listVersionsStmt *sql.Stmt + deleteHashinStmt *sql.Stmt + deleteNameStmt *sql.Stmt + upsertStmt *sql.Stmt + err error } -func (s *SQLCacheDB) pools() (*sql.DB, *sql.DB, error) { - ctx := context.Background() - - s.once.Do(func() { - sqlite.RegisterConnectionHook(func(conn sqlite.ExecQuerierContext, _ string) error { - _, err := conn.ExecContext(ctx, ` +func (s *SQLCacheDB) openInner(ctx context.Context) error { + sqlite.RegisterConnectionHook(func(conn sqlite.ExecQuerierContext, _ string) error { + _, err := conn.ExecContext(ctx, ` PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 10000; PRAGMA synchronous = NORMAL; @@ -50,86 +52,128 @@ func (s *SQLCacheDB) pools() (*sql.DB, *sql.DB, error) { PRAGMA mmap_size = 268435456; PRAGMA temp_store = MEMORY; `, nil) - return err - }) + return err + }) - if err := os.MkdirAll(filepath.Dir(s.path), os.ModePerm); err != nil { - s.err = fmt.Errorf("OpenSQLCacheDB mkdir: %w", err) - return - } + if err := os.MkdirAll(filepath.Dir(s.path), os.ModePerm); err != nil { + return fmt.Errorf("OpenSQLCacheDB mkdir: %w", err) + } - wdb, err := openSQLiteDB(s.path) - if err != nil { - s.err = fmt.Errorf("OpenSQLCacheDB open wdb: %w", err) - return - } - // One writer at a time — prevents SQLITE_BUSY. - wdb.SetMaxOpenConns(1) + wdb, err := openSQLiteDB(s.path) + if err != nil { + return fmt.Errorf("OpenSQLCacheDB open wdb: %w", err) + } + // One writer at a time — prevents SQLITE_BUSY. + wdb.SetMaxOpenConns(1) - if err := initSQLCacheDB(ctx, wdb); err != nil { - _ = wdb.Close() - s.err = fmt.Errorf("OpenSQLCacheDB init: %w", err) - return - } + if err := initSQLCacheDB(ctx, wdb); err != nil { + _ = wdb.Close() + return fmt.Errorf("OpenSQLCacheDB init: %w", err) + } - rdb, err := openSQLiteDB(s.path) - if err != nil { - _ = wdb.Close() - s.err = fmt.Errorf("OpenSQLCacheDB open rdb: %w", err) - return - } - // No cap — WAL lets concurrent readers run in parallel. - rdb.SetMaxIdleConns(100) - rdb.SetMaxOpenConns(100) + rdb, err := openSQLiteDB(s.path) + if err != nil { + _ = wdb.Close() + return fmt.Errorf("OpenSQLCacheDB open rdb: %w", err) + } + // No cap — WAL lets concurrent readers run in parallel. + rdb.SetMaxIdleConns(100) + rdb.SetMaxOpenConns(100) - readerStmt, err := rdb.PrepareContext(ctx, ` + s.rdb = rdb + s.wdb = wdb + + s.readerStmt, err = rdb.PrepareContext(ctx, ` SELECT data FROM cache_blobs WHERE target_addr = ? AND hashin = ? AND artifact_name = ? `) - if err != nil { - _ = rdb.Close() - _ = wdb.Close() - s.err = fmt.Errorf("OpenSQLCacheDB prepare reader: %w", err) - return - } + if err != nil { + return fmt.Errorf("OpenSQLCacheDB prepare reader: %w", err) + } + + s.existsStmt, err = rdb.PrepareContext(ctx, `SELECT 1 FROM cache_blobs WHERE target_addr = ? AND hashin = ? AND artifact_name = ? LIMIT 1`) + if err != nil { + return fmt.Errorf("OpenSQLCacheDB prepare exists: %w", err) + } + + s.listArtifactsStmt, err = rdb.PrepareContext(ctx, "SELECT artifact_name FROM cache_blobs WHERE target_addr = ? AND hashin = ?") + if err != nil { + return fmt.Errorf("OpenSQLCacheDB prepare list artifacts: %w", err) + } + + s.listVersionsStmt, err = rdb.PrepareContext(ctx, "SELECT DISTINCT hashin FROM cache_blobs WHERE target_addr = ?") + if err != nil { + return fmt.Errorf("OpenSQLCacheDB prepare list versions: %w", err) + } - existsStmt, err := rdb.PrepareContext(ctx, `SELECT 1 FROM cache_blobs WHERE target_addr = ? AND hashin = ? AND artifact_name = ? LIMIT 1`) + s.deleteHashinStmt, err = wdb.PrepareContext(ctx, `DELETE FROM cache_blobs WHERE target_addr = ? AND hashin = ?`) + if err != nil { + return fmt.Errorf("OpenSQLCacheDB prepare delete hashin: %w", err) + } + + s.deleteNameStmt, err = wdb.PrepareContext(ctx, `DELETE FROM cache_blobs WHERE target_addr = ? AND hashin = ? AND artifact_name = ?`) + if err != nil { + return fmt.Errorf("OpenSQLCacheDB prepare delete name: %w", err) + } + + s.upsertStmt, err = wdb.PrepareContext(ctx, ` + INSERT INTO cache_blobs (target_addr, hashin, artifact_name, data, created_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(target_addr, hashin, artifact_name) DO UPDATE SET + data = excluded.data, + created_at = excluded.created_at + `) + if err != nil { + return fmt.Errorf("OpenSQLCacheDB prepare upsert: %w", err) + } + return nil +} +func (s *SQLCacheDB) open() (*sql.DB, *sql.DB, error) { + ctx := context.Background() + + s.once.Do(func() { + err := s.openInner(ctx) if err != nil { - _ = readerStmt.Close() //nolint:sqlclosecheck - _ = rdb.Close() - _ = wdb.Close() - s.err = fmt.Errorf("OpenSQLCacheDB prepare exists: %w", err) + s.err = fmt.Errorf("sqlcache open: %w", err) + _ = s.Close() return } - - s.rdb = rdb - s.wdb = wdb - s.readerStmt = readerStmt - s.existsStmt = existsStmt }) return s.rdb, s.wdb, s.err } func (s *SQLCacheDB) Close() error { - // If pools() was never called, once.Do has never run and rdb/wdb are nil. - if s.rdb == nil { - return nil + var errs []error + if s.readerStmt != nil { + errs = append(errs, s.readerStmt.Close()) + } + if s.existsStmt != nil { + errs = append(errs, s.existsStmt.Close()) + } + if s.listArtifactsStmt != nil { + errs = append(errs, s.listArtifactsStmt.Close()) + } + if s.listVersionsStmt != nil { + errs = append(errs, s.listVersionsStmt.Close()) + } + if s.deleteHashinStmt != nil { + errs = append(errs, s.deleteHashinStmt.Close()) + } + if s.deleteNameStmt != nil { + errs = append(errs, s.deleteNameStmt.Close()) } - errR := s.rdb.Close() - errW := s.wdb.Close() - errS := s.readerStmt.Close() - errE := s.existsStmt.Close() - if errR != nil { - return errR + if s.upsertStmt != nil { + errs = append(errs, s.upsertStmt.Close()) } - if errS != nil { - return errS + if s.rdb != nil { + errs = append(errs, s.rdb.Close()) } - if errE != nil { - return errE + if s.wdb != nil { + errs = append(errs, s.wdb.Close()) } - return errW + + return errors.Join(errs...) } type SQLCache struct { @@ -139,7 +183,7 @@ type SQLCache struct { } func (c *SQLCache) rwdb(ctx context.Context) (*sql.DB, *sql.DB, error) { - rdb, wdb, err := c.db.pools() + rdb, wdb, err := c.db.open() if err != nil { return nil, nil, fmt.Errorf("sqlcache open db: %w", err) } @@ -168,7 +212,7 @@ func (c *SQLCache) Exists(ctx context.Context, ref *pluginv1.TargetRef, hashin, } func (c *SQLCache) Delete(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) error { - _, wdb, err := c.rwdb(ctx) + _, _, err := c.rwdb(ctx) if err != nil { return err } @@ -176,15 +220,13 @@ func (c *SQLCache) Delete(ctx context.Context, ref *pluginv1.TargetRef, hashin, targetAddr := c.targetKey(ref) if name == "" { - _, err = wdb.ExecContext( + _, err = c.db.deleteHashinStmt.ExecContext( ctx, - `DELETE FROM cache_blobs WHERE target_addr = ? AND hashin = ?`, targetAddr, hashin, ) } else { - _, err = wdb.ExecContext( + _, err = c.db.deleteNameStmt.ExecContext( ctx, - `DELETE FROM cache_blobs WHERE target_addr = ? AND hashin = ? AND artifact_name = ?`, targetAddr, hashin, name, ) } @@ -299,7 +341,7 @@ func (c *SQLCache) Reader(ctx context.Context, ref *pluginv1.TargetRef, hashin, } func (c *SQLCache) writeEntry(ctx context.Context, targetAddr, hashin, name string, data io.Reader) error { - _, wdb, err := c.rwdb(ctx) + _, _, err := c.rwdb(ctx) if err != nil { return err } @@ -310,15 +352,8 @@ func (c *SQLCache) writeEntry(ctx context.Context, targetAddr, hashin, name stri } // Single UPSERT — write lock held for exactly one statement. - _, err = wdb.ExecContext( + _, err = c.db.upsertStmt.ExecContext( ctx, - ` - INSERT INTO cache_blobs (target_addr, hashin, artifact_name, data, created_at) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(target_addr, hashin, artifact_name) DO UPDATE SET - data = excluded.data, - created_at = excluded.created_at - `, targetAddr, hashin, name, payload, time.Now().UnixNano(), ) if err != nil { @@ -376,7 +411,7 @@ func (c *SQLCache) Writer(ctx context.Context, ref *pluginv1.TargetRef, hashin, func (c *SQLCache) ListArtifacts(ctx context.Context, ref *pluginv1.TargetRef, hashin string) iter.Seq2[string, error] { return func(yield func(string, error) bool) { - rdb, _, err := c.rwdb(ctx) + _, _, err := c.rwdb(ctx) if err != nil { yield("", err) return @@ -384,8 +419,7 @@ func (c *SQLCache) ListArtifacts(ctx context.Context, ref *pluginv1.TargetRef, h targetAddr := c.targetKey(ref) - rows, err := rdb.QueryContext(ctx, - "SELECT artifact_name FROM cache_blobs WHERE target_addr = ? AND hashin = ?", + rows, err := c.db.listArtifactsStmt.QueryContext(ctx, targetAddr, hashin, ) if err != nil { @@ -414,7 +448,7 @@ func (c *SQLCache) ListArtifacts(ctx context.Context, ref *pluginv1.TargetRef, h func (c *SQLCache) ListVersions(ctx context.Context, ref *pluginv1.TargetRef) iter.Seq2[string, error] { return func(yield func(string, error) bool) { - rdb, _, err := c.rwdb(ctx) + _, _, err := c.rwdb(ctx) if err != nil { yield("", err) return @@ -422,8 +456,7 @@ func (c *SQLCache) ListVersions(ctx context.Context, ref *pluginv1.TargetRef) it targetAddr := c.targetKey(ref) - rows, err := rdb.QueryContext(ctx, - "SELECT DISTINCT hashin FROM cache_blobs WHERE target_addr = ?", + rows, err := c.db.listVersionsStmt.QueryContext(ctx, targetAddr, ) if err != nil {