diff --git a/go.mod b/go.mod index e270d536..1e6a2135 100644 --- a/go.mod +++ b/go.mod @@ -58,10 +58,11 @@ require ( go.opentelemetry.io/otel/trace v1.35.0 go.starlark.net v0.0.0-20241226192728-8dfa5b98479f go.uber.org/mock v0.5.2 - golang.org/x/net v0.39.0 + golang.org/x/net v0.46.0 golang.org/x/sync v0.19.0 golang.org/x/sys v0.41.0 google.golang.org/protobuf v1.36.7-0.20250625222701-8e8926ef675d + modernc.org/sqlite v1.46.1 ) require ( @@ -87,6 +88,7 @@ require ( github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect github.com/colega/zeropool v0.0.0-20230505084239-6fb4a4f75381 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect @@ -105,9 +107,11 @@ require ( github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect github.com/muesli/cancelreader v0.2.2 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect @@ -120,17 +124,20 @@ require ( go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect go.opentelemetry.io/proto/otlp v1.5.0 // indirect - golang.org/x/crypto v0.37.0 // indirect - golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 // indirect - golang.org/x/mod v0.23.0 // indirect + golang.org/x/crypto v0.43.0 // indirect + golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect + golang.org/x/mod v0.29.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/text v0.24.0 // indirect + golang.org/x/text v0.30.0 // indirect golang.org/x/time v0.11.0 // indirect - golang.org/x/tools v0.30.0 // indirect + golang.org/x/tools v0.38.0 // indirect google.golang.org/api v0.232.0 // indirect google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250505200425-f936aa4a68b2 // indirect google.golang.org/grpc v1.72.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + modernc.org/libc v1.67.6 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect ) diff --git a/go.sum b/go.sum index aaf3b7a5..1b953c3c 100644 --- a/go.sum +++ b/go.sum @@ -84,6 +84,8 @@ github.com/creack/pty v1.1.20/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= @@ -123,6 +125,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -136,6 +140,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 h1:e9Rjr40Z98/clHv5Yg79Is0Ntos github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1/go.mod h1:tIxuGz/9mpox++sgp9fJjHO0+q1X9/UOWd798aAm22M= github.com/hanwen/go-fuse/v2 v2.7.2 h1:SbJP1sUP+n1UF8NXBA14BuojmTez+mDgOk0bC057HQw= github.com/hanwen/go-fuse/v2 v2.7.2/go.mod h1:ugNaD/iv5JYyS1Rcvi57Wz7/vrLQJo10mmketmoef48= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/heimdalr/dag v1.5.0 h1:hqVtijvY776P5OKP3QbdVBRt3Xxq6BYopz3XgklsGvo= github.com/heimdalr/dag v1.5.0/go.mod h1:lthekrHl01dddmzqyBQ1YZbi7XcVGGzjFo0jIky5knc= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -164,6 +170,8 @@ github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vyg github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= github.com/pkg/xattr v0.4.12 h1:rRTkSyFNTRElv6pkA3zpjHpQ90p/OdHQC1GmGh1aTjM= @@ -174,6 +182,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= @@ -260,20 +270,20 @@ go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= -golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 h1:1UoZQm6f0P/ZO0w1Ri+f+ifG/gXhegadRdwBIXEFWDo= -golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= +golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= +golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY= +golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM= -golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= -golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= +golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -303,16 +313,16 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= +golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY= -golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.232.0 h1:qGnmaIMf7KcuwHOlF3mERVzChloDYwRfOJOrHt8YC3I= google.golang.org/api v0.232.0/go.mod h1:p9QCfBWZk1IJETUdbTKloR5ToFdKbYh2fkjsUL6vNoY= @@ -332,3 +342,31 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis= +modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.30.1 h1:4r4U1J6Fhj98NKfSjnPUN7Ze2c6MnAdL0hWw6+LrJpc= +modernc.org/ccgo/v4 v4.30.1/go.mod h1:bIOeI1JL54Utlxn+LwrFyjCx2n2RDiYEaJVSrgdrRfM= +modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA= +modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.1 h1:k8T3gkXWY9sEiytKhcgyiZ2L0DTyCQ/nvX+LoCljoRE= +modernc.org/gc/v3 v3.1.1/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.67.6 h1:eVOQvpModVLKOdT+LvBPjdQqfrZq+pC39BygcT+E7OI= +modernc.org/libc v1.67.6/go.mod h1:JAhxUVlolfYDErnwiqaLvUqc8nfb2r6S6slAgZOnaiE= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.46.1 h1:eFJ2ShBLIEnUWlLy12raN0Z1plqmFX9Qe3rjQTKt6sU= +modernc.org/sqlite v1.46.1/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/cmd/inspect_cache.go b/internal/cmd/inspect_cache.go new file mode 100644 index 00000000..07bb33de --- /dev/null +++ b/internal/cmd/inspect_cache.go @@ -0,0 +1,131 @@ +package cmd + +import ( + "errors" + "fmt" + + "github.com/hephbuild/heph/internal/engine" + "github.com/spf13/cobra" +) + +var cacheCmd *cobra.Command + +func init() { + cacheCmd = &cobra.Command{ + Use: "cache", + Short: "Cache utilities", + } + + inspectCmd.AddCommand(cacheCmd) +} + +func init() { + cmdArgs := parseRefArgs{cmdName: "list-versions"} + + cmd := &cobra.Command{ + Use: cmdArgs.Use(), + Short: "List versions for hashin", + Args: cmdArgs.Args(), + ValidArgsFunction: cmdArgs.ValidArgsFunction(), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + ctx, stop := newSignalNotifyContext(ctx) + defer stop() + + cwd, err := engine.Cwd() + if err != nil { + return err + } + + root, err := engine.Root() + if err != nil { + return err + } + + ref, err := cmdArgs.Parse(args[0], cwd, root) + if err != nil { + return err + } + + e, err := newEngine(ctx, root) + if err != nil { + return err + } + + for hashin, err := range e.CacheSmall.ListVersions(ctx, ref) { + if err != nil { + return err + } + + fmt.Println(hashin) + } + + return nil + }, + } + + cacheCmd.AddCommand(cmd) +} + +func init() { + cmdArgs := parseRefArgs{cmdName: "list-artifacts"} + + cmd := &cobra.Command{ + Use: cmdArgs.Use(), + Short: "List artifacts at revision", + Args: func(cmd *cobra.Command, args []string) error { + switch len(args) { + case 2: + return nil + default: + return errors.New("must be `list-artifacts `") + } + }, + ValidArgsFunction: cmdArgs.ValidArgsFunction(), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + ctx, stop := newSignalNotifyContext(ctx) + defer stop() + + cwd, err := engine.Cwd() + if err != nil { + return err + } + + root, err := engine.Root() + if err != nil { + return err + } + + ref, err := cmdArgs.Parse(args[0], cwd, root) + if err != nil { + return err + } + + hashin := args[1] + + e, err := newEngine(ctx, root) + if err != nil { + return err + } + + for i, cache := range [...]engine.LocalCache{e.CacheSmall, e.CacheLarge} { + fmt.Println("======", i) + + for name, err := range cache.ListArtifacts(ctx, ref, hashin) { + if err != nil { + return err + } + + fmt.Println(name) + } + } + + return nil + }, + } + + cacheCmd.AddCommand(cmd) +} diff --git a/internal/cmd/run.go b/internal/cmd/run.go index 9cb9a6be..e132f319 100644 --- a/internal/cmd/run.go +++ b/internal/cmd/run.go @@ -200,35 +200,27 @@ func init() { _ = f.Close() } + + if stdout.Written && stdout.LastByte != '\n' { + _, _ = io.WriteString(stdout, "\n") + } } } case listArtifacts: for _, re := range res { for _, output := range re.Artifacts { fmt.Println(output.GetName()) - fmt.Println(" group: ", output.GetGroup()) - switch output.WhichContent() { - case pluginv1.Artifact_File_case: - fmt.Println(" content:", output.GetFile()) - case pluginv1.Artifact_Raw_case: - fmt.Println(" content:", output.GetRaw()) - case pluginv1.Artifact_TarPath_case: - fmt.Println(" content:", output.GetTarPath()) - case pluginv1.Artifact_TargzPath_case: - fmt.Println(" content:", output.GetTargzPath()) - case pluginv1.Artifact_Content_not_set_case: - fmt.Println(" content: ") - } - fmt.Println(" type: ", output.GetType().String()) + fmt.Println(" group: ", output.GetGroup()) + fmt.Println(" type: ", output.GetType().String()) } } case hashOut: for _, re := range res { for _, output := range re.Artifacts { if output.GetGroup() == "" { - fmt.Println(output.Hashout) + fmt.Println(output.GetHashout()) } else { - fmt.Println(output.GetGroup(), output.Hashout) + fmt.Println(output.GetGroup(), output.GetHashout()) } } } diff --git a/internal/engine/artifact_group.go b/internal/engine/artifact_group.go new file mode 100644 index 00000000..6298736b --- /dev/null +++ b/internal/engine/artifact_group.go @@ -0,0 +1,12 @@ +package engine + +import "github.com/hephbuild/heph/lib/pluginsdk" + +type artifactGroupMap struct { + pluginsdk.Artifact + group string +} + +func (a artifactGroupMap) GetGroup() string { + return a.group +} diff --git a/internal/engine/cache_fs.go b/internal/engine/cache_fs.go new file mode 100644 index 00000000..812515cd --- /dev/null +++ b/internal/engine/cache_fs.go @@ -0,0 +1,105 @@ +package engine + +import ( + "context" + "encoding/hex" + "errors" + "io" + "iter" + + "github.com/hephbuild/heph/internal/hfs" + "github.com/hephbuild/heph/internal/hproto/hashpb" + "github.com/hephbuild/heph/lib/tref" + pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" + "github.com/zeebo/xxh3" +) + +type FSCache struct { + root hfs.OS // absolute cache root +} + +func (c FSCache) Exists(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (bool, error) { + _, err := c.path(ref, hashin, name).Lstat() + if err != nil { + if errors.Is(err, hfs.ErrNotExist) { + return false, nil + } + + return false, err + } + + return true, nil +} + +var _ LocalCache = (*FSCache)(nil) + +func NewFSCache(root hfs.OS) *FSCache { + return &FSCache{root: root} +} + +func (c FSCache) targetDirName(ref *pluginv1.TargetRef) string { + if len(ref.GetArgs()) == 0 { + return "__" + ref.GetName() + } + + h := xxh3.New() + hashpb.Hash(h, ref, tref.OmitHashPb) + + return "__" + ref.GetName() + "_" + hex.EncodeToString(h.Sum(nil)) +} + +func (c FSCache) path(ref *pluginv1.TargetRef, hashin, name string) hfs.Node { + return c.root.At(ref.GetPackage(), c.targetDirName(ref), hashin, name) +} + +func (c FSCache) Reader(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.ReadCloser, error) { + return hfs.Open(c.path(ref, hashin, name)) +} + +func (c FSCache) Writer(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.WriteCloser, error) { + return hfs.Create(c.path(ref, hashin, name)) +} + +func (c FSCache) Delete(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) error { + return c.path(ref, hashin, name).RemoveAll() +} + +func (c FSCache) ListArtifacts(ctx context.Context, ref *pluginv1.TargetRef, hashin string) iter.Seq2[string, error] { + return func(yield func(string, error) bool) { + entries, err := c.path(ref, hashin, "").ReadDir() + if err != nil { + yield("", err) + return + } + + for _, entry := range entries { + if entry.IsDir() { + continue + } + + if !yield(entry.Name(), nil) { + return + } + } + } +} + +func (c FSCache) ListVersions(ctx context.Context, ref *pluginv1.TargetRef) iter.Seq2[string, error] { + return func(yield func(string, error) bool) { + entries, err := c.path(ref, "", "").ReadDir() + if err != nil { + yield("", err) + return + } + + for _, entry := range entries { + if !entry.IsDir() { + continue + } + + if !yield(entry.Name(), nil) { + return + } + } + } +} diff --git a/internal/engine/cache_sql.go b/internal/engine/cache_sql.go new file mode 100644 index 00000000..79ffe8fd --- /dev/null +++ b/internal/engine/cache_sql.go @@ -0,0 +1,484 @@ +package engine + +import ( + "bytes" + "context" + "database/sql" + "encoding/hex" + "errors" + "fmt" + "io" + "iter" + "os" + "path/filepath" + "sync" + "time" + + "github.com/hephbuild/heph/internal/hio" + "github.com/hephbuild/heph/internal/hproto/hashpb" + "github.com/hephbuild/heph/internal/hsync" + "github.com/hephbuild/heph/lib/tref" + pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" + "github.com/zeebo/xxh3" + "modernc.org/sqlite" +) + +// SQLCacheDB holds the path and lazily opens both connection pools on first use. +// Call Close only if the DB was actually used; it is safe to call regardless. +type SQLCacheDB struct { + path string + once sync.Once + rdb *sql.DB + wdb *sql.DB + readerStmt *sql.Stmt + existsStmt *sql.Stmt + listArtifactsStmt *sql.Stmt + listVersionsStmt *sql.Stmt + deleteHashinStmt *sql.Stmt + deleteNameStmt *sql.Stmt + upsertStmt *sql.Stmt + err error +} + +func (s *SQLCacheDB) openInner(ctx context.Context) error { + sqlite.RegisterConnectionHook(func(conn sqlite.ExecQuerierContext, _ string) error { + _, err := conn.ExecContext(ctx, ` + PRAGMA journal_mode = WAL; + PRAGMA busy_timeout = 10000; + PRAGMA synchronous = NORMAL; + PRAGMA foreign_keys = ON; + PRAGMA cache_size = -64000; + PRAGMA page_size = 8192; + PRAGMA mmap_size = 268435456; + PRAGMA temp_store = MEMORY; + `, nil) + return err + }) + + if err := os.MkdirAll(filepath.Dir(s.path), os.ModePerm); err != nil { + return fmt.Errorf("OpenSQLCacheDB mkdir: %w", err) + } + + wdb, err := openSQLiteDB(s.path) + if err != nil { + return fmt.Errorf("OpenSQLCacheDB open wdb: %w", err) + } + // One writer at a time — prevents SQLITE_BUSY. + wdb.SetMaxOpenConns(1) + + if err := initSQLCacheDB(ctx, wdb); err != nil { + _ = wdb.Close() + return fmt.Errorf("OpenSQLCacheDB init: %w", err) + } + + rdb, err := openSQLiteDB(s.path) + if err != nil { + _ = wdb.Close() + return fmt.Errorf("OpenSQLCacheDB open rdb: %w", err) + } + // No cap — WAL lets concurrent readers run in parallel. + rdb.SetMaxIdleConns(100) + rdb.SetMaxOpenConns(100) + + s.rdb = rdb + s.wdb = wdb + + s.readerStmt, err = rdb.PrepareContext(ctx, ` + SELECT data + FROM cache_blobs + WHERE target_addr = ? AND hashin = ? AND artifact_name = ? + `) + if err != nil { + return fmt.Errorf("OpenSQLCacheDB prepare reader: %w", err) + } + + s.existsStmt, err = rdb.PrepareContext(ctx, `SELECT 1 FROM cache_blobs WHERE target_addr = ? AND hashin = ? AND artifact_name = ? LIMIT 1`) + if err != nil { + return fmt.Errorf("OpenSQLCacheDB prepare exists: %w", err) + } + + s.listArtifactsStmt, err = rdb.PrepareContext(ctx, "SELECT artifact_name FROM cache_blobs WHERE target_addr = ? AND hashin = ?") + if err != nil { + return fmt.Errorf("OpenSQLCacheDB prepare list artifacts: %w", err) + } + + s.listVersionsStmt, err = rdb.PrepareContext(ctx, "SELECT DISTINCT hashin FROM cache_blobs WHERE target_addr = ?") + if err != nil { + return fmt.Errorf("OpenSQLCacheDB prepare list versions: %w", err) + } + + s.deleteHashinStmt, err = wdb.PrepareContext(ctx, `DELETE FROM cache_blobs WHERE target_addr = ? AND hashin = ?`) + if err != nil { + return fmt.Errorf("OpenSQLCacheDB prepare delete hashin: %w", err) + } + + s.deleteNameStmt, err = wdb.PrepareContext(ctx, `DELETE FROM cache_blobs WHERE target_addr = ? AND hashin = ? AND artifact_name = ?`) + if err != nil { + return fmt.Errorf("OpenSQLCacheDB prepare delete name: %w", err) + } + + s.upsertStmt, err = wdb.PrepareContext(ctx, ` + INSERT INTO cache_blobs (target_addr, hashin, artifact_name, data, created_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(target_addr, hashin, artifact_name) DO UPDATE SET + data = excluded.data, + created_at = excluded.created_at + `) + if err != nil { + return fmt.Errorf("OpenSQLCacheDB prepare upsert: %w", err) + } + return nil +} +func (s *SQLCacheDB) open() (*sql.DB, *sql.DB, error) { + ctx := context.Background() + + s.once.Do(func() { + err := s.openInner(ctx) + if err != nil { + s.err = fmt.Errorf("sqlcache open: %w", err) + _ = s.Close() + return + } + }) + return s.rdb, s.wdb, s.err +} + +func (s *SQLCacheDB) Close() error { + var errs []error + if s.readerStmt != nil { + errs = append(errs, s.readerStmt.Close()) + } + if s.existsStmt != nil { + errs = append(errs, s.existsStmt.Close()) + } + if s.listArtifactsStmt != nil { + errs = append(errs, s.listArtifactsStmt.Close()) + } + if s.listVersionsStmt != nil { + errs = append(errs, s.listVersionsStmt.Close()) + } + if s.deleteHashinStmt != nil { + errs = append(errs, s.deleteHashinStmt.Close()) + } + if s.deleteNameStmt != nil { + errs = append(errs, s.deleteNameStmt.Close()) + } + if s.upsertStmt != nil { + errs = append(errs, s.upsertStmt.Close()) + } + if s.rdb != nil { + errs = append(errs, s.rdb.Close()) + } + if s.wdb != nil { + errs = append(errs, s.wdb.Close()) + } + + return errors.Join(errs...) +} + +type SQLCache struct { + db *SQLCacheDB + + rpool hsync.Pool[[]byte] +} + +func (c *SQLCache) rwdb(ctx context.Context) (*sql.DB, *sql.DB, error) { + rdb, wdb, err := c.db.open() + if err != nil { + return nil, nil, fmt.Errorf("sqlcache open db: %w", err) + } + + return rdb, wdb, nil +} + +func (c *SQLCache) Exists(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (bool, error) { + _, _, err := c.rwdb(ctx) + if err != nil { + return false, err + } + + targetAddr := c.targetKey(ref) + + var exists bool + err = c.db.existsStmt.QueryRowContext( + ctx, + targetAddr, hashin, name, + ).Scan(&exists) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return false, fmt.Errorf("exists: %w", err) + } + + return exists, nil +} + +func (c *SQLCache) Delete(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) error { + _, _, err := c.rwdb(ctx) + if err != nil { + return err + } + + targetAddr := c.targetKey(ref) + + if name == "" { + _, err = c.db.deleteHashinStmt.ExecContext( + ctx, + targetAddr, hashin, + ) + } else { + _, err = c.db.deleteNameStmt.ExecContext( + ctx, + targetAddr, hashin, name, + ) + } + if err != nil { + return fmt.Errorf("delete: %w", err) + } + + return nil +} + +var _ LocalCache = (*SQLCache)(nil) + +func initSQLCacheDB(ctx context.Context, db *sql.DB) error { + _, err := db.ExecContext(ctx, ` + CREATE TABLE IF NOT EXISTS cache_blobs ( + target_addr TEXT NOT NULL, + hashin TEXT NOT NULL, + artifact_name TEXT NOT NULL, + data BLOB, + created_at INTEGER NOT NULL, + PRIMARY KEY (target_addr, hashin, artifact_name) + ); + + CREATE INDEX IF NOT EXISTS cache_blobs_target_hashin + ON cache_blobs (target_addr, hashin); + `) + + return err +} + +// openSQLiteDB opens a connection pool to a SQLite file with common pragmas. +func openSQLiteDB(path string) (*sql.DB, error) { + db, err := sql.Open("sqlite", path) + if err != nil { + return nil, err + } + return db, nil +} + +// OpenSQLCacheDB returns a SQLCacheDB that opens its connection pools lazily +// on first use. No file I/O happens until the first cache operation. +func OpenSQLCacheDB(path string) (*SQLCacheDB, error) { + return &SQLCacheDB{path: path}, nil +} + +func NewSQLCache(db *SQLCacheDB) *SQLCache { + return &SQLCache{ + db: db, + rpool: hsync.Pool[[]byte]{New: func() []byte { + return make([]byte, 100_000) + }}, + } +} + +// targetAddr computes a stable filesystem-safe address for a TargetRef. +// When the ref has no args, this is just "__". When args are present, +// a hash of the full ref is appended to disambiguate. +func (c *SQLCache) targetAddr(ref *pluginv1.TargetRef) string { + if len(ref.GetArgs()) == 0 { + return ref.GetName() + } + + h := xxh3.New() + hashpb.Hash(h, ref, tref.OmitHashPb) + + return ref.GetName() + "@" + hex.EncodeToString(h.Sum(nil)) +} + +// targetKey returns the compound target address used as target_addr in the DB. +func (c *SQLCache) targetKey(ref *pluginv1.TargetRef) string { + return ref.GetPackage() + ":" + c.targetAddr(ref) +} + +func (c *SQLCache) Reader(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.ReadCloser, error) { + _, _, err := c.rwdb(ctx) + if err != nil { + return nil, err + } + + targetAddr := c.targetKey(ref) + + rows, err := c.db.readerStmt.QueryContext( + ctx, + targetAddr, hashin, name, + ) + if err != nil { + return nil, fmt.Errorf("reader query: %w", err) + } + defer rows.Close() + + if !rows.Next() { + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("reader next: %w", err) + } + return nil, ErrLocalCacheNotFound + } + + var raw sql.RawBytes + err = rows.Scan(&raw) + if err != nil { + return nil, fmt.Errorf("reader scan: %w", err) + } + + buf := c.rpool.Get() + buf = append(buf[:0], raw...) + + return hio.NewReadCloserFunc(bytes.NewReader(buf), func() error { + c.rpool.Put(buf) + + return nil + }), nil +} + +func (c *SQLCache) writeEntry(ctx context.Context, targetAddr, hashin, name string, data io.Reader) error { + _, _, err := c.rwdb(ctx) + if err != nil { + return err + } + + payload, err := io.ReadAll(data) + if err != nil { + return fmt.Errorf("writeEntry read: %w", err) + } + + // Single UPSERT — write lock held for exactly one statement. + _, err = c.db.upsertStmt.ExecContext( + ctx, + targetAddr, hashin, name, payload, time.Now().UnixNano(), + ) + if err != nil { + return fmt.Errorf("writeEntry upsert: %w", err) + } + + return nil +} + +type sqlCacheWriter struct { + pw *io.PipeWriter + done <-chan error + closeOnce sync.Once + closeErr error +} + +func (w *sqlCacheWriter) Write(p []byte) (int, error) { + return w.pw.Write(p) +} + +func (w *sqlCacheWriter) Close() error { + w.closeOnce.Do(func() { + // Close the write end; this unblocks the goroutine's reader. + if err := w.pw.Close(); err != nil { + w.closeErr = err + return + } + // Wait for the goroutine to finish and return any write error. + w.closeErr = <-w.done + }) + return w.closeErr +} + +func (c *SQLCache) Writer(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.WriteCloser, error) { + targetAddr := c.targetKey(ref) + + pr, pw := io.Pipe() + done := make(chan error, 1) + + go func() { + defer pr.Close() + err := c.writeEntry(ctx, targetAddr, hashin, name, pr) + if err != nil { + wrappedErr := fmt.Errorf("writer write: %q %q %q %w", tref.Format(ref), hashin, name, err) + _ = pr.CloseWithError(wrappedErr) + done <- wrappedErr + } else { + done <- nil + } + close(done) + }() + + return &sqlCacheWriter{pw: pw, done: done}, nil +} + +func (c *SQLCache) ListArtifacts(ctx context.Context, ref *pluginv1.TargetRef, hashin string) iter.Seq2[string, error] { + return func(yield func(string, error) bool) { + _, _, err := c.rwdb(ctx) + if err != nil { + yield("", err) + return + } + + targetAddr := c.targetKey(ref) + + rows, err := c.db.listArtifactsStmt.QueryContext(ctx, + targetAddr, hashin, + ) + if err != nil { + yield("", err) + return + } + defer rows.Close() + + for rows.Next() { + var artifactName string + if err := rows.Scan(&artifactName); err != nil { + if !yield("", err) { + return + } + continue + } + if !yield(artifactName, nil) { + return + } + } + if err := rows.Err(); err != nil { + yield("", err) + } + } +} + +func (c *SQLCache) ListVersions(ctx context.Context, ref *pluginv1.TargetRef) iter.Seq2[string, error] { + return func(yield func(string, error) bool) { + _, _, err := c.rwdb(ctx) + if err != nil { + yield("", err) + return + } + + targetAddr := c.targetKey(ref) + + rows, err := c.db.listVersionsStmt.QueryContext(ctx, + targetAddr, + ) + if err != nil { + yield("", err) + return + } + defer rows.Close() + + for rows.Next() { + var hashin string + if err := rows.Scan(&hashin); err != nil { + if !yield("", err) { + return + } + continue + } + if !yield(hashin, nil) { + return + } + } + if err := rows.Err(); err != nil { + yield("", err) + } + } +} diff --git a/internal/engine/cache_sql_test.go b/internal/engine/cache_sql_test.go new file mode 100644 index 00000000..1b627cd8 --- /dev/null +++ b/internal/engine/cache_sql_test.go @@ -0,0 +1,79 @@ +package engine_test + +import ( + "context" + "io" + "path/filepath" + "testing" + + "github.com/hephbuild/heph/internal/engine" + pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" + "github.com/stretchr/testify/require" +) + +func TestSQLCache(t *testing.T) { + tempDir := t.TempDir() + dbPath := filepath.Join(tempDir, "cache.db") + db, err := engine.OpenSQLCacheDB(dbPath) + require.NoError(t, err) + defer db.Close() + + cache := engine.NewSQLCache(db) + + ctx := context.Background() + pkg := "pkg" + name := "target" + ref := (&pluginv1.TargetRef_builder{ + Package: &pkg, + Name: &name, + }).Build() + hashin := "hash123" + + // Test Write & Read + w, err := cache.Writer(ctx, ref, hashin, "art1") + require.NoError(t, err) + _, err = io.WriteString(w, "hello world") + require.NoError(t, err) + require.NoError(t, w.Close()) + + exists, err := cache.Exists(ctx, ref, hashin, "art1") + require.NoError(t, err) + require.True(t, exists) + r, err := cache.Reader(ctx, ref, hashin, "art1") + require.NoError(t, err) + b, err := io.ReadAll(r) + require.NoError(t, err) + require.NoError(t, r.Close()) + require.Equal(t, "hello world", string(b)) + + // Test Read Not Exist + exists, err = cache.Exists(ctx, ref, hashin, "art2") + require.NoError(t, err) + require.False(t, exists) + _, err = cache.Reader(ctx, ref, hashin, "art2") + require.ErrorIs(t, err, engine.ErrLocalCacheNotFound) + + // Test ListArtifacts + w, err = cache.Writer(ctx, ref, hashin, "art2") + require.NoError(t, err) + require.NoError(t, w.Close()) + + var artifacts []string + for a, e := range cache.ListArtifacts(ctx, ref, hashin) { + require.NoError(t, e) + artifacts = append(artifacts, a) + } + require.ElementsMatch(t, []string{"art1", "art2"}, artifacts) + + // Test ListVersions + w, err = cache.Writer(ctx, ref, "hash456", "art1") + require.NoError(t, err) + require.NoError(t, w.Close()) + + var versions []string + for v, e := range cache.ListVersions(ctx, ref) { + require.NoError(t, e) + versions = append(versions, v) + } + require.ElementsMatch(t, []string{"hash123", "hash456"}, versions) +} diff --git a/internal/engine/codegen.go b/internal/engine/codegen.go index 8012df75..41b3c78d 100644 --- a/internal/engine/codegen.go +++ b/internal/engine/codegen.go @@ -15,7 +15,7 @@ import ( "github.com/hephbuild/heph/plugin/pluginfs" ) -func (e *Engine) codegenTree(ctx context.Context, def *LightLinkedTarget, outputs []ExecuteResultArtifact) error { +func (e *Engine) codegenTree(ctx context.Context, def *LightLinkedTarget, outputs []*ResultArtifact) error { step, ctx := hstep.New(ctx, "Copying to tree...") defer step.Done() @@ -32,7 +32,7 @@ func (e *Engine) codegenTree(ctx context.Context, def *LightLinkedTarget, output return nil } -func (e *Engine) codegenTreeCopy(ctx context.Context, def *LightLinkedTarget, outputs []ExecuteResultArtifact, mode pluginv1.TargetDef_Path_CodegenMode) error { +func (e *Engine) codegenTreeCopy(ctx context.Context, def *LightLinkedTarget, outputs []*ResultArtifact, mode pluginv1.TargetDef_Path_CodegenMode) error { codegenPaths := make([]string, 0) for _, output := range def.GetOutputs() { for _, path := range output.GetPaths() { diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 66c2339d..efc6976b 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -90,6 +90,9 @@ type Engine struct { Sandbox hfs.OS RootSpan trace.Span + CacheSmall LocalCache + CacheLarge LocalCache + WellKnownPackages []string // For testing, for now... CoreHandle EngineHandle @@ -131,6 +134,13 @@ func New(ctx context.Context, root string, cfg Config) (*Engine, error) { FSLock: cfg.LockDriver == "fs", } + db, err := OpenSQLCacheDB(filepath.Join(cachefs.Path(), "cache.db")) + if err != nil { + return nil, fmt.Errorf("open cache db: %w", err) + } + e.CacheSmall = NewSQLCache(db) + e.CacheLarge = NewFSCache(cachefs) + for _, s := range cfg.Packages.Exclude { e.PackagesExclude = append(e.PackagesExclude, filepath.Join(root, s)) } @@ -173,7 +183,6 @@ func New(ctx context.Context, root string, cfg Config) (*Engine, error) { srvh.Mux.Handle(corev1connect.NewLogServiceHandler(hlog.NewLoggerHandler(hlog.From(ctx)))) srvh.Mux.Handle(corev1connect.NewStepServiceHandler(hstep.NewHandler(hstep.HandlerFromContext(ctx)), handlerOpts...)) - srvh.Mux.Handle(corev1connect.NewResultServiceHandler(e.ResultHandler(), handlerOpts...)) e.CoreHandle = EngineHandle{ ServerHandle: srvh, @@ -181,7 +190,7 @@ func New(ctx context.Context, root string, cfg Config) (*Engine, error) { return pluginsdk.Engine{ LogClient: corev1connect.NewLogServiceClient(srvh.HTTPClient(), srvh.GetBaseURL()), StepClient: corev1connect.NewStepServiceClient(srvh.HTTPClient(), srvh.GetBaseURL(), clientOpts...), - ResultClient: e.Resulter(), + ResultClient: pluginsdk.Resulter{Engine: e.Resulter()}, } }, } diff --git a/internal/engine/handler_result.go b/internal/engine/handler_result.go deleted file mode 100644 index 51768409..00000000 --- a/internal/engine/handler_result.go +++ /dev/null @@ -1,26 +0,0 @@ -package engine - -import ( - "context" - - "connectrpc.com/connect" - corev1 "github.com/hephbuild/heph/plugin/gen/heph/core/v1" - "github.com/hephbuild/heph/plugin/gen/heph/core/v1/corev1connect" -) - -func (e *Engine) ResultHandler() corev1connect.ResultServiceHandler { - return &resultServiceHandler{Engine: e} -} - -type resultServiceHandler struct { - *Engine -} - -func (r resultServiceHandler) Get(ctx context.Context, req *connect.Request[corev1.ResultRequest]) (*connect.Response[corev1.ResultResponse], error) { - res, err := r.Resulter().Get(ctx, req.Msg) - if err != nil { - return nil, err - } - - return connect.NewResponse(res), nil -} diff --git a/internal/engine/handler_resulter.go b/internal/engine/handler_resulter.go index a60db858..79f59149 100644 --- a/internal/engine/handler_resulter.go +++ b/internal/engine/handler_resulter.go @@ -5,23 +5,26 @@ import ( "errors" "fmt" + "github.com/google/uuid" "github.com/hephbuild/heph/lib/pluginsdk" corev1 "github.com/hephbuild/heph/plugin/gen/heph/core/v1" - pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" + sync_map "github.com/zolstein/sync-map" "go.opentelemetry.io/otel" ) -func (e *Engine) Resulter() pluginsdk.Resulter { +func (e *Engine) Resulter() pluginsdk.EngineResulter { return &resulterHandler{Engine: e} } type resulterHandler struct { *Engine + + m sync_map.Map[string, *ExecuteResultLocks] } var tracer = otel.Tracer("heph/engine") -func (r resulterHandler) Get(ctx context.Context, req *corev1.ResultRequest) (*corev1.ResultResponse, error) { +func (r *resulterHandler) Get(ctx context.Context, req *corev1.ResultRequest) (*pluginsdk.GetResult, error) { rs, err := r.GetRequestState(req.GetRequestId()) if err != nil { return nil, err @@ -52,16 +55,20 @@ func (r resulterHandler) Get(ctx context.Context, req *corev1.ResultRequest) (*c default: return nil, fmt.Errorf("unexpected message type: %v", kind) } - // TODO: this is in the wrong place, the caller should be responsible for releasing the locks - defer res.Unlock(ctx) - artifacts := make([]*pluginv1.Artifact, 0, len(res.Artifacts)) + id := uuid.New().String() + r.m.Store(id, res) + + artifacts := make([]pluginsdk.Artifact, 0, len(res.Artifacts)) for _, artifact := range res.Artifacts { - artifacts = append(artifacts, artifact.Artifact) + artifacts = append(artifacts, artifact) } - return corev1.ResultResponse_builder{ + return &pluginsdk.GetResult{ + Release: func() { + res.Unlock(ctx) + }, Artifacts: artifacts, Def: res.Def.TargetDef.TargetDef, - }.Build(), nil + }, nil } diff --git a/internal/engine/hash_utils.go b/internal/engine/hash_utils.go index 25d717ec..8fea0841 100644 --- a/internal/engine/hash_utils.go +++ b/internal/engine/hash_utils.go @@ -25,7 +25,7 @@ func newHashWithDebug(w *xxh3.Hasher, name, hint string) hashWithDebug { c, _ := debugCounter.GetOrSet(name, &atomic.Int32{}) id := c.Add(1) - path := filepath.Join("/tmp/hashdebug", hinstance.UID, name, fmt.Sprintf("%d_%s.txt", id, hint)) + path := filepath.Join("/tmp/hashdebug", hinstance.LocalUID, name, fmt.Sprintf("%d_%s.txt", id, hint)) return hashWithDebug{Hasher: w, path: path} } diff --git a/internal/engine/link.go b/internal/engine/link.go index 4d0c3d09..c8544203 100644 --- a/internal/engine/link.go +++ b/internal/engine/link.go @@ -193,6 +193,10 @@ func (e *Engine) resolveSpecQuery(ctx context.Context, rs *RequestState, ref *pl deps = append(deps, structpb.NewStringValue(tref.Format(qref))) } + slices.SortFunc(deps, func(a, b *structpb.Value) int { + return strings.Compare(a.GetStringValue(), b.GetStringValue()) + }) + return pluginv1.TargetSpec_builder{ Ref: ref, Driver: htypes.Ptr(plugingroup.Name), diff --git a/internal/engine/local_cache.go b/internal/engine/local_cache.go index 73c79be3..1113eefc 100644 --- a/internal/engine/local_cache.go +++ b/internal/engine/local_cache.go @@ -1,15 +1,13 @@ package engine import ( - "archive/tar" - "bytes" "context" "encoding/hex" "errors" "fmt" "hash" "io" - "os" + "iter" "slices" "strconv" "strings" @@ -17,18 +15,28 @@ import ( "github.com/hephbuild/heph/internal/hcore/hstep" "github.com/hephbuild/heph/internal/hproto/hashpb" + "github.com/hephbuild/heph/lib/pluginsdk" "github.com/hephbuild/heph/lib/tref" - "github.com/hephbuild/heph/internal/htar" - "github.com/hephbuild/heph/internal/hartifact" "github.com/hephbuild/heph/internal/hfs" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" "github.com/zeebo/xxh3" ) -func (e *Engine) hashout(ctx context.Context, ref *pluginv1.TargetRef, artifact *pluginv1.Artifact) (string, error) { +var ErrLocalCacheNotFound = errors.New("artifact not found") + +type LocalCache interface { + Reader(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.ReadCloser, error) + Exists(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (bool, error) + Writer(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.WriteCloser, error) + Delete(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) error + ListArtifacts(ctx context.Context, ref *pluginv1.TargetRef, hashin string) iter.Seq2[string, error] + ListVersions(ctx context.Context, ref *pluginv1.TargetRef) iter.Seq2[string, error] +} + +func (e *Engine) hashout(ctx context.Context, ref *pluginv1.TargetRef, artifact pluginsdk.Artifact) (string, error) { var h interface { hash.Hash io.StringWriter @@ -72,160 +80,123 @@ func (e *Engine) targetDirName(ref *pluginv1.TargetRef) string { return "__" + ref.GetName() + "_" + hex.EncodeToString(h.Sum(nil)) } -func (e *Engine) CacheLocally( +func (e *Engine) cacheLocally( ctx context.Context, def *LightLinkedTarget, hashin string, - sandboxArtifacts []ExecuteResultArtifact, -) ([]ExecuteResultArtifact, *hartifact.Manifest, error) { + sandboxArtifacts []*ExecuteArtifact, +) ([]*ResultArtifact, error) { step, ctx := hstep.New(ctx, "Caching...") defer step.Done() - cachedir := hfs.At(e.Cache, def.GetRef().GetPackage(), e.targetDirName(def.GetRef()), hashin) - - cacheArtifacts := make([]ExecuteResultArtifact, 0, len(sandboxArtifacts)) + cacheArtifacts := make([]*ResultArtifact, 0, len(sandboxArtifacts)) for _, artifact := range sandboxArtifacts { - if artifact.GetType() == pluginv1.Artifact_TYPE_MANIFEST_V1 { - continue - } - - if artifact.HasFile() { - content := artifact.GetFile() - - artifact.SetName(artifact.GetName() + ".tar") - tarf, err := hfs.Create(cachedir.At(artifact.GetName())) - if err != nil { - return nil, nil, err - } - defer tarf.Close() - p := htar.NewPacker(tarf) - - sourcefs := hfs.NewOS(content.GetSourcePath()) - - f, err := hfs.Open(sourcefs) - if err != nil { - return nil, nil, err - } - defer f.Close() - - err = p.WriteFile(f, content.GetOutPath()) - if err != nil { - return nil, nil, err - } - - _ = f.Close() - _ = tarf.Close() - - artifact.SetTarPath(tarf.Name()) - } - if artifact.HasRaw() { - content := artifact.GetRaw() - - artifact.SetName(artifact.GetName() + ".tar") - tarf, err := hfs.Create(cachedir.At(artifact.GetName())) - if err != nil { - return nil, nil, err - } - defer tarf.Close() - p := htar.NewPacker(tarf) - - mode := int64(os.ModePerm) - if content.GetX() { - mode |= 0111 // executable - } - - err = p.Write(bytes.NewReader(content.GetData()), &tar.Header{ - Typeflag: tar.TypeReg, - Name: content.GetPath(), - Size: int64(len(content.GetData())), - Mode: mode, - }) - if err != nil { - return nil, nil, err - } - - _ = tarf.Close() - - artifact.SetTarPath(tarf.Name()) - } - - fromPath, err := hartifact.Path(artifact.Artifact) + contentType, err := artifact.GetContentType() if err != nil { - return nil, nil, err - } - - if fromPath == "" { - return nil, nil, fmt.Errorf("artifact %s has no path", artifact.GetName()) + return nil, err } - var prefix string - switch artifact.GetType() { - case pluginv1.Artifact_TYPE_OUTPUT: - prefix = "out_" - case pluginv1.Artifact_TYPE_SUPPORT_FILE: - prefix = "support_" - case pluginv1.Artifact_TYPE_LOG: - prefix = "log_" - case pluginv1.Artifact_TYPE_OUTPUT_LIST_V1, pluginv1.Artifact_TYPE_MANIFEST_V1, pluginv1.Artifact_TYPE_UNSPECIFIED: - fallthrough - default: - return nil, nil, fmt.Errorf("invalid artifact type: %s", artifact.GetType()) + contentSize, err := artifact.GetContentSize() + if err != nil { + return nil, err } - artifact.SetName(prefix + artifact.GetName()) - fromfs := hfs.NewOS(fromPath) - tofs := hfs.At(cachedir, artifact.GetName()) - - if false && strings.HasPrefix(fromfs.Path(), e.Home.Path()) { - // TODO: there was a bug here where when a target was running in tree, it would move things out from tree - err = hfs.Move(fromfs, tofs) - if err != nil { - return nil, nil, fmt.Errorf("move: %w", err) - } - } else { - err = hfs.Copy(fromfs, tofs) - if err != nil { - return nil, nil, fmt.Errorf("copy: %w", err) - } + src, err := artifact.GetContentReader() + if err != nil { + return nil, err } - - cachedArtifact, err := hartifact.Relocated(artifact.Artifact, tofs.Path()) + defer src.Close() + + cacheArtifact, err := e.cacheArtifactLocally(ctx, def.GetRef(), hashin, CacheLocallyArtifact{ + Reader: src, + Size: contentSize, + Type: artifact.GetType(), + Group: artifact.GetGroup(), + Name: artifact.GetName(), + ContentType: hartifact.ManifestArtifactContentType(contentType), + }, artifact.Hashout) if err != nil { - return nil, nil, fmt.Errorf("relocated: %w", err) + return nil, fmt.Errorf("%q/%q: %w", artifact.GetGroup(), artifact.GetName(), err) } - cacheArtifacts = append(cacheArtifacts, ExecuteResultArtifact{ - Hashout: artifact.Hashout, - Artifact: cachedArtifact, - }) + cacheArtifacts = append(cacheArtifacts, cacheArtifact) + + _ = src.Close() } - m := hartifact.Manifest{ + return cacheArtifacts, nil +} + +func (e *Engine) createLocalCacheManifest(ctx context.Context, ref *pluginv1.TargetRef, hashin string, artifacts []*ResultArtifact) (*hartifact.Manifest, error) { + m := &hartifact.Manifest{ Version: "v1", - Target: tref.Format(def.GetRef()), + Target: tref.Format(ref), CreatedAt: time.Now(), Hashin: hashin, } - for _, artifact := range cacheArtifacts { - martifact, err := hartifact.ProtoArtifactToManifest(artifact.Hashout, artifact.Artifact) + for _, artifact := range artifacts { + martifact, err := hartifact.ProtoArtifactToManifest(artifact.GetHashout(), artifact.Artifact) if err != nil { - return nil, nil, err + return nil, err } m.Artifacts = append(m.Artifacts, martifact) } - manifestArtifact, err := hartifact.WriteManifest(cachedir, m) + w, err := e.CacheSmall.Writer(ctx, ref, hashin, hartifact.ManifestName) + if err != nil { + return nil, err + } + defer w.Close() + + err = hartifact.EncodeManifest(w, m) + if err != nil { + return nil, err + } + + err = w.Close() if err != nil { - return nil, nil, err + return nil, err } - cacheArtifacts = append(cacheArtifacts, ExecuteResultArtifact{ - Artifact: manifestArtifact, - }) + return m, nil +} + +type cachePluginArtifact struct { + artifact hartifact.ManifestArtifact + ref *pluginv1.TargetRef + hashin string + cache LocalCache +} + +func (e cachePluginArtifact) GetGroup() string { + return e.artifact.Group +} +func (e cachePluginArtifact) GetName() string { + return e.artifact.Name +} +func (e cachePluginArtifact) GetType() pluginv1.Artifact_Type { + return pluginv1.Artifact_Type(e.artifact.Type) +} + +func (e cachePluginArtifact) GetContentReader() (io.ReadCloser, error) { + return e.cache.Reader(context.TODO(), e.ref, e.hashin, e.artifact.Name) +} + +func (e cachePluginArtifact) GetContentSize() (int64, error) { + return e.artifact.Size, nil +} + +func (e cachePluginArtifact) GetContentType() (pluginsdk.ArtifactContentType, error) { + switch e.artifact.ContentType { + case hartifact.ManifestArtifactContentTypeTar: + return pluginsdk.ArtifactContentTypeTar, nil + case hartifact.ManifestArtifactContentTypeTarGz: + return pluginsdk.ArtifactContentTypeTarGz, nil + } - return cacheArtifacts, &m, nil + return "", fmt.Errorf("invalid artifact content type: %q", e.artifact.ContentType) } func (e *Engine) ClearCacheLocally( @@ -236,9 +207,15 @@ func (e *Engine) ClearCacheLocally( step, ctx := hstep.New(ctx, "Clearing...") defer step.Done() - cachedir := hfs.At(e.Cache, ref.GetPackage(), e.targetDirName(ref), hashin) + if err := e.CacheLarge.Delete(ctx, ref, hashin, ""); err != nil { + return fmt.Errorf("fs: %w", err) + } - return cachedir.RemoveAll() + if err := e.CacheSmall.Delete(ctx, ref, hashin, ""); err != nil { + return fmt.Errorf("db: %w", err) + } + + return nil } func keyRefOutputs(ref *pluginv1.TargetRef, outputs []string) string { @@ -255,62 +232,167 @@ func keyRefOutputs(ref *pluginv1.TargetRef, outputs []string) string { return refKey(ref) + fmt.Sprintf("%#v", outputs) } -func (e *Engine) ResultFromLocalCache(ctx context.Context, def *LightLinkedTarget, outputs []string, hashin string) (*ExecuteResult, bool, error) { +func (e *Engine) ResultFromLocalCache(ctx context.Context, def *LightLinkedTarget, outputs []string, hashin string) (*Result, bool, error) { ctx, span := tracer.Start(ctx, "ResultFromLocalCache") defer span.End() res, ok, err := e.resultFromLocalCacheInner(ctx, def, outputs, hashin) if err != nil { - // if the file doesnt exist, thats not an error, just means the cache doesnt exist locally if errors.Is(err, hfs.ErrNotExist) { return nil, false, nil } - return nil, false, err } return res, ok, nil } -func (e *Engine) resultFromLocalCacheInner(ctx context.Context, def *LightLinkedTarget, outputs []string, hashin string) (*ExecuteResult, bool, error) { - dirfs := hfs.At(e.Cache, def.GetRef().GetPackage(), e.targetDirName(def.GetRef()), hashin) +func (e *Engine) readAnyCache(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (io.ReadCloser, error) { + for _, c := range [...]LocalCache{e.CacheSmall, e.CacheLarge} { + r, err := c.Reader(ctx, ref, hashin, name) + if err != nil { + if errors.Is(err, ErrLocalCacheNotFound) { + continue + } + + return nil, err + } + + return r, nil + } + + return nil, ErrLocalCacheNotFound +} + +func (e *Engine) existsAnyCache(ctx context.Context, ref *pluginv1.TargetRef, hashin, name string) (bool, LocalCache, error) { + for _, c := range [...]LocalCache{e.CacheSmall, e.CacheLarge} { + exists, err := c.Exists(ctx, ref, hashin, name) + if err != nil { + return false, c, err + } + + if exists { + return true, c, nil + } + } + + return false, nil, nil +} + +func (e *Engine) resultFromLocalCacheInner(ctx context.Context, def *LightLinkedTarget, outputs []string, hashin string) (*Result, bool, error) { + r, err := e.readAnyCache(ctx, def.GetRef(), hashin, hartifact.ManifestName) + if err != nil { + if errors.Is(err, ErrLocalCacheNotFound) { + return nil, false, nil + } + + return nil, false, err + } + defer r.Close() - manifest, manifestArtifact, err := hartifact.ManifestFromFS(dirfs) + m, err := hartifact.DecodeManifest(r) if err != nil { - return nil, false, fmt.Errorf("ManifestFromFS: %w", err) + return nil, false, err } artifacts := make([]hartifact.ManifestArtifact, 0, len(outputs)) for _, output := range outputs { - outputArtifacts := manifest.GetArtifacts(output) + outputArtifacts := m.GetArtifacts(output) artifacts = append(artifacts, outputArtifacts...) } - execArtifacts := make([]ExecuteResultArtifact, 0, len(artifacts)) + execArtifacts := make([]*ResultArtifact, 0, len(artifacts)) for _, artifact := range artifacts { - if !hfs.Exists(dirfs.At(artifact.Name)) { - return nil, false, nil + exists, cache, err := e.existsAnyCache(ctx, def.GetRef(), hashin, artifact.Name) + if err != nil { + return nil, false, err } - partifact, err := hartifact.ManifestArtifactToProto(artifact, dirfs.At(artifact.Name).Path()) - if err != nil { - return nil, false, fmt.Errorf("ManifestArtifactToProto: %w", err) + if !exists { + return nil, false, nil } - execArtifacts = append(execArtifacts, ExecuteResultArtifact{ - Hashout: artifact.Hashout, - Artifact: partifact, + execArtifacts = append(execArtifacts, &ResultArtifact{ + Artifact: cachePluginArtifact{ + artifact: artifact, + ref: def.GetRef(), + hashin: hashin, + cache: cache, + }, + Manifest: artifact, }) } - execArtifacts = append(execArtifacts, ExecuteResultArtifact{ - Artifact: manifestArtifact, - }) - - return ExecuteResult{ + return Result{ Def: def, - Hashin: manifest.Hashin, + Hashin: m.Hashin, Artifacts: execArtifacts, + Manifest: m, }.Sorted(), true, nil } + +type CacheLocallyArtifact struct { + Reader io.Reader + Size int64 + Type pluginv1.Artifact_Type + Group string + Name string + ContentType hartifact.ManifestArtifactContentType +} + +func (e *Engine) cacheArtifactLocally(ctx context.Context, ref *pluginv1.TargetRef, hashin string, art CacheLocallyArtifact, hashout string) (*ResultArtifact, error) { + cache := e.CacheSmall + if art.Size > 100_000 { // 100kb + cache = e.CacheLarge + } + + var prefix string + switch art.Type { + case pluginv1.Artifact_TYPE_OUTPUT: + prefix = "out_" + case pluginv1.Artifact_TYPE_SUPPORT_FILE: + prefix = "support_" + case pluginv1.Artifact_TYPE_LOG: + prefix = "log_" + case pluginv1.Artifact_TYPE_OUTPUT_LIST_V1, pluginv1.Artifact_TYPE_UNSPECIFIED: + fallthrough + default: + return nil, fmt.Errorf("invalid artifact type: %s", art.Type) + } + + dst, err := cache.Writer(ctx, ref, hashin, prefix+art.Name) + if err != nil { + return nil, err + } + defer dst.Close() + + _, err = io.Copy(dst, art.Reader) + if err != nil { + return nil, fmt.Errorf("copy to local: %w", err) + } + + err = dst.Close() + if err != nil { + return nil, err + } + + manifestArtifact := hartifact.ManifestArtifact{ + Hashout: hashout, + Group: art.Group, + Name: prefix + art.Name, + Size: art.Size, + Type: hartifact.ManifestArtifactType(art.Type), + ContentType: art.ContentType, + } + + return &ResultArtifact{ + Artifact: cachePluginArtifact{ + artifact: manifestArtifact, + ref: ref, + hashin: hashin, + cache: cache, + }, + Manifest: manifestArtifact, + }, nil +} diff --git a/internal/engine/query.go b/internal/engine/query.go index 90bf1f6f..cc9443f2 100644 --- a/internal/engine/query.go +++ b/internal/engine/query.go @@ -150,7 +150,7 @@ func (e *Engine) query(ctx context.Context, rs *RequestState, matcher *pluginv1. wg := hiter.NewGroup(ctx, yield) for pkg, err := range e.Packages(ctx, matcher) { if err != nil { - yield(nil, err) + wg.Yield(nil, err) return } diff --git a/internal/engine/remote_cache.go b/internal/engine/remote_cache.go index 129fc59b..694978a5 100644 --- a/internal/engine/remote_cache.go +++ b/internal/engine/remote_cache.go @@ -6,27 +6,21 @@ import ( "fmt" "io" "log/slog" - "os" "path" - "sync" "github.com/hephbuild/heph/internal/herrgroup" "github.com/hephbuild/heph/internal/hartifact" "github.com/hephbuild/heph/internal/hcore/hlog" "github.com/hephbuild/heph/internal/hcore/hstep" - "github.com/hephbuild/heph/internal/hfs" - "github.com/hephbuild/heph/internal/hinstance" - "github.com/hephbuild/heph/internal/hrand" "github.com/hephbuild/heph/internal/hslices" "github.com/hephbuild/heph/lib/pluginsdk" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - "golang.org/x/sync/errgroup" ) -func (e *Engine) CacheRemotely(ctx context.Context, def *LightLinkedTarget, hashin string, manifest *hartifact.Manifest, artifacts []ExecuteResultArtifact) { +func (e *Engine) CacheRemotely(ctx context.Context, def *LightLinkedTarget, hashin string, manifest *hartifact.Manifest, artifacts []*ResultArtifact) { if def.GetDisableRemoteCache() { return } @@ -71,7 +65,7 @@ func (e *Engine) cacheRemotelyInner(ctx context.Context, ref *pluginv1.TargetRef, hashin string, manifest *hartifact.Manifest, - artifacts []ExecuteResultArtifact, + artifacts []*ResultArtifact, cache CacheHandle, ) error { step, ctx := hstep.New(ctx, fmt.Sprintf("Caching %q...", cache.Name)) @@ -79,32 +73,55 @@ func (e *Engine) cacheRemotelyInner(ctx context.Context, // TODO: remote lock ? + g := herrgroup.NewContext(ctx, true) + for _, artifact := range artifacts { - r, err := hartifact.Reader(ctx, artifact.Artifact) - if err != nil { - return err - } - defer r.Close() + g.Go(func(ctx context.Context) error { + r, err := artifact.GetContentReader() + if err != nil { + return err + } + defer r.Close() - artifactName := artifact.GetName() - if artifact.GetType() == pluginv1.Artifact_TYPE_MANIFEST_V1 { - artifactName = hartifact.ManifestName - } + key := e.remoteCacheKey(ref, hashin, artifact.GetName()) - key := e.remoteCacheKey(ref, hashin, artifactName) + err = cache.Client.Store(ctx, key, r) + if err != nil { + return err + } - err = cache.Client.Store(ctx, key, r) + return nil + }) + } + + err := g.Wait() + if err != nil { + return err + } + + key := e.remoteCacheKey(ref, hashin, hartifact.ManifestName) + + pr, pw := io.Pipe() + + go func() { + defer pw.Close() + err := hartifact.EncodeManifest(pw, manifest) if err != nil { - return err + _ = pw.CloseWithError(err) + + return } + }() - _ = r.Close() + err = cache.Client.Store(ctx, key, pr) + if err != nil { + return err } return nil } -func (e *Engine) ResultFromRemoteCache(ctx context.Context, rs *RequestState, def *LightLinkedTarget, outputs []string, hashin string) (*ExecuteResult, bool, error) { +func (e *Engine) ResultFromRemoteCache(ctx context.Context, rs *RequestState, def *LightLinkedTarget, outputs []string, hashin string) (*Result, bool, error) { ref := def.GetRef() if def.GetDisableRemoteCache() { @@ -134,81 +151,50 @@ func (e *Engine) ResultFromRemoteCache(ctx context.Context, rs *RequestState, de continue } - tmpCacheDir := hfs.At( - e.Cache, - def.GetRef().GetPackage(), - e.targetDirName(def.GetRef())+"_remote_tmp_"+hinstance.UID+"_"+hrand.Str(7)+"_"+hashin, - ) - err := tmpCacheDir.MkdirAll(os.ModePerm) - if err != nil { - return nil, false, err - } - - defer tmpCacheDir.RemoveAll() - - cacheDir := hfs.At(e.Cache, def.GetRef().GetPackage(), e.targetDirName(def.GetRef()), hashin) - - artifacts, ok, err := e.resultFromRemoteCacheInner(ctx, ref, outputs, hashin, cache, tmpCacheDir) + ok, err := e.resultFromRemoteCacheInner(ctx, ref, outputs, hashin, cache) if err != nil { hlog.From(ctx).With(slog.String("cache", cache.Name), slog.String("err", err.Error())).Error("failed to get from cache") continue } + if !ok { + continue + } - if ok { - localArtifacts := make([]ExecuteResultArtifact, 0, len(artifacts)) - for _, artifact := range artifacts { - to := cacheDir.At(artifact.GetName()) - - err = hfs.Move(tmpCacheDir.At(artifact.GetName()), to) - if err != nil { - return nil, false, err - } - - rartifact, err := hartifact.Relocated(artifact.Artifact, to.Path()) - if err != nil { - return nil, false, err - } - - localArtifacts = append(localArtifacts, ExecuteResultArtifact{ - Hashout: artifact.Hashout, - Artifact: rartifact, - }) - } + res, ok, err := e.ResultFromLocalCache(ctx, def, outputs, hashin) + if err != nil { + // this is really not supposed to happen... - return ExecuteResult{ - Def: def, - Executed: false, - Hashin: hashin, - Artifacts: localArtifacts, - }.Sorted(), true, nil + return nil, false, fmt.Errorf("malformed local cache from remote cache: %w", err) } - _ = tmpCacheDir.RemoveAll() + if ok { + return res, true, nil + } } return nil, false, nil } -func (e *Engine) manifestFromRemoteCache(ctx context.Context, ref *pluginv1.TargetRef, hashin string, cache CacheHandle) (hartifact.Manifest, bool, error) { +func (e *Engine) manifestFromRemoteCache(ctx context.Context, ref *pluginv1.TargetRef, hashin string, cache CacheHandle) (*hartifact.Manifest, bool, error) { manifestKey := e.remoteCacheKey(ref, hashin, hartifact.ManifestName) r, err := cache.Client.Get(ctx, manifestKey) if err != nil { if errors.Is(err, pluginsdk.ErrCacheNotFound) { - return hartifact.Manifest{}, false, nil + return nil, false, nil } - return hartifact.Manifest{}, false, nil + return nil, false, nil } defer r.Close() m, err := hartifact.DecodeManifest(r) if err != nil { if errors.Is(err, pluginsdk.ErrCacheNotFound) { - return hartifact.Manifest{}, false, nil + return nil, false, nil } - return hartifact.Manifest{}, false, err + return nil, false, err } return m, true, nil @@ -226,17 +212,16 @@ func (e *Engine) resultFromRemoteCacheInner( outputs []string, hashin string, cache CacheHandle, - cachedir hfs.OS, -) ([]ExecuteResultArtifact, bool, error) { +) (bool, error) { ctx, span := tracer.Start(ctx, "ResultFromLocalCacheInner", trace.WithAttributes(attribute.String("cache", cache.Name))) defer span.End() manifest, ok, err := e.manifestFromRemoteCache(ctx, ref, hashin, cache) if err != nil { - return nil, false, err + return false, err } if !ok { - return nil, false, nil + return false, nil } remoteArtifacts := make([]hartifact.ManifestArtifact, 0, len(outputs)) @@ -249,55 +234,39 @@ func (e *Engine) resultFromRemoteCacheInner( return artifact.Group == output }) if !ok { - return nil, false, nil + return false, nil } remoteArtifacts = append(remoteArtifacts, artifact) } - var localArtifactsm sync.Mutex - localArtifacts := make([]ExecuteResultArtifact, 0, len(outputs)) + localArtifacts := make([]*ResultArtifact, len(outputs)) - g, ctx := errgroup.WithContext(ctx) + g := herrgroup.NewContext(ctx, true) - for _, artifact := range remoteArtifacts { + for i, artifact := range remoteArtifacts { key := e.remoteCacheKey(ref, hashin, artifact.Name) - tofs := cachedir.At(artifact.Name) - - g.Go(func() error { + g.Go(func(ctx context.Context) error { r, err := cache.Client.Get(ctx, key) - if err != nil { - if errors.Is(err, pluginsdk.ErrCacheNotFound) { - return pluginsdk.ErrCacheNotFound - } - - return err - } - - f, err := hfs.Create(tofs) - if err != nil { - return err - } - defer f.Close() - - _, err = io.Copy(f, r) if err != nil { return err } - _ = f.Close() - - lartifact, err := hartifact.ManifestArtifactToProto(artifact, tofs.Path()) + defer r.Close() + + localArtifact, err := e.cacheArtifactLocally(ctx, ref, hashin, CacheLocallyArtifact{ + Reader: r, + Size: artifact.Size, + Type: pluginv1.Artifact_Type(artifact.Type), + Group: artifact.Group, + Name: artifact.Name, + ContentType: artifact.ContentType, + }, artifact.Hashout) if err != nil { return err } - localArtifactsm.Lock() - localArtifacts = append(localArtifacts, ExecuteResultArtifact{ - Hashout: artifact.Hashout, - Artifact: lartifact, - }) - localArtifactsm.Unlock() + localArtifacts[i] = localArtifact return nil }) @@ -305,20 +274,16 @@ func (e *Engine) resultFromRemoteCacheInner( err = g.Wait() if err != nil { if errors.Is(err, pluginsdk.ErrCacheNotFound) { - return nil, false, nil + return false, nil } - return nil, false, err + return false, err } - manifestArtifact, err := hartifact.WriteManifest(cachedir, manifest) + _, err = e.createLocalCacheManifest(ctx, ref, hashin, localArtifacts) if err != nil { - return nil, false, err + return false, err } - localArtifacts = append(localArtifacts, ExecuteResultArtifact{ - Artifact: manifestArtifact, - }) - - return localArtifacts, true, nil + return true, nil } diff --git a/internal/engine/caches.go b/internal/engine/remote_caches.go similarity index 100% rename from internal/engine/caches.go rename to internal/engine/remote_caches.go diff --git a/internal/engine/schedule.go b/internal/engine/schedule.go index 7209d241..eb987b91 100644 --- a/internal/engine/schedule.go +++ b/internal/engine/schedule.go @@ -23,7 +23,6 @@ import ( "github.com/hephbuild/heph/internal/herrgroup" "github.com/hephbuild/heph/internal/hinstance" "github.com/hephbuild/heph/internal/hpanic" - "github.com/hephbuild/heph/internal/hproto" "github.com/hephbuild/heph/internal/hproto/hashpb" "github.com/hephbuild/heph/internal/htypes" "github.com/hephbuild/heph/internal/tmatch" @@ -57,7 +56,9 @@ type ExecuteOptions struct { shell bool force bool interactive bool - hashin string + metaHashin string + getCache bool + storeCache bool } type InteractiveExecOptions struct { @@ -278,15 +279,9 @@ func (e *Engine) result(ctx context.Context, rs *RequestState, c DefContainer, o } if onlyManifest { - res.Artifacts = slices.DeleteFunc(res.Artifacts, func(artifact ExecuteResultArtifact) bool { - return artifact.GetType() != pluginv1.Artifact_TYPE_MANIFEST_V1 - }) + res.Artifacts = nil } else { - res.Artifacts = slices.DeleteFunc(res.Artifacts, func(artifact ExecuteResultArtifact) bool { - if artifact.GetType() == pluginv1.Artifact_TYPE_MANIFEST_V1 { - return true - } - + res.Artifacts = slices.DeleteFunc(res.Artifacts, func(artifact *ResultArtifact) bool { if artifact.GetType() == pluginv1.Artifact_TYPE_SUPPORT_FILE { return false } @@ -341,12 +336,12 @@ func (e *Engine) depsResults(ctx context.Context, rs *RequestState, t *LightLink return err } - res.Artifacts = slices.DeleteFunc(res.Artifacts, func(output ExecuteResultArtifact) bool { + res.Artifacts = slices.DeleteFunc(res.Artifacts, func(output *ResultArtifact) bool { return output.GetType() != pluginv1.Artifact_TYPE_OUTPUT && output.GetType() != pluginv1.Artifact_TYPE_SUPPORT_FILE }) for _, artifact := range res.Artifacts { - if artifact.Hashout == "" { + if artifact.GetHashout() == "" { return fmt.Errorf("%v: output %q has empty hashout", tref.Format(dep.GetRef()), artifact.GetGroup()) } } @@ -451,19 +446,11 @@ func (e *Engine) ResultMetaFromDef(ctx context.Context, rs *RequestState, def *T } defer res.Unlock(ctx) - manifestArtifact, ok := res.FindManifest() - if !ok { - return ResultMeta{}, errors.New("no manifest") - } - - manifest, err := hartifact.ManifestFromArtifact(ctx, manifestArtifact.Artifact) - if err != nil { - return ResultMeta{}, fmt.Errorf("manifest from artifact: %w", err) - } + manifest := res.Manifest m := ResultMeta{ Hashin: manifest.Hashin, - CreatedAt: manifest.CreatedAt, + CreatedAt: manifest.CreatedAt.UTC(), } if len(outputs) == 1 && outputs[0] == AllOutputs { @@ -563,7 +550,7 @@ func (e *Engine) depsResultMetas(ctx context.Context, rs *RequestState, def *Lig const AllOutputs = "__all_outputs__" -func (e *Engine) resultFromCache(ctx context.Context, rs *RequestState, def *LightLinkedTarget, outputs []string, hashin string) (*ExecuteResult, bool, error) { +func (e *Engine) resultFromCache(ctx context.Context, rs *RequestState, def *LightLinkedTarget, outputs []string, hashin string) (*Result, bool, error) { res, ok, err := e.ResultFromLocalCache(ctx, def, outputs, hashin) if err != nil { return nil, false, fmt.Errorf("result from local cache: %w", err) @@ -624,14 +611,14 @@ func (e *Engine) innerResult(ctx context.Context, rs *RequestState, def *LightLi } res, ok, err := e.resultFromCache(ctx, rs, def, outputs, hashin) - if err != nil { + if err != nil && !errors.Is(err, context.Canceled) { hlog.From(ctx).With(slog.String("target", tref.Format(def.GetRef())), slog.String("err", err.Error())).Warn("failed to get result from local cache") } if ok { return &ExecuteResultLocks{ - ExecuteResult: res, - Locks: locks, + Result: res, + Locks: locks, }, nil } @@ -649,56 +636,46 @@ func (e *Engine) innerResult(ctx context.Context, rs *RequestState, def *LightLi } defer results.Unlock(ctx) - manifest := hartifact.Manifest{ + manifest := &hartifact.Manifest{ Version: "v1", Target: tref.Format(def.GetRef()), CreatedAt: time.Now(), Hashin: hashin, } - var artifacts []ExecuteResultArtifact + var artifacts []*ResultArtifact var locks CacheLocks for _, result := range results { for _, artifact := range result.Artifacts { - gartifact := hproto.Clone(artifact.Artifact) - gartifact.ClearGroup() // TODO support output group - - artifacts = append(artifacts, ExecuteResultArtifact{ - Hashout: artifact.Hashout, - Artifact: gartifact, - }) + partifact := artifactGroupMap{Artifact: artifact, group: ""} // TODO support output group - martifact, err := hartifact.ProtoArtifactToManifest(artifact.Hashout, gartifact) + martifact, err := hartifact.ProtoArtifactToManifest(artifact.GetHashout(), partifact) if err != nil { return nil, fmt.Errorf("proto artifact to manifest: %w", err) } + artifacts = append(artifacts, &ResultArtifact{ + Artifact: partifact, + Manifest: martifact, + }) + manifest.Artifacts = append(manifest.Artifacts, martifact) locks.AddFrom(result.Locks) } } - martifact, err := hartifact.NewManifestArtifact(manifest) - if err != nil { - return nil, fmt.Errorf("new manifest artifact: %w", err) - } - - artifacts = append(artifacts, ExecuteResultArtifact{ - Artifact: martifact, - }) - err = locks.RLock(ctx) if err != nil { return nil, err } return &ExecuteResultLocks{ - ExecuteResult: ExecuteResult{ + Result: Result{ Def: def, - Executed: true, Hashin: hashin, Artifacts: artifacts, + Manifest: manifest, }.Sorted(), Locks: &locks, }, nil @@ -713,52 +690,16 @@ func (e *Engine) innerResult(ctx context.Context, rs *RequestState, def *LightLi shell: shouldShell, force: shouldForce, interactive: tref.Equal(rs.Shell, def.GetRef()) || tref.Equal(rs.Interactive, def.GetRef()), - hashin: hashin, + metaHashin: hashin, + getCache: getCache, + storeCache: storeCache, } - var res *ExecuteResult - if storeCache { - res, err = e.ExecuteAndCache(ctx, rs, def, execOptions) - if err != nil { - err = errors.Join(err, locks.Unlock()) - - return nil, err - } - } else { - res, err = e.Execute(ctx, rs, def, execOptions) - if err != nil { - err = errors.Join(err, locks.Unlock()) - - return nil, err - } - - m := hartifact.Manifest{ - Version: "v1", - Target: tref.Format(def.GetRef()), - CreatedAt: time.Now(), - Hashin: hashin, - } - for _, artifact := range res.Artifacts { - martifact, err := hartifact.ProtoArtifactToManifest(artifact.Hashout, artifact.Artifact) - if err != nil { - err = errors.Join(err, locks.Unlock()) - - return nil, err - } - - m.Artifacts = append(m.Artifacts, martifact) - } - - manifestArtifact, err := hartifact.NewManifestArtifact(m) - if err != nil { - err = errors.Join(err, locks.Unlock()) - - return nil, err - } + res, err := e.executeAndCacheInner(ctx, rs, def, execOptions) + if err != nil { + err = errors.Join(err, locks.Unlock()) - res.Artifacts = append(res.Artifacts, ExecuteResultArtifact{ - Artifact: manifestArtifact, - }) + return nil, err } err = locks.Lock2RLock(ctx) @@ -769,8 +710,8 @@ func (e *Engine) innerResult(ctx context.Context, rs *RequestState, def *LightLi } return &ExecuteResultLocks{ - ExecuteResult: res, - Locks: locks, + Result: res, + Locks: locks, }, nil }) if err != nil { @@ -863,7 +804,7 @@ func (e *Engine) hashin(ctx context.Context, def *LightLinkedTarget, results []* artifacts := make([]DepMetaArtifact, 0, len(result.Artifacts)) for _, artifact := range result.Artifacts { artifacts = append(artifacts, DepMetaArtifact{ - Hashout: artifact.Hashout, + Hashout: artifact.GetHashout(), }) } @@ -881,32 +822,39 @@ func (e *Engine) hashin(ctx context.Context, def *LightLinkedTarget, results []* return e.hashin2(ctx, def, metas, "res") } -type ExecuteResultArtifact struct { - Hashout string - *pluginv1.Artifact +type ExecuteResult struct { + Def *LightLinkedTarget + Hashin string + Artifacts []*ExecuteArtifact + AfterCache func() } -type ExecuteResult struct { +type Result struct { Def *LightLinkedTarget - Executed bool Hashin string - Artifacts []ExecuteResultArtifact + Artifacts []*ResultArtifact + Manifest *hartifact.Manifest } -func (r ExecuteResult) FindManifest() (ExecuteResultArtifact, bool) { - for _, artifact := range r.Artifacts { - if artifact.GetType() != pluginv1.Artifact_TYPE_MANIFEST_V1 { - continue - } +func (r Result) Sorted() *Result { + slices.SortFunc(r.Artifacts, func(a, b *ResultArtifact) int { + return strings.Compare(a.GetHashout(), b.GetHashout()) + }) - return artifact, true - } + return &r +} - return ExecuteResultArtifact{}, false +func (r Result) Clone() *Result { + return &Result{ + Def: r.Def, + Hashin: r.Hashin, + Artifacts: slices.Clone(r.Artifacts), + Manifest: r.Manifest, + } } -func (r ExecuteResult) FindOutputs(group string) []ExecuteResultArtifact { - res := make([]ExecuteResultArtifact, 0, len(r.Artifacts)) +func (r Result) FindOutputs(group string) []*ResultArtifact { + res := make([]*ResultArtifact, 0, len(r.Artifacts)) for _, artifact := range r.Artifacts { if artifact.GetType() != pluginv1.Artifact_TYPE_OUTPUT { continue @@ -921,8 +869,8 @@ func (r ExecuteResult) FindOutputs(group string) []ExecuteResultArtifact { return res } -func (r ExecuteResult) FindSupport() []ExecuteResultArtifact { - res := make([]ExecuteResultArtifact, 0, len(r.Artifacts)) +func (r Result) FindSupport() []*ResultArtifact { + res := make([]*ResultArtifact, 0, len(r.Artifacts)) for _, artifact := range r.Artifacts { if artifact.GetType() != pluginv1.Artifact_TYPE_SUPPORT_FILE { continue @@ -934,22 +882,37 @@ func (r ExecuteResult) FindSupport() []ExecuteResultArtifact { return res } +type ExecuteArtifact struct { + Hashout string + pluginsdk.Artifact +} + +type ResultArtifact struct { + Manifest hartifact.ManifestArtifact + + pluginsdk.Artifact +} + +func (r ResultArtifact) GetHashout() string { + return r.Manifest.Hashout +} + type ExecuteResultLocks struct { - *ExecuteResult + *Result Locks *CacheLocks } func (r *ExecuteResultLocks) Clone() *ExecuteResultLocks { return &ExecuteResultLocks{ - ExecuteResult: r.ExecuteResult.Clone(), - Locks: r.Locks.Clone(), + Result: r.Result.Clone(), + Locks: r.Locks.Clone(), } } func (r *ExecuteResultLocks) CloneWithoutLocks() *ExecuteResultLocks { return &ExecuteResultLocks{ - ExecuteResult: r.ExecuteResult.Clone(), - Locks: r.Locks, + Result: r.Result.Clone(), + Locks: r.Locks, } } @@ -965,7 +928,7 @@ func (r *ExecuteResultLocks) Unlock(ctx context.Context) { } func (r ExecuteResult) Sorted() *ExecuteResult { - slices.SortFunc(r.Artifacts, func(a, b ExecuteResultArtifact) int { + slices.SortFunc(r.Artifacts, func(a, b *ExecuteArtifact) int { return strings.Compare(a.Hashout, b.Hashout) }) @@ -974,10 +937,10 @@ func (r ExecuteResult) Sorted() *ExecuteResult { func (r ExecuteResult) Clone() *ExecuteResult { return &ExecuteResult{ - Def: r.Def.Clone(), - Executed: r.Executed, - Hashin: r.Hashin, - Artifacts: slices.Clone(r.Artifacts), + Def: r.Def.Clone(), + Hashin: r.Hashin, + Artifacts: slices.Clone(r.Artifacts), + AfterCache: r.AfterCache, } } @@ -1157,7 +1120,9 @@ func (e *Engine) pickShellDriver(ctx context.Context, def *LightLinkedTarget) (p var sem = semaphore.NewWeighted(1000 * int64(runtime.GOMAXPROCS(-1))) -func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinkedTarget, options ExecuteOptions) (*ExecuteResult, error) { +func (e *Engine) execute(ctx context.Context, rs *RequestState, def *LightLinkedTarget, results DepsResults, options ExecuteOptions) (*ExecuteResult, error) { + hashin := options.metaHashin + ctx, span := tracer.Start(ctx, "Execute") defer span.End() @@ -1168,18 +1133,13 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked }) defer cleanLabels() - results, err := e.depsResults(ctx, rs, def) - if err != nil { - return nil, fmt.Errorf("deps results: %w", err) - } - defer results.Unlock(ctx) - driver, ok := e.DriversByName[def.GetDriver()] if !ok { return nil, fmt.Errorf("driver not found: %v", def.GetDriver()) } if options.shell { + var err error driver, err = e.pickShellDriver(ctx, def) if err != nil { return nil, err @@ -1194,30 +1154,7 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked targetfolder = fmt.Sprintf("__%v__%v", e.targetDirName(def.GetRef()), time.Now().UnixNano()) } - hashin, err := e.hashin(ctx, def, results) - if err != nil { - return nil, fmt.Errorf("hashin1: %w", err) - } - - if hashin != options.hashin { - return nil, fmt.Errorf("results hashin (%v) != meta hashin (%v)", hashin, options.hashin) - } - - if def.GetCache() && !options.force && !options.shell { - res, ok, err := e.ResultFromLocalCache(ctx, def, def.OutputNames(), hashin) - if err != nil { - hlog.From(ctx).With(slog.String("target", tref.Format(def.GetRef())), slog.String("err", err.Error())).Warn("failed to get result from local cache") - } - - if ok { - step := hstep.From(ctx) - step.SetText(fmt.Sprintf("%v: cached", tref.Format(def.GetRef()))) - - return res, nil - } - } - - err = sem.Acquire(ctx, 1) + err := sem.Acquire(ctx, 1) if err != nil { return nil, err } @@ -1252,13 +1189,31 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked } } - var inputs []*pluginv1.ArtifactWithOrigin + var inputLen int + for _, result := range results { + inputLen += len(result.Artifacts) + } + + inputs := make([]*pluginv1.RunRequest_Input, 0, inputLen) + inputsSdk := make([]*pluginsdk.ArtifactWithOrigin, 0, inputLen) for _, result := range results { for _, artifact := range result.Artifacts { - inputs = append(inputs, pluginv1.ArtifactWithOrigin_builder{ - Artifact: artifact.Artifact, + partifact := pluginv1.RunRequest_Input_Artifact_builder{ + Group: htypes.Ptr(artifact.GetGroup()), + Name: htypes.Ptr(artifact.GetName()), + Type: htypes.Ptr(artifact.GetType()), + Id: htypes.Ptr("TODO"), // to be implemented along with the pluginsdkconnect.ProtoArtifact + }.Build() + + inputs = append(inputs, pluginv1.RunRequest_Input_builder{ + Artifact: partifact, Origin: result.InputOrigin, }.Build()) + + inputsSdk = append(inputsSdk, &pluginsdk.ArtifactWithOrigin{ + Artifact: artifact.Artifact, + Origin: result.InputOrigin, + }) } } @@ -1288,15 +1243,18 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked }() runRes, runErr = hpanic.RecoverV(func() (*pluginv1.RunResponse, error) { - return driver.Run(ctx, pluginv1.RunRequest_builder{ - RequestId: htypes.Ptr(rs.ID), - Target: def.TargetDef.TargetDef, - SandboxPath: htypes.Ptr(sandboxfs.Path()), - TreeRootPath: htypes.Ptr(e.Root.Path()), - Inputs: inputs, - Pipes: pipes, - Hashin: htypes.Ptr(hashin), - }.Build()) + return driver.Run(ctx, &pluginsdk.RunRequest{ + RunRequest: pluginv1.RunRequest_builder{ + RequestId: htypes.Ptr(rs.ID), + Target: def.TargetDef.TargetDef, + SandboxPath: htypes.Ptr(sandboxfs.Path()), + TreeRootPath: htypes.Ptr(e.Root.Path()), + Inputs: inputs, + Pipes: pipes, + Hashin: htypes.Ptr(hashin), + }.Build(), + Inputs: inputsSdk, + }) }) }, Pty: def.GetPty(), @@ -1321,14 +1279,18 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked if options.shell { return &ExecuteResult{ - Def: def, - Hashin: hashin, - Executed: true, + Def: def, + Hashin: hashin, + AfterCache: func() { + err := sandboxfs.RemoveAll() + if err != nil { + hlog.From(ctx).Error(fmt.Sprintf("failed to remove sandboxfs: %v", err)) + } + }, }, nil } - cachefs := hfs.At(e.Cache, def.GetRef().GetPackage(), e.targetDirName(def.GetRef()), hashin) - execArtifacts := make([]ExecuteResultArtifact, 0, len(def.OutputNames())) + execArtifacts := make([]*ExecuteArtifact, 0, len(def.OutputNames())) for _, output := range def.GetOutputs() { shouldCollect := false @@ -1340,11 +1302,11 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked } if !shouldCollect { - break + continue } tarname := output.GetGroup() + ".tar" - tarf, err := hfs.Create(cachefs.At(tarname)) + tarf, err := hfs.Create(sandboxfs.At("collect", tarname)) if err != nil { return nil, err } @@ -1394,19 +1356,26 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked return nil, err } - execArtifact := pluginv1.Artifact_builder{ - Group: htypes.Ptr(output.GetGroup()), - Name: htypes.Ptr(tarname), - Type: htypes.Ptr(pluginv1.Artifact_TYPE_OUTPUT), - TarPath: proto.String(tarf.Name()), - }.Build() + err = tarf.Close() + if err != nil { + return nil, err + } + + execArtifact := pluginsdk.ProtoArtifact{ + Artifact: pluginv1.Artifact_builder{ + Group: htypes.Ptr(output.GetGroup()), + Name: htypes.Ptr(tarname), + Type: htypes.Ptr(pluginv1.Artifact_TYPE_OUTPUT), + TarPath: proto.String(tarf.Name()), + }.Build(), + } hashout, err := e.hashout(ctx, def.GetRef(), execArtifact) if err != nil { return nil, err } - execArtifacts = append(execArtifacts, ExecuteResultArtifact{ + execArtifacts = append(execArtifacts, &ExecuteArtifact{ Hashout: hashout, Artifact: execArtifact, }) @@ -1423,7 +1392,7 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked if shouldCollect { tarname := "support.tar" - tarf, err := hfs.Create(cachefs.At(tarname)) + tarf, err := hfs.Create(sandboxfs.At("collect", tarname)) if err != nil { return nil, err } @@ -1468,90 +1437,165 @@ func (e *Engine) Execute(ctx context.Context, rs *RequestState, def *LightLinked return nil, err } - execArtifact := pluginv1.Artifact_builder{ - Name: htypes.Ptr(tarname), - Type: htypes.Ptr(pluginv1.Artifact_TYPE_SUPPORT_FILE), - TarPath: proto.String(tarf.Name()), - }.Build() + execArtifact := pluginsdk.ProtoArtifact{ + Artifact: pluginv1.Artifact_builder{ + Name: htypes.Ptr(tarname), + Type: htypes.Ptr(pluginv1.Artifact_TYPE_SUPPORT_FILE), + TarPath: proto.String(tarf.Name()), + }.Build(), + } hashout, err := e.hashout(ctx, def.GetRef(), execArtifact) if err != nil { return nil, err } - execArtifacts = append(execArtifacts, ExecuteResultArtifact{ + execArtifacts = append(execArtifacts, &ExecuteArtifact{ Hashout: hashout, Artifact: execArtifact, }) } } - for _, artifact := range runRes.GetArtifacts() { - if artifact.GetType() != pluginv1.Artifact_TYPE_OUTPUT { + for _, partifact := range runRes.GetArtifacts() { + if partifact.GetType() != pluginv1.Artifact_TYPE_OUTPUT { continue } + artifact := pluginsdk.ProtoArtifact{ + Artifact: partifact, + } + hashout, err := e.hashout(ctx, def.GetRef(), artifact) if err != nil { return nil, fmt.Errorf("hashout: %w", err) } - execArtifacts = append(execArtifacts, ExecuteResultArtifact{ + execArtifacts = append(execArtifacts, &ExecuteArtifact{ Hashout: hashout, Artifact: artifact, }) - - // panic("copy to cache not implemented yet") - - // TODO: copy to cache - // hfs.Copy() - // - // artifact.Uri - // - // execOutputs = append(execOutputs, ExecuteResultOutput{ - // Name: artifact.Group, - // Hashout: "", - // TarPath: "", - // }) - } - - err = sandboxfs.RemoveAll() - if err != nil { - return nil, err } return ExecuteResult{ Hashin: hashin, Def: def, - Executed: true, Artifacts: execArtifacts, + AfterCache: func() { + err := sandboxfs.RemoveAll() + if err != nil { + hlog.From(ctx).Error(fmt.Sprintf("failed to remove sandboxfs: %v", err)) + } + }, }.Sorted(), nil } -func (e *Engine) ExecuteAndCache(ctx context.Context, rs *RequestState, def *LightLinkedTarget, options ExecuteOptions) (*ExecuteResult, error) { - res, err := e.Execute(ctx, rs, def, options) +func (e *Engine) executeAndCacheInner(ctx context.Context, rs *RequestState, def *LightLinkedTarget, options ExecuteOptions) (*Result, error) { + results, err := e.depsResults(ctx, rs, def) + if err != nil { + return nil, fmt.Errorf("deps results: %w", err) + } + defer results.Unlock(ctx) + + hashin, err := e.hashin(ctx, def, results) + if err != nil { + return nil, fmt.Errorf("results hashin: %w", err) + } + + if hashin != options.metaHashin { + return nil, fmt.Errorf("results hashin (%v) != meta hashin (%v)", hashin, options.metaHashin) + } + + cacheHashin := hashin + if !options.storeCache { + cacheHashin = hinstance.UID + "_" + hashin + } + + if options.storeCache { + // One last cache check after the deps have completed + res, ok, err := e.ResultFromLocalCache(ctx, def, def.OutputNames(), cacheHashin) + if err != nil && !errors.Is(err, context.Canceled) { + hlog.From(ctx).With(slog.String("target", tref.Format(def.GetRef())), slog.String("err", err.Error())).Warn("failed to get result from local cache") + } + + if ok { + return res, nil + } + } + + res, err := e.execute(ctx, rs, def, results, options) if err != nil { return nil, fmt.Errorf("execute: %w", err) } - var cachedArtifacts []ExecuteResultArtifact - if res.Executed { - artifacts, manifest, err := e.CacheLocally(ctx, def, res.Hashin, res.Artifacts) - if err != nil { - return nil, fmt.Errorf("cache locally: %w", err) + var artifactsToCache []*ExecuteArtifact + var artifactsToPassthrough []*ResultArtifact + if options.storeCache { + artifactsToCache = res.Artifacts + } else { + // this caters for the pluginfs case where it doesnt make sense to copy the tree into the cache, if it's + // an uncached target + artifactsToPassthrough = make([]*ResultArtifact, 0, len(res.Artifacts)) + for _, artifact := range res.Artifacts { + if e.isPassthroughArtifact(artifact) { + m, err := hartifact.ProtoArtifactToManifest(artifact.Hashout, artifact) + if err != nil { + return nil, fmt.Errorf("protoartifacttomanifest: %w", err) + } + + artifactsToPassthrough = append(artifactsToPassthrough, &ResultArtifact{ + Manifest: m, + Artifact: artifact, + }) + } else { + artifactsToCache = append(artifactsToCache, artifact) + } } + } + + cachedArtifacts, err := e.cacheLocally(ctx, def, cacheHashin, artifactsToCache) + if err != nil { + return nil, fmt.Errorf("cache locally: %w", err) + } + + cachedArtifacts = append(cachedArtifacts, artifactsToPassthrough...) + + manifest, err := e.createLocalCacheManifest(ctx, def.GetRef(), hashin, cachedArtifacts) + if err != nil { + return nil, fmt.Errorf("create local cache manifest: %w", err) + } - cachedArtifacts = artifacts + if res.AfterCache != nil { + res.AfterCache() + } - // TODO: move this to a background execution model , so that local build can proceed, while this is uploading in the background + if options.storeCache { + // TODO: move this to a background execution so that local build can proceed, while this is uploading in the background e.CacheRemotely(ctx, def, res.Hashin, manifest, cachedArtifacts) - } else { - cachedArtifacts = res.Artifacts } - return ExecuteResult{ + return Result{ Def: def, Hashin: res.Hashin, Artifacts: cachedArtifacts, + Manifest: manifest, }.Sorted(), nil } + +func (e *Engine) isPassthroughArtifact(artifact *ExecuteArtifact) bool { + fsartifact, ok := artifact.Artifact.(pluginsdk.FSArtifact) + if !ok { + return false + } + + node := fsartifact.FSNode() + if node == nil { + return false + } + + if hfs.HasPathPrefix(node.Path(), e.Home.Path()) { + return false + } + + return true +} diff --git a/internal/enginee2e/cache_test.go b/internal/enginee2e/cache_test.go new file mode 100644 index 00000000..74c40f54 --- /dev/null +++ b/internal/enginee2e/cache_test.go @@ -0,0 +1,90 @@ +package enginee2e + +import ( + "testing" + + "github.com/hephbuild/heph/internal/htypes" + + "github.com/hephbuild/heph/internal/hproto/hstructpb" + + "github.com/hephbuild/heph/internal/engine" + "github.com/hephbuild/heph/internal/hartifact" + "github.com/hephbuild/heph/internal/hfs" + "github.com/hephbuild/heph/internal/hfs/hfstest" + pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" + "github.com/hephbuild/heph/plugin/pluginexec" + "github.com/hephbuild/heph/plugin/pluginstaticprovider" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +func TestCache(t *testing.T) { + ctx := t.Context() + + dir := t.TempDir() + + e, err := engine.New(ctx, dir, engine.Config{}) + require.NoError(t, err) + + staticprovider := pluginstaticprovider.New([]pluginstaticprovider.Target{ + { + Spec: pluginv1.TargetSpec_builder{ + Ref: pluginv1.TargetRef_builder{ + Package: htypes.Ptr(""), + Name: htypes.Ptr("child"), + }.Build(), + Driver: htypes.Ptr("bash"), + Config: map[string]*structpb.Value{ + "run": hstructpb.NewStringsValue([]string{`echo hello > $OUT`}), + "out": hstructpb.NewStringsValue([]string{"out_child"}), + }, + }.Build(), + }, + }) + + _, err = e.RegisterProvider(ctx, staticprovider, engine.RegisterProviderConfig{}) + require.NoError(t, err) + + _, err = e.RegisterDriver(ctx, pluginexec.NewExec(), nil) + require.NoError(t, err) + _, err = e.RegisterDriver(ctx, pluginexec.NewSh(), nil) + require.NoError(t, err) + _, err = e.RegisterDriver(ctx, pluginexec.NewBash(), nil) + require.NoError(t, err) + + assertOut := func(res *engine.ExecuteResultLocks) { + fs := hfstest.New(t) + err = hartifact.Unpack(ctx, res.FindOutputs("")[0].Artifact, fs) + require.NoError(t, err) + + b, err := hfs.ReadFile(fs.At("out_child")) + require.NoError(t, err) + + assert.Equal(t, "hello\n", string(b)) + } + + { // This will run all + rs, clean := e.NewRequestState() + defer clean() + + res, err := e.Result(ctx, rs, "", "child", []string{engine.AllOutputs}) + require.NoError(t, err) + defer res.Unlock(ctx) + + assertOut(res) + res.Unlock(ctx) + } + + { // this should reuse cache from deps + rs, clean := e.NewRequestState() + defer clean() + + res, err := e.Result(ctx, rs, "", "child", []string{engine.AllOutputs}) + require.NoError(t, err) + defer res.Unlock(ctx) + + assertOut(res) + res.Unlock(ctx) + } +} diff --git a/internal/enginee2e/deps_cache_test.go b/internal/enginee2e/deps_cache_test.go index 05da0345..42aea1a3 100644 --- a/internal/enginee2e/deps_cache_test.go +++ b/internal/enginee2e/deps_cache_test.go @@ -111,7 +111,91 @@ func TestDepsCache(t *testing.T) { } } -func TestDepsCache2(t *testing.T) { +func TestDepsCacheLarge(t *testing.T) { + ctx := t.Context() + + dir := t.TempDir() + + e, err := engine.New(ctx, dir, engine.Config{}) + require.NoError(t, err) + + staticprovider := pluginstaticprovider.New([]pluginstaticprovider.Target{ + { + Spec: pluginv1.TargetSpec_builder{ + Ref: pluginv1.TargetRef_builder{ + Package: htypes.Ptr(""), + Name: htypes.Ptr("child"), + }.Build(), + Driver: htypes.Ptr("bash"), + Config: map[string]*structpb.Value{ + "run": hstructpb.NewStringsValue([]string{`dd if=/dev/zero of=$OUT bs=1k count=200`}), + "out": hstructpb.NewStringsValue([]string{"out_child"}), + }, + }.Build(), + }, + { + Spec: pluginv1.TargetSpec_builder{ + Ref: pluginv1.TargetRef_builder{ + Package: htypes.Ptr(""), + Name: htypes.Ptr("parent"), + }.Build(), + Driver: htypes.Ptr("bash"), + Config: map[string]*structpb.Value{ + "run": hstructpb.NewStringsValue([]string{`wc -c < "${SRC}" | tr -d ' ' > $OUT`}), + "deps": hstructpb.NewStringsValue([]string{"//:child"}), + "out": hstructpb.NewStringsValue([]string{"out_parent"}), + }, + }.Build(), + }, + }) + + _, err = e.RegisterProvider(ctx, staticprovider, engine.RegisterProviderConfig{}) + require.NoError(t, err) + + _, err = e.RegisterDriver(ctx, pluginexec.NewExec(), nil) + require.NoError(t, err) + _, err = e.RegisterDriver(ctx, pluginexec.NewSh(), nil) + require.NoError(t, err) + _, err = e.RegisterDriver(ctx, pluginexec.NewBash(), nil) + require.NoError(t, err) + + assertOut := func(res *engine.ExecuteResultLocks) { + fs := hfstest.New(t) + err = hartifact.Unpack(ctx, res.FindOutputs("")[0].Artifact, fs) + require.NoError(t, err) + + b, err := hfs.ReadFile(fs.At("out_parent")) + require.NoError(t, err) + + assert.Equal(t, "204800\n", string(b)) + } + + { // This will run all + rs, clean := e.NewRequestState() + defer clean() + + res, err := e.Result(ctx, rs, "", "parent", []string{engine.AllOutputs}) + require.NoError(t, err) + defer res.Unlock(ctx) + + assertOut(res) + res.Unlock(ctx) + } + + { // this should reuse cache from deps + rs, clean := e.NewRequestState() + defer clean() + + res, err := e.Result(ctx, rs, "", "parent", []string{engine.AllOutputs}) + require.NoError(t, err) + defer res.Unlock(ctx) + + assertOut(res) + res.Unlock(ctx) + } +} + +func TestDepsCacheRemote(t *testing.T) { ctx := t.Context() c := gomock.NewController(t) @@ -170,7 +254,7 @@ func TestDepsCache2(t *testing.T) { driver.EXPECT(). Run(gomock.Any(), gomock.Any()). - DoAndReturn(func(ctx context.Context, request *pluginv1.RunRequest) (*pluginv1.RunResponse, error) { + DoAndReturn(func(ctx context.Context, request *pluginsdk.RunRequest) (*pluginv1.RunResponse, error) { return pluginv1.RunResponse_builder{ Artifacts: []*pluginv1.Artifact{ pluginv1.Artifact_builder{ @@ -192,10 +276,10 @@ func TestDepsCache2(t *testing.T) { cache := pluginsdk.NewMockCache(c) cache.EXPECT(). - Get(gomock.Any(), "__child/e5d50f4b478a3687/manifest.v1.json"). + Get(gomock.Any(), "__child/16f2c56460420e9a/manifest.v1.json"). Return(nil, pluginsdk.ErrNotFound).Times(1) - for _, key := range []string{"__child/e5d50f4b478a3687/manifest.v1.json", "__child/e5d50f4b478a3687/out_out.tar"} { + for _, key := range []string{"__child/16f2c56460420e9a/manifest.v1.json", "__child/16f2c56460420e9a/out_out.tar"} { cache.EXPECT(). Store(gomock.Any(), key, gomock.Any()). DoAndReturn(func(ctx context.Context, key string, reader io.Reader) error { @@ -206,7 +290,7 @@ func TestDepsCache2(t *testing.T) { cache.EXPECT(). Get(gomock.Any(), key). - Return(io.NopCloser(bytes.NewReader(b)), nil).Times(1) + Return(io.NopCloser(bytes.NewReader(b)), nil).AnyTimes() return nil }).Times(1) diff --git a/internal/enginee2e/pluginscyclicprovider/provider.go b/internal/enginee2e/pluginscyclicprovider/provider.go index 46b996e7..b3e2e4da 100644 --- a/internal/enginee2e/pluginscyclicprovider/provider.go +++ b/internal/enginee2e/pluginscyclicprovider/provider.go @@ -82,7 +82,7 @@ func (p *Provider) List(ctx context.Context, req *pluginv1.ListRequest) (plugins } if p.resultOnList { - _, err := p.resultClient.ResultClient.Get(ctx, corev1.ResultRequest_builder{ + res, err := p.resultClient.ResultClient.Get(ctx, corev1.ResultRequest_builder{ RequestId: htypes.Ptr(req.GetRequestId()), Spec: pluginv1.TargetSpec_builder{ Ref: tref.New(hephPackage, "think", nil), @@ -100,6 +100,8 @@ func (p *Provider) List(ctx context.Context, req *pluginv1.ListRequest) (plugins if err != nil { return err } + + defer res.Release() } for _, spec := range p.targets() { @@ -125,7 +127,7 @@ func (p *Provider) Get(ctx context.Context, req *pluginv1.GetRequest) (*pluginv1 } if p.resultOnGet { - _, err := p.resultClient.ResultClient.Get(ctx, corev1.ResultRequest_builder{ + res, err := p.resultClient.ResultClient.Get(ctx, corev1.ResultRequest_builder{ RequestId: htypes.Ptr(req.GetRequestId()), Spec: pluginv1.TargetSpec_builder{ Ref: tref.New(hephPackage, "think", nil), @@ -143,6 +145,8 @@ func (p *Provider) Get(ctx context.Context, req *pluginv1.GetRequest) (*pluginv1 if err != nil { return nil, err } + + defer res.Release() } for _, spec := range p.targets() { diff --git a/internal/enginee2e/pluginsmartprovidertest/provider.go b/internal/enginee2e/pluginsmartprovidertest/provider.go index 9371aee6..b3875ce8 100644 --- a/internal/enginee2e/pluginsmartprovidertest/provider.go +++ b/internal/enginee2e/pluginsmartprovidertest/provider.go @@ -63,8 +63,9 @@ func (p *Provider) Get(ctx context.Context, req *pluginv1.GetRequest) (*pluginv1 if err != nil { return nil, err } + defer res.Release() - artifacts := hartifact.FindOutputs(res.GetArtifacts(), "") + artifacts := hartifact.FindOutputs(res.Artifacts, "") r, err := hartifact.FileReader(ctx, artifacts[0]) if err != nil { diff --git a/internal/enginee2e/sanity_remotecache_test.go b/internal/enginee2e/sanity_remotecache_test.go index 1df3cdeb..3947b7d7 100644 --- a/internal/enginee2e/sanity_remotecache_test.go +++ b/internal/enginee2e/sanity_remotecache_test.go @@ -3,6 +3,7 @@ package enginee2e import ( "bytes" "context" + "fmt" "io" "testing" @@ -112,76 +113,76 @@ func TestSanityRemoteCache(t *testing.T) { // Simulates 2 independent runs, with the same cache for i := 1; i <= 2; i++ { - t.Log("RUN", i) + t.Run(fmt.Sprintf("run %v", i), func(t *testing.T) { + dir := t.TempDir() - dir := t.TempDir() - - e, err := engine.New(ctx, dir, engine.Config{}) - require.NoError(t, err) + e, err := engine.New(ctx, dir, engine.Config{}) + require.NoError(t, err) - _, err = e.RegisterProvider(ctx, staticprovider, engine.RegisterProviderConfig{}) - require.NoError(t, err) + _, err = e.RegisterProvider(ctx, staticprovider, engine.RegisterProviderConfig{}) + require.NoError(t, err) - _, err = e.RegisterDriver(ctx, pluginexec.NewSh(), nil) - require.NoError(t, err) + _, err = e.RegisterDriver(ctx, pluginexec.NewSh(), nil) + require.NoError(t, err) - _, err = e.RegisterCache("test", cache, true, true) - require.NoError(t, err) + _, err = e.RegisterCache("test", cache, true, true) + require.NoError(t, err) - { - rs, clean := e.NewRequestState() - defer clean() + { + rs, clean := e.NewRequestState() + defer clean() - res, err := e.Result(ctx, rs, pkg, "t1", []string{""}) - require.NoError(t, err) - defer res.Unlock(ctx) + res, err := e.Result(ctx, rs, pkg, "t1", []string{""}) + require.NoError(t, err) + defer res.Unlock(ctx) - require.Len(t, res.Artifacts, 1) + require.Len(t, res.Artifacts, 1) - manifest, err := e.ResultMetaFromRef(ctx, rs, tref.New(pkg, "t1", nil), []string{""}) - require.NoError(t, err) + manifest, err := e.ResultMetaFromRef(ctx, rs, tref.New(pkg, "t1", nil), []string{""}) + require.NoError(t, err) - assert.Equal(t, "e52c3f7fe43c3c02", manifest.Hashin) - assert.Equal(t, "d4fd9c2c4c50146f", manifest.Artifacts[0].Hashout) - } + assert.Equal(t, "9d12ac88089ebc06", manifest.Hashin) + assert.Equal(t, "d4fd9c2c4c50146f", manifest.Artifacts[0].Hashout) + } - { - rs, clean := e.NewRequestState() - defer clean() + { + rs, clean := e.NewRequestState() + defer clean() - res, err := e.Result(ctx, rs, pkg, "t2", []string{""}) - require.NoError(t, err) - defer res.Unlock(ctx) + res, err := e.Result(ctx, rs, pkg, "t2", []string{""}) + require.NoError(t, err) + defer res.Unlock(ctx) - require.Len(t, res.Artifacts, 1) + require.Len(t, res.Artifacts, 1) - manifest, err := e.ResultMetaFromRef(ctx, rs, tref.New(pkg, "t2", nil), []string{""}) - require.NoError(t, err) + manifest, err := e.ResultMetaFromRef(ctx, rs, tref.New(pkg, "t2", nil), []string{""}) + require.NoError(t, err) - assert.Equal(t, "7103b83d26040e7a", manifest.Hashin) - assert.Equal(t, "3b0f519635c52211", manifest.Artifacts[0].Hashout) - } + assert.Equal(t, "c4bc4a188996ffc8", manifest.Hashin) + assert.Equal(t, "3b0f519635c52211", manifest.Artifacts[0].Hashout) + } - { - rs, clean := e.NewRequestState() - defer clean() + { + rs, clean := e.NewRequestState() + defer clean() - res, err := e.Result(ctx, rs, pkg, "t3", []string{""}) - require.NoError(t, err) - defer res.Unlock(ctx) + res, err := e.Result(ctx, rs, pkg, "t3", []string{""}) + require.NoError(t, err) + defer res.Unlock(ctx) - require.Len(t, res.Artifacts, 1) + require.Len(t, res.Artifacts, 1) - manifest, err := e.ResultMetaFromRef(ctx, rs, tref.New(pkg, "t3", nil), []string{""}) - require.NoError(t, err) + manifest, err := e.ResultMetaFromRef(ctx, rs, tref.New(pkg, "t3", nil), []string{""}) + require.NoError(t, err) - assert.Equal(t, "7a65fe455c8b6178", manifest.Hashin) - assert.Equal(t, "3b0f519635c52211", manifest.Artifacts[0].Hashout) - } + assert.Equal(t, "7e2b9b570c832965", manifest.Hashin) + assert.Equal(t, "3b0f519635c52211", manifest.Artifacts[0].Hashout) + } - assert.Len(t, cache.storeWrites, 6) - for k, c := range cache.storeWrites { - assert.Equalf(t, 1, c, "cache hit count for %v", k) - } + assert.Len(t, cache.storeWrites, 6) + for k, c := range cache.storeWrites { + assert.Equalf(t, 1, c, "cache hit count for %v", k) + } + }) } } diff --git a/internal/hartifact/manifest.go b/internal/hartifact/manifest.go index a9929d1f..976f0712 100644 --- a/internal/hartifact/manifest.go +++ b/internal/hartifact/manifest.go @@ -1,16 +1,12 @@ package hartifact import ( - "context" - "encoding/base64" "encoding/json" "fmt" "io" "time" - "github.com/hephbuild/heph/internal/htypes" - - "github.com/hephbuild/heph/internal/hfs" + "github.com/hephbuild/heph/lib/pluginsdk" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" ) @@ -49,8 +45,6 @@ type ManifestArtifactContentType string const ( ManifestArtifactContentTypeTar ManifestArtifactContentType = "application/x-tar" ManifestArtifactContentTypeTarGz ManifestArtifactContentType = "application/x-gtar" - ManifestArtifactContentTypeFile ManifestArtifactContentType = "file" - ManifestArtifactContentTypeRaw ManifestArtifactContentType = "raw" ) type ManifestArtifact struct { @@ -58,10 +52,9 @@ type ManifestArtifact struct { Group string Name string + Size int64 Type ManifestArtifactType ContentType ManifestArtifactContentType - OutPath string `json:",omitempty"` // set for file & raw - X bool `json:",omitempty"` // set for file & raw } type Manifest struct { @@ -85,158 +78,59 @@ func (m Manifest) GetArtifacts(output string) []ManifestArtifact { return a } -func WriteManifest(node hfs.Node, m Manifest) (*pluginv1.Artifact, error) { - b, err := json.Marshal(m) //nolint:musttag - if err != nil { - return nil, err - } - - err = hfs.WriteFile(node.At(ManifestName), b) +func EncodeManifest(w io.Writer, m *Manifest) error { + err := json.NewEncoder(w).Encode(m) //nolint:musttag if err != nil { - return nil, err + return err } - return newManifestArtifact(node), nil + return nil } -func NewManifestArtifact(m Manifest) (*pluginv1.Artifact, error) { - b, err := json.Marshal(m) //nolint:musttag +func DecodeManifest(r io.Reader) (*Manifest, error) { + var manifest Manifest + err := json.NewDecoder(r).Decode(&manifest) //nolint:musttag if err != nil { return nil, err } - return pluginv1.Artifact_builder{ - Name: htypes.Ptr(ManifestName), - Type: htypes.Ptr(pluginv1.Artifact_TYPE_MANIFEST_V1), - Raw: pluginv1.Artifact_ContentRaw_builder{ - Data: b, - Path: htypes.Ptr(ManifestName), - }.Build(), - }.Build(), nil -} - -func newManifestArtifact(node hfs.Node) *pluginv1.Artifact { - return pluginv1.Artifact_builder{ - Name: htypes.Ptr(ManifestName), - Type: htypes.Ptr(pluginv1.Artifact_TYPE_MANIFEST_V1), - File: pluginv1.Artifact_ContentFile_builder{ - SourcePath: htypes.Ptr(node.At(ManifestName).Path()), - OutPath: htypes.Ptr(ManifestName), - }.Build(), - }.Build() -} - -func ManifestFromArtifact(ctx context.Context, a *pluginv1.Artifact) (Manifest, error) { - r, err := FileReader(ctx, a) - if err != nil { - return Manifest{}, err - } - defer r.Close() - - return DecodeManifest(r) -} - -func ManifestFromFS(node hfs.Node) (Manifest, *pluginv1.Artifact, error) { - f, err := hfs.Open(node.At(ManifestName)) - if err != nil { - return Manifest{}, nil, err - } - defer f.Close() - - m, err := DecodeManifest(f) - if err != nil { - return Manifest{}, nil, err - } - - return m, newManifestArtifact(node), nil + return &manifest, nil } -func DecodeManifest(r io.Reader) (Manifest, error) { - var manifest Manifest - err := json.NewDecoder(r).Decode(&manifest) //nolint:musttag +func ManifestContentType(a pluginsdk.Artifact) (ManifestArtifactContentType, error) { + contentType, err := a.GetContentType() if err != nil { - return Manifest{}, err + return "", err } - return manifest, nil -} - -func ManifestContentType(a *pluginv1.Artifact) (ManifestArtifactContentType, error) { - switch a.WhichContent() { - case pluginv1.Artifact_TargzPath_case: + switch contentType { + case pluginsdk.ArtifactContentTypeTarGz: return ManifestArtifactContentTypeTarGz, nil - case pluginv1.Artifact_TarPath_case: + case pluginsdk.ArtifactContentTypeTar: return ManifestArtifactContentTypeTar, nil - case pluginv1.Artifact_File_case: - return ManifestArtifactContentTypeFile, nil - case pluginv1.Artifact_Raw_case: - return ManifestArtifactContentTypeRaw, nil default: } - return "", fmt.Errorf("unsupported content %v", a.WhichContent().String()) + return "", fmt.Errorf("unsupported content %v", contentType) } -func ProtoArtifactToManifest(hashout string, artifact *pluginv1.Artifact) (ManifestArtifact, error) { - contentType, err := ManifestContentType(artifact) +func ProtoArtifactToManifest(hashout string, a pluginsdk.Artifact) (ManifestArtifact, error) { + contentType, err := ManifestContentType(a) if err != nil { return ManifestArtifact{}, err } - var outPath string - var x bool - switch artifact.WhichContent() { //nolint:exhaustive - case pluginv1.Artifact_File_case: - outPath = artifact.GetFile().GetOutPath() - x = artifact.GetFile().GetX() - case pluginv1.Artifact_Raw_case: - outPath = artifact.GetRaw().GetPath() - x = artifact.GetRaw().GetX() + size, err := a.GetContentSize() + if err != nil { + return ManifestArtifact{}, err } return ManifestArtifact{ Hashout: hashout, - Group: artifact.GetGroup(), - Name: artifact.GetName(), - Type: ManifestArtifactType(artifact.GetType()), + Group: a.GetGroup(), + Name: a.GetName(), + Size: size, + Type: ManifestArtifactType(a.GetType()), ContentType: contentType, - OutPath: outPath, - X: x, }, nil } - -func ManifestArtifactToProto(artifact ManifestArtifact, path string) (*pluginv1.Artifact, error) { - partifact := pluginv1.Artifact_builder{ - Group: htypes.Ptr(artifact.Group), - Name: htypes.Ptr(artifact.Name), - Type: htypes.Ptr(pluginv1.Artifact_Type(artifact.Type)), - }.Build() - - switch artifact.ContentType { - case ManifestArtifactContentTypeTar: - partifact.SetTarPath(path) - case ManifestArtifactContentTypeTarGz: - partifact.SetTargzPath(path) - case ManifestArtifactContentTypeFile: - partifact.SetFile(pluginv1.Artifact_ContentFile_builder{ - SourcePath: &path, - OutPath: &artifact.OutPath, - X: &artifact.X, - }.Build()) - case ManifestArtifactContentTypeRaw: - b, err := base64.StdEncoding.DecodeString(path) - if err != nil { - return nil, err - } - - partifact.SetRaw(pluginv1.Artifact_ContentRaw_builder{ - Data: b, - Path: &artifact.OutPath, - X: &artifact.X, - }.Build()) - default: - return nil, fmt.Errorf("unsupported content type %q", artifact.ContentType) - } - - return partifact, nil -} diff --git a/internal/hartifact/reader.go b/internal/hartifact/reader.go index 21add7c1..706f9518 100644 --- a/internal/hartifact/reader.go +++ b/internal/hartifact/reader.go @@ -2,44 +2,26 @@ package hartifact import ( "archive/tar" - "bytes" "context" "fmt" "io" "iter" - "os" "github.com/hephbuild/heph/internal/hio" "github.com/hephbuild/heph/internal/htar" - - pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" + "github.com/hephbuild/heph/lib/pluginsdk" ) -// Reader gives a raw io.Reader of an artifact, useful for things like hashing. -func Reader(ctx context.Context, a *pluginv1.Artifact) (io.ReadCloser, error) { - switch a.WhichContent() { - case pluginv1.Artifact_File_case: - return os.Open(a.GetFile().GetSourcePath()) - case pluginv1.Artifact_Raw_case: - return io.NopCloser(bytes.NewReader(a.GetRaw().GetData())), nil - case pluginv1.Artifact_TargzPath_case: - return os.Open(a.GetTargzPath()) - case pluginv1.Artifact_TarPath_case: - return os.Open(a.GetTarPath()) - default: - return nil, fmt.Errorf("unsupported encoding %v", a.WhichContent()) +// FileReader Assumes the output has a single file, and provides a reader for it (no matter the packaging). +func FileReader(ctx context.Context, a pluginsdk.Artifact) (io.ReadCloser, error) { + contentType, err := a.GetContentType() + if err != nil { + return nil, fmt.Errorf("get content type: %w", err) } -} -// FileReader Assumes the output has a single file, and provides a reader for it (no matter the packaging). -func FileReader(ctx context.Context, a *pluginv1.Artifact) (io.ReadCloser, error) { - switch a.WhichContent() { - case pluginv1.Artifact_File_case: - return os.Open(a.GetFile().GetSourcePath()) - case pluginv1.Artifact_Raw_case: - return io.NopCloser(bytes.NewReader(a.GetRaw().GetData())), nil - case pluginv1.Artifact_TarPath_case: - r, err := Reader(ctx, a) + switch contentType { + case pluginsdk.ArtifactContentTypeTar: + r, err := a.GetContentReader() if err != nil { return nil, err } @@ -52,14 +34,14 @@ func FileReader(ctx context.Context, a *pluginv1.Artifact) (io.ReadCloser, error } return hio.NewReadCloser(tr, r), nil - // case *pluginv1.Artifact_TargzPath: + // case pluginsdk.ArtifactContentTypeTarGz: default: - return nil, fmt.Errorf("unsupported encoding %v", a.WhichContent()) + return nil, fmt.Errorf("unsupported encoding %v", contentType) } } // FileReader Assumes the output has a single file, and provides the bytes for it (no matter the packaging). -func FileReadAll(ctx context.Context, a *pluginv1.Artifact) ([]byte, error) { +func FileReadAll(ctx context.Context, a pluginsdk.Artifact) ([]byte, error) { f, err := FileReader(ctx, a) if err != nil { return nil, err @@ -75,46 +57,26 @@ type File struct { } // FilesReader provides a reader for each file it (no matter the packaging). -func FilesReader(ctx context.Context, a *pluginv1.Artifact) iter.Seq2[*File, error] { +func FilesReader(ctx context.Context, a pluginsdk.Artifact) iter.Seq2[*File, error] { return func(yield func(*File, error) bool) { - switch a.WhichContent() { - case pluginv1.Artifact_File_case: - f, err := os.Open(a.GetFile().GetSourcePath()) - if err != nil { - if !yield(nil, err) { - return - } - return - } - - if !yield(&File{ - ReadCloser: f, - Path: a.GetFile().GetOutPath(), - }, nil) { - return - } - case pluginv1.Artifact_Raw_case: - f := io.NopCloser(bytes.NewReader(a.GetRaw().GetData())) + contentType, err := a.GetContentType() + if err != nil { + yield(nil, fmt.Errorf("get content type: %w", err)) + return + } - if !yield(&File{ - ReadCloser: f, - Path: a.GetRaw().GetPath(), - }, nil) { - return - } - case pluginv1.Artifact_TarPath_case: - r, err := Reader(ctx, a) + switch contentType { + case pluginsdk.ArtifactContentTypeTar: + r, err := a.GetContentReader() if err != nil { - if !yield(nil, err) { - return - } + yield(nil, err) return } defer r.Close() tr := tar.NewReader(r) - err = htar.Walk(tr, func(header *tar.Header, reader *tar.Reader) error { + err = htar.Walk(tr, func(header *tar.Header, reader io.Reader) error { if header.Typeflag != tar.TypeReg { return nil } @@ -129,16 +91,15 @@ func FilesReader(ctx context.Context, a *pluginv1.Artifact) iter.Seq2[*File, err return nil }) if err != nil { - if !yield(nil, err) { - return - } + yield(nil, err) return } - // case *pluginv1.Artifact_TargzPath: + + return + // case pluginsdk.Artifact_TargzPath: default: - if !yield(nil, fmt.Errorf("unsupported encoding %v", a.WhichContent())) { - return - } + yield(nil, fmt.Errorf("unsupported encoding %v", contentType)) + return } } } diff --git a/internal/hartifact/unpack.go b/internal/hartifact/unpack.go index 7e3cc913..4fe13390 100644 --- a/internal/hartifact/unpack.go +++ b/internal/hartifact/unpack.go @@ -3,12 +3,11 @@ package hartifact import ( "context" "fmt" - "io" "github.com/hephbuild/heph/internal/htar" + "github.com/hephbuild/heph/lib/pluginsdk" "github.com/hephbuild/heph/internal/hfs" - pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" ) type unpackConfig struct { @@ -30,7 +29,7 @@ func WithFilter(filter func(from string) bool) UnpackOption { } } -func Unpack(ctx context.Context, artifact *pluginv1.Artifact, node hfs.Node, options ...UnpackOption) error { +func Unpack(ctx context.Context, artifact pluginsdk.Artifact, node hfs.Node, options ...UnpackOption) error { var cfg unpackConfig for _, option := range options { option(&cfg) @@ -44,74 +43,27 @@ func Unpack(ctx context.Context, artifact *pluginv1.Artifact, node hfs.Node, opt } } - r, err := Reader(ctx, artifact) + r, err := artifact.GetContentReader() if err != nil { return err } defer r.Close() - switch artifact.WhichContent() { - case pluginv1.Artifact_File_case: - if !cfg.filter(artifact.GetFile().GetOutPath()) { - return nil - } - - create := hfs.Create - if artifact.GetFile().GetX() { - create = hfs.CreateExec - } - - f, err := create(node.At(artifact.GetFile().GetOutPath())) - if err != nil { - return fmt.Errorf("file: create: %w", err) - } - defer cfg.onFile(f.Name()) - defer f.Close() - - _, err = io.Copy(f, r) - if err != nil { - return err - } - - err = hfs.CloseEnsureROFD(f) - if err != nil { - return err - } - case pluginv1.Artifact_Raw_case: - if !cfg.filter(artifact.GetRaw().GetPath()) { - return nil - } - - create := hfs.Create - if artifact.GetRaw().GetX() { - create = hfs.CreateExec - } - - f, err := create(node.At(artifact.GetRaw().GetPath())) - if err != nil { - return fmt.Errorf("raw: create: %w", err) - } - defer cfg.onFile(f.Name()) - defer f.Close() - - _, err = io.Copy(f, r) - if err != nil { - return err - } + contentType, err := artifact.GetContentType() + if err != nil { + return err + } - err = hfs.CloseEnsureROFD(f) - if err != nil { - return err - } - case pluginv1.Artifact_TarPath_case: + switch contentType { + case pluginsdk.ArtifactContentTypeTar: err = htar.Unpack(ctx, r, node, htar.WithOnFile(cfg.onFile), htar.WithFilter(cfg.filter)) if err != nil { return fmt.Errorf("tar: %w", err) } + + return nil // case *pluginv1.Artifact_TargzPath: default: - return fmt.Errorf("unsupported encoding %v", artifact.WhichContent()) + return fmt.Errorf("unsupported encoding %v", contentType) } - - return nil } diff --git a/internal/hartifact/well_known.go b/internal/hartifact/well_known.go index cd4ca386..d5870897 100644 --- a/internal/hartifact/well_known.go +++ b/internal/hartifact/well_known.go @@ -1,18 +1,12 @@ package hartifact import ( - "iter" - "slices" - + "github.com/hephbuild/heph/lib/pluginsdk" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" ) -func FindOutputs(artifacts []*pluginv1.Artifact, group string) []*pluginv1.Artifact { - return findOutputs(slices.All(artifacts), group) -} - -func findOutputs(artifacts iter.Seq2[int, *pluginv1.Artifact], group string) []*pluginv1.Artifact { - out := make([]*pluginv1.Artifact, 0, 1) +func FindOutputs(artifacts []pluginsdk.Artifact, group string) []pluginsdk.Artifact { + out := make([]pluginsdk.Artifact, 0, 1) for _, artifact := range artifacts { if artifact.GetType() != pluginv1.Artifact_TYPE_OUTPUT { continue diff --git a/internal/hfs/atomic.go b/internal/hfs/atomic.go index 6596d3e7..6a9e4bfb 100644 --- a/internal/hfs/atomic.go +++ b/internal/hfs/atomic.go @@ -6,7 +6,7 @@ import ( ) func processUniquePath(p string) string { - return p + "_tmp_" + hinstance.UID + "_" + hrand.Str(7) + return p + "_tmp_" + hinstance.LocalUID + "_" + hrand.Str(7) } type AtomicFile struct { diff --git a/internal/hinstance/uid.go b/internal/hinstance/uid.go index e4abc915..02b34b92 100644 --- a/internal/hinstance/uid.go +++ b/internal/hinstance/uid.go @@ -12,12 +12,18 @@ import ( func gen() string { host, _ := os.Hostname() - return fmt.Sprintf("%v_%v_%v", os.Getpid(), host, time.Now().UnixNano()) + return fmt.Sprintf("%v_%v_%v", host, os.Getpid(), time.Now().UnixNano()) } var UID = gen() -var Hash = sync.OnceValue(func() string { +func localgen() string { + return fmt.Sprintf("%v_%v", os.Getpid(), time.Now().UnixNano()) +} + +var LocalUID = localgen() + +var HashExec = sync.OnceValue(func() string { p, err := os.Executable() if err != nil { panic(err) diff --git a/internal/hio/closer.go b/internal/hio/closer.go index 358a65bc..9865c83d 100644 --- a/internal/hio/closer.go +++ b/internal/hio/closer.go @@ -13,3 +13,19 @@ func NewReadCloser(r io.Reader, c io.Closer) io.ReadCloser { Closer: c, } } + +type readCloserFunc struct { + io.Reader + CloserFunc func() error +} + +func (r readCloserFunc) Close() error { + return r.CloserFunc() +} + +func NewReadCloserFunc(r io.Reader, f func() error) io.ReadCloser { + return readCloserFunc{ + Reader: r, + CloserFunc: f, + } +} diff --git a/internal/hiter/group.go b/internal/hiter/group.go index b4bbf384..aa4caf92 100644 --- a/internal/hiter/group.go +++ b/internal/hiter/group.go @@ -34,7 +34,7 @@ func (g *Group[K, V]) Go(f func(ctx context.Context, yield func(K, V) bool)) { return errYieldExited } - f(ctx, g.gyield) + f(ctx, g.Yield) if g.yieldExited { return errYieldExited @@ -52,7 +52,7 @@ func (g *Group[K, V]) SetLimit(n int) { g.wg.SetLimit(n) } -func (g *Group[K, V]) gyield(k K, v V) bool { +func (g *Group[K, V]) Yield(k K, v V) bool { g.yieldmu.Lock() defer g.yieldmu.Unlock() diff --git a/internal/hmaps/sorted.go b/internal/hmaps/sorted.go index afec6383..f0eef0fe 100644 --- a/internal/hmaps/sorted.go +++ b/internal/hmaps/sorted.go @@ -14,9 +14,8 @@ func Sorted[Map ~map[K]V, K cmp.Ordered, V any](m Map) iter.Seq2[K, V] { return case 1: for k, v := range m { - if !yield(k, v) { - return - } + yield(k, v) + break } return diff --git a/internal/htar/pack.go b/internal/htar/pack.go index 1261f740..afe8a187 100644 --- a/internal/htar/pack.go +++ b/internal/htar/pack.go @@ -4,6 +4,7 @@ import ( "archive/tar" "fmt" "io" + "io/fs" "os" "path/filepath" @@ -11,7 +12,8 @@ import ( ) type Packer struct { - tw *tar.Writer + tw *tar.Writer + AllowAbsLink bool } func NewPacker(w io.Writer) *Packer { @@ -25,7 +27,7 @@ func (p *Packer) WriteFile(f hfs.File, path string) error { } var link string - if info.Mode().Type()&os.ModeSymlink != 0 { + if info.Mode().Type()&fs.ModeSymlink != 0 { l, err := os.Readlink(f.Name()) if err != nil { return err @@ -33,7 +35,7 @@ func (p *Packer) WriteFile(f hfs.File, path string) error { link = l - if filepath.IsAbs(link) { + if !p.AllowAbsLink && filepath.IsAbs(link) { return fmt.Errorf("absolute link not allowed: %v -> %v", f.Name(), link) } } @@ -45,19 +47,7 @@ func (p *Packer) WriteFile(f hfs.File, path string) error { hdr.Name = path - if err := p.tw.WriteHeader(hdr); err != nil { - return err - } - - if !info.Mode().IsRegular() { // nothing more to do for non-regular - return nil - } - - if _, err := io.Copy(p.tw, io.LimitReader(f, info.Size())); err != nil { - return err - } - - return nil + return p.Write(f, hdr) } func (p *Packer) Write(r io.Reader, hdr *tar.Header) error { diff --git a/internal/htar/unpack.go b/internal/htar/unpack.go index c66b3cc2..1dd32af8 100644 --- a/internal/htar/unpack.go +++ b/internal/htar/unpack.go @@ -43,31 +43,44 @@ type config struct { type Matcher = func(hdr *tar.Header) bool func FileReader(ctx context.Context, r io.Reader, match Matcher) (io.Reader, error) { - tr := tar.NewReader(r) + pr, pw := io.Pipe() - var fileReader io.Reader - err := Walk(tr, func(hdr *tar.Header, r *tar.Reader) error { - if !match(hdr) { - return nil - } + go func() { + defer pw.Close() - switch hdr.Typeflag { - case tar.TypeReg: - fileReader = r - return ErrStopWalk - default: - return fmt.Errorf("is not a file, is %v: %s", hdr.Typeflag, hdr.Name) + tr := tar.NewReader(r) + + var matched bool + err := Walk(tr, func(hdr *tar.Header, r io.Reader) error { + if !match(hdr) { + return nil + } + + switch hdr.Typeflag { + case tar.TypeReg: + matched = true + _, err := io.Copy(pw, r) + if err != nil { + return err + } + + return ErrStopWalk + default: + return fmt.Errorf("is not a file, is %v: %s", hdr.Typeflag, hdr.Name) + } + }) + if err != nil { + _ = pw.CloseWithError(err) + return } - }) - if err != nil { - return nil, err - } - if fileReader == nil { - return nil, errors.New("file not found") - } + if !matched { + _ = pw.CloseWithError(errors.New("tar is empty")) + return + } + }() - return fileReader, nil + return pr, nil } func Unpack(ctx context.Context, r io.Reader, to hfs.Node, options ...Option) error { @@ -86,7 +99,7 @@ func Unpack(ctx context.Context, r io.Reader, to hfs.Node, options ...Option) er tr := tar.NewReader(r) - return Walk(tr, func(hdr *tar.Header, r *tar.Reader) error { + return Walk(tr, func(hdr *tar.Header, r io.Reader) error { if !cfg.filter(hdr.Name) { return nil } @@ -134,7 +147,7 @@ func Unpack(ctx context.Context, r io.Reader, to hfs.Node, options ...Option) er }) } -func unpackFile(hdr *tar.Header, tr *tar.Reader, to hfs.Node, ro bool, onFile func(to string)) error { +func unpackFile(hdr *tar.Header, tr io.Reader, to hfs.Node, ro bool, onFile func(to string)) error { fileNode := to.At(hdr.Name) info, err := fileNode.Lstat() @@ -196,7 +209,7 @@ func unpackFile(hdr *tar.Header, tr *tar.Reader, to hfs.Node, ro bool, onFile fu var ErrStopWalk = errors.New("stop walk") -func Walk(tr *tar.Reader, fs ...func(*tar.Header, *tar.Reader) error) error { +func Walk(tr *tar.Reader, f func(*tar.Header, io.Reader) error) error { for { hdr, err := tr.Next() if err != nil { @@ -207,15 +220,13 @@ func Walk(tr *tar.Reader, fs ...func(*tar.Header, *tar.Reader) error) error { return fmt.Errorf("walk: %w", err) } - for _, f := range fs { - err = f(hdr, tr) - if err != nil { - if errors.Is(err, ErrStopWalk) { - return nil - } - - return err + err = f(hdr, io.LimitReader(tr, hdr.Size)) + if err != nil { + if errors.Is(err, ErrStopWalk) { + break } + + return err } } diff --git a/internal/termui/interactive.go b/internal/termui/interactive.go index 1a093c01..a46bc959 100644 --- a/internal/termui/interactive.go +++ b/internal/termui/interactive.go @@ -12,6 +12,7 @@ import ( "sync/atomic" "time" + "charm.land/lipgloss/v2" "github.com/hephbuild/heph/internal/hsoftcontext" "github.com/hephbuild/heph/internal/htime" @@ -125,12 +126,13 @@ func (m Model) View() tea.View { failed = fmt.Sprintf(" Failed jobs: %-6d", m.stepsState.failed) } - sb.WriteString(fmt.Sprintf( + _, _ = fmt.Fprintf( + &sb, "Total jobs: %-6d Completed jobs: %-6d%v Total time: %v\n", m.stepsState.total, m.stepsState.completed, failed, htime.FormatFixedWidthDuration(time.Since(m.startedAt)), - )) + ) if !m.finalRendering { sb.WriteString(strings.Repeat("-", m.width)) @@ -139,7 +141,7 @@ func (m Model) View() tea.View { buildStepsTree(m.stepsState.steps, m.stepsState.leafs, &sb, m.width, (m.height-2)/2) } - return tea.NewView(sb.String()) + return tea.NewView(lipgloss.NewStyle().MaxWidth(m.width).Render(sb.String())) } func NewStepsStore(ctx context.Context, p *tea.Program) (func(*corev1.Step), func()) { diff --git a/lib/pluginsdk/artifact.go b/lib/pluginsdk/artifact.go new file mode 100644 index 00000000..ca40b3f6 --- /dev/null +++ b/lib/pluginsdk/artifact.go @@ -0,0 +1,42 @@ +package pluginsdk + +import ( + "io" + + "github.com/hephbuild/heph/internal/hfs" + pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" +) + +type ArtifactContentType string + +const ( + ArtifactContentTypeTar ArtifactContentType = "application/x-tar" + ArtifactContentTypeTarGz ArtifactContentType = "application/x-gtar" +) + +type Artifact interface { + GetGroup() string + GetName() string + GetType() pluginv1.Artifact_Type + GetContentReader() (io.ReadCloser, error) + GetContentSize() (int64, error) + GetContentType() (ArtifactContentType, error) +} + +type FSArtifact interface { + Artifact + FSNode() hfs.Node +} + +type ArtifactWithOrigin struct { + Artifact + Origin *pluginv1.TargetDef_InputOrigin +} + +func (a ArtifactWithOrigin) GetArtifact() Artifact { + return a.Artifact +} + +func (a ArtifactWithOrigin) GetOrigin() *pluginv1.TargetDef_InputOrigin { + return a.Origin +} diff --git a/lib/pluginsdk/artifact_proto.go b/lib/pluginsdk/artifact_proto.go new file mode 100644 index 00000000..af668f21 --- /dev/null +++ b/lib/pluginsdk/artifact_proto.go @@ -0,0 +1,176 @@ +package pluginsdk + +import ( + "archive/tar" + "bytes" + "fmt" + "io" + "os" + + "github.com/hephbuild/heph/internal/hfs" + "github.com/hephbuild/heph/internal/htar" + pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" +) + +type ProtoArtifact struct { + *pluginv1.Artifact +} + +var _ Artifact = (*ProtoArtifact)(nil) + +func (a ProtoArtifact) GetContentReader() (io.ReadCloser, error) { + partifact := a.Artifact + + switch partifact.WhichContent() { + case pluginv1.Artifact_File_case: + content := partifact.GetFile() + + pr, pw := io.Pipe() + + go func() { + defer pw.Close() + + tarPacker := htar.NewPacker(pw) + tarPacker.AllowAbsLink = true // TODO: This is wrong., but let's ignore for now... + + f, err := hfs.Open(hfs.NewOS(content.GetSourcePath())) + if err != nil { + _ = pw.CloseWithError(err) + + return + } + defer f.Close() + + err = tarPacker.WriteFile(f, content.GetOutPath()) + if err != nil { + _ = pw.CloseWithError(err) + + return + } + }() + + return pr, nil + case pluginv1.Artifact_Raw_case: + content := partifact.GetRaw() + + pr, pw := io.Pipe() + + go func() { + defer pw.Close() + + mode := int64(os.ModePerm) + if content.GetX() { + mode |= 0111 // executable + } + + tarPacker := htar.NewPacker(pw) + + err := tarPacker.Write(bytes.NewReader(content.GetData()), &tar.Header{ + Typeflag: tar.TypeReg, + Name: content.GetPath(), + Size: int64(len(content.GetData())), + Mode: mode, + }) + if err != nil { + _ = pw.CloseWithError(err) + + return + } + }() + + return pr, nil + case pluginv1.Artifact_TargzPath_case: + return os.Open(partifact.GetTargzPath()) + case pluginv1.Artifact_TarPath_case: + return os.Open(partifact.GetTarPath()) + default: + return nil, fmt.Errorf("unsupported encoding %v", partifact.WhichContent()) + } +} + +func (a ProtoArtifact) GetContentSize() (int64, error) { + partifact := a.Artifact + + switch partifact.WhichContent() { + case pluginv1.Artifact_File_case: + content := partifact.GetFile() + + sourcefs := hfs.NewOS(content.GetSourcePath()) + + info, err := sourcefs.Lstat() + if err != nil { + return 0, err + } + + return info.Size(), nil + case pluginv1.Artifact_Raw_case: + content := partifact.GetRaw() + + return int64(len(content.GetData())), nil + case pluginv1.Artifact_TargzPath_case: + sourcefs := hfs.NewOS(partifact.GetTargzPath()) + + info, err := sourcefs.Lstat() + if err != nil { + return 0, err + } + + return info.Size(), nil + case pluginv1.Artifact_TarPath_case: + sourcefs := hfs.NewOS(partifact.GetTarPath()) + + info, err := sourcefs.Lstat() + if err != nil { + return 0, err + } + + return info.Size(), nil + default: + return -1, fmt.Errorf("unsupported encoding %v", partifact.WhichContent()) + } +} + +func (a ProtoArtifact) GetContentType() (ArtifactContentType, error) { + partifact := a.Artifact + + switch partifact.WhichContent() { + case pluginv1.Artifact_File_case: + return ArtifactContentTypeTar, nil + case pluginv1.Artifact_Raw_case: + return ArtifactContentTypeTar, nil + case pluginv1.Artifact_TargzPath_case: + return ArtifactContentTypeTarGz, nil + case pluginv1.Artifact_TarPath_case: + return ArtifactContentTypeTar, nil + default: + return "", fmt.Errorf("unsupported encoding %v", partifact.WhichContent()) + } +} + +func (a ProtoArtifact) GetName() string { + partifact := a.Artifact + + switch partifact.WhichContent() { + case pluginv1.Artifact_File_case, pluginv1.Artifact_Raw_case: + return a.Artifact.GetName() + ".tar" + default: + return a.Artifact.GetName() + } +} + +func (a ProtoArtifact) FSNode() hfs.Node { + partifact := a.Artifact + + switch partifact.WhichContent() { + case pluginv1.Artifact_File_case: + return hfs.NewOS(partifact.GetFile().GetSourcePath()) + case pluginv1.Artifact_Raw_case: + return nil + case pluginv1.Artifact_TargzPath_case: + return hfs.NewOS(partifact.GetTargzPath()) + case pluginv1.Artifact_TarPath_case: + return hfs.NewOS(partifact.GetTarPath()) + default: + return nil + } +} diff --git a/lib/pluginsdk/init.go b/lib/pluginsdk/init.go index f7b525e9..79f9d437 100644 --- a/lib/pluginsdk/init.go +++ b/lib/pluginsdk/init.go @@ -5,10 +5,30 @@ import ( corev1 "github.com/hephbuild/heph/plugin/gen/heph/core/v1" "github.com/hephbuild/heph/plugin/gen/heph/core/v1/corev1connect" + pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" ) -type Resulter interface { - Get(context.Context, *corev1.ResultRequest) (*corev1.ResultResponse, error) +type GetResult struct { + Release func() + Artifacts []Artifact + Def *pluginv1.TargetDef +} + +type EngineResulter interface { + Get(context.Context, *corev1.ResultRequest) (*GetResult, error) +} + +type Resulter struct { + Engine EngineResulter +} + +func (r *Resulter) Get(ctx context.Context, req *corev1.ResultRequest) (*GetResult, error) { + res, err := r.Engine.Get(ctx, req) + if err != nil { + return nil, err + } + + return res, nil } type InitPayload struct { diff --git a/lib/pluginsdk/plugin_driver.go b/lib/pluginsdk/plugin_driver.go index b198d955..33d99fa3 100644 --- a/lib/pluginsdk/plugin_driver.go +++ b/lib/pluginsdk/plugin_driver.go @@ -6,10 +6,20 @@ import ( pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" ) +type RunRequest struct { + *pluginv1.RunRequest + + Inputs []*ArtifactWithOrigin +} + +func (r *RunRequest) GetInputs() []*ArtifactWithOrigin { + return r.Inputs +} + type Driver interface { Config(context.Context, *pluginv1.ConfigRequest) (*pluginv1.ConfigResponse, error) Parse(context.Context, *pluginv1.ParseRequest) (*pluginv1.ParseResponse, error) ApplyTransitive(context.Context, *pluginv1.ApplyTransitiveRequest) (*pluginv1.ApplyTransitiveResponse, error) - Run(context.Context, *pluginv1.RunRequest) (*pluginv1.RunResponse, error) + Run(context.Context, *RunRequest) (*pluginv1.RunResponse, error) Pipe(context.Context, *pluginv1.PipeRequest) (*pluginv1.PipeResponse, error) } diff --git a/lib/pluginsdk/pluginsdkconnect/artifact_proto.go b/lib/pluginsdk/pluginsdkconnect/artifact_proto.go new file mode 100644 index 00000000..5db2b904 --- /dev/null +++ b/lib/pluginsdk/pluginsdkconnect/artifact_proto.go @@ -0,0 +1,27 @@ +package pluginsdkconnect + +import ( + "io" + "math" + + "github.com/hephbuild/heph/lib/pluginsdk" + pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" +) + +type ProtoArtifact struct { + *pluginv1.RunRequest_Input_Artifact +} + +var _ pluginsdk.Artifact = (*ProtoArtifact)(nil) + +func (a ProtoArtifact) GetContentReader() (io.ReadCloser, error) { + panic("TODO: implement call from plugin to core to get the content") +} + +func (a ProtoArtifact) GetContentSize() (int64, error) { + return math.MaxInt64, nil // will not be used +} + +func (a ProtoArtifact) GetContentType() (pluginsdk.ArtifactContentType, error) { + panic("TODO: implement call from plugin to core to get the content") +} diff --git a/lib/pluginsdk/pluginsdkconnect/plugin_driver.go b/lib/pluginsdk/pluginsdkconnect/plugin_driver.go index dabe9d5a..d14d4e87 100644 --- a/lib/pluginsdk/pluginsdkconnect/plugin_driver.go +++ b/lib/pluginsdk/pluginsdkconnect/plugin_driver.go @@ -49,7 +49,7 @@ func (p driverConnectClient) Parse(ctx context.Context, req *pluginv1.ParseReque return res.Msg, nil } -func (p driverConnectClient) Run(ctx context.Context, req *pluginv1.RunRequest) (*pluginv1.RunResponse, error) { +func (p driverConnectClient) Run(ctx context.Context, req *pluginsdk.RunRequest) (*pluginv1.RunResponse, error) { hardCtx, cancelHardCtx := context.WithCancel(context.WithoutCancel(ctx)) defer cancelHardCtx() @@ -69,7 +69,7 @@ func (p driverConnectClient) Run(ctx context.Context, req *pluginv1.RunRequest) } }() - err := strm.Send(pluginv1.RunContainer_builder{Start: proto.ValueOrDefault(req)}.Build()) + err := strm.Send(pluginv1.RunContainer_builder{Start: req.RunRequest}.Build()) if err != nil { return nil, p.handleErr(ctx, err) } @@ -137,11 +137,28 @@ func (p driverConnectHandler) Parse(ctx context.Context, req *connect.Request[pl return connect.NewResponse(res), nil } +func (p driverConnectHandler) runRequest(rr *pluginv1.RunRequest) *pluginsdk.RunRequest { + inputs := make([]*pluginsdk.ArtifactWithOrigin, 0, len(rr.GetInputs())) + for _, input := range rr.GetInputs() { + inputs = append(inputs, &pluginsdk.ArtifactWithOrigin{ + Artifact: ProtoArtifact{ + RunRequest_Input_Artifact: input.GetArtifact(), + }, + Origin: input.GetOrigin(), + }) + } + + return &pluginsdk.RunRequest{ + RunRequest: rr, + Inputs: inputs, + } +} + func (p driverConnectHandler) Run(ctx context.Context, strm *connect.BidiStream[pluginv1.RunContainer, pluginv1.RunResponse]) error { ctx, cancel := context.WithCancel(ctx) defer cancel() - startCh := make(chan *pluginv1.RunRequest, 1) + startCh := make(chan *pluginsdk.RunRequest, 1) errCh := make(chan error, 1) go func() { @@ -156,7 +173,7 @@ func (p driverConnectHandler) Run(ctx context.Context, strm *connect.BidiStream[ switch msg.WhichMsg() { case pluginv1.RunContainer_Start_case: - startCh <- msg.GetStart() + startCh <- p.runRequest(msg.GetStart()) close(startCh) case pluginv1.RunContainer_Cancel_case: cancel() diff --git a/lib/tref/utils.go b/lib/tref/utils.go index 940801eb..ff269127 100644 --- a/lib/tref/utils.go +++ b/lib/tref/utils.go @@ -115,6 +115,6 @@ func WithoutOut(ref *pluginv1.TargetRefWithOutput) *pluginv1.TargetRef { } var OmitHashPb = map[string]struct{}{ - string((&pluginv1.TargetRefWithOutput{}).ProtoReflect().Descriptor().Name()) + ".hash": {}, - string((&pluginv1.TargetRef{}).ProtoReflect().Descriptor().Name()) + ".hash": {}, + string((&pluginv1.TargetRefWithOutput{}).ProtoReflect().Descriptor().FullName()) + ".hash": {}, + string((&pluginv1.TargetRef{}).ProtoReflect().Descriptor().FullName()) + ".hash": {}, } diff --git a/plugin/pluginbin/plugin.go b/plugin/pluginbin/plugin.go index 18ac5e10..518a518b 100644 --- a/plugin/pluginbin/plugin.go +++ b/plugin/pluginbin/plugin.go @@ -22,7 +22,7 @@ type Plugin struct { const Name = "bin" func (p Plugin) Pipe(ctx context.Context, request *pluginv1.PipeRequest) (*pluginv1.PipeResponse, error) { - return nil, connect.NewError(connect.CodeUnimplemented, errors.New("not implemented")) + return nil, pluginsdk.ErrNotImplemented } func (p Plugin) Config(ctx context.Context, request *pluginv1.ConfigRequest) (*pluginv1.ConfigResponse, error) { @@ -71,7 +71,7 @@ const wrapperScript = ` exec "$@" ` -func (p Plugin) Run(ctx context.Context, req *pluginv1.RunRequest) (*pluginv1.RunResponse, error) { +func (p Plugin) Run(ctx context.Context, req *pluginsdk.RunRequest) (*pluginv1.RunResponse, error) { t := &binv1.Target{} err := req.GetTarget().GetDef().UnmarshalTo(t) if err != nil { diff --git a/plugin/pluginexec/parse_config.go b/plugin/pluginexec/parse_config.go index 944c7ca5..066b61a5 100644 --- a/plugin/pluginexec/parse_config.go +++ b/plugin/pluginexec/parse_config.go @@ -3,16 +3,13 @@ package pluginexec import ( "context" "fmt" - "maps" "slices" "strings" "github.com/hephbuild/heph/internal/hfs" "github.com/hephbuild/heph/internal/hmaps" - "github.com/hephbuild/heph/internal/hproto" "github.com/hephbuild/heph/internal/hproto/hashpb" "github.com/hephbuild/heph/internal/hproto/hstructpb" - "github.com/hephbuild/heph/internal/hslices" "github.com/hephbuild/heph/internal/htypes" "github.com/hephbuild/heph/lib/tref" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" @@ -190,42 +187,33 @@ func ToDef[S proto.Message](ref *pluginv1.TargetRef, target S, getTarget func(S) return def, nil } -func hashTarget(target *execv1.Target) []byte { - var cloned bool - clonedOnce := func() { - if !cloned { - cloned = true - target = hproto.Clone(target) - } - } +var targetProtoName = string((&execv1.Target{}).ProtoReflect().Descriptor().FullName()) - if hmaps.Has(target.GetEnv(), func(s string, env *execv1.Target_Env) bool { - return !env.GetHash() - }) { - clonedOnce() +func hashTarget(target *execv1.Target) []byte { + omit := hmaps.Concat(tref.OmitHashPb, map[string]struct{}{ + targetProtoName + ".env": {}, + targetProtoName + ".deps": {}, + }) + h := xxh3.New() + hashpb.Hash(h, target, omit) - env := target.GetEnv() + for k, v := range hmaps.Sorted(target.GetEnv()) { + if !v.GetHash() { + continue + } - maps.DeleteFunc(env, func(s string, env *execv1.Target_Env) bool { - return !env.GetHash() - }) - target.SetEnv(env) + _, _ = h.WriteString(k) + hashpb.Hash(h, v, omit) } - if hslices.Has(target.GetDeps(), func(dep *execv1.Target_Dep) bool { - return !dep.GetHash() - }) { - clonedOnce() + for _, v := range target.GetDeps() { + if !v.GetHash() { + continue + } - deps := slices.DeleteFunc(target.GetDeps(), func(dep *execv1.Target_Dep) bool { - return !dep.GetHash() - }) - target.SetDeps(deps) + hashpb.Hash(h, v, omit) } - h := xxh3.New() - hashpb.Hash(h, target, tref.OmitHashPb) - return h.Sum(nil) } diff --git a/plugin/pluginexec/plugin_test.go b/plugin/pluginexec/plugin_test.go index 27836eef..aff7e238 100644 --- a/plugin/pluginexec/plugin_test.go +++ b/plugin/pluginexec/plugin_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/hephbuild/heph/internal/htypes" + "github.com/hephbuild/heph/lib/pluginsdk" "github.com/hephbuild/heph/internal/herrgroup" "github.com/hephbuild/heph/lib/hpipe" @@ -67,10 +68,12 @@ func TestSanity(t *testing.T) { } { - res, err := p.Run(ctx, pluginv1.RunRequest_builder{ - Target: def, - SandboxPath: htypes.Ptr(sandboxPath), - }.Build()) + res, err := p.Run(ctx, &pluginsdk.RunRequest{ + RunRequest: pluginv1.RunRequest_builder{ + Target: def, + SandboxPath: htypes.Ptr(sandboxPath), + }.Build(), + }) require.NoError(t, err) assert.Len(t, res.GetArtifacts(), 1) @@ -110,17 +113,19 @@ func TestPipeStdout(t *testing.T) { require.NoError(t, err) go func() { - _, err = p.Run(ctx, pluginv1.RunRequest_builder{ - Target: pluginv1.TargetDef_builder{ - Ref: pluginv1.TargetRef_builder{ - Package: htypes.Ptr("some/pkg"), - Name: htypes.Ptr("target"), + _, err = p.Run(ctx, &pluginsdk.RunRequest{ + RunRequest: pluginv1.RunRequest_builder{ + Target: pluginv1.TargetDef_builder{ + Ref: pluginv1.TargetRef_builder{ + Package: htypes.Ptr("some/pkg"), + Name: htypes.Ptr("target"), + }.Build(), + Def: def, }.Build(), - Def: def, + SandboxPath: htypes.Ptr(sandboxPath), + Pipes: []string{"", res.Msg.GetId(), ""}, }.Build(), - SandboxPath: htypes.Ptr(sandboxPath), - Pipes: []string{"", res.Msg.GetId(), ""}, - }.Build()) + }) if err != nil { panic(err) } @@ -182,17 +187,19 @@ func TestPipeStdin(t *testing.T) { require.NoError(t, err) go func() { - _, err = p.Run(ctx, pluginv1.RunRequest_builder{ - Target: pluginv1.TargetDef_builder{ - Ref: pluginv1.TargetRef_builder{ - Package: htypes.Ptr("some/pkg"), - Name: htypes.Ptr("target"), + _, err = p.Run(ctx, &pluginsdk.RunRequest{ + RunRequest: pluginv1.RunRequest_builder{ + Target: pluginv1.TargetDef_builder{ + Ref: pluginv1.TargetRef_builder{ + Package: htypes.Ptr("some/pkg"), + Name: htypes.Ptr("target"), + }.Build(), + Def: def, }.Build(), - Def: def, + SandboxPath: htypes.Ptr(sandboxPath), + Pipes: []string{pipeIn.Msg.GetId(), pipeOut.Msg.GetId(), ""}, }.Build(), - SandboxPath: htypes.Ptr(sandboxPath), - Pipes: []string{pipeIn.Msg.GetId(), pipeOut.Msg.GetId(), ""}, - }.Build()) + }) if err != nil { panic(err) } @@ -266,17 +273,19 @@ func TestPipeStdinLargeAndSlow(t *testing.T) { require.NoError(t, err) go func() { - _, err = p.Run(ctx, pluginv1.RunRequest_builder{ - Target: pluginv1.TargetDef_builder{ - Ref: pluginv1.TargetRef_builder{ - Package: htypes.Ptr("some/pkg"), - Name: htypes.Ptr("target"), + _, err = p.Run(ctx, &pluginsdk.RunRequest{ + RunRequest: pluginv1.RunRequest_builder{ + Target: pluginv1.TargetDef_builder{ + Ref: pluginv1.TargetRef_builder{ + Package: htypes.Ptr("some/pkg"), + Name: htypes.Ptr("target"), + }.Build(), + Def: def, }.Build(), - Def: def, + SandboxPath: htypes.Ptr(sandboxPath), + Pipes: []string{pipeIn.Msg.GetId(), pipeOut.Msg.GetId(), ""}, }.Build(), - SandboxPath: htypes.Ptr(sandboxPath), - Pipes: []string{pipeIn.Msg.GetId(), pipeOut.Msg.GetId(), ""}, - }.Build()) + }) if err != nil { panic(err) } @@ -336,17 +345,19 @@ func TestPipe404(t *testing.T) { return err }) - _, err = p.Run(ctx, pluginv1.RunRequest_builder{ - Target: pluginv1.TargetDef_builder{ - Ref: pluginv1.TargetRef_builder{ - Package: htypes.Ptr("some/pkg"), - Name: htypes.Ptr("target"), + _, err = p.Run(ctx, &pluginsdk.RunRequest{ + RunRequest: pluginv1.RunRequest_builder{ + Target: pluginv1.TargetDef_builder{ + Ref: pluginv1.TargetRef_builder{ + Package: htypes.Ptr("some/pkg"), + Name: htypes.Ptr("target"), + }.Build(), + Def: def, }.Build(), - Def: def, + SandboxPath: htypes.Ptr(sandboxPath), + Pipes: []string{"", res.Msg.GetId(), ""}, }.Build(), - SandboxPath: htypes.Ptr(sandboxPath), - Pipes: []string{"", res.Msg.GetId(), ""}, - }.Build()) + }) require.NoError(t, err) err = eg.Wait() diff --git a/plugin/pluginexec/run.go b/plugin/pluginexec/run.go index 49491805..69b8c9cc 100644 --- a/plugin/pluginexec/run.go +++ b/plugin/pluginexec/run.go @@ -17,6 +17,7 @@ import ( "time" "github.com/hephbuild/heph/internal/htypes" + "github.com/hephbuild/heph/lib/pluginsdk" "github.com/hephbuild/heph/internal/hdebug" "github.com/hephbuild/heph/lib/tref" @@ -40,7 +41,7 @@ var tracer = otel.Tracer("heph/pluginexec") var sem = semaphore.NewWeighted(int64(runtime.GOMAXPROCS(-1))) -func (p *Plugin[S]) Run(ctx context.Context, req *pluginv1.RunRequest) (*pluginv1.RunResponse, error) { +func (p *Plugin[S]) Run(ctx context.Context, req *pluginsdk.RunRequest) (*pluginv1.RunResponse, error) { ctx, cleanLabels := hdebug.SetLabels(ctx, func() []string { return []string{ "where", fmt.Sprintf("hephpluginexec %v: %v", p.name, tref.Format(req.GetTarget().GetRef())), @@ -355,8 +356,8 @@ func getEnvEntryWithName(name, value string) string { return name + "=" + value } -func (p *Plugin[S]) inputEnv(ctx context.Context, inputs []*pluginv1.ArtifactWithOrigin, t *execv1.Target, listfs hfs.Node, basePath string) ([]string, error) { - m := map[string][]*pluginv1.ArtifactWithOrigin{} +func (p *Plugin[S]) inputEnv(ctx context.Context, inputs []*pluginsdk.ArtifactWithOrigin, t *execv1.Target, listfs hfs.Node, basePath string) ([]string, error) { + m := map[string][]*pluginsdk.ArtifactWithOrigin{} for _, dep := range t.GetDeps() { id := dep.GetId() @@ -402,7 +403,7 @@ func (p *Plugin[S]) inputEnv(ctx context.Context, inputs []*pluginv1.ArtifactWit seenFiles := map[string]struct{}{} envSb.Reset() - slices.SortFunc(artifacts, func(a, b *pluginv1.ArtifactWithOrigin) int { + slices.SortFunc(artifacts, func(a, b *pluginsdk.ArtifactWithOrigin) int { if v := strings.Compare(a.GetOrigin().GetId(), b.GetOrigin().GetId()); v != 0 { return v } @@ -415,10 +416,6 @@ func (p *Plugin[S]) inputEnv(ctx context.Context, inputs []*pluginv1.ArtifactWit return v } - if v := hproto.Compare(a, b, nil); v != 0 { - return v - } - return 0 }) @@ -439,6 +436,9 @@ func (p *Plugin[S]) inputEnv(ctx context.Context, inputs []*pluginv1.ArtifactWit for sc.Scan() { line := sc.Text() + if line == "" { + continue + } if _, ok := seenFiles[line]; ok { continue diff --git a/plugin/pluginexec/sandbox.go b/plugin/pluginexec/sandbox.go index 69741f69..0e1e618a 100644 --- a/plugin/pluginexec/sandbox.go +++ b/plugin/pluginexec/sandbox.go @@ -8,9 +8,10 @@ import ( "os" "path/filepath" "slices" + "sync/atomic" - "github.com/hephbuild/heph/internal/hproto/hashpb" "github.com/hephbuild/heph/internal/htypes" + "github.com/hephbuild/heph/lib/pluginsdk" "github.com/hephbuild/heph/lib/tref" @@ -24,14 +25,14 @@ import ( ) type SetupSandboxResult struct { - ListArtifacts []*pluginv1.ArtifactWithOrigin + ListArtifacts []*pluginsdk.ArtifactWithOrigin Sourcemap map[string]string } func SetupSandbox( ctx context.Context, t *execv1.Target, - results []*pluginv1.ArtifactWithOrigin, + results []*pluginsdk.ArtifactWithOrigin, workfs, binfs, cwdfs, @@ -53,7 +54,7 @@ func SetupSandbox( sourcemap := map[string]string{} - var listArtifacts []*pluginv1.ArtifactWithOrigin + var listArtifacts []*pluginsdk.ArtifactWithOrigin if setupWd { err = cwdfs.MkdirAll(os.ModePerm) if err != nil { @@ -62,58 +63,58 @@ func SetupSandbox( for _, target := range t.GetDeps() { for artifact := range ArtifactsForId(results, target.GetId(), pluginv1.Artifact_TYPE_OUTPUT) { - listArtifact, err := SetupSandboxArtifact(ctx, artifact.GetArtifact(), target, workfs, target.GetRef().GetFilters(), sourcemap) + listArtifact, err := SetupSandboxArtifact(ctx, artifact, target, workfs, target.GetRef().GetFilters(), sourcemap) if err != nil { return nil, fmt.Errorf("setup artifact: %v: %w", target.GetId(), err) } - listArtifacts = append(listArtifacts, pluginv1.ArtifactWithOrigin_builder{ + listArtifacts = append(listArtifacts, &pluginsdk.ArtifactWithOrigin{ Artifact: listArtifact, Origin: pluginv1.TargetDef_InputOrigin_builder{ Id: htypes.Ptr(target.GetId()), }.Build(), - }.Build()) + }) } for artifact := range ArtifactsForId(results, target.GetId(), pluginv1.Artifact_TYPE_SUPPORT_FILE) { - listArtifact, err := SetupSandboxArtifact(ctx, artifact.GetArtifact(), target, workfs, nil, nil) + listArtifact, err := SetupSandboxArtifact(ctx, artifact, target, workfs, nil, nil) if err != nil { return nil, fmt.Errorf("setup support file artifact: %v: %w", target.GetId(), err) } - listArtifacts = append(listArtifacts, pluginv1.ArtifactWithOrigin_builder{ + listArtifacts = append(listArtifacts, &pluginsdk.ArtifactWithOrigin{ Artifact: listArtifact, Origin: pluginv1.TargetDef_InputOrigin_builder{ Id: htypes.Ptr(target.GetId()), }.Build(), - }.Build()) + }) } } } for _, tool := range t.GetTools() { for artifact := range ArtifactsForId(results, tool.GetId(), pluginv1.Artifact_TYPE_OUTPUT) { - listArtifact, err := SetupSandboxBinArtifact(ctx, artifact.GetArtifact(), binfs) + listArtifact, err := SetupSandboxBinArtifact(ctx, artifact, binfs) if err != nil { return nil, fmt.Errorf("%v: %w", tref.FormatOut(tool.GetRef()), err) } - listArtifacts = append(listArtifacts, pluginv1.ArtifactWithOrigin_builder{ + listArtifacts = append(listArtifacts, &pluginsdk.ArtifactWithOrigin{ Artifact: listArtifact, Origin: pluginv1.TargetDef_InputOrigin_builder{ Id: htypes.Ptr(tool.GetId()), }.Build(), - }.Build()) + }) } for artifact := range ArtifactsForId(results, tool.GetId(), pluginv1.Artifact_TYPE_SUPPORT_FILE) { - listArtifact, err := SetupSandboxArtifact(ctx, artifact.GetArtifact(), nil, workfs, nil, nil) + listArtifact, err := SetupSandboxArtifact(ctx, artifact, nil, workfs, nil, nil) if err != nil { return nil, fmt.Errorf("setup support file artifact: %v: %w", tref.FormatOut(tool.GetRef()), err) } - listArtifacts = append(listArtifacts, pluginv1.ArtifactWithOrigin_builder{ + listArtifacts = append(listArtifacts, &pluginsdk.ArtifactWithOrigin{ Artifact: listArtifact, Origin: pluginv1.TargetDef_InputOrigin_builder{ Id: htypes.Ptr(tool.GetId()), }.Build(), - }.Build()) + }) } } @@ -132,8 +133,8 @@ func SetupSandbox( }, nil } -func ArtifactsForId(inputs []*pluginv1.ArtifactWithOrigin, id string, typ pluginv1.Artifact_Type) iter.Seq[*pluginv1.ArtifactWithOrigin] { - return func(yield func(origin *pluginv1.ArtifactWithOrigin) bool) { +func ArtifactsForId(inputs []*pluginsdk.ArtifactWithOrigin, id string, typ pluginv1.Artifact_Type) iter.Seq[*pluginsdk.ArtifactWithOrigin] { + return func(yield func(origin *pluginsdk.ArtifactWithOrigin) bool) { for _, input := range inputs { if input.GetArtifact().GetType() != typ { continue @@ -150,17 +151,16 @@ func ArtifactsForId(inputs []*pluginv1.ArtifactWithOrigin, id string, typ plugin } } -func SetupSandboxArtifact(ctx context.Context, artifact *pluginv1.Artifact, source *execv1.Target_Dep, node hfs.Node, filters []string, sourcemap map[string]string) (*pluginv1.Artifact, error) { +var artifactId atomic.Int64 + +func SetupSandboxArtifact(ctx context.Context, artifact pluginsdk.Artifact, source *execv1.Target_Dep, workfs hfs.Node, filters []string, sourcemap map[string]string) (pluginsdk.Artifact, error) { ctx, span := tracer.Start(ctx, "SetupSandboxArtifact") defer span.End() h := xxh3.New() - hashpb.Hash(h, artifact, tref.OmitHashPb) - for _, f := range filters { - _, _ = h.WriteString(f) - } + _, _ = fmt.Fprintf(h, "%d", artifactId.Add(1)) - listf, err := hfs.Create(node.At(hex.EncodeToString(h.Sum(nil)) + ".list")) + listf, err := hfs.Create(workfs.At(hex.EncodeToString(h.Sum(nil)) + ".list")) if err != nil { return nil, fmt.Errorf("create list file: %w", err) } @@ -168,7 +168,7 @@ func SetupSandboxArtifact(ctx context.Context, artifact *pluginv1.Artifact, sour writeNl := false - err = hartifact.Unpack(ctx, artifact, node, hartifact.WithOnFile(func(to string) { + err = hartifact.Unpack(ctx, artifact, workfs, hartifact.WithOnFile(func(to string) { if writeNl { _, _ = listf.Write([]byte("\n")) } @@ -176,7 +176,7 @@ func SetupSandboxArtifact(ctx context.Context, artifact *pluginv1.Artifact, sour writeNl = true if sourcemap != nil { - rel, err := filepath.Rel(node.Path(), to) + rel, err := filepath.Rel(workfs.Path(), to) if err != nil { panic(err) } @@ -205,17 +205,19 @@ func SetupSandboxArtifact(ctx context.Context, artifact *pluginv1.Artifact, sour listType = pluginv1.Artifact_TYPE_SUPPORT_FILE_LIST_V1 } - return pluginv1.Artifact_builder{ - Group: htypes.Ptr(artifact.GetGroup()), - Name: htypes.Ptr(artifact.GetName() + ".list"), - Type: htypes.Ptr(listType), - File: pluginv1.Artifact_ContentFile_builder{ - SourcePath: htypes.Ptr(listf.Name()), + return &pluginsdk.ProtoArtifact{ + Artifact: pluginv1.Artifact_builder{ + Group: htypes.Ptr(artifact.GetGroup()), + Name: htypes.Ptr(artifact.GetName() + ".list"), + Type: htypes.Ptr(listType), + File: pluginv1.Artifact_ContentFile_builder{ + SourcePath: htypes.Ptr(listf.Name()), + }.Build(), }.Build(), - }.Build(), nil + }, nil } -func SetupSandboxBinArtifact(ctx context.Context, artifact *pluginv1.Artifact, node hfs.Node) (*pluginv1.Artifact, error) { +func SetupSandboxBinArtifact(ctx context.Context, artifact pluginsdk.Artifact, node hfs.Node) (pluginsdk.Artifact, error) { ctx, span := tracer.Start(ctx, "SetupSandboxBinArtifact") defer span.End() @@ -251,13 +253,15 @@ func SetupSandboxBinArtifact(ctx context.Context, artifact *pluginv1.Artifact, n return nil, err } - return pluginv1.Artifact_builder{ - Group: htypes.Ptr(artifact.GetGroup()), - Name: htypes.Ptr(artifact.GetName() + ".list"), - Type: htypes.Ptr(pluginv1.Artifact_TYPE_OUTPUT_LIST_V1), - Raw: pluginv1.Artifact_ContentRaw_builder{ - Data: []byte(dest.Path()), - Path: htypes.Ptr(artifact.GetName() + ".list"), + return &pluginsdk.ProtoArtifact{ + Artifact: pluginv1.Artifact_builder{ + Group: htypes.Ptr(artifact.GetGroup()), + Name: htypes.Ptr(artifact.GetName() + ".list"), + Type: htypes.Ptr(pluginv1.Artifact_TYPE_OUTPUT_LIST_V1), + Raw: pluginv1.Artifact_ContentRaw_builder{ + Data: []byte(dest.Path()), + Path: htypes.Ptr(artifact.GetName() + ".list"), + }.Build(), }.Build(), - }.Build(), nil + }, nil } diff --git a/plugin/pluginfs/driver.go b/plugin/pluginfs/driver.go index 75ae4e62..039e5540 100644 --- a/plugin/pluginfs/driver.go +++ b/plugin/pluginfs/driver.go @@ -103,7 +103,7 @@ func (p *Driver) Parse(ctx context.Context, req *pluginv1.ParseRequest) (*plugin }.Build(), nil } -func (p *Driver) Run(ctx context.Context, req *pluginv1.RunRequest) (*pluginv1.RunResponse, error) { +func (p *Driver) Run(ctx context.Context, req *pluginsdk.RunRequest) (*pluginv1.RunResponse, error) { t := &fsv1.Target{} err := req.GetTarget().GetDef().UnmarshalTo(t) if err != nil { @@ -153,7 +153,12 @@ func (p *Driver) Run(ctx context.Context, req *pluginv1.RunRequest) (*pluginv1.R }.Build(), nil } - exclude := []string{} // TODO: exclude home, cache, git etc... + // TODO: exclude home, cache, git etc... + exclude := []string{ + ".git/**/*", + ".heph/**/*", + ".heph2/**/*", + } exclude = append(exclude, t.GetExclude()...) var artifacts []*pluginv1.Artifact diff --git a/plugin/plugingo/get_std.go b/plugin/plugingo/get_std.go index 7145f137..193448ee 100644 --- a/plugin/plugingo/get_std.go +++ b/plugin/plugingo/get_std.go @@ -36,8 +36,9 @@ func (p *Plugin) resultStdListInner(ctx context.Context, factors Factors, reques if err != nil { return nil, err } + defer res.Release() - outputs := hartifact.FindOutputs(res.GetArtifacts(), "list") + outputs := hartifact.FindOutputs(res.Artifacts, "list") if len(outputs) == 0 { return nil, errors.New("no install artifact found") diff --git a/plugin/plugingo/pkg_analysis.go b/plugin/plugingo/pkg_analysis.go index a207ac94..bfc492cc 100644 --- a/plugin/plugingo/pkg_analysis.go +++ b/plugin/plugingo/pkg_analysis.go @@ -33,108 +33,117 @@ import ( var errNoGoFiles = errors.New("no Go files in package") -func (p *Plugin) goListPkg(ctx context.Context, pkg string, factors Factors, imp, requestId string) ([]*pluginv1.Artifact, *pluginv1.TargetRef, error) { - res, err := p.resultClient.ResultClient.Get(ctx, corev1.ResultRequest_builder{ - RequestId: htypes.Ptr(requestId), - Ref: tref.New(pkg, "_golist", hmaps.Concat(factors.Args(), map[string]string{ - "imp": imp, - })), - }.Build()) - if err != nil { - return nil, nil, fmt.Errorf("golist: %v (in %v): %w", imp, pkg, err) - } - - return res.GetArtifacts(), res.GetDef().GetRef(), nil -} - func (p *Plugin) goListPkgResult(ctx context.Context, basePkg, runPkg, imp string, factors Factors, requestId string) (Package, error) { - artifacts, _, err := p.goListPkg(ctx, runPkg, factors, imp, requestId) - if err != nil { - return Package{}, fmt.Errorf("go list: %w", err) + key := goListPkgResultKey{ + RequestId: requestId, + Factors: factors, + BasePkg: basePkg, + RunPkg: runPkg, + Imp: imp, } - jsonArtifacts := hartifact.FindOutputs(artifacts, "json") - rootArtifacts := hartifact.FindOutputs(artifacts, "root") - smArtifacts := hartifact.FindOutputs(artifacts, "sm") - nogoArtifacts := hartifact.FindOutputs(artifacts, "nogo") + res, err, _ := p.goListPkgCache.Do(ctx, key, func(ctx context.Context) (Package, error) { + res, err := p.resultClient.ResultClient.Get(ctx, corev1.ResultRequest_builder{ + RequestId: htypes.Ptr(requestId), + Ref: tref.New(runPkg, "_golist", hmaps.Concat(factors.Args(), map[string]string{ + "imp": imp, + })), + }.Build()) + if err != nil { + return Package{}, fmt.Errorf("golist: %v (in %v): %w", imp, runPkg, err) + } + defer res.Release() - if len(jsonArtifacts) == 0 || len(rootArtifacts) == 0 || len(smArtifacts) == 0 || len(nogoArtifacts) == 0 { - return Package{}, connect.NewError(connect.CodeInternal, errors.New("golist: no json found")) - } + artifacts := res.Artifacts - jsonArtifact := jsonArtifacts[0] - rootArtifact := rootArtifacts[0] - smArtifact := smArtifacts[0] - nogoArtifact := nogoArtifacts[0] + jsonArtifacts := hartifact.FindOutputs(artifacts, "json") + rootArtifacts := hartifact.FindOutputs(artifacts, "root") + smArtifacts := hartifact.FindOutputs(artifacts, "sm") + nogoArtifacts := hartifact.FindOutputs(artifacts, "nogo") - { - b, err := hartifact.FileReadAll(ctx, nogoArtifact) - if err != nil { - return Package{}, err + if len(jsonArtifacts) == 0 || len(rootArtifacts) == 0 || len(smArtifacts) == 0 || len(nogoArtifacts) == 0 { + return Package{}, connect.NewError(connect.CodeInternal, errors.New("golist: no json found")) } - if ok, _ := strconv.ParseBool(strings.TrimSpace(string(b))); ok { - return Package{}, errNoGoFiles - } - } + jsonArtifact := jsonArtifacts[0] + rootArtifact := rootArtifacts[0] + smArtifact := smArtifacts[0] + nogoArtifact := nogoArtifacts[0] - rootb, err := hartifact.FileReadAll(ctx, rootArtifact) - if err != nil { - return Package{}, fmt.Errorf("root artifact: %w", err) - } + { + b, err := hartifact.FileReadAll(ctx, nogoArtifact) + if err != nil { + return Package{}, err + } - root := strings.TrimSpace(string(rootb)) + if ok, _ := strconv.ParseBool(strings.TrimSpace(string(b))); ok { + return Package{}, errNoGoFiles + } + } - var goPkg Package - { - f, err := hartifact.FileReader(ctx, jsonArtifact) + rootb, err := hartifact.FileReadAll(ctx, rootArtifact) if err != nil { - return Package{}, err + return Package{}, fmt.Errorf("root artifact: %w", err) } - defer f.Close() - err = json.NewDecoder(f).Decode(&goPkg) - if err != nil { - return Package{}, fmt.Errorf("gopkg decode: %w", err) + root := strings.TrimSpace(string(rootb)) + + var goPkg Package + { + f, err := hartifact.FileReader(ctx, jsonArtifact) + if err != nil { + return Package{}, err + } + defer f.Close() + + err = json.NewDecoder(f).Decode(&goPkg) + if err != nil { + return Package{}, fmt.Errorf("gopkg decode %q: %w", tref.Format(res.Def.GetRef()), err) + } } - } - var sourcemap map[string]string - { - f, err := hartifact.FileReader(ctx, smArtifact) - if err != nil { - return Package{}, err + var sourcemap map[string]string + { + f, err := hartifact.FileReader(ctx, smArtifact) + if err != nil { + return Package{}, err + } + defer f.Close() + + err = json.NewDecoder(f).Decode(&sourcemap) + if err != nil { + return Package{}, fmt.Errorf("sourcemap decode: %q: %w", tref.Format(res.Def.GetRef()), err) + } } - defer f.Close() - err = json.NewDecoder(f).Decode(&sourcemap) + goPkg.Sourcemap = sourcemap + + relPkg, err := tref.DirToPackage(goPkg.Dir, root) if err != nil { - return Package{}, fmt.Errorf("sourcemap decode: %w", err) - } - } + goPkg.Is3rdParty = true - goPkg.Sourcemap = sourcemap + if goPkg.Module == nil { + return Package{}, fmt.Errorf("%v: not in a module", imp) + } - relPkg, err := tref.DirToPackage(goPkg.Dir, root) - if err != nil { - goPkg.Is3rdParty = true + modPath := strings.ReplaceAll(goPkg.ImportPath, goPkg.Module.Path, "") + modPath = strings.TrimPrefix(modPath, "/") - if goPkg.Module == nil { - return Package{}, fmt.Errorf("%v: not in a module", imp) + goPkg.HephPackage = ThirdpartyBuildPackage(basePkg, goPkg.Module.Path, goPkg.Module.Version, modPath) + goPkg.HephBuildPackage = ThirdpartyBuildPackage(basePkg, goPkg.Module.Path, goPkg.Module.Version, modPath) + } else { + goPkg.Dir = filepath.Join(p.root, relPkg) + goPkg.HephPackage = relPkg } + goPkg.Factors = factors - modPath := strings.ReplaceAll(goPkg.ImportPath, goPkg.Module.Path, "") - modPath = strings.TrimPrefix(modPath, "/") - - goPkg.HephPackage = ThirdpartyBuildPackage(basePkg, goPkg.Module.Path, goPkg.Module.Version, modPath) - goPkg.HephBuildPackage = ThirdpartyBuildPackage(basePkg, goPkg.Module.Path, goPkg.Module.Version, modPath) - } else { - goPkg.Dir = filepath.Join(p.root, relPkg) - goPkg.HephPackage = relPkg + return goPkg, nil + }) + if err != nil { + return Package{}, err } - goPkg.Factors = factors - return goPkg, nil + return res, nil } type GetGoPackageCache struct { @@ -699,8 +708,10 @@ func (p *Plugin) goModules(ctx context.Context, pkg, requestId string) ([]Module return nil, fmt.Errorf("gomod: %w", err) } - jsonArtifacts := hartifact.FindOutputs(res.GetArtifacts(), "json") - rootArtifacts := hartifact.FindOutputs(res.GetArtifacts(), "root") + defer res.Release() + + jsonArtifacts := hartifact.FindOutputs(res.Artifacts, "json") + rootArtifacts := hartifact.FindOutputs(res.Artifacts, "root") if len(jsonArtifacts) == 0 || len(rootArtifacts) == 0 { return nil, connect.NewError(connect.CodeInternal, errors.New("gomodules: no output found")) diff --git a/plugin/plugingo/plugin.go b/plugin/plugingo/plugin.go index 3fd37c30..2dc4b734 100644 --- a/plugin/plugingo/plugin.go +++ b/plugin/plugingo/plugin.go @@ -69,6 +69,14 @@ type moduleCacheKey struct { BasePkg string } +type goListPkgResultKey struct { + RequestId string + Factors Factors + BasePkg string + RunPkg string + Imp string +} + type Plugin struct { goTool string resultClient pluginsdk.Engine @@ -79,6 +87,7 @@ type Plugin struct { moduleCache hsingleflight.GroupMem[moduleCacheKey, []Module] stdCache hsingleflight.GroupMem[stdCacheKey, map[string]Package] goModGoWorkCache hsingleflight.GroupMemContext[string, goModRoot] + goListPkgCache hsingleflight.GroupMemContext[goListPkgResultKey, Package] } func (p *Plugin) getGoToolStructpb() *structpb.Value { diff --git a/plugin/plugingroup/plugin.go b/plugin/plugingroup/plugin.go index cec8eaac..0fff7455 100644 --- a/plugin/plugingroup/plugin.go +++ b/plugin/plugingroup/plugin.go @@ -75,7 +75,7 @@ func (p Plugin) Parse(ctx context.Context, req *pluginv1.ParseRequest) (*pluginv }.Build(), nil } -func (p Plugin) Run(ctx context.Context, request *pluginv1.RunRequest) (*pluginv1.RunResponse, error) { +func (p Plugin) Run(ctx context.Context, request *pluginsdk.RunRequest) (*pluginv1.RunResponse, error) { panic("group: run should not be called") // custom handling in engine } diff --git a/plugin/pluginnix/driver.go b/plugin/pluginnix/driver.go index 4e30daa6..3e766fc6 100644 --- a/plugin/pluginnix/driver.go +++ b/plugin/pluginnix/driver.go @@ -71,7 +71,7 @@ func parseConfig(ctx context.Context, ref *pluginv1.TargetRef, config map[string } var omitHashPb = hmaps.Concat(map[string]struct{}{ - string((&nixv1.Target{}).ProtoReflect().Descriptor().Name()) + ".target": {}, + string((&nixv1.Target{}).ProtoReflect().Descriptor().FullName()) + ".target": {}, }, tref.OmitHashPb) func hashTarget(nixTarget *nixv1.Target, execTargetHash []byte) []byte { diff --git a/plugin/pluginnix/driver_test.go b/plugin/pluginnix/driver_test.go index d7d72a16..a57306f2 100644 --- a/plugin/pluginnix/driver_test.go +++ b/plugin/pluginnix/driver_test.go @@ -7,6 +7,7 @@ import ( "github.com/hephbuild/heph/internal/hproto/hstructpb" "github.com/hephbuild/heph/internal/htypes" + "github.com/hephbuild/heph/lib/pluginsdk" pluginv1 "github.com/hephbuild/heph/plugin/gen/heph/plugin/v1" "github.com/stretchr/testify/assert" @@ -64,10 +65,12 @@ func TestSanity(t *testing.T) { } { - res, err := p.Run(ctx, pluginv1.RunRequest_builder{ - Target: def, - SandboxPath: htypes.Ptr(sandboxPath), - }.Build()) + res, err := p.Run(ctx, &pluginsdk.RunRequest{ + RunRequest: pluginv1.RunRequest_builder{ + Target: def, + SandboxPath: htypes.Ptr(sandboxPath), + }.Build(), + }) require.NoError(t, err) assert.Len(t, res.GetArtifacts(), 1) @@ -108,10 +111,12 @@ func TestToolSanity(t *testing.T) { } { - res, err := p.Run(ctx, pluginv1.RunRequest_builder{ - Target: def, - SandboxPath: htypes.Ptr(sandboxPath), - }.Build()) + res, err := p.Run(ctx, &pluginsdk.RunRequest{ + RunRequest: pluginv1.RunRequest_builder{ + Target: def, + SandboxPath: htypes.Ptr(sandboxPath), + }.Build(), + }) require.NoError(t, err) assert.Len(t, res.GetArtifacts(), 1) diff --git a/plugin/plugintextfile/plugin.go b/plugin/plugintextfile/plugin.go index 0482b7e2..6626f6fa 100644 --- a/plugin/plugintextfile/plugin.go +++ b/plugin/plugintextfile/plugin.go @@ -75,7 +75,7 @@ func (p Plugin) Parse(ctx context.Context, req *pluginv1.ParseRequest) (*pluginv }.Build(), nil } -func (p Plugin) Run(ctx context.Context, req *pluginv1.RunRequest) (*pluginv1.RunResponse, error) { +func (p Plugin) Run(ctx context.Context, req *pluginsdk.RunRequest) (*pluginv1.RunResponse, error) { t := &textfilev1.Target{} err := req.GetTarget().GetDef().UnmarshalTo(t) if err != nil { diff --git a/plugin/proto/heph/core/v1/result.proto b/plugin/proto/heph/core/v1/result.proto index 88af3304..bdb1bdda 100644 --- a/plugin/proto/heph/core/v1/result.proto +++ b/plugin/proto/heph/core/v1/result.proto @@ -18,10 +18,7 @@ message ResultRequest { } message ResultResponse { - repeated heph.plugin.v1.Artifact artifacts = 1; - heph.plugin.v1.TargetDef def = 2; -} - -service ResultService { - rpc Get(ResultRequest) returns (ResultResponse) {} + string id = 1; + repeated heph.plugin.v1.Artifact artifacts = 2; + heph.plugin.v1.TargetDef def = 3; } diff --git a/plugin/proto/heph/plugin/v1/driver.proto b/plugin/proto/heph/plugin/v1/driver.proto index 5af59c7e..365dcb66 100644 --- a/plugin/proto/heph/plugin/v1/driver.proto +++ b/plugin/proto/heph/plugin/v1/driver.proto @@ -25,17 +25,24 @@ message ApplyTransitiveResponse { TargetDef target = 1; } -message ArtifactWithOrigin { - Artifact artifact = 1; - TargetDef.InputOrigin origin = 2; -} - message RunRequest { + message Input { + message Artifact { + string group = 1; + string name = 2; + heph.plugin.v1.Artifact.Type type = 3; + string id = 4; + } + + Artifact artifact = 1; + TargetDef.InputOrigin origin = 2; + } + string request_id = 1; TargetDef target = 2; string sandbox_path = 3; string tree_root_path = 4; - repeated ArtifactWithOrigin inputs = 5; + repeated Input inputs = 5; repeated string pipes = 6; string hashin = 7; } @@ -53,7 +60,6 @@ message Artifact { TYPE_OUTPUT = 1; TYPE_OUTPUT_LIST_V1 = 2; TYPE_LOG = 3; - TYPE_MANIFEST_V1 = 4; TYPE_SUPPORT_FILE = 5; TYPE_SUPPORT_FILE_LIST_V1 = 6; }