diff --git a/go.mod b/go.mod index db15dbd7f7..b705059752 100644 --- a/go.mod +++ b/go.mod @@ -103,6 +103,7 @@ require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect github.com/agext/levenshtein v1.2.3 // indirect + github.com/apache/arrow/go/v15 v15.0.2 // indirect github.com/apparentlymart/go-cidr v1.1.0 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect @@ -136,11 +137,13 @@ require ( github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/gobuffalo/flect v0.2.3 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.2.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.3 // indirect + github.com/google/flatbuffers v23.5.26+incompatible // indirect github.com/google/gnostic v0.6.9 // indirect github.com/google/go-cpy v0.0.0-20211218193943-a9c933c06932 // indirect github.com/google/gofuzz v1.2.0 // indirect @@ -177,6 +180,8 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -200,6 +205,7 @@ require ( github.com/oklog/run v1.0.0 // indirect github.com/onsi/ginkgo v1.16.5 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/prometheus/client_model v0.6.0 // indirect @@ -214,6 +220,7 @@ require ( github.com/vmihailenco/tagparser v0.1.2 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xlab/treeprint v1.1.0 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect go.opentelemetry.io/otel v1.29.0 // indirect @@ -224,12 +231,14 @@ require ( go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect go.uber.org/multierr v1.10.0 // indirect golang.org/x/crypto v0.28.0 // indirect + golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/mod v0.20.0 // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/term v0.25.0 // indirect golang.org/x/text v0.19.0 // indirect golang.org/x/tools v0.24.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.8 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index c715574541..b5a4bbf118 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ cloud.google.com/go/compute v1.28.1 h1:XwPcZjgMCnU2tkwY10VleUjSAfpTj9RDn+kGrbYsi cloud.google.com/go/compute v1.28.1/go.mod h1:b72iXMY4FucVry3NR3Li4kVyyTvbMDE7x5WsqvxjsYk= cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo= cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= +cloud.google.com/go/datacatalog v1.22.1 h1:i0DyKb/o7j+0vgaFtimcRFjYsD6wFw1jpnODYUyiYRs= +cloud.google.com/go/datacatalog v1.22.1/go.mod h1:MscnJl9B2lpYlFoxRjicw19kFTwEke8ReKL5Y/6TWg8= cloud.google.com/go/dataflow v0.10.1 h1:RoVpCZ1BjJBH/5mzaXCgNg+l9FgTIYQ7C9xBRGvhkzo= cloud.google.com/go/dataflow v0.10.1/go.mod h1:zP4/tNjONFRcS4NcI9R94YDQEkPalimdbPkijVNJt/g= cloud.google.com/go/dataform v0.10.1 h1:FkOPrxf8sN9J2TMc4CIBhVivhMiO8D0eYN33s5A5Uo4= @@ -152,6 +154,8 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYU github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE= +github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA= github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU= github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM= @@ -405,6 +409,8 @@ github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/gobuffalo/flect v0.2.3 h1:f/ZukRnSNA/DUpSNDadko7Qc0PhGvsew35p/2tu+CRY= github.com/gobuffalo/flect v0.2.3/go.mod h1:vmkQwuZYhN5Pc4ljYQZzP+1sq+NEkK+lh20jmEmX3jc= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -459,6 +465,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= +github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -484,9 +492,12 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -658,6 +669,10 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -792,6 +807,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -952,6 +969,10 @@ github.com/zclconf/go-cty v1.10.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uU github.com/zclconf/go-cty v1.13.0 h1:It5dfKTTZHe9aeppbNOda3mN7Ag7sg6QkBNm6TkyFa0= github.com/zclconf/go-cty v1.13.0/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0= github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -1042,6 +1063,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1352,10 +1375,14 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gomodules.xyz/jsonpatch/v2 v2.3.0 h1:8NFhfS6gzxNqjLIYnZxg319wZ5Qjnx4m/CcX+Klzazc= gomodules.xyz/jsonpatch/v2 v2.3.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= +gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= +gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= diff --git a/mockgcp/mockbigquery/datasets.go b/mockgcp/mockbigquery/datasets.go index 6dc1d09efe..e158f22872 100644 --- a/mockgcp/mockbigquery/datasets.go +++ b/mockgcp/mockbigquery/datasets.go @@ -209,6 +209,43 @@ func (s *datasetsServer) UpdateDataset(ctx context.Context, req *pb.UpdateDatase return updated, err } +func (s *datasetsServer) PatchDataset(ctx context.Context, req *pb.PatchDatasetRequest) (*pb.Dataset, error) { + name, err := s.buildDatasetName(req.GetProjectId(), req.GetDatasetId()) + if err != nil { + return nil, err + } + + fqn := name.String() + + existing := &pb.Dataset{} + if err := s.storage.Get(ctx, fqn, existing); err != nil { + return nil, err + } + + now := time.Now() + + updated := req.GetDataset() + updated.DatasetReference = existing.DatasetReference + + updated.CreationTime = existing.CreationTime + updated.LastModifiedTime = PtrTo(now.UnixMilli()) + updated.Id = PtrTo(existing.GetDatasetReference().GetProjectId() + ":" + existing.GetDatasetReference().GetDatasetId()) + updated.Kind = PtrTo("bigquery#dataset") + updated.Location = existing.Location + updated.Type = existing.Type + updated.SelfLink = PtrTo("https://bigquery.googleapis.com/bigquery/v2/" + name.String()) + + sortAccess(updated) + + updated.Etag = PtrTo(computeEtag(updated)) + + if err := s.storage.Update(ctx, fqn, updated); err != nil { + return nil, err + } + + return updated, err +} + func (s *datasetsServer) DeleteDataset(ctx context.Context, req *pb.DeleteDatasetRequest) (*empty.Empty, error) { name, err := s.buildDatasetName(req.GetProjectId(), req.GetDatasetId()) if err != nil { diff --git a/pkg/controller/direct/bigquerydataset/bigquerydataset_mappings.go b/pkg/controller/direct/bigquerydataset/bigquerydataset_mappings.go index 05f6649eda..113f59c6a3 100644 --- a/pkg/controller/direct/bigquerydataset/bigquerydataset_mappings.go +++ b/pkg/controller/direct/bigquerydataset/bigquerydataset_mappings.go @@ -21,208 +21,309 @@ package bigquerydataset import ( + "fmt" "strconv" + "strings" + "time" + pb "cloud.google.com/go/bigquery" krm "github.com/GoogleCloudPlatform/k8s-config-connector/apis/bigquery/v1beta1" "github.com/GoogleCloudPlatform/k8s-config-connector/apis/refs/v1beta1" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct" - api "google.golang.org/api/bigquery/v2" ) -func BigQueryDatasetSpec_ToAPI(mapCtx *direct.MapContext, in *krm.BigQueryDatasetSpec, name string) *api.Dataset { +func BigQueryDatasetSpec_ToProtoMetadata(mapCtx *direct.MapContext, in *krm.BigQueryDatasetSpec) *pb.DatasetMetadata { if in == nil { return nil } - out := &api.Dataset{} - acccessList := []*api.DatasetAccess{} + out := &pb.DatasetMetadata{} + acccessList := []*pb.AccessEntry{} for _, access := range in.Access { - curAccess := Access_ToAPI(mapCtx, direct.LazyPtr(access)) + curAccess := AccessEntry_ToProto(mapCtx, direct.LazyPtr(access)) acccessList = append(acccessList, curAccess) } out.Access = acccessList out.DefaultCollation = direct.ValueOf(in.DefaultCollation) - out.DefaultPartitionExpirationMs = direct.ValueOf(in.DefaultPartitionExpirationMs) - out.DefaultTableExpirationMs = direct.ValueOf(in.DefaultTableExpirationMs) - out.DefaultEncryptionConfiguration = EncryptionConfiguration_ToAPI(mapCtx, in.DefaultEncryptionConfiguration) + out.DefaultPartitionExpiration = time.Duration(direct.ValueOf(in.DefaultPartitionExpirationMs)) * time.Millisecond + out.DefaultTableExpiration = time.Duration(direct.ValueOf(in.DefaultTableExpirationMs)) * time.Millisecond + out.DefaultEncryptionConfig = EncryptionConfiguration_ToProto(mapCtx, in.DefaultEncryptionConfiguration) out.Description = direct.ValueOf(in.Description) - out.FriendlyName = direct.ValueOf(in.FriendlyName) - out.DatasetReference = DatasetReference_ToAPI(mapCtx, in, name) + out.ExternalDatasetReference = DatasetSpec_ToExternalDatasetReference(mapCtx, in) out.Location = direct.ValueOf(in.Location) - out.IsCaseInsensitive = direct.ValueOf(in.IsCaseInsensitive) if in.MaxTimeTravelHours != nil { - out.MaxTimeTravelHours, _ = strconv.ParseInt(direct.ValueOf(in.MaxTimeTravelHours), 10, 64) + maxHours, _ := strconv.ParseInt(direct.ValueOf(in.MaxTimeTravelHours), 10, 64) + out.MaxTimeTravel = time.Duration(maxHours) * time.Hour } + out.Name = direct.ValueOf(in.FriendlyName) out.StorageBillingModel = direct.ValueOf(in.StorageBillingModel) return out } -func BigQueryDatasetSpec_FromAPI(mapCtx *direct.MapContext, in *api.Dataset) *krm.BigQueryDatasetSpec { +func BigQueryDatasetSpec_FromProtoMetadata(mapCtx *direct.MapContext, in *pb.DatasetMetadata) *krm.BigQueryDatasetSpec { if in == nil { return nil } out := &krm.BigQueryDatasetSpec{} accessList := []krm.Access{} for _, access := range in.Access { - curAccess := Access_FromAPI(mapCtx, access) + curAccess := AccessEntry_FromProto(mapCtx, access) accessList = append(accessList, direct.ValueOf(curAccess)) } out.Access = accessList out.DefaultCollation = direct.LazyPtr(in.DefaultCollation) - out.DefaultPartitionExpirationMs = direct.LazyPtr(in.DefaultPartitionExpirationMs) - out.DefaultTableExpirationMs = direct.LazyPtr(in.DefaultTableExpirationMs) - out.DefaultEncryptionConfiguration = EncryptionConfiguration_FromAPI(mapCtx, in.DefaultEncryptionConfiguration) + out.DefaultPartitionExpirationMs = direct.LazyPtr(in.DefaultPartitionExpiration.Milliseconds()) + out.DefaultTableExpirationMs = direct.LazyPtr(in.DefaultTableExpiration.Milliseconds()) + out.DefaultEncryptionConfiguration = EncryptionConfiguration_FromProto(mapCtx, in.DefaultEncryptionConfig) out.Description = direct.LazyPtr(in.Description) - out.FriendlyName = direct.LazyPtr(in.FriendlyName) + out.FriendlyName = direct.LazyPtr(in.Name) out.Location = direct.LazyPtr(in.Location) - out.IsCaseInsensitive = direct.LazyPtr(in.IsCaseInsensitive) - maxTime := strconv.FormatInt(in.MaxTimeTravelHours, 10) - out.MaxTimeTravelHours = direct.LazyPtr(maxTime) + maxTimeInHours := fmt.Sprintf("%v", in.MaxTimeTravel.Hours()) + out.MaxTimeTravelHours = &maxTimeInHours out.StorageBillingModel = direct.LazyPtr(in.StorageBillingModel) return out } -func BigQueryDatasetStatus_FromAPI(mapCtx *direct.MapContext, in *api.Dataset) *krm.BigQueryDatasetStatus { +func BigQueryDatasetStatus_FromProtoMetadata(mapCtx *direct.MapContext, in *pb.DatasetMetadata) *krm.BigQueryDatasetStatus { if in == nil { return nil } out := &krm.BigQueryDatasetStatus{} - out.Etag = direct.LazyPtr(in.Etag) - out.CreationTime = direct.LazyPtr(in.CreationTime) - out.LastModifiedTime = direct.LazyPtr(in.LastModifiedTime) - out.SelfLink = direct.LazyPtr(in.SelfLink) + out.Etag = direct.LazyPtr(in.ETag) + out.CreationTime = direct.LazyPtr(in.CreationTime.UnixMilli()) + out.LastModifiedTime = direct.LazyPtr(in.LastModifiedTime.UnixMilli()) + // The full dataset ID in the form projectID:datasetID + tokens := strings.Split(in.FullID, ":") + if len(tokens) == 2 { + out.SelfLink = direct.LazyPtr(fmt.Sprintf("https://bigquery.googleapis.com/bigquery/v2/projects/%s/datasets/%s", tokens[0], tokens[1])) + } return out } -func BigQueryDatasetStatusObservedState_ToAPI(mapCtx *direct.MapContext, in *krm.BigQueryDatasetStatus) *api.Dataset { +func BigQueryDatasetStatus_ToProto(mapCtx *direct.MapContext, in *krm.BigQueryDatasetStatus) *pb.DatasetMetadata { if in == nil { return nil } - out := &api.Dataset{} - out.Etag = direct.ValueOf(in.Etag) - out.CreationTime = direct.ValueOf(in.CreationTime) - out.LastModifiedTime = direct.ValueOf(in.LastModifiedTime) - out.SelfLink = direct.ValueOf(in.SelfLink) + out := &pb.DatasetMetadata{} + out.ETag = direct.ValueOf(in.Etag) + out.CreationTime = direct.UnixMillisToTime(direct.ValueOf(in.CreationTime)) + out.LastModifiedTime = direct.UnixMillisToTime(direct.ValueOf(in.LastModifiedTime)) + // The full dataset ID in the form projectID:datasetID + if in.SelfLink != nil { + selfLink := strings.Trim(direct.ValueOf(in.SelfLink), "https://bigquery.googleapis.com/bigquery/v2/") + tokens := strings.Split(selfLink, "/") + if len(tokens) == 4 && tokens[0] == "projects" && tokens[2] == "datasets" { + out.FullID = fmt.Sprintf("%s:%s", tokens[1], tokens[3]) + } + } return out } -func Access_ToAPI(mapCtx *direct.MapContext, in *krm.Access) *api.DatasetAccess { +func AccessEntry_ToProto(mapCtx *direct.MapContext, in *krm.Access) *pb.AccessEntry { if in == nil { return nil } - out := &api.DatasetAccess{} - out.Domain = direct.ValueOf(in.Domain) - out.GroupByEmail = direct.ValueOf(in.GroupByEmail) - out.IamMember = direct.ValueOf(in.IamMember) - out.UserByEmail = direct.ValueOf(in.UserByEmail) - out.SpecialGroup = direct.ValueOf(in.SpecialGroup) - out.Role = direct.ValueOf(in.Role) - out.Dataset = DatasetAccessEntry_ToAPI(mapCtx, in.Dataset) - out.Routine = RoutineReference_ToAPI(mapCtx, in.Routine) - out.View = TableReference_ToAPI(mapCtx, in.View) + out := &pb.AccessEntry{} + out.Role = pb.AccessRole(direct.ValueOf(in.Role)) + switch { + case in.Domain != nil: + out.Entity = direct.ValueOf(in.Domain) + out.EntityType = pb.DomainEntity + case in.GroupByEmail != nil: + out.Entity = direct.ValueOf(in.GroupByEmail) + out.EntityType = pb.GroupEmailEntity + case in.UserByEmail != nil: + out.Entity = direct.ValueOf(in.UserByEmail) + out.EntityType = pb.UserEmailEntity + case in.SpecialGroup != nil: + out.Entity = direct.ValueOf(in.SpecialGroup) + out.EntityType = pb.SpecialGroupEntity + case in.View != nil: + out.View = TableReference_ToProto(mapCtx, in.View) + out.EntityType = pb.ViewEntity + case in.IamMember != nil: + out.Entity = direct.ValueOf(in.IamMember) + out.EntityType = pb.IAMMemberEntity + case in.Routine != nil: + out.Routine = RoutineReference_ToProto(mapCtx, in.Routine) + out.EntityType = pb.ViewEntity + case in.Dataset != nil: + out.Dataset = DatasetAccessEntry_ToProto(mapCtx, in.Dataset) + out.EntityType = pb.ViewEntity + } return out } -func Access_FromAPI(mapCtx *direct.MapContext, in *api.DatasetAccess) *krm.Access { +func AccessEntry_FromProto(mapCtx *direct.MapContext, in *pb.AccessEntry) *krm.Access { if in == nil { return nil } out := &krm.Access{} - out.Domain = direct.LazyPtr(in.Domain) - out.GroupByEmail = direct.LazyPtr(in.GroupByEmail) - out.IamMember = direct.LazyPtr(in.IamMember) - out.UserByEmail = direct.LazyPtr(in.UserByEmail) - out.SpecialGroup = direct.LazyPtr(in.SpecialGroup) - out.Role = direct.LazyPtr(in.Role) - out.Dataset = DatasetAccessEntry_FromAPI(mapCtx, in.Dataset) - out.Routine = RoutineReference_FromAPI(mapCtx, in.Routine) - out.View = TableReference_FromAPI(mapCtx, in.View) + out.Role = direct.LazyPtr(string(in.Role)) + switch in.EntityType { + case pb.DomainEntity: + out.Domain = direct.LazyPtr(in.Entity) + case pb.GroupEmailEntity: + out.GroupByEmail = direct.LazyPtr(in.Entity) + case pb.UserEmailEntity: + out.UserByEmail = direct.LazyPtr(in.Entity) + case pb.SpecialGroupEntity: + out.SpecialGroup = direct.LazyPtr(in.Entity) + case pb.ViewEntity: + out.View = TableReference_FromProto(mapCtx, in.View) + case pb.IAMMemberEntity: + out.IamMember = direct.LazyPtr(in.Entity) + case pb.RoutineEntity: + out.Routine = RoutineReference_FromProto(mapCtx, in.Routine) + case pb.DatasetEntity: + out.Dataset = DatasetAccessEntry_FromProto(mapCtx, in.Dataset) + } + return out +} +func BigQueryDatasetMetadata_ToMetadataToUpdate(mapCtx *direct.MapContext, in *pb.DatasetMetadata) *pb.DatasetMetadataToUpdate { + if in == nil { + return nil + } + out := &pb.DatasetMetadataToUpdate{} + acccessList := []*pb.AccessEntry{} + for _, access := range in.Access { + acccessList = append(acccessList, access) + } + out.Access = acccessList + if in.DefaultCollation != "" { + out.DefaultCollation = in.DefaultCollation + } + if in.DefaultEncryptionConfig != nil { + out.DefaultEncryptionConfig = in.DefaultEncryptionConfig + } + if in.DefaultPartitionExpiration != 0 { + out.DefaultPartitionExpiration = in.DefaultPartitionExpiration + } + if in.DefaultTableExpiration != 0 { + out.DefaultTableExpiration = in.DefaultTableExpiration + } + if in.Description != "" { + out.Description = in.Description + } + if in.ExternalDatasetReference != nil { + out.ExternalDatasetReference = in.ExternalDatasetReference + } + if in.MaxTimeTravel != 0 { + out.MaxTimeTravel = in.MaxTimeTravel + } + if in.Name != "" { + out.Name = in.Name + } + if in.StorageBillingModel != "" { + out.StorageBillingModel = in.StorageBillingModel + } + if in.Labels != nil { + for k, v := range in.Labels { + out.SetLabel(k, v) + } + } return out } -func DatasetAccessEntry_FromAPI(mapCtx *direct.MapContext, in *api.DatasetAccessEntry) *krm.DatasetAccessEntry { +func DatasetAccessEntry_FromProto(mapCtx *direct.MapContext, in *pb.DatasetAccessEntry) *krm.DatasetAccessEntry { if in == nil { return nil } out := &krm.DatasetAccessEntry{} - out.Dataset = &krm.DatasetReference{ - DatasetId: direct.LazyPtr(in.Dataset.DatasetId), - ProjectId: direct.LazyPtr(in.Dataset.ProjectId), + out.Dataset = DatasetReference_FromProto(mapCtx, in.Dataset) + for _, targetType := range in.TargetTypes { + out.TargetTypes = append(out.TargetTypes, targetType) } - out.TargetTypes = in.TargetTypes return out } -func DatasetAccessEntry_ToAPI(mapCtx *direct.MapContext, in *krm.DatasetAccessEntry) *api.DatasetAccessEntry { +func DatasetAccessEntry_ToProto(mapCtx *direct.MapContext, in *krm.DatasetAccessEntry) *pb.DatasetAccessEntry { if in == nil { return nil } - out := &api.DatasetAccessEntry{} - out.Dataset = &api.DatasetReference{ - DatasetId: direct.ValueOf(in.Dataset.DatasetId), - ProjectId: direct.ValueOf(in.Dataset.ProjectId), + out := &pb.DatasetAccessEntry{} + out.Dataset = DatasetReference_ToProto(mapCtx, in.Dataset) + out.TargetTypes = make([]string, len(in.TargetTypes)) + for _, targetType := range in.TargetTypes { + out.TargetTypes = append(out.TargetTypes, targetType) } - out.TargetTypes = in.TargetTypes return out } -func DatasetReference_ToAPI(mapCtx *direct.MapContext, in *krm.BigQueryDatasetSpec, name string) *api.DatasetReference { +func DatasetReference_FromProto(mapCtx *direct.MapContext, in *pb.Dataset) *krm.DatasetReference { if in == nil { return nil } - out := &api.DatasetReference{} - out.DatasetId = name + out := &krm.DatasetReference{} + out.DatasetId = direct.LazyPtr(in.DatasetID) + out.ProjectId = direct.LazyPtr(in.ProjectID) return out } -func EncryptionConfiguration_ToAPI(mapCtx *direct.MapContext, in *krm.EncryptionConfiguration) *api.EncryptionConfiguration { +func DatasetReference_ToProto(mapCtx *direct.MapContext, in *krm.DatasetReference) *pb.Dataset { + if in == nil { + return nil + } + out := &pb.Dataset{} + out.DatasetID = direct.ValueOf(in.DatasetId) + out.ProjectID = direct.ValueOf(in.ProjectId) + return out +} +func DatasetSpec_ToExternalDatasetReference(mapCtx *direct.MapContext, in *krm.BigQueryDatasetSpec) *pb.ExternalDatasetReference { + // **NOTYET** + // There are no matching fields in KRM for now. + // out.Connection + // out.ExternalSource + return nil +} +func EncryptionConfiguration_ToProto(mapCtx *direct.MapContext, in *krm.EncryptionConfiguration) *pb.EncryptionConfig { if in == nil { return nil } - out := &api.EncryptionConfiguration{} + out := &pb.EncryptionConfig{} if in.KmsKeyRef != nil { - out.KmsKeyName = in.KmsKeyRef.External + out.KMSKeyName = in.KmsKeyRef.External } return out } -func EncryptionConfiguration_FromAPI(mapCtx *direct.MapContext, in *api.EncryptionConfiguration) *krm.EncryptionConfiguration { +func EncryptionConfiguration_FromProto(mapCtx *direct.MapContext, in *pb.EncryptionConfig) *krm.EncryptionConfiguration { if in == nil { return nil } out := &krm.EncryptionConfiguration{} out.KmsKeyRef = &v1beta1.KMSCryptoKeyRef{ - External: in.KmsKeyName, + External: in.KMSKeyName, } return out } -func RoutineReference_FromAPI(mapCtx *direct.MapContext, in *api.RoutineReference) *krm.RoutineReference { +func RoutineReference_FromProto(mapCtx *direct.MapContext, in *pb.Routine) *krm.RoutineReference { if in == nil { return nil } out := &krm.RoutineReference{} - out.DatasetId = direct.LazyPtr(in.DatasetId) - out.ProjectId = direct.LazyPtr(in.ProjectId) - out.RoutineId = direct.LazyPtr(in.RoutineId) + out.DatasetId = direct.LazyPtr(in.DatasetID) + out.ProjectId = direct.LazyPtr(in.ProjectID) + out.RoutineId = direct.LazyPtr(in.RoutineID) return out } -func RoutineReference_ToAPI(mapCtx *direct.MapContext, in *krm.RoutineReference) *api.RoutineReference { +func RoutineReference_ToProto(mapCtx *direct.MapContext, in *krm.RoutineReference) *pb.Routine { if in == nil { return nil } - out := &api.RoutineReference{} - out.DatasetId = direct.ValueOf(in.DatasetId) - out.ProjectId = direct.ValueOf(in.ProjectId) - out.RoutineId = direct.ValueOf(in.RoutineId) + out := &pb.Routine{} + out.DatasetID = direct.ValueOf(in.DatasetId) + out.ProjectID = direct.ValueOf(in.ProjectId) + out.RoutineID = direct.ValueOf(in.RoutineId) return out } -func TableReference_FromAPI(mapCtx *direct.MapContext, in *api.TableReference) *krm.TableReference { +func TableReference_FromProto(mapCtx *direct.MapContext, in *pb.Table) *krm.TableReference { if in == nil { return nil } out := &krm.TableReference{} - out.DatasetId = direct.LazyPtr(in.DatasetId) - out.ProjectId = direct.LazyPtr(in.ProjectId) - out.TableId = direct.LazyPtr(in.TableId) + out.DatasetId = direct.LazyPtr(in.DatasetID) + out.ProjectId = direct.LazyPtr(in.ProjectID) + out.TableId = direct.LazyPtr(in.TableID) return out } -func TableReference_ToAPI(mapCtx *direct.MapContext, in *krm.TableReference) *api.TableReference { +func TableReference_ToProto(mapCtx *direct.MapContext, in *krm.TableReference) *pb.Table { if in == nil { return nil } - out := &api.TableReference{} - out.DatasetId = direct.ValueOf(in.DatasetId) - out.ProjectId = direct.ValueOf(in.ProjectId) - out.TableId = direct.ValueOf(in.TableId) + out := &pb.Table{} + out.DatasetID = direct.ValueOf(in.DatasetId) + out.ProjectID = direct.ValueOf(in.ProjectId) + out.TableID = direct.ValueOf(in.TableId) return out } diff --git a/pkg/controller/direct/bigquerydataset/dataset_controller.go b/pkg/controller/direct/bigquerydataset/dataset_controller.go index 9b3c36786f..5afa5c6513 100644 --- a/pkg/controller/direct/bigquerydataset/dataset_controller.go +++ b/pkg/controller/direct/bigquerydataset/dataset_controller.go @@ -25,9 +25,10 @@ import ( "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct/directbase" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct/registry" + "github.com/huandu/go-clone" - clone "github.com/huandu/go-clone" - api "google.golang.org/api/bigquery/v2" + bigquery "cloud.google.com/go/bigquery" + pb "cloud.google.com/go/bigquery" "google.golang.org/api/option" "google.golang.org/protobuf/types/known/fieldmaskpb" @@ -56,13 +57,13 @@ type model struct { config config.ControllerConfig } -func (m *model) service(ctx context.Context) (*api.Service, error) { +func (m *model) service(ctx context.Context, projectID string) (*bigquery.Client, error) { var opts []option.ClientOption opts, err := m.config.RESTClientOptions() if err != nil { return nil, err } - gcpService, err := api.NewService(ctx, opts...) + gcpService, err := bigquery.NewClient(ctx, projectID, opts...) if err != nil { return nil, fmt.Errorf("building Dataset client: %w", err) } @@ -80,8 +81,14 @@ func (m *model) AdapterForObject(ctx context.Context, reader client.Reader, u *u return nil, err } + parent, _, err := krm.ParseBigQueryDatasetExternal(id.External) + projectID := parent.ProjectID + if projectID == "" { + return nil, fmt.Errorf("cannot resolve project ID") + } + // Get bigquery GCP client - gcpService, err := m.service(ctx) + gcpService, err := m.service(ctx, projectID) if err != nil { return nil, err } @@ -100,9 +107,9 @@ func (m *model) AdapterForURL(ctx context.Context, url string) (directbase.Adapt type Adapter struct { id *krm.BigQueryDatasetRef - gcpService *api.Service + gcpService *bigquery.Client desired *krm.BigQueryDataset - actual *api.Dataset + actual *bigquery.DatasetMetadata reader client.Reader } @@ -116,8 +123,8 @@ func (a *Adapter) Find(ctx context.Context) (bool, error) { if err != nil { return false, fmt.Errorf("failed to parse bigquery dataset full name, %w", err) } - datasetGetCall := a.gcpService.Datasets.Get(parent.ProjectID, datasetId) - datasetpb, err := datasetGetCall.Do() + dsHandler := a.gcpService.DatasetInProject(parent.ProjectID, datasetId) + datasetpb, err := dsHandler.Metadata(ctx) if err != nil { if direct.IsNotFound(err) { return false, nil @@ -134,13 +141,13 @@ func (a *Adapter) Create(ctx context.Context, createOp *directbase.CreateOperati log.V(2).Info("creating Dataset", "name", a.id.External) mapCtx := &direct.MapContext{} - desiredDataset := BigQueryDatasetSpec_ToAPI(mapCtx, &a.desired.Spec, a.desired.Name) + desiredDataset := BigQueryDatasetSpec_ToProtoMetadata(mapCtx, &a.desired.Spec) desiredDataset.Labels = make(map[string]string) for k, v := range a.desired.GetObjectMeta().GetLabels() { desiredDataset.Labels[k] = v } desiredDataset.Labels["managed-by-cnrm"] = "true" - parent, _, err := krm.ParseBigQueryDatasetExternal(a.id.External) + parent, datasetId, err := krm.ParseBigQueryDatasetExternal(a.id.External) if err != nil { return fmt.Errorf("failed to parse bigquery dataset full name, %w", err) } @@ -150,17 +157,25 @@ func (a *Adapter) Create(ctx context.Context, createOp *directbase.CreateOperati if err != nil { return err } - desiredDataset.DefaultEncryptionConfiguration.KmsKeyName = kmsRef.External + desiredDataset.DefaultEncryptionConfig.KMSKeyName = kmsRef.External } - insertDatasetCall := a.gcpService.Datasets.Insert(parent.ProjectID, desiredDataset) - inserted, err := insertDatasetCall.Do() - if err != nil { - return fmt.Errorf("inserting Dataset %s: %w", a.id.External, err) + dsHandler := a.gcpService.DatasetInProject(parent.ProjectID, datasetId) + if err := dsHandler.Create(ctx, desiredDataset); err != nil { + return fmt.Errorf("Error creating Dataset %s: %w", a.id.External, err) } - log.V(2).Info("successfully inserted Dataset", "name", a.id.External) + log.V(2).Info("successfully created Dataset", "name", a.id.External) + // The bigquery go client Create() does not return the created dataset. + // Fetching the dataset metadata + createdMetadata, err := dsHandler.Metadata(ctx) + if err != nil { + if direct.IsNotFound(err) { + return nil + } + return fmt.Errorf("Error getting the created BigQueryDataset %q: %w", a.id.External, err) + } status := &krm.BigQueryDatasetStatus{} - status = BigQueryDatasetStatus_FromAPI(mapCtx, inserted) + status = BigQueryDatasetStatus_FromProtoMetadata(mapCtx, createdMetadata) if mapCtx.Err() != nil { return mapCtx.Err() } @@ -177,12 +192,12 @@ func (a *Adapter) Update(ctx context.Context, updateOp *directbase.UpdateOperati // Convert KRM object to proto message desiredKRM := a.desired.DeepCopy() - desired := BigQueryDatasetSpec_ToAPI(mapCtx, &desiredKRM.Spec, desiredKRM.Name) + desired := BigQueryDatasetSpec_ToProtoMetadata(mapCtx, &desiredKRM.Spec) if mapCtx.Err() != nil { return mapCtx.Err() } - resource := clone.Clone(a.actual).(*api.Dataset) + resource := clone.Clone(a.actual).(*pb.DatasetMetadata) // Check for immutable fields if !reflect.DeepEqual(desired.Location, resource.Location) { @@ -195,40 +210,32 @@ func (a *Adapter) Update(ctx context.Context, updateOp *directbase.UpdateOperati resource.Description = desired.Description updateMask.Paths = append(updateMask.Paths, "description") } - if !reflect.DeepEqual(desired.FriendlyName, resource.FriendlyName) { - resource.FriendlyName = desired.FriendlyName - updateMask.Paths = append(updateMask.Paths, "friendly_name") - } - if !reflect.DeepEqual(desired.DefaultPartitionExpirationMs, resource.DefaultPartitionExpirationMs) { - resource.DefaultPartitionExpirationMs = desired.DefaultPartitionExpirationMs + if !reflect.DeepEqual(desired.DefaultPartitionExpiration, resource.DefaultPartitionExpiration) { + resource.DefaultPartitionExpiration = desired.DefaultPartitionExpiration updateMask.Paths = append(updateMask.Paths, "default_partition_expirationMs") } - if !reflect.DeepEqual(desired.DefaultTableExpirationMs, resource.DefaultTableExpirationMs) { - resource.DefaultTableExpirationMs = desired.DefaultTableExpirationMs + if !reflect.DeepEqual(desired.DefaultTableExpiration, resource.DefaultTableExpiration) { + resource.DefaultTableExpiration = desired.DefaultTableExpiration updateMask.Paths = append(updateMask.Paths, "default_table_expirationMs") } if !reflect.DeepEqual(desired.DefaultCollation, resource.DefaultCollation) { resource.DefaultCollation = desired.DefaultCollation updateMask.Paths = append(updateMask.Paths, "default_collation") } - if desired.DefaultEncryptionConfiguration != nil && resource.DefaultEncryptionConfiguration != nil && !reflect.DeepEqual(desired.DefaultEncryptionConfiguration, resource.DefaultEncryptionConfiguration) { + if desired.DefaultEncryptionConfig != nil && resource.DefaultEncryptionConfig != nil && !reflect.DeepEqual(desired.DefaultEncryptionConfig, resource.DefaultEncryptionConfig) { // Resolve KMS key reference if a.desired.Spec.DefaultEncryptionConfiguration != nil { kmsRef, err := refs.ResolveKMSCryptoKeyRef(ctx, a.reader, a.desired, a.desired.Spec.DefaultEncryptionConfiguration.KmsKeyRef) if err != nil { return err } - desired.DefaultEncryptionConfiguration.KmsKeyName = kmsRef.External + desired.DefaultEncryptionConfig.KMSKeyName = kmsRef.External } - resource.DefaultEncryptionConfiguration.KmsKeyName = desired.DefaultEncryptionConfiguration.KmsKeyName + resource.DefaultEncryptionConfig.KMSKeyName = desired.DefaultEncryptionConfig.KMSKeyName updateMask.Paths = append(updateMask.Paths, "default_encryption_configuration") } - if !reflect.DeepEqual(desired.IsCaseInsensitive, resource.IsCaseInsensitive) { - resource.IsCaseInsensitive = desired.IsCaseInsensitive - updateMask.Paths = append(updateMask.Paths, "is_case_sensitive") - } - if !reflect.DeepEqual(desired.MaxTimeTravelHours, resource.MaxTimeTravelHours) { - resource.MaxTimeTravelHours = desired.MaxTimeTravelHours + if !reflect.DeepEqual(desired.MaxTimeTravel, resource.MaxTimeTravel) { + resource.MaxTimeTravel = desired.MaxTimeTravel updateMask.Paths = append(updateMask.Paths, "max_time_interval_hours") } if desired.Access != nil && resource.Access != nil && len(desired.Access) > 0 && !reflect.DeepEqual(desired.Access, resource.Access) { @@ -245,6 +252,11 @@ func (a *Adapter) Update(ctx context.Context, updateOp *directbase.UpdateOperati if len(updateMask.Paths) == 0 { return nil } + resource.Labels = make(map[string]string) + for k, v := range a.desired.GetObjectMeta().GetLabels() { + resource.Labels[k] = v + } + resource.Labels["managed-by-cnrm"] = "true" parent, datasetId, err := krm.ParseBigQueryDatasetExternal(a.id.External) if err != nil { return fmt.Errorf("failed to parse bigquery dataset full name, %w", err) @@ -253,15 +265,16 @@ func (a *Adapter) Update(ctx context.Context, updateOp *directbase.UpdateOperati if desired.Access == nil || len(desired.Access) == 0 { resource.Access = a.actual.Access } - updateDatasetCall := a.gcpService.Datasets.Update(parent.ProjectID, datasetId, resource) - updated, err := updateDatasetCall.Do() + datasetMetadataToUpdate := BigQueryDatasetMetadata_ToMetadataToUpdate(mapCtx, resource) + dsHandler := a.gcpService.DatasetInProject(parent.ProjectID, datasetId) + updated, err := dsHandler.Update(ctx, *datasetMetadataToUpdate, resource.ETag) if err != nil { return fmt.Errorf("updating Dataset %s: %w", a.id.External, err) } log.V(2).Info("successfully updated Dataset", "name", a.id.External) status := &krm.BigQueryDatasetStatus{} - status = BigQueryDatasetStatus_FromAPI(mapCtx, updated) + status = BigQueryDatasetStatus_FromProtoMetadata(mapCtx, updated) if mapCtx.Err() != nil { return mapCtx.Err() } @@ -276,7 +289,7 @@ func (a *Adapter) Export(ctx context.Context) (*unstructured.Unstructured, error obj := &krm.BigQueryDataset{} mapCtx := &direct.MapContext{} - obj.Spec = direct.ValueOf(BigQueryDatasetSpec_FromAPI(mapCtx, a.actual)) + obj.Spec = direct.ValueOf(BigQueryDatasetSpec_FromProtoMetadata(mapCtx, a.actual)) if mapCtx.Err() != nil { return nil, mapCtx.Err() } @@ -304,9 +317,8 @@ func (a *Adapter) Delete(ctx context.Context, deleteOp *directbase.DeleteOperati if err != nil { return false, fmt.Errorf("failed to parse bigquery dataset full name, %w", err) } - deleteDatasetCall := a.gcpService.Datasets.Delete(parent.ProjectID, datasetId) - err = deleteDatasetCall.Do() - if err != nil { + dsHandler := a.gcpService.DatasetInProject(parent.ProjectID, datasetId) + if err := dsHandler.Delete(ctx); err != nil { return false, fmt.Errorf("deleting Dataset %s: %w", a.id.External, err) } log.V(2).Info("successfully deleted Dataset", "name", a.id.External) diff --git a/pkg/controller/direct/maputils.go b/pkg/controller/direct/maputils.go index 34ba5a03f5..fc09883996 100644 --- a/pkg/controller/direct/maputils.go +++ b/pkg/controller/direct/maputils.go @@ -334,3 +334,13 @@ func Int64Value_FromProto(mapCtx *MapContext, ts *wrapperspb.Int64Value) int64 { func Int64Value_ToProto(mapCtx *MapContext, s int64) *wrapperspb.Int64Value { return wrapperspb.Int64(s) } + +// Convert a number of milliseconds since the Unix epoch to a time.Time. +// Treat an input of zero specially: convert it to the zero time, +// rather than the start of the epoch. +func UnixMillisToTime(m int64) time.Time { + if m == 0 { + return time.Time{} + } + return time.Unix(0, m*1e6) +} diff --git a/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/basicbigquerydataset-direct/_generated_export_basicbigquerydataset-direct.golden b/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/basicbigquerydataset-direct/_generated_export_basicbigquerydataset-direct.golden index 6c64b9259f..2f5704c501 100644 --- a/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/basicbigquerydataset-direct/_generated_export_basicbigquerydataset-direct.golden +++ b/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/basicbigquerydataset-direct/_generated_export_basicbigquerydataset-direct.golden @@ -17,7 +17,7 @@ spec: specialGroup: projectReaders - role: WRITER specialGroup: projectWriters - friendlyName: bigquerydataset-sample-updated + friendlyName: bigquerydataset-sample location: us-central1 maxTimeTravelHours: "168" projectRef: diff --git a/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/basicbigquerydataset-direct/_http.log b/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/basicbigquerydataset-direct/_http.log index 6b3fc1f9c8..b77cee540d 100644 --- a/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/basicbigquerydataset-direct/_http.log +++ b/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/basicbigquerydataset-direct/_http.log @@ -151,8 +151,9 @@ X-Xss-Protection: 0 --- -PUT https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false +PATCH https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false Content-Type: application/json +If-Match: 1111111 User-Agent: kcc/controller-manager { @@ -174,23 +175,11 @@ User-Agent: kcc/controller-manager "specialGroup": "projectReaders" } ], - "creationTime": "123456789", - "datasetReference": { - "datasetId": "bigquerydatasetsample${uniqueId}", - "projectId": "${projectId}" - }, - "etag": "abcdef0123A=", - "friendlyName": "bigquerydataset-sample-updated", - "id": "000000000000000000000", - "kind": "bigquery#dataset", + "friendlyName": "bigquerydataset-sample", "labels": { "cnrm-test": "true", "managed-by-cnrm": "true" - }, - "lastModifiedTime": "123456789", - "location": "us-central1", - "selfLink": "https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/bigquerydatasetsample${uniqueId}", - "type": "DEFAULT" + } } 200 OK @@ -229,7 +218,7 @@ X-Xss-Protection: 0 "projectId": "${projectId}" }, "etag": "abcdef0123A=", - "friendlyName": "bigquerydataset-sample-updated", + "friendlyName": "bigquerydataset-sample", "id": "000000000000000000000", "kind": "bigquery#dataset", "labels": { @@ -284,7 +273,7 @@ X-Xss-Protection: 0 "projectId": "${projectId}" }, "etag": "abcdef0123A=", - "friendlyName": "bigquerydataset-sample-updated", + "friendlyName": "bigquerydataset-sample", "id": "000000000000000000000", "kind": "bigquery#dataset", "labels": { @@ -339,7 +328,7 @@ X-Xss-Protection: 0 "projectId": "${projectId}" }, "etag": "abcdef0123A=", - "friendlyName": "bigquerydataset-sample-updated", + "friendlyName": "bigquerydataset-sample", "id": "000000000000000000000", "kind": "bigquery#dataset", "labels": { @@ -355,7 +344,7 @@ X-Xss-Protection: 0 --- -DELETE https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false +DELETE https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&deleteContents=false&prettyPrint=false User-Agent: kcc/controller-manager 204 No Content diff --git a/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/bigquerydatasetaccessblock-direct/_http.log b/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/bigquerydatasetaccessblock-direct/_http.log index 5dec562437..da6d0b38f6 100644 --- a/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/bigquerydatasetaccessblock-direct/_http.log +++ b/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/bigquerydatasetaccessblock-direct/_http.log @@ -151,8 +151,9 @@ X-Xss-Protection: 0 --- -PUT https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false +PATCH https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false Content-Type: application/json +If-Match: 1111111 User-Agent: kcc/controller-manager { @@ -178,25 +179,13 @@ User-Agent: kcc/controller-manager "role": "READER" } ], - "creationTime": "123456789", - "datasetReference": { - "datasetId": "bigquerydataset${uniqueId}", - "projectId": "${projectId}" - }, "defaultTableExpirationMs": "7200000", "description": "BigQuery Dataset With Access Block v2", - "etag": "abcdef0123A=", "friendlyName": "bigquerydataset-accessblock", - "id": "000000000000000000000", - "kind": "bigquery#dataset", "labels": { "cnrm-test": "true", "managed-by-cnrm": "true" - }, - "lastModifiedTime": "123456789", - "location": "US", - "selfLink": "https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/bigquerydataset${uniqueId}", - "type": "DEFAULT" + } } 200 OK @@ -379,7 +368,7 @@ X-Xss-Protection: 0 --- -DELETE https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false +DELETE https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&deleteContents=false&prettyPrint=false User-Agent: kcc/controller-manager 204 No Content diff --git a/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/fullybigquerydataset-direct/_generated_export_fullybigquerydataset-direct.golden b/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/fullybigquerydataset-direct/_generated_export_fullybigquerydataset-direct.golden index 0523cbdbdf..1e7dade694 100644 --- a/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/fullybigquerydataset-direct/_generated_export_fullybigquerydataset-direct.golden +++ b/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/fullybigquerydataset-direct/_generated_export_fullybigquerydataset-direct.golden @@ -21,7 +21,7 @@ spec: defaultPartitionExpirationMs: 3800000 defaultTableExpirationMs: 3800000 description: Fully Configured BigQuery Dataset updated - friendlyName: bigquerydataset-fullyconfigured-updated + friendlyName: bigquerydataset-fullyconfigured location: US maxTimeTravelHours: "96" projectRef: diff --git a/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/fullybigquerydataset-direct/_http.log b/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/fullybigquerydataset-direct/_http.log index 4e7cf5c240..0c7ca3d52e 100644 --- a/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/fullybigquerydataset-direct/_http.log +++ b/pkg/test/resourcefixture/testdata/basic/bigquery/v1beta1/bigquerydataset/fullybigquerydataset-direct/_http.log @@ -416,7 +416,6 @@ User-Agent: kcc/controller-manager "defaultTableExpirationMs": "3600000", "description": "Fully Configured BigQuery Dataset", "friendlyName": "bigquerydataset-fullyconfigured", - "isCaseInsensitive": true, "labels": { "cnrm-test": "true", "managed-by-cnrm": "true" @@ -459,7 +458,6 @@ X-Xss-Protection: 0 "etag": "abcdef0123A=", "friendlyName": "bigquerydataset-fullyconfigured", "id": "000000000000000000000", - "isCaseInsensitive": true, "kind": "bigquery#dataset", "labels": { "cnrm-test": "true", @@ -511,7 +509,6 @@ X-Xss-Protection: 0 "etag": "abcdef0123A=", "friendlyName": "bigquerydataset-fullyconfigured", "id": "000000000000000000000", - "isCaseInsensitive": true, "kind": "bigquery#dataset", "labels": { "cnrm-test": "true", @@ -527,8 +524,9 @@ X-Xss-Protection: 0 --- -PUT https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false +PATCH https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false Content-Type: application/json +If-Match: 1111111 User-Agent: kcc/controller-manager { @@ -550,31 +548,19 @@ User-Agent: kcc/controller-manager "userByEmail": "user@google.com" } ], - "creationTime": "123456789", - "datasetReference": { - "datasetId": "bigquerydataset${uniqueId}", - "projectId": "${projectId}" - }, "defaultEncryptionConfiguration": { "kmsKeyName": "projects/${projectId}/locations/us/keyRings/kmskeyring-${uniqueId}/cryptoKeys/kmscryptokey-${uniqueId}" }, "defaultPartitionExpirationMs": "3800000", "defaultTableExpirationMs": "3800000", "description": "Fully Configured BigQuery Dataset updated", - "etag": "abcdef0123A=", - "friendlyName": "bigquerydataset-fullyconfigured-updated", - "id": "000000000000000000000", - "kind": "bigquery#dataset", + "friendlyName": "bigquerydataset-fullyconfigured", "labels": { "cnrm-test": "true", "managed-by-cnrm": "true" }, - "lastModifiedTime": "123456789", - "location": "US", "maxTimeTravelHours": "96", - "selfLink": "https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/bigquerydataset${uniqueId}", - "storageBillingModel": "LOGICAL", - "type": "DEFAULT" + "storageBillingModel": "LOGICAL" } 200 OK @@ -619,7 +605,7 @@ X-Xss-Protection: 0 "defaultTableExpirationMs": "3800000", "description": "Fully Configured BigQuery Dataset updated", "etag": "abcdef0123A=", - "friendlyName": "bigquerydataset-fullyconfigured-updated", + "friendlyName": "bigquerydataset-fullyconfigured", "id": "000000000000000000000", "kind": "bigquery#dataset", "labels": { @@ -682,7 +668,7 @@ X-Xss-Protection: 0 "defaultTableExpirationMs": "3800000", "description": "Fully Configured BigQuery Dataset updated", "etag": "abcdef0123A=", - "friendlyName": "bigquerydataset-fullyconfigured-updated", + "friendlyName": "bigquerydataset-fullyconfigured", "id": "000000000000000000000", "kind": "bigquery#dataset", "labels": { @@ -744,7 +730,7 @@ X-Xss-Protection: 0 "defaultTableExpirationMs": "3800000", "description": "Fully Configured BigQuery Dataset updated", "etag": "abcdef0123A=", - "friendlyName": "bigquerydataset-fullyconfigured-updated", + "friendlyName": "bigquerydataset-fullyconfigured", "id": "000000000000000000000", "kind": "bigquery#dataset", "labels": { @@ -761,7 +747,7 @@ X-Xss-Protection: 0 --- -DELETE https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&prettyPrint=false +DELETE https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&deleteContents=false&prettyPrint=false User-Agent: kcc/controller-manager 204 No Content diff --git a/tests/e2e/unified_test.go b/tests/e2e/unified_test.go index 3436857df5..7432190b47 100644 --- a/tests/e2e/unified_test.go +++ b/tests/e2e/unified_test.go @@ -663,6 +663,11 @@ func runScenario(ctx context.Context, t *testing.T, testPause bool, fixture reso // Specific to BigQuery addSetStringReplacement(".access[].userByEmail", "user@google.com") + for _, event := range events { + if event.Request.Method == "PATCH" && strings.Contains(event.Request.URL, "bigquery.googleapis.com/bigquery/v2/") { + event.Request.ReplaceHeader("If-Match", "1111111") + } + } // Specific to BigTable addSetStringReplacement(".instances[].createTime", "2024-04-01T12:34:56.123456Z")