root
/
grafana
mirror of https://github.com/grafana/grafana.git
1
0
Fork 0
Code Issues Actions 83 Packages Projects Releases Wiki Activity

WIP: Spawn backend plugins v2 (#19835) 

* WIP: Spawn backend plugins v2

* Add test for plugin version

* Fix support for SDK plugins

Co-authored-by: Kyle Brandt <kyle@kbrandt.com>
Co-authored-by: Marcus Olsson <olsson.e.marcus@gmail.com>
Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com>

* WIP: Draft PR for fork of V2 sdk / bi-directional support (#19890)

* temporary use export-datasource-plugin branch of grafana-plugin-sdk

* fix failing test

* remove debug (spew) lines

* misc cleanup

* add expressions feature toggle

* use latest grafana-plugin-sdk-go
This commit is contained in:
Sofia Papagiannaki 2019-10-24 18:15:27 +03:00 committed by Marcus Efraimsson
parent b0198e7c9c
commit fc08c26025
276 changed files with 30794 additions and 2504 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -29,10 +29,10 @@ require (
github.com/go-xorm/core v0.6.2
github.com/go-xorm/xorm v0.7.1
github.com/gobwas/glob v0.2.3
github.com/google/go-cmp v0.2.0 // indirect
github.com/gorilla/websocket v1.4.0
github.com/gosimple/slug v1.4.2
github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4
github.com/grafana/grafana-plugin-sdk-go v0.0.0-20191024130641-6756418f682c
github.com/hashicorp/go-hclog v0.8.0
github.com/hashicorp/go-plugin v1.0.1
github.com/hashicorp/go-version v1.1.0

20
go.sum
View File

@ -1,3 +1,4 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
@ -6,6 +7,8 @@ github.com/VividCortex/mysqlerr v0.0.0-20170204212430-6c6b55f8796f h1:HR5nRmUQgX
github.com/VividCortex/mysqlerr v0.0.0-20170204212430-6c6b55f8796f/go.mod h1:f3HiCrHjHBdcm6E83vGaXh1KomZMA2P6aeo3hKx/wg0=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/apache/arrow/go/arrow v0.0.0-20190716210558-5f564424c71c h1:iHUHzx3S1TU5xt+D7vLb0PAk3e+RfayF9IhR6+hyO/k=
github.com/apache/arrow/go/arrow v0.0.0-20190716210558-5f564424c71c/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0=
github.com/aws/aws-sdk-go v1.25.6 h1:Rmg2pgKXoCfNe0KQb4LNSNmHqMdcgBjpMeXK9IjHWq8=
github.com/aws/aws-sdk-go v1.25.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
@ -17,6 +20,7 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668 h1:U/lr3Dgy4WK+hNk4tyD+nuGjpVLPEHuJSFXMw11/HPA=
github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
@ -81,6 +85,7 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@ -88,6 +93,8 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/flatbuffers v1.11.0 h1:O7CEyB8Cb3/DmtxODGtLHcEvpr81Jm5qLg/hsHnxA2A=
github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
@ -101,6 +108,8 @@ github.com/gosimple/slug v1.4.2 h1:jDmprx3q/9Lfk4FkGZtvzDQ9Cj9eAmsjzeQGp24PeiQ=
github.com/gosimple/slug v1.4.2/go.mod h1:ER78kgg1Mv0NQGlXiDe57DpCyfbNywXXZ9mIorhxAf0=
github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4 h1:SPdxCL9BChFTlyi0Khv64vdCW4TMna8+sxL7+Chx+Ag=
github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4/go.mod h1:nc0XxBzjeGcrMltCDw269LoWF9S8ibhgxolCdA1R8To=
github.com/grafana/grafana-plugin-sdk-go v0.0.0-20191024130641-6756418f682c h1:VqEGTH0daEXSpqoLNv87ohtflLi0nuvZsSsrmw3uNnY=
github.com/grafana/grafana-plugin-sdk-go v0.0.0-20191024130641-6756418f682c/go.mod h1:nH8fL+JRTcD8u34ZZw8mYq5mVNtJlVl7AfvFUZ4fzUc=
github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI=
github.com/hashicorp/go-hclog v0.8.0 h1:z3ollgGRg8RjfJH6UVBaG54R70GFd++QOkvnJH3VSBY=
github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
@ -150,6 +159,8 @@ github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lunny/log v0.0.0-20160921050905-7887c61bf0de/go.mod h1:3q8WtuPQsoRbatJuy3nvq/hRSvuBJrHHr+ybPPiNvHQ=
github.com/lunny/nodb v0.0.0-20160621015157-fc1ef06ad4af/go.mod h1:Cqz6pqow14VObJ7peltM+2n3PWOz7yTrfUuGbVFkzN0=
github.com/mattetti/filebuffer v1.0.0 h1:ixTvQ0JjBTwWbdpDZ98lLrydo7KRi8xNRIi5RFszsbY=
github.com/mattetti/filebuffer v1.0.0/go.mod h1:X6nyAIge2JGVmuJt2MFCqmHrb/5IHiphfHtot0s5cnI=
github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
@ -228,6 +239,7 @@ github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:s
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@ -265,6 +277,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/image v0.0.0-20190507092727-e4e5bf290fec/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -277,6 +290,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190319182350-c85d3e98c914 h1:jIOcLT9BZzyJ9ce+IwwZ+aF9yeCqzrR+NrD68a/SHKw=
golang.org/x/oauth2 v0.0.0-20190319182350-c85d3e98c914/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -300,12 +314,15 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190802220118-1d1727260058/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
golang.org/x/tools v0.0.0-20190805222050-c5a2fd39b72a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
@ -314,6 +331,8 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/grpc v1.14.0 h1:ArxJuB1NWfPY6r9Gp9gqwplT0Ge7nqv9msgu03lHLmo=
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.23.1 h1:q4XQuHFC6I28BKZpo6IYyb3mNO+l7lSOxRuYTCiDfXk=
google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
@ -344,3 +363,4 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

176
pkg/plugins/datasource/wrapper/datasource_plugin_wrapper_v2.go Normal file
View File

@ -0,0 +1,176 @@
package wrapper
import (
"context"
"errors"
"fmt"
sdk "github.com/grafana/grafana-plugin-sdk-go"
"github.com/grafana/grafana-plugin-sdk-go/dataframe"
"github.com/grafana/grafana-plugin-sdk-go/genproto/datasource"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb"
)
type grafanaAPI struct {
logger log.Logger
}
func (s *grafanaAPI) QueryDatasource(ctx context.Context, req *datasource.QueryDatasourceRequest) (*datasource.QueryDatasourceResponse, error) {
if len(req.Queries) == 0 {
return nil, fmt.Errorf("zero queries found in datasource request")
}
getDsInfo := &models.GetDataSourceByIdQuery{
Id: req.DatasourceId,
OrgId: req.OrgId,
}
if err := bus.Dispatch(getDsInfo); err != nil {
return nil, fmt.Errorf("Could not find datasource %v", err)
}
// Convert plugin-model (datasource) queries to tsdb queries
queries := make([]*tsdb.Query, len(req.Queries))
for i, query := range req.Queries {
sj, err := simplejson.NewJson([]byte(query.ModelJson))
if err != nil {
return nil, err
}
queries[i] = &tsdb.Query{
RefId: query.RefId,
IntervalMs: query.IntervalMs,
MaxDataPoints: query.MaxDataPoints,
DataSource: getDsInfo.Result,
Model: sj,
}
}
timeRange := tsdb.NewTimeRange(req.TimeRange.FromRaw, req.TimeRange.ToRaw)
tQ := &tsdb.TsdbQuery{
TimeRange: timeRange,
Queries: queries,
}
// Execute the converted queries
tsdbRes, err := tsdb.HandleRequest(ctx, getDsInfo.Result, tQ)
if err != nil {
return nil, err
}
// Convert tsdb results (map) to plugin-model/datasource (slice) results
// Only error and Series responses mapped.
results := make([]*datasource.QueryResult, len(tsdbRes.Results))
resIdx := 0
for refID, res := range tsdbRes.Results {
qr := &datasource.QueryResult{
RefId: refID,
}
if res.Error != nil {
qr.Error = res.ErrorString
results[resIdx] = qr
resIdx++
continue
}
encodedFrames := make([][]byte, len(res.Series))
for sIdx, series := range res.Series {
frame, err := tsdb.SeriesToFrame(series)
if err != nil {
return nil, err
}
encodedFrames[sIdx], err = dataframe.MarshalArrow(frame)
if err != nil {
return nil, err
}
}
qr.Dataframes = encodedFrames
results[resIdx] = qr
resIdx++
}
return &datasource.QueryDatasourceResponse{Results: results}, nil
}
func NewDatasourcePluginWrapperV2(log log.Logger, plugin sdk.DatasourcePlugin) *DatasourcePluginWrapperV2 {
return &DatasourcePluginWrapperV2{DatasourcePlugin: plugin, logger: log}
}
type DatasourcePluginWrapperV2 struct {
sdk.DatasourcePlugin
logger log.Logger
}
func (tw *DatasourcePluginWrapperV2) Query(ctx context.Context, ds *models.DataSource, query *tsdb.TsdbQuery) (*tsdb.Response, error) {
jsonData, err := ds.JsonData.MarshalJSON()
if err != nil {
return nil, err
}
pbQuery := &datasource.DatasourceRequest{
Datasource: &datasource.DatasourceInfo{
Name: ds.Name,
Type: ds.Type,
Url: ds.Url,
Id: ds.Id,
OrgId: ds.OrgId,
JsonData: string(jsonData),
DecryptedSecureJsonData: ds.SecureJsonData.Decrypt(),
},
TimeRange: &datasource.TimeRange{
FromRaw: query.TimeRange.From,
ToRaw: query.TimeRange.To,
ToEpochMs: query.TimeRange.GetToAsMsEpoch(),
FromEpochMs: query.TimeRange.GetFromAsMsEpoch(),
},
Queries: []*datasource.Query{},
}
for _, q := range query.Queries {
modelJSON, err := q.Model.MarshalJSON()
if err != nil {
return nil, err
}
pbQuery.Queries = append(pbQuery.Queries, &datasource.Query{
ModelJson: string(modelJSON),
IntervalMs: q.IntervalMs,
RefId: q.RefId,
MaxDataPoints: q.MaxDataPoints,
})
}
pbres, err := tw.DatasourcePlugin.Query(ctx, pbQuery, &grafanaAPI{logger: tw.logger})
if err != nil {
return nil, err
}
res := &tsdb.Response{
Results: map[string]*tsdb.QueryResult{},
}
for _, r := range pbres.Results {
qr := &tsdb.QueryResult{
RefId: r.RefId,
}
if r.Error != "" {
qr.Error = errors.New(r.Error)
qr.ErrorString = r.Error
}
if r.MetaJson != "" {
metaJSON, err := simplejson.NewJson([]byte(r.MetaJson))
if err != nil {
tw.logger.Error("Error parsing JSON Meta field: " + err.Error())
}
qr.Meta = metaJSON
}
qr.Dataframes = r.Dataframes
res.Results[r.RefId] = qr
}
return res, nil
}

67
pkg/plugins/datasource_plugin.go
View File

@ -3,11 +3,16 @@ package plugins
import (
"context"
"encoding/json"
"errors"
"fmt"
"os/exec"
"path"
"time"
"github.com/grafana/grafana-plugin-model/go/datasource"
"github.com/grafana/grafana/pkg/setting"
datasourceV1 "github.com/grafana/grafana-plugin-model/go/datasource"
sdk "github.com/grafana/grafana-plugin-sdk-go"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins/datasource/wrapper"
@ -34,16 +39,29 @@ type DataSourcePlugin struct {
Backend bool `json:"backend,omitempty"`
Executable string `json:"executable,omitempty"`
SDK bool `json:"sdk,omitempty"`
log log.Logger
client *plugin.Client
}
func isExpressionsEnabled() bool {
v, ok := setting.FeatureToggles["expressions"]
if !ok {
return false
}
return v
}
func (p *DataSourcePlugin) Load(decoder *json.Decoder, pluginDir string) error {
if err := decoder.Decode(&p); err != nil {
return err
}
if !p.isVersionOne() && !isExpressionsEnabled() {
return errors.New("A plugin version 2 was found but expressions feature toggle are not enabled")
}
if err := p.registerPlugin(pluginDir); err != nil {
return err
}
@ -73,18 +91,35 @@ func (p *DataSourcePlugin) startBackendPlugin(ctx context.Context, log log.Logge
return nil
}
func (p *DataSourcePlugin) isVersionOne() bool {
return !p.SDK
}
func (p *DataSourcePlugin) spawnSubProcess() error {
cmd := ComposePluginStartCommmand(p.Executable)
fullpath := path.Join(p.PluginDir, cmd)
p.client = plugin.NewClient(&plugin.ClientConfig{
HandshakeConfig: handshakeConfig,
Plugins: map[string]plugin.Plugin{p.Id: &datasource.DatasourcePluginImpl{}},
Cmd: exec.Command(fullpath),
AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC},
Logger: LogWrapper{Logger: p.log},
})
var newClient *plugin.Client
if p.isVersionOne() {
newClient = plugin.NewClient(&plugin.ClientConfig{
HandshakeConfig: handshakeConfig,
Plugins: map[string]plugin.Plugin{p.Id: &datasourceV1.DatasourcePluginImpl{}},
Cmd: exec.Command(fullpath),
AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC},
Logger: LogWrapper{Logger: p.log},
})
} else {
newClient = plugin.NewClient(&plugin.ClientConfig{
HandshakeConfig: handshakeConfig,
Plugins: map[string]plugin.Plugin{p.Id: &sdk.DatasourcePluginImpl{}},
Cmd: exec.Command(fullpath),
AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC},
Logger: LogWrapper{Logger: p.log},
})
}
p.client = newClient
rpcClient, err := p.client.Client()
if err != nil {
@ -96,10 +131,22 @@ func (p *DataSourcePlugin) spawnSubProcess() error {
return err
}
plugin := raw.(datasource.DatasourcePlugin)
if p.isVersionOne() {
plugin := raw.(datasourceV1.DatasourcePlugin)
tsdb.RegisterTsdbQueryEndpoint(p.Id, func(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
return wrapper.NewDatasourcePluginWrapper(p.log, plugin), nil
})
return nil
}
plugin, ok := raw.(sdk.DatasourcePlugin)
if !ok {
return fmt.Errorf("unxpected type %T, expeced sdk.DatasourcePlugin", raw)
}
tsdb.RegisterTsdbQueryEndpoint(p.Id, func(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
return wrapper.NewDatasourcePluginWrapper(p.log, plugin), nil
return wrapper.NewDatasourcePluginWrapperV2(p.log, plugin), nil
})
return nil

34
pkg/plugins/datasource_plugin_test.go Normal file
View File

@ -0,0 +1,34 @@
package plugins
import (
"bytes"
"encoding/json"
"testing"
"github.com/stretchr/testify/assert"
)
func TestLoadDatasourceVersion(t *testing.T) {
t.Run("If plugin version is not set, it should be treated as plugin version one", func(t *testing.T) {
pluginJSON, _ := json.Marshal(DataSourcePlugin{})
datasourcePlugin := DataSourcePlugin{}
(&datasourcePlugin).Load(json.NewDecoder(bytes.NewReader(pluginJSON)), "/tmp")
assert.True(t, datasourcePlugin.isVersionOne())
})
t.Run("If plugin version is set to one, it should be treated as plugin version one", func(t *testing.T) {
pluginJSON, _ := json.Marshal(DataSourcePlugin{SDK: false})
datasourcePlugin := DataSourcePlugin{}
(&datasourcePlugin).Load(json.NewDecoder(bytes.NewReader(pluginJSON)), "/tmp")
assert.True(t, datasourcePlugin.isVersionOne())
assert.False(t, datasourcePlugin.SDK)
})
t.Run("If plugin version is set to two, it should not be treated as plugin version one", func(t *testing.T) {
pluginJSON, _ := json.Marshal(DataSourcePlugin{SDK: true})
datasourcePlugin := DataSourcePlugin{}
(&datasourcePlugin).Load(json.NewDecoder(bytes.NewReader(pluginJSON)), "/tmp")
assert.False(t, datasourcePlugin.isVersionOne())
assert.True(t, datasourcePlugin.SDK)
})
}

2
pkg/setting/setting.go
View File

@ -206,6 +206,7 @@ var (
S3TempImageStoreSecretKey string
ImageUploadProvider string
FeatureToggles map[string]bool
)
// TODO move all global vars to this struct
@ -955,6 +956,7 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error {
for _, feature := range util.SplitString(featuresTogglesStr) {
cfg.FeatureToggles[feature] = true
}
FeatureToggles = cfg.FeatureToggles
// check old location for this option
if panelsSection.Key("enable_alpha").MustBool(false) {

37
pkg/tsdb/frame_util.go Normal file
View File

@ -0,0 +1,37 @@
package tsdb
import (
"time"
"github.com/grafana/grafana-plugin-sdk-go/dataframe"
)
// SeriesToFrame converts a TimeSeries to a sdk Frame
func SeriesToFrame(series *TimeSeries) (*dataframe.Frame, error) {
timeVec := make([]*time.Time, len(series.Points))
floatVec := make([]*float64, len(series.Points))
for idx, point := range series.Points {
timeVec[idx], floatVec[idx] = convertTSDBTimePoint(point)
}
frame := dataframe.New(series.Name, dataframe.Labels(series.Tags),
dataframe.NewField("time", dataframe.FieldTypeTime, timeVec),
dataframe.NewField("value", dataframe.FieldTypeNumber, floatVec),
)
return frame, nil
}
// convertTSDBTimePoint coverts a tsdb.TimePoint into two values appropriate
// for Series values.
func convertTSDBTimePoint(point TimePoint) (t *time.Time, f *float64) {
timeIdx, valueIdx := 1, 0
if point[timeIdx].Valid { // Assuming valid is null?
tI := int64(point[timeIdx].Float64)
uT := time.Unix(tI/int64(1e+3), (tI%int64(1e+3))*int64(1e+6)) // time.Time from millisecond unix ts
t = &uT
}
if point[valueIdx].Valid {
f = &point[valueIdx].Float64
}
return
}

1
pkg/tsdb/models.go
View File

@ -32,6 +32,7 @@ type QueryResult struct {
Meta *simplejson.Json `json:"meta,omitempty"`
Series TimeSeriesSlice `json:"series"`
Tables []*Table `json:"tables"`
Dataframes [][]byte `json:"dataframes"`
}
type TimeSeries struct {

35
vendor/github.com/apache/arrow/go/arrow/.gitignore generated vendored Normal file
View File

@ -0,0 +1,35 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
### Go template
# Binaries for programs and plugins
*.exe
*.dll
*.so
*.dylib
*.o
# Test binary, build with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
.glide/
bin/
vendor/

44
vendor/github.com/apache/arrow/go/arrow/Gopkg.lock generated vendored Normal file
View File

@ -0,0 +1,44 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
digest = "1:56c130d885a4aacae1dd9c7b71cfe39912c7ebc1ff7d2b46083c8812996dc43b"
name = "github.com/davecgh/go-spew"
packages = ["spew"]
pruneopts = ""
revision = "346938d642f2ec3594ed81d874461961cd0faa76"
version = "v1.1.0"
[[projects]]
digest = "1:1d7e1867c49a6dd9856598ef7c3123604ea3daabf5b83f303ff457bcbc410b1d"
name = "github.com/pkg/errors"
packages = ["."]
pruneopts = ""
revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4"
version = "v0.8.1"
[[projects]]
digest = "1:256484dbbcd271f9ecebc6795b2df8cad4c458dd0f5fd82a8c2fa0c29f233411"
name = "github.com/pmezard/go-difflib"
packages = ["difflib"]
pruneopts = ""
revision = "792786c7400a136282c1664665ae0a8db921c6c2"
version = "v1.0.0"
[[projects]]
digest = "1:2d0dc026c4aef5e2f3a0e06a4dabe268b840d8f63190cf6894e02134a03f52c5"
name = "github.com/stretchr/testify"
packages = ["assert"]
pruneopts = ""
revision = "b91bfb9ebec76498946beb6af7c0230c7cc7ba6c"
version = "v1.2.0"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
input-imports = [
"github.com/pkg/errors",
"github.com/stretchr/testify/assert",
]
solver-name = "gps-cdcl"
solver-version = 1

23
vendor/github.com/apache/arrow/go/arrow/Gopkg.toml generated vendored Normal file
View File

@ -0,0 +1,23 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[[constraint]]
name = "github.com/stretchr/testify"
version = "1.2.0"
[[constraint]]
name = "github.com/pkg/errors"
version = "0.8.1"

54
vendor/github.com/apache/arrow/go/arrow/Makefile generated vendored Normal file
View File

@ -0,0 +1,54 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
GO_BUILD=go build
GO_GEN=go generate
GO_TEST?=go test
GOPATH=$(realpath ../../../../../..)
GO_SOURCES := $(shell find . -path ./_lib -prune -o -name '*.go' -not -name '*_test.go')
ALL_SOURCES := $(shell find . -path ./_lib -prune -o -name '*.go' -name '*.s' -not -name '*_test.go')
SOURCES_NO_VENDOR := $(shell find . -path ./vendor -prune -o -name "*.go" -not -name '*_test.go' -print)
.PHONEY: test bench assembly generate
assembly:
@$(MAKE) -C memory assembly
@$(MAKE) -C math assembly
generate: bin/tmpl
bin/tmpl -i -data=numeric.tmpldata type_traits_numeric.gen.go.tmpl type_traits_numeric.gen_test.go.tmpl array/numeric.gen.go.tmpl array/numericbuilder.gen_test.go.tmpl array/numericbuilder.gen.go.tmpl array/bufferbuilder_numeric.gen.go.tmpl
bin/tmpl -i -data=datatype_numeric.gen.go.tmpldata datatype_numeric.gen.go.tmpl
@$(MAKE) -C math generate
fmt: $(SOURCES_NO_VENDOR)
goimports -w $^
bench: $(GO_SOURCES) | assembly
$(GO_TEST) $(GO_TEST_ARGS) -bench=. -run=- ./...
bench-noasm: $(GO_SOURCES)
$(GO_TEST) $(GO_TEST_ARGS) -tags='noasm' -bench=. -run=- ./...
test: $(GO_SOURCES) | assembly
$(GO_TEST) $(GO_TEST_ARGS) ./...
test-noasm: $(GO_SOURCES)
$(GO_TEST) $(GO_TEST_ARGS) -tags='noasm' ./...
bin/tmpl: _tools/tmpl/main.go
$(GO_BUILD) -o $@ ./_tools/tmpl

206
vendor/github.com/apache/arrow/go/arrow/array/array.go generated vendored Normal file
View File

@ -0,0 +1,206 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array // import "github.com/apache/arrow/go/arrow/array"
import (
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
)
// A type which satisfies array.Interface represents an immutable sequence of values.
type Interface interface {
// DataType returns the type metadata for this instance.
DataType() arrow.DataType
// NullN returns the number of null values in the array.
NullN() int
// NullBitmapBytes returns a byte slice of the validity bitmap.
NullBitmapBytes() []byte
// IsNull returns true if value at index is null.
// NOTE: IsNull will panic if NullBitmapBytes is not empty and 0 > i ≥ Len.
IsNull(i int) bool
// IsValid returns true if value at index is not null.
// NOTE: IsValid will panic if NullBitmapBytes is not empty and 0 > i ≥ Len.
IsValid(i int) bool
Data() *Data
// Len returns the number of elements in the array.
Len() int
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
Retain()
// Release decreases the reference count by 1.
// Release may be called simultaneously from multiple goroutines.
// When the reference count goes to zero, the memory is freed.
Release()
}
const (
// UnknownNullCount specifies the NullN should be calculated from the null bitmap buffer.
UnknownNullCount = -1
)
type array struct {
refCount int64
data *Data
nullBitmapBytes []byte
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (a *array) Retain() {
atomic.AddInt64(&a.refCount, 1)
}
// Release decreases the reference count by 1.
// Release may be called simultaneously from multiple goroutines.
// When the reference count goes to zero, the memory is freed.
func (a *array) Release() {
debug.Assert(atomic.LoadInt64(&a.refCount) > 0, "too many releases")
if atomic.AddInt64(&a.refCount, -1) == 0 {
a.data.Release()
a.data, a.nullBitmapBytes = nil, nil
}
}
// DataType returns the type metadata for this instance.
func (a *array) DataType() arrow.DataType { return a.data.dtype }
// NullN returns the number of null values in the array.
func (a *array) NullN() int {
if a.data.nulls < 0 {
a.data.nulls = a.data.length - bitutil.CountSetBits(a.nullBitmapBytes, a.data.offset, a.data.length)
}
return a.data.nulls
}
// NullBitmapBytes returns a byte slice of the validity bitmap.
func (a *array) NullBitmapBytes() []byte { return a.nullBitmapBytes }
func (a *array) Data() *Data { return a.data }
// Len returns the number of elements in the array.
func (a *array) Len() int { return a.data.length }
// IsNull returns true if value at index is null.
// NOTE: IsNull will panic if NullBitmapBytes is not empty and 0 > i ≥ Len.
func (a *array) IsNull(i int) bool {
return len(a.nullBitmapBytes) != 0 && bitutil.BitIsNotSet(a.nullBitmapBytes, a.data.offset+i)
}
// IsValid returns true if value at index is not null.
// NOTE: IsValid will panic if NullBitmapBytes is not empty and 0 > i ≥ Len.
func (a *array) IsValid(i int) bool {
return len(a.nullBitmapBytes) == 0 || bitutil.BitIsSet(a.nullBitmapBytes, a.data.offset+i)
}
func (a *array) setData(data *Data) {
if a.data != nil {
a.data.Release()
}
data.Retain()
if len(data.buffers) > 0 && data.buffers[0] != nil {
a.nullBitmapBytes = data.buffers[0].Bytes()
}
a.data = data
}
func (a *array) Offset() int {
return a.data.Offset()
}
type arrayConstructorFn func(*Data) Interface
var (
makeArrayFn [32]arrayConstructorFn
)
func unsupportedArrayType(data *Data) Interface {
panic("unsupported data type: " + data.dtype.ID().String())
}
func invalidDataType(data *Data) Interface {
panic("invalid data type: " + data.dtype.ID().String())
}
// MakeFromData constructs a strongly-typed array instance from generic Data.
func MakeFromData(data *Data) Interface {
return makeArrayFn[byte(data.dtype.ID()&0x1f)](data)
}
// NewSlice constructs a zero-copy slice of the array with the indicated
// indices i and j, corresponding to array[i:j].
// The returned array must be Release()'d after use.
//
// NewSlice panics if the slice is outside the valid range of the input array.
// NewSlice panics if j < i.
func NewSlice(arr Interface, i, j int64) Interface {
data := NewSliceData(arr.Data(), i, j)
slice := MakeFromData(data)
data.Release()
return slice
}
func init() {
makeArrayFn = [...]arrayConstructorFn{
arrow.NULL: func(data *Data) Interface { return NewNullData(data) },
arrow.BOOL: func(data *Data) Interface { return NewBooleanData(data) },
arrow.UINT8: func(data *Data) Interface { return NewUint8Data(data) },
arrow.INT8: func(data *Data) Interface { return NewInt8Data(data) },
arrow.UINT16: func(data *Data) Interface { return NewUint16Data(data) },
arrow.INT16: func(data *Data) Interface { return NewInt16Data(data) },
arrow.UINT32: func(data *Data) Interface { return NewUint32Data(data) },
arrow.INT32: func(data *Data) Interface { return NewInt32Data(data) },
arrow.UINT64: func(data *Data) Interface { return NewUint64Data(data) },
arrow.INT64: func(data *Data) Interface { return NewInt64Data(data) },
arrow.FLOAT16: func(data *Data) Interface { return NewFloat16Data(data) },
arrow.FLOAT32: func(data *Data) Interface { return NewFloat32Data(data) },
arrow.FLOAT64: func(data *Data) Interface { return NewFloat64Data(data) },
arrow.STRING: func(data *Data) Interface { return NewStringData(data) },
arrow.BINARY: func(data *Data) Interface { return NewBinaryData(data) },
arrow.FIXED_SIZE_BINARY: func(data *Data) Interface { return NewFixedSizeBinaryData(data) },
arrow.DATE32: func(data *Data) Interface { return NewDate32Data(data) },
arrow.DATE64: func(data *Data) Interface { return NewDate64Data(data) },
arrow.TIMESTAMP: func(data *Data) Interface { return NewTimestampData(data) },
arrow.TIME32: func(data *Data) Interface { return NewTime32Data(data) },
arrow.TIME64: func(data *Data) Interface { return NewTime64Data(data) },
arrow.INTERVAL: func(data *Data) Interface { return NewIntervalData(data) },
arrow.DECIMAL: func(data *Data) Interface { return NewDecimal128Data(data) },
arrow.LIST: func(data *Data) Interface { return NewListData(data) },
arrow.STRUCT: func(data *Data) Interface { return NewStructData(data) },
arrow.UNION: unsupportedArrayType,
arrow.DICTIONARY: unsupportedArrayType,
arrow.MAP: unsupportedArrayType,
arrow.EXTENSION: unsupportedArrayType,
arrow.FIXED_SIZE_LIST: func(data *Data) Interface { return NewFixedSizeListData(data) },
arrow.DURATION: func(data *Data) Interface { return NewDurationData(data) },
// invalid data types to fill out array size 2⁵-1
31: invalidDataType,
}
}

134
vendor/github.com/apache/arrow/go/arrow/array/binary.go generated vendored Normal file
View File

@ -0,0 +1,134 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"bytes"
"fmt"
"strings"
"unsafe"
"github.com/apache/arrow/go/arrow"
)
// A type which represents an immutable sequence of variable-length binary strings.
type Binary struct {
array
valueOffsets []int32
valueBytes []byte
}
// NewBinaryData constructs a new Binary array from data.
func NewBinaryData(data *Data) *Binary {
a := &Binary{}
a.refCount = 1
a.setData(data)
return a
}
// Value returns the slice at index i. This value should not be mutated.
func (a *Binary) Value(i int) []byte {
if i < 0 || i >= a.array.data.length {
panic("arrow/array: index out of range")
}
idx := a.array.data.offset + i
return a.valueBytes[a.valueOffsets[idx]:a.valueOffsets[idx+1]]
}
// ValueString returns the string at index i without performing additional allocations.
// The string is only valid for the lifetime of the Binary array.
func (a *Binary) ValueString(i int) string {
b := a.Value(i)
return *(*string)(unsafe.Pointer(&b))
}
func (a *Binary) ValueOffset(i int) int {
if i < 0 || i >= a.array.data.length {
panic("arrow/array: index out of range")
}
return int(a.valueOffsets[a.array.data.offset+i])
}
func (a *Binary) ValueLen(i int) int {
if i < 0 || i >= a.array.data.length {
panic("arrow/array: index out of range")
}
beg := a.array.data.offset + i
return int(a.valueOffsets[beg+1] - a.valueOffsets[beg])
}
func (a *Binary) ValueOffsets() []int32 {
beg := a.array.data.offset
end := beg + a.array.data.length + 1
return a.valueOffsets[beg:end]
}
func (a *Binary) ValueBytes() []byte {
beg := a.array.data.offset
end := beg + a.array.data.length
return a.valueBytes[a.valueOffsets[beg]:a.valueOffsets[end]]
}
func (a *Binary) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i := 0; i < a.Len(); i++ {
if i > 0 {
o.WriteString(" ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%q", a.ValueString(i))
}
}
o.WriteString("]")
return o.String()
}
func (a *Binary) setData(data *Data) {
if len(data.buffers) != 3 {
panic("len(data.buffers) != 3")
}
a.array.setData(data)
if valueData := data.buffers[2]; valueData != nil {
a.valueBytes = valueData.Bytes()
}
if valueOffsets := data.buffers[1]; valueOffsets != nil {
a.valueOffsets = arrow.Int32Traits.CastFromBytes(valueOffsets.Bytes())
}
}
func arrayEqualBinary(left, right *Binary) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if bytes.Compare(left.Value(i), right.Value(i)) != 0 {
return false
}
}
return true
}
var (
_ Interface = (*Binary)(nil)
)

217
vendor/github.com/apache/arrow/go/arrow/array/binarybuilder.go generated vendored Normal file
View File

@ -0,0 +1,217 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"math"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
const (
binaryArrayMaximumCapacity = math.MaxInt32
)
// A BinaryBuilder is used to build a Binary array using the Append methods.
type BinaryBuilder struct {
builder
dtype arrow.BinaryDataType
offsets *int32BufferBuilder
values *byteBufferBuilder
}
func NewBinaryBuilder(mem memory.Allocator, dtype arrow.BinaryDataType) *BinaryBuilder {
b := &BinaryBuilder{
builder: builder{refCount: 1, mem: mem},
dtype: dtype,
offsets: newInt32BufferBuilder(mem),
values: newByteBufferBuilder(mem),
}
return b
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (b *BinaryBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
if b.offsets != nil {
b.offsets.Release()
b.offsets = nil
}
if b.values != nil {
b.values.Release()
b.values = nil
}
}
}
func (b *BinaryBuilder) Append(v []byte) {
b.Reserve(1)
b.appendNextOffset()
b.values.Append(v)
b.UnsafeAppendBoolToBitmap(true)
}
func (b *BinaryBuilder) AppendString(v string) {
b.Append([]byte(v))
}
func (b *BinaryBuilder) AppendNull() {
b.Reserve(1)
b.appendNextOffset()
b.UnsafeAppendBoolToBitmap(false)
}
// AppendValues will append the values in the v slice. The valid slice determines which values
// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
// all values in v are appended and considered valid.
func (b *BinaryBuilder) AppendValues(v [][]byte, valid []bool) {
if len(v) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
if len(v) == 0 {
return
}
b.Reserve(len(v))
for _, vv := range v {
b.appendNextOffset()
b.values.Append(vv)
}
b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
}
// AppendStringValues will append the values in the v slice. The valid slice determines which values
// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
// all values in v are appended and considered valid.
func (b *BinaryBuilder) AppendStringValues(v []string, valid []bool) {
if len(v) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
if len(v) == 0 {
return
}
b.Reserve(len(v))
for _, vv := range v {
b.appendNextOffset()
b.values.Append([]byte(vv))
}
b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
}
func (b *BinaryBuilder) Value(i int) []byte {
offsets := b.offsets.Values()
start := int(offsets[i])
var end int
if i == (b.length - 1) {
end = b.values.Len()
} else {
end = int(offsets[i+1])
}
return b.values.Bytes()[start:end]
}
func (b *BinaryBuilder) init(capacity int) {
b.builder.init(capacity)
b.offsets.resize((capacity + 1) * arrow.Int32SizeBytes)
}
// DataLen returns the number of bytes in the data array.
func (b *BinaryBuilder) DataLen() int { return b.values.length }
// DataCap returns the total number of bytes that can be stored
// without allocating additional memory.
func (b *BinaryBuilder) DataCap() int { return b.values.capacity }
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *BinaryBuilder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// ReserveData ensures there is enough space for appending n bytes
// by checking the capacity and resizing the data buffer if necessary.
func (b *BinaryBuilder) ReserveData(n int) {
if b.values.capacity < b.values.length+n {
b.values.resize(b.values.Len() + n)
}
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may be reduced.
func (b *BinaryBuilder) Resize(n int) {
b.offsets.resize((n + 1) * arrow.Int32SizeBytes)
b.builder.resize(n, b.init)
}
// NewArray creates a Binary array from the memory buffers used by the builder and resets the BinaryBuilder
// so it can be used to build a new array.
func (b *BinaryBuilder) NewArray() Interface {
return b.NewBinaryArray()
}
// NewBinaryArray creates a Binary array from the memory buffers used by the builder and resets the BinaryBuilder
// so it can be used to build a new array.
func (b *BinaryBuilder) NewBinaryArray() (a *Binary) {
data := b.newData()
a = NewBinaryData(data)
data.Release()
return
}
func (b *BinaryBuilder) newData() (data *Data) {
b.appendNextOffset()
offsets, values := b.offsets.Finish(), b.values.Finish()
data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, offsets, values}, nil, b.nulls, 0)
if offsets != nil {
offsets.Release()
}
if values != nil {
values.Release()
}
b.builder.reset()
return
}
func (b *BinaryBuilder) appendNextOffset() {
numBytes := b.values.Len()
// TODO(sgc): check binaryArrayMaximumCapacity?
b.offsets.AppendValue(int32(numBytes))
}
var (
_ Builder = (*BinaryBuilder)(nil)
)

95
vendor/github.com/apache/arrow/go/arrow/array/boolean.go generated vendored Normal file
View File

@ -0,0 +1,95 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"fmt"
"strings"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/bitutil"
"github.com/apache/arrow/go/arrow/memory"
)
// A type which represents an immutable sequence of boolean values.
type Boolean struct {
array
values []byte
}
// NewBoolean creates a boolean array from the data memory.Buffer and contains length elements.
// The nullBitmap buffer can be nil of there are no null values.
// If nulls is not known, use UnknownNullCount to calculate the value of NullN at runtime from the nullBitmap buffer.
func NewBoolean(length int, data *memory.Buffer, nullBitmap *memory.Buffer, nulls int) *Boolean {
return NewBooleanData(NewData(arrow.FixedWidthTypes.Boolean, length, []*memory.Buffer{nullBitmap, data}, nil, nulls, 0))
}
func NewBooleanData(data *Data) *Boolean {
a := &Boolean{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Boolean) Value(i int) bool {
if i < 0 || i >= a.array.data.length {
panic("arrow/array: index out of range")
}
return bitutil.BitIsSet(a.values, a.array.data.offset+i)
}
func (a *Boolean) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i := 0; i < a.Len(); i++ {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", a.Value(i))
}
}
o.WriteString("]")
return o.String()
}
func (a *Boolean) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = vals.Bytes()
}
}
func arrayEqualBoolean(left, right *Boolean) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
var (
_ Interface = (*Boolean)(nil)
)

165
vendor/github.com/apache/arrow/go/arrow/array/booleanbuilder.go generated vendored Normal file
View File

@ -0,0 +1,165 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
type BooleanBuilder struct {
builder
data *memory.Buffer
rawData []byte
}
func NewBooleanBuilder(mem memory.Allocator) *BooleanBuilder {
return &BooleanBuilder{builder: builder{refCount: 1, mem: mem}}
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (b *BooleanBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
}
}
func (b *BooleanBuilder) Append(v bool) {
b.Reserve(1)
b.UnsafeAppend(v)
}
func (b *BooleanBuilder) AppendByte(v byte) {
b.Reserve(1)
b.UnsafeAppend(v != 0)
}
func (b *BooleanBuilder) AppendNull() {
b.Reserve(1)
b.UnsafeAppendBoolToBitmap(false)
}
func (b *BooleanBuilder) UnsafeAppend(v bool) {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
if v {
bitutil.SetBit(b.rawData, b.length)
} else {
bitutil.ClearBit(b.rawData, b.length)
}
b.length++
}
func (b *BooleanBuilder) AppendValues(v []bool, valid []bool) {
if len(v) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
if len(v) == 0 {
return
}
b.Reserve(len(v))
for i, vv := range v {
bitutil.SetBitTo(b.rawData, b.length+i, vv)
}
b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
}
func (b *BooleanBuilder) init(capacity int) {
b.builder.init(capacity)
b.data = memory.NewResizableBuffer(b.mem)
bytesN := arrow.BooleanTraits.BytesRequired(capacity)
b.data.Resize(bytesN)
b.rawData = b.data.Bytes()
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *BooleanBuilder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *BooleanBuilder) Resize(n int) {
if n < minBuilderCapacity {
n = minBuilderCapacity
}
if b.capacity == 0 {
b.init(n)
} else {
b.builder.resize(n, b.init)
b.data.Resize(arrow.BooleanTraits.BytesRequired(n))
b.rawData = b.data.Bytes()
}
}
// NewArray creates a Boolean array from the memory buffers used by the builder and resets the BooleanBuilder
// so it can be used to build a new array.
func (b *BooleanBuilder) NewArray() Interface {
return b.NewBooleanArray()
}
// NewBooleanArray creates a Boolean array from the memory buffers used by the builder and resets the BooleanBuilder
// so it can be used to build a new array.
func (b *BooleanBuilder) NewBooleanArray() (a *Boolean) {
data := b.newData()
a = NewBooleanData(data)
data.Release()
return
}
func (b *BooleanBuilder) newData() *Data {
bytesRequired := arrow.BooleanTraits.BytesRequired(b.length)
if bytesRequired > 0 && bytesRequired < b.data.Len() {
// trim buffers
b.data.Resize(bytesRequired)
}
res := NewData(arrow.FixedWidthTypes.Boolean, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0)
b.reset()
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
return res
}
var (
_ Builder = (*BooleanBuilder)(nil)
)

127
vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder.go generated vendored Normal file
View File

@ -0,0 +1,127 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"sync/atomic"
"github.com/apache/arrow/go/arrow/internal/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
// A bufferBuilder provides common functionality for populating memory with a sequence of type-specific values.
// Specialized implementations provide type-safe APIs for appending and accessing the memory.
type bufferBuilder struct {
refCount int64
mem memory.Allocator
buffer *memory.Buffer
length int
capacity int
bytes []byte
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (b *bufferBuilder) Retain() {
atomic.AddInt64(&b.refCount, 1)
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (b *bufferBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.buffer != nil {
b.buffer.Release()
b.buffer, b.bytes = nil, nil
}
}
}
// Len returns the length of the memory buffer in bytes.
func (b *bufferBuilder) Len() int { return b.length }
// Cap returns the total number of bytes that can be stored without allocating additional memory.
func (b *bufferBuilder) Cap() int { return b.capacity }
// Bytes returns a slice of length b.Len().
// The slice is only valid for use until the next buffer modification. That is, until the next call
// to Advance, Reset, Finish or any Append function. The slice aliases the buffer content at least until the next
// buffer modification.
func (b *bufferBuilder) Bytes() []byte { return b.bytes[:b.length] }
func (b *bufferBuilder) resize(elements int) {
if b.buffer == nil {
b.buffer = memory.NewResizableBuffer(b.mem)
}
b.buffer.Resize(elements)
oldCapacity := b.capacity
b.capacity = b.buffer.Cap()
b.bytes = b.buffer.Buf()
if b.capacity > oldCapacity {
memory.Set(b.bytes[oldCapacity:], 0)
}
}
// Advance increases the buffer by length and initializes the skipped bytes to zero.
func (b *bufferBuilder) Advance(length int) {
if b.capacity < b.length+length {
newCapacity := bitutil.NextPowerOf2(b.length + length)
b.resize(newCapacity)
}
b.length += length
}
// Append appends the contents of v to the buffer, resizing it if necessary.
func (b *bufferBuilder) Append(v []byte) {
if b.capacity < b.length+len(v) {
newCapacity := bitutil.NextPowerOf2(b.length + len(v))
b.resize(newCapacity)
}
b.unsafeAppend(v)
}
// Reset returns the buffer to an empty state. Reset releases the memory and sets the length and capacity to zero.
func (b *bufferBuilder) Reset() {
if b.buffer != nil {
b.buffer.Release()
}
b.buffer, b.bytes = nil, nil
b.capacity, b.length = 0, 0
}
// Finish TODO(sgc)
func (b *bufferBuilder) Finish() (buffer *memory.Buffer) {
if b.length > 0 {
b.buffer.ResizeNoShrink(b.length)
}
buffer = b.buffer
b.buffer = nil
b.Reset()
return
}
func (b *bufferBuilder) unsafeAppend(data []byte) {
copy(b.bytes[b.length:], data)
b.length += len(data)
}

30
vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder_byte.go generated vendored Normal file
View File

@ -0,0 +1,30 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import "github.com/apache/arrow/go/arrow/memory"
type byteBufferBuilder struct {
bufferBuilder
}
func newByteBufferBuilder(mem memory.Allocator) *byteBufferBuilder {
return &byteBufferBuilder{bufferBuilder: bufferBuilder{refCount: 1, mem: mem}}
}
func (b *byteBufferBuilder) Values() []byte { return b.Bytes() }
func (b *byteBufferBuilder) Value(i int) byte { return b.bytes[i] }

58
vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder_numeric.gen.go generated vendored Normal file
View File

@ -0,0 +1,58 @@
// Code generated by array/bufferbuilder_numeric.gen.go.tmpl. DO NOT EDIT.
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/bitutil"
"github.com/apache/arrow/go/arrow/memory"
)
type int32BufferBuilder struct {
bufferBuilder
}
func newInt32BufferBuilder(mem memory.Allocator) *int32BufferBuilder {
return &int32BufferBuilder{bufferBuilder: bufferBuilder{refCount: 1, mem: mem}}
}
// AppendValues appends the contents of v to the buffer, growing the buffer as needed.
func (b *int32BufferBuilder) AppendValues(v []int32) { b.Append(arrow.Int32Traits.CastToBytes(v)) }
// Values returns a slice of length b.Len().
// The slice is only valid for use until the next buffer modification. That is, until the next call
// to Advance, Reset, Finish or any Append function. The slice aliases the buffer content at least until the next
// buffer modification.
func (b *int32BufferBuilder) Values() []int32 { return arrow.Int32Traits.CastFromBytes(b.Bytes()) }
// Value returns the int32 element at the index i. Value will panic if i is negative or ≥ Len.
func (b *int32BufferBuilder) Value(i int) int32 { return b.Values()[i] }
// Len returns the number of int32 elements in the buffer.
func (b *int32BufferBuilder) Len() int { return b.length / arrow.Int32SizeBytes }
// AppendValue appends v to the buffer, growing the buffer as needed.
func (b *int32BufferBuilder) AppendValue(v int32) {
if b.capacity < b.length+arrow.Int32SizeBytes {
newCapacity := bitutil.NextPowerOf2(b.length + arrow.Int32SizeBytes)
b.resize(newCapacity)
}
arrow.Int32Traits.PutValue(b.bytes[b.length:], v)
b.length += arrow.Int32SizeBytes
}

61
vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder_numeric.gen.go.tmpl generated vendored Normal file
View File

@ -0,0 +1,61 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/bitutil"
"github.com/apache/arrow/go/arrow/memory"
)
{{range .In}}
{{$TypeNamePrefix := .name}}
{{if .Opt.BufferBuilder}}
type {{$TypeNamePrefix}}BufferBuilder struct {
bufferBuilder
}
func new{{.Name}}BufferBuilder(mem memory.Allocator) *{{$TypeNamePrefix}}BufferBuilder {
return &{{$TypeNamePrefix}}BufferBuilder{bufferBuilder:bufferBuilder{refCount: 1, mem:mem}}
}
// AppendValues appends the contents of v to the buffer, growing the buffer as needed.
func (b *{{$TypeNamePrefix}}BufferBuilder) AppendValues(v []{{.Type}}) { b.Append(arrow.{{.Name}}Traits.CastToBytes(v)) }
// Values returns a slice of length b.Len().
// The slice is only valid for use until the next buffer modification. That is, until the next call
// to Advance, Reset, Finish or any Append function. The slice aliases the buffer content at least until the next
// buffer modification.
func (b *{{$TypeNamePrefix}}BufferBuilder) Values() []{{.Type}} { return arrow.{{.Name}}Traits.CastFromBytes(b.Bytes()) }
// Value returns the {{.Type}} element at the index i. Value will panic if i is negative or ≥ Len.
func (b *{{$TypeNamePrefix}}BufferBuilder) Value(i int) {{.Type}} { return b.Values()[i] }
// Len returns the number of {{.Type}} elements in the buffer.
func (b *{{$TypeNamePrefix}}BufferBuilder) Len() int { return b.length/arrow.{{.Name}}SizeBytes }
// AppendValue appends v to the buffer, growing the buffer as needed.
func (b *{{$TypeNamePrefix}}BufferBuilder) AppendValue(v {{.Type}}) {
if b.capacity < b.length+arrow.{{.Name}}SizeBytes {
newCapacity := bitutil.NextPowerOf2(b.length + arrow.{{.Name}}SizeBytes)
b.resize(newCapacity)
}
arrow.{{.Name}}Traits.PutValue(b.bytes[b.length:], v)
b.length+=arrow.{{.Name}}SizeBytes
}
{{end}}
{{end}}

273
vendor/github.com/apache/arrow/go/arrow/array/builder.go generated vendored Normal file
View File

@ -0,0 +1,273 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"fmt"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/bitutil"
"github.com/apache/arrow/go/arrow/memory"
)
const (
minBuilderCapacity = 1 << 5
)
// Builder provides an interface to build arrow arrays.
type Builder interface {
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
Retain()
// Release decreases the reference count by 1.
Release()
// Len returns the number of elements in the array builder.
Len() int
// Cap returns the total number of elements that can be stored
// without allocating additional memory.
Cap() int
// NullN returns the number of null values in the array builder.
NullN() int
// AppendNull adds a new null value to the array being built.
AppendNull()
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
Reserve(n int)
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
Resize(n int)
// NewArray creates a new array from the memory buffers used
// by the builder and resets the Builder so it can be used to build
// a new array.
NewArray() Interface
init(capacity int)
resize(newBits int, init func(int))
}
// builder provides common functionality for managing the validity bitmap (nulls) when building arrays.
type builder struct {
refCount int64
mem memory.Allocator
nullBitmap *memory.Buffer
nulls int
length int
capacity int
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (b *builder) Retain() {
atomic.AddInt64(&b.refCount, 1)
}
// Len returns the number of elements in the array builder.
func (b *builder) Len() int { return b.length }
// Cap returns the total number of elements that can be stored without allocating additional memory.
func (b *builder) Cap() int { return b.capacity }
// NullN returns the number of null values in the array builder.
func (b *builder) NullN() int { return b.nulls }
func (b *builder) init(capacity int) {
toAlloc := bitutil.CeilByte(capacity) / 8
b.nullBitmap = memory.NewResizableBuffer(b.mem)
b.nullBitmap.Resize(toAlloc)
b.capacity = capacity
memory.Set(b.nullBitmap.Buf(), 0)
}
func (b *builder) reset() {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
b.nulls = 0
b.length = 0
b.capacity = 0
}
func (b *builder) resize(newBits int, init func(int)) {
if b.nullBitmap == nil {
init(newBits)
return
}
newBytesN := bitutil.CeilByte(newBits) / 8
oldBytesN := b.nullBitmap.Len()
b.nullBitmap.Resize(newBytesN)
b.capacity = newBits
if oldBytesN < newBytesN {
// TODO(sgc): necessary?
memory.Set(b.nullBitmap.Buf()[oldBytesN:], 0)
}
if newBits < b.length {
b.length = newBits
b.nulls = newBits - bitutil.CountSetBits(b.nullBitmap.Buf(), 0, newBits)
}
}
func (b *builder) reserve(elements int, resize func(int)) {
if b.length+elements > b.capacity {
newCap := bitutil.NextPowerOf2(b.length + elements)
resize(newCap)
}
}
// unsafeAppendBoolsToBitmap appends the contents of valid to the validity bitmap.
// As an optimization, if the valid slice is empty, the next length bits will be set to valid (not null).
func (b *builder) unsafeAppendBoolsToBitmap(valid []bool, length int) {
if len(valid) == 0 {
b.unsafeSetValid(length)
return
}
byteOffset := b.length / 8
bitOffset := byte(b.length % 8)
nullBitmap := b.nullBitmap.Bytes()
bitSet := nullBitmap[byteOffset]
for _, v := range valid {
if bitOffset == 8 {
bitOffset = 0
nullBitmap[byteOffset] = bitSet
byteOffset++
bitSet = nullBitmap[byteOffset]
}
if v {
bitSet |= bitutil.BitMask[bitOffset]
} else {
bitSet &= bitutil.FlippedBitMask[bitOffset]
b.nulls++
}
bitOffset++
}
if bitOffset != 0 {
nullBitmap[byteOffset] = bitSet
}
b.length += len(valid)
}
// unsafeSetValid sets the next length bits to valid in the validity bitmap.
func (b *builder) unsafeSetValid(length int) {
padToByte := min(8-(b.length%8), length)
if padToByte == 8 {
padToByte = 0
}
bits := b.nullBitmap.Bytes()
for i := b.length; i < b.length+padToByte; i++ {
bitutil.SetBit(bits, i)
}
start := (b.length + padToByte) / 8
fastLength := (length - padToByte) / 8
memory.Set(bits[start:start+fastLength], 0xff)
newLength := b.length + length
// trailing bytes
for i := b.length + padToByte + (fastLength * 8); i < newLength; i++ {
bitutil.SetBit(bits, i)
}
b.length = newLength
}
func (b *builder) UnsafeAppendBoolToBitmap(isValid bool) {
if isValid {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
} else {
b.nulls++
}
b.length++
}
func newBuilder(mem memory.Allocator, dtype arrow.DataType) Builder {
// FIXME(sbinet): use a type switch on dtype instead?
switch dtype.ID() {
case arrow.NULL:
case arrow.BOOL:
return NewBooleanBuilder(mem)
case arrow.UINT8:
return NewUint8Builder(mem)
case arrow.INT8:
return NewInt8Builder(mem)
case arrow.UINT16:
return NewUint16Builder(mem)
case arrow.INT16:
return NewInt16Builder(mem)
case arrow.UINT32:
return NewUint32Builder(mem)
case arrow.INT32:
return NewInt32Builder(mem)
case arrow.UINT64:
return NewUint64Builder(mem)
case arrow.INT64:
return NewInt64Builder(mem)
case arrow.FLOAT16:
return NewFloat16Builder(mem)
case arrow.FLOAT32:
return NewFloat32Builder(mem)
case arrow.FLOAT64:
return NewFloat64Builder(mem)
case arrow.STRING:
return NewStringBuilder(mem)
case arrow.BINARY:
return NewBinaryBuilder(mem, arrow.BinaryTypes.Binary)
case arrow.FIXED_SIZE_BINARY:
typ := dtype.(*arrow.FixedSizeBinaryType)
return NewFixedSizeBinaryBuilder(mem, typ)
case arrow.DATE32:
case arrow.DATE64:
case arrow.TIMESTAMP:
case arrow.TIME32:
typ := dtype.(*arrow.Time32Type)
return NewTime32Builder(mem, typ)
case arrow.TIME64:
typ := dtype.(*arrow.Time64Type)
return NewTime64Builder(mem, typ)
case arrow.INTERVAL:
case arrow.DECIMAL:
case arrow.LIST:
typ := dtype.(*arrow.ListType)
return NewListBuilder(mem, typ.Elem())
case arrow.STRUCT:
typ := dtype.(*arrow.StructType)
return NewStructBuilder(mem, typ)
case arrow.UNION:
case arrow.DICTIONARY:
case arrow.MAP:
case arrow.EXTENSION:
case arrow.FIXED_SIZE_LIST:
typ := dtype.(*arrow.FixedSizeListType)
return NewFixedSizeListBuilder(mem, typ.Len(), typ.Elem())
case arrow.DURATION:
}
panic(fmt.Errorf("arrow/array: unsupported builder for %T", dtype))
}

474
vendor/github.com/apache/arrow/go/arrow/array/compare.go generated vendored Normal file
View File

@ -0,0 +1,474 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"math"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/float16"
"github.com/pkg/errors"
)
// RecordEqual reports whether the two provided records are equal.
func RecordEqual(left, right Record) bool {
switch {
case left.NumCols() != right.NumCols():
return false
case left.NumRows() != right.NumRows():
return false
}
for i := range left.Columns() {
lc := left.Column(i)
rc := right.Column(i)
if !ArrayEqual(lc, rc) {
return false
}
}
return true
}
// RecordApproxEqual reports whether the two provided records are approximately equal.
// For non-floating point columns, it is equivalent to RecordEqual.
func RecordApproxEqual(left, right Record, opts ...EqualOption) bool {
switch {
case left.NumCols() != right.NumCols():
return false
case left.NumRows() != right.NumRows():
return false
}
opt := newEqualOption(opts...)
for i := range left.Columns() {
lc := left.Column(i)
rc := right.Column(i)
if !arrayApproxEqual(lc, rc, opt) {
return false
}
}
return true
}
// ArrayEqual reports whether the two provided arrays are equal.
func ArrayEqual(left, right Interface) bool {
switch {
case !baseArrayEqual(left, right):
return false
case left.Len() == 0:
return true
case left.NullN() == left.Len():
return true
}
// at this point, we know both arrays have same type, same length, same number of nulls
// and nulls at the same place.
// compare the values.
switch l := left.(type) {
case *Null:
return true
case *Boolean:
r := right.(*Boolean)
return arrayEqualBoolean(l, r)
case *FixedSizeBinary:
r := right.(*FixedSizeBinary)
return arrayEqualFixedSizeBinary(l, r)
case *Binary:
r := right.(*Binary)
return arrayEqualBinary(l, r)
case *String:
r := right.(*String)
return arrayEqualString(l, r)
case *Int8:
r := right.(*Int8)
return arrayEqualInt8(l, r)
case *Int16:
r := right.(*Int16)
return arrayEqualInt16(l, r)
case *Int32:
r := right.(*Int32)
return arrayEqualInt32(l, r)
case *Int64:
r := right.(*Int64)
return arrayEqualInt64(l, r)
case *Uint8:
r := right.(*Uint8)
return arrayEqualUint8(l, r)
case *Uint16:
r := right.(*Uint16)
return arrayEqualUint16(l, r)
case *Uint32:
r := right.(*Uint32)
return arrayEqualUint32(l, r)
case *Uint64:
r := right.(*Uint64)
return arrayEqualUint64(l, r)
case *Float16:
r := right.(*Float16)
return arrayEqualFloat16(l, r)
case *Float32:
r := right.(*Float32)
return arrayEqualFloat32(l, r)
case *Float64:
r := right.(*Float64)
return arrayEqualFloat64(l, r)
case *Decimal128:
r := right.(*Decimal128)
return arrayEqualDecimal128(l, r)
case *Date32:
r := right.(*Date32)
return arrayEqualDate32(l, r)
case *Date64:
r := right.(*Date64)
return arrayEqualDate64(l, r)
case *Time32:
r := right.(*Time32)
return arrayEqualTime32(l, r)
case *Time64:
r := right.(*Time64)
return arrayEqualTime64(l, r)
case *Timestamp:
r := right.(*Timestamp)
return arrayEqualTimestamp(l, r)
case *List:
r := right.(*List)
return arrayEqualList(l, r)
case *FixedSizeList:
r := right.(*FixedSizeList)
return arrayEqualFixedSizeList(l, r)
case *Struct:
r := right.(*Struct)
return arrayEqualStruct(l, r)
case *MonthInterval:
r := right.(*MonthInterval)
return arrayEqualMonthInterval(l, r)
case *DayTimeInterval:
r := right.(*DayTimeInterval)
return arrayEqualDayTimeInterval(l, r)
case *Duration:
r := right.(*Duration)
return arrayEqualDuration(l, r)
default:
panic(errors.Errorf("arrow/array: unknown array type %T", l))
}
}
// ArraySliceEqual reports whether slices left[lbeg:lend] and right[rbeg:rend] are equal.
func ArraySliceEqual(left Interface, lbeg, lend int64, right Interface, rbeg, rend int64) bool {
l := NewSlice(left, lbeg, lend)
defer l.Release()
r := NewSlice(right, rbeg, rend)
defer r.Release()
return ArrayEqual(l, r)
}
const defaultAbsoluteTolerance = 1e-5
type equalOption struct {
atol float64 // absolute tolerance
nansEq bool // whether NaNs are considered equal.
}
func (eq equalOption) f16(f1, f2 float16.Num) bool {
v1 := float64(f1.Float32())
v2 := float64(f2.Float32())
switch {
case eq.nansEq:
return math.Abs(v1-v2) <= eq.atol || (math.IsNaN(v1) && math.IsNaN(v2))
default:
return math.Abs(v1-v2) <= eq.atol
}
}
func (eq equalOption) f32(f1, f2 float32) bool {
v1 := float64(f1)
v2 := float64(f2)
switch {
case eq.nansEq:
return math.Abs(v1-v2) <= eq.atol || (math.IsNaN(v1) && math.IsNaN(v2))
default:
return math.Abs(v1-v2) <= eq.atol
}
}
func (eq equalOption) f64(v1, v2 float64) bool {
switch {
case eq.nansEq:
return math.Abs(v1-v2) <= eq.atol || (math.IsNaN(v1) && math.IsNaN(v2))
default:
return math.Abs(v1-v2) <= eq.atol
}
}
func newEqualOption(opts ...EqualOption) equalOption {
eq := equalOption{
atol: defaultAbsoluteTolerance,
nansEq: false,
}
for _, opt := range opts {
opt(&eq)
}
return eq
}
// EqualOption is a functional option type used to configure how Records and Arrays are compared.
type EqualOption func(*equalOption)
// WithNaNsEqual configures the comparison functions so that NaNs are considered equal.
func WithNaNsEqual(v bool) EqualOption {
return func(o *equalOption) {
o.nansEq = v
}
}
// WithAbsTolerance configures the comparison functions so that 2 floating point values
// v1 and v2 are considered equal if |v1-v2| <= atol.
func WithAbsTolerance(atol float64) EqualOption {
return func(o *equalOption) {
o.atol = atol
}
}
// ArrayApproxEqual reports whether the two provided arrays are approximately equal.
// For non-floating point arrays, it is equivalent to ArrayEqual.
func ArrayApproxEqual(left, right Interface, opts ...EqualOption) bool {
opt := newEqualOption(opts...)
return arrayApproxEqual(left, right, opt)
}
func arrayApproxEqual(left, right Interface, opt equalOption) bool {
switch {
case !baseArrayEqual(left, right):
return false
case left.Len() == 0:
return true
case left.NullN() == left.Len():
return true
}
// at this point, we know both arrays have same type, same length, same number of nulls
// and nulls at the same place.
// compare the values.
switch l := left.(type) {
case *Null:
return true
case *Boolean:
r := right.(*Boolean)
return arrayEqualBoolean(l, r)
case *FixedSizeBinary:
r := right.(*FixedSizeBinary)
return arrayEqualFixedSizeBinary(l, r)
case *Binary:
r := right.(*Binary)
return arrayEqualBinary(l, r)
case *String:
r := right.(*String)
return arrayEqualString(l, r)
case *Int8:
r := right.(*Int8)
return arrayEqualInt8(l, r)
case *Int16:
r := right.(*Int16)
return arrayEqualInt16(l, r)
case *Int32:
r := right.(*Int32)
return arrayEqualInt32(l, r)
case *Int64:
r := right.(*Int64)
return arrayEqualInt64(l, r)
case *Uint8:
r := right.(*Uint8)
return arrayEqualUint8(l, r)
case *Uint16:
r := right.(*Uint16)
return arrayEqualUint16(l, r)
case *Uint32:
r := right.(*Uint32)
return arrayEqualUint32(l, r)
case *Uint64:
r := right.(*Uint64)
return arrayEqualUint64(l, r)
case *Float16:
r := right.(*Float16)
return arrayApproxEqualFloat16(l, r, opt)
case *Float32:
r := right.(*Float32)
return arrayApproxEqualFloat32(l, r, opt)
case *Float64:
r := right.(*Float64)
return arrayApproxEqualFloat64(l, r, opt)
case *Decimal128:
r := right.(*Decimal128)
return arrayEqualDecimal128(l, r)
case *Date32:
r := right.(*Date32)
return arrayEqualDate32(l, r)
case *Date64:
r := right.(*Date64)
return arrayEqualDate64(l, r)
case *Time32:
r := right.(*Time32)
return arrayEqualTime32(l, r)
case *Time64:
r := right.(*Time64)
return arrayEqualTime64(l, r)
case *Timestamp:
r := right.(*Timestamp)
return arrayEqualTimestamp(l, r)
case *List:
r := right.(*List)
return arrayApproxEqualList(l, r, opt)
case *FixedSizeList:
r := right.(*FixedSizeList)
return arrayApproxEqualFixedSizeList(l, r, opt)
case *Struct:
r := right.(*Struct)
return arrayApproxEqualStruct(l, r, opt)
case *MonthInterval:
r := right.(*MonthInterval)
return arrayEqualMonthInterval(l, r)
case *DayTimeInterval:
r := right.(*DayTimeInterval)
return arrayEqualDayTimeInterval(l, r)
case *Duration:
r := right.(*Duration)
return arrayEqualDuration(l, r)
default:
panic(errors.Errorf("arrow/array: unknown array type %T", l))
}
return false
}
func baseArrayEqual(left, right Interface) bool {
switch {
case left.Len() != right.Len():
return false
case left.NullN() != right.NullN():
return false
case !arrow.TypeEquals(left.DataType(), right.DataType()): // We do not check for metadata as in the C++ implementation.
return false
case !validityBitmapEqual(left, right):
return false
}
return true
}
func validityBitmapEqual(left, right Interface) bool {
// TODO(alexandreyc): make it faster by comparing byte slices of the validity bitmap?
n := left.Len()
if n != right.Len() {
return false
}
for i := 0; i < n; i++ {
if left.IsNull(i) != right.IsNull(i) {
return false
}
}
return true
}
func arrayApproxEqualFloat16(left, right *Float16, opt equalOption) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if !opt.f16(left.Value(i), right.Value(i)) {
return false
}
}
return true
}
func arrayApproxEqualFloat32(left, right *Float32, opt equalOption) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if !opt.f32(left.Value(i), right.Value(i)) {
return false
}
}
return true
}
func arrayApproxEqualFloat64(left, right *Float64, opt equalOption) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if !opt.f64(left.Value(i), right.Value(i)) {
return false
}
}
return true
}
func arrayApproxEqualList(left, right *List, opt equalOption) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
o := func() bool {
l := left.newListValue(i)
defer l.Release()
r := right.newListValue(i)
defer r.Release()
return arrayApproxEqual(l, r, opt)
}()
if !o {
return false
}
}
return true
}
func arrayApproxEqualFixedSizeList(left, right *FixedSizeList, opt equalOption) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
o := func() bool {
l := left.newListValue(i)
defer l.Release()
r := right.newListValue(i)
defer r.Release()
return arrayApproxEqual(l, r, opt)
}()
if !o {
return false
}
}
return true
}
func arrayApproxEqualStruct(left, right *Struct, opt equalOption) bool {
for i, lf := range left.fields {
rf := right.fields[i]
if !arrayApproxEqual(lf, rf, opt) {
return false
}
}
return true
}

133
vendor/github.com/apache/arrow/go/arrow/array/data.go generated vendored Normal file
View File

@ -0,0 +1,133 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
// A type which represents the memory and metadata for an Arrow array.
type Data struct {
refCount int64
dtype arrow.DataType
nulls int
offset int
length int
buffers []*memory.Buffer // TODO(sgc): should this be an interface?
childData []*Data // TODO(sgc): managed by ListArray, StructArray and UnionArray types
}
func NewData(dtype arrow.DataType, length int, buffers []*memory.Buffer, childData []*Data, nulls, offset int) *Data {
for _, b := range buffers {
if b != nil {
b.Retain()
}
}
for _, child := range childData {
if child != nil {
child.Retain()
}
}
return &Data{
refCount: 1,
dtype: dtype,
nulls: nulls,
length: length,
offset: offset,
buffers: buffers,
childData: childData,
}
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (d *Data) Retain() {
atomic.AddInt64(&d.refCount, 1)
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (d *Data) Release() {
debug.Assert(atomic.LoadInt64(&d.refCount) > 0, "too many releases")
if atomic.AddInt64(&d.refCount, -1) == 0 {
for _, b := range d.buffers {
if b != nil {
b.Release()
}
}
for _, b := range d.childData {
b.Release()
}
d.buffers, d.childData = nil, nil
}
}
func (d *Data) DataType() arrow.DataType { return d.dtype }
func (d *Data) NullN() int { return d.nulls }
func (d *Data) Len() int { return d.length }
func (d *Data) Offset() int { return d.offset }
func (d *Data) Buffers() []*memory.Buffer { return d.buffers }
// NewSliceData returns a new slice that shares backing data with the input.
// The returned Data slice starts at i and extends j-i elements, such as:
// slice := data[i:j]
// The returned value must be Release'd after use.
//
// NewSliceData panics if the slice is outside the valid range of the input Data.
// NewSliceData panics if j < i.
func NewSliceData(data *Data, i, j int64) *Data {
if j > int64(data.length) || i > j || data.offset+int(i) > data.length {
panic("arrow/array: index out of range")
}
for _, b := range data.buffers {
if b != nil {
b.Retain()
}
}
for _, child := range data.childData {
if child != nil {
child.Retain()
}
}
o := &Data{
refCount: 1,
dtype: data.dtype,
nulls: UnknownNullCount,
length: int(j - i),
offset: data.offset + int(i),
buffers: data.buffers,
childData: data.childData,
}
if data.nulls == 0 {
o.nulls = 0
}
return o
}

235
vendor/github.com/apache/arrow/go/arrow/array/decimal128.go generated vendored Normal file
View File

@ -0,0 +1,235 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array // import "github.com/apache/arrow/go/arrow/array"
import (
"fmt"
"strings"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/decimal128"
"github.com/apache/arrow/go/arrow/internal/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
// A type which represents an immutable sequence of 128-bit decimal values.
type Decimal128 struct {
array
values []decimal128.Num
}
func NewDecimal128Data(data *Data) *Decimal128 {
a := &Decimal128{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Decimal128) Value(i int) decimal128.Num { return a.values[i] }
func (a *Decimal128) Values() []decimal128.Num { return a.values }
func (a *Decimal128) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i := 0; i < a.Len(); i++ {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", a.Value(i))
}
}
o.WriteString("]")
return o.String()
}
func (a *Decimal128) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.Decimal128Traits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualDecimal128(left, right *Decimal128) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
type Decimal128Builder struct {
builder
dtype *arrow.Decimal128Type
data *memory.Buffer
rawData []decimal128.Num
}
func NewDecimal128Builder(mem memory.Allocator, dtype *arrow.Decimal128Type) *Decimal128Builder {
return &Decimal128Builder{
builder: builder{refCount: 1, mem: mem},
dtype: dtype,
}
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
func (b *Decimal128Builder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
}
}
func (b *Decimal128Builder) Append(v decimal128.Num) {
b.Reserve(1)
b.UnsafeAppend(v)
}
func (b *Decimal128Builder) UnsafeAppend(v decimal128.Num) {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
b.rawData[b.length] = v
b.length++
}
func (b *Decimal128Builder) AppendNull() {
b.Reserve(1)
b.UnsafeAppendBoolToBitmap(false)
}
func (b *Decimal128Builder) UnsafeAppendBoolToBitmap(isValid bool) {
if isValid {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
} else {
b.nulls++
}
b.length++
}
// AppendValues will append the values in the v slice. The valid slice determines which values
// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
// all values in v are appended and considered valid.
func (b *Decimal128Builder) AppendValues(v []decimal128.Num, valid []bool) {
if len(v) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
if len(v) == 0 {
return
}
b.Reserve(len(v))
if len(v) > 0 {
arrow.Decimal128Traits.Copy(b.rawData[b.length:], v)
}
b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
}
func (b *Decimal128Builder) init(capacity int) {
b.builder.init(capacity)
b.data = memory.NewResizableBuffer(b.mem)
bytesN := arrow.Decimal128Traits.BytesRequired(capacity)
b.data.Resize(bytesN)
b.rawData = arrow.Decimal128Traits.CastFromBytes(b.data.Bytes())
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *Decimal128Builder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *Decimal128Builder) Resize(n int) {
nBuilder := n
if n < minBuilderCapacity {
n = minBuilderCapacity
}
if b.capacity == 0 {
b.init(n)
} else {
b.builder.resize(nBuilder, b.init)
b.data.Resize(arrow.Decimal128Traits.BytesRequired(n))
b.rawData = arrow.Decimal128Traits.CastFromBytes(b.data.Bytes())
}
}
// NewArray creates a Decimal128 array from the memory buffers used by the builder and resets the Decimal128Builder
// so it can be used to build a new array.
func (b *Decimal128Builder) NewArray() Interface {
return b.NewDecimal128Array()
}
// NewDecimal128Array creates a Decimal128 array from the memory buffers used by the builder and resets the Decimal128Builder
// so it can be used to build a new array.
func (b *Decimal128Builder) NewDecimal128Array() (a *Decimal128) {
data := b.newData()
a = NewDecimal128Data(data)
data.Release()
return
}
func (b *Decimal128Builder) newData() (data *Data) {
bytesRequired := arrow.Decimal128Traits.BytesRequired(b.length)
if bytesRequired > 0 && bytesRequired < b.data.Len() {
// trim buffers
b.data.Resize(bytesRequired)
}
data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0)
b.reset()
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
return
}
var (
_ Interface = (*Decimal128)(nil)
_ Builder = (*Decimal128Builder)(nil)
)

20
vendor/github.com/apache/arrow/go/arrow/array/doc.go generated vendored Normal file
View File

@ -0,0 +1,20 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
Package array provides implementations of various Arrow array types.
*/
package array

240
vendor/github.com/apache/arrow/go/arrow/array/fixed_size_list.go generated vendored Normal file
View File

@ -0,0 +1,240 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"fmt"
"strings"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
// FixedSizeList represents an immutable sequence of N array values.
type FixedSizeList struct {
array
n int32
values Interface
}
// NewFixedSizeListData returns a new List array value, from data.
func NewFixedSizeListData(data *Data) *FixedSizeList {
a := &FixedSizeList{}
a.refCount = 1
a.setData(data)
return a
}
func (a *FixedSizeList) ListValues() Interface { return a.values }
func (a *FixedSizeList) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i := 0; i < a.Len(); i++ {
if i > 0 {
o.WriteString(" ")
}
if !a.IsValid(i) {
o.WriteString("(null)")
continue
}
sub := a.newListValue(i)
fmt.Fprintf(o, "%v", sub)
sub.Release()
}
o.WriteString("]")
return o.String()
}
func (a *FixedSizeList) newListValue(i int) Interface {
n := int64(a.n)
off := int64(a.array.data.offset)
beg := (off + int64(i)) * n
end := (off + int64(i+1)) * n
sli := NewSlice(a.values, beg, end)
return sli
}
func (a *FixedSizeList) setData(data *Data) {
a.array.setData(data)
a.n = a.DataType().(*arrow.FixedSizeListType).Len()
a.values = MakeFromData(data.childData[0])
}
func arrayEqualFixedSizeList(left, right *FixedSizeList) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
o := func() bool {
l := left.newListValue(i)
defer l.Release()
r := right.newListValue(i)
defer r.Release()
return ArrayEqual(l, r)
}()
if !o {
return false
}
}
return true
}
// Len returns the number of elements in the array.
func (a *FixedSizeList) Len() int { return a.array.Len() }
func (a *FixedSizeList) Retain() {
a.array.Retain()
a.values.Retain()
}
func (a *FixedSizeList) Release() {
a.array.Release()
a.values.Release()
}
type FixedSizeListBuilder struct {
builder
etype arrow.DataType // data type of the list's elements.
n int32 // number of elements in the fixed-size list.
values Builder // value builder for the list's elements.
}
// NewFixedSizeListBuilder returns a builder, using the provided memory allocator.
// The created list builder will create a list whose elements will be of type etype.
func NewFixedSizeListBuilder(mem memory.Allocator, n int32, etype arrow.DataType) *FixedSizeListBuilder {
return &FixedSizeListBuilder{
builder: builder{refCount: 1, mem: mem},
etype: etype,
n: n,
values: newBuilder(mem, etype),
}
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
func (b *FixedSizeListBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
if b.values != nil {
b.values.Release()
b.values = nil
}
}
}
func (b *FixedSizeListBuilder) Append(v bool) {
b.Reserve(1)
b.unsafeAppendBoolToBitmap(v)
}
func (b *FixedSizeListBuilder) AppendNull() {
b.Reserve(1)
b.unsafeAppendBoolToBitmap(false)
}
func (b *FixedSizeListBuilder) AppendValues(valid []bool) {
b.Reserve(len(valid))
b.builder.unsafeAppendBoolsToBitmap(valid, len(valid))
}
func (b *FixedSizeListBuilder) unsafeAppend(v bool) {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
b.length++
}
func (b *FixedSizeListBuilder) unsafeAppendBoolToBitmap(isValid bool) {
if isValid {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
} else {
b.nulls++
}
b.length++
}
func (b *FixedSizeListBuilder) init(capacity int) {
b.builder.init(capacity)
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *FixedSizeListBuilder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *FixedSizeListBuilder) Resize(n int) {
if n < minBuilderCapacity {
n = minBuilderCapacity
}
if b.capacity == 0 {
b.init(n)
} else {
b.builder.resize(n, b.builder.init)
}
}
func (b *FixedSizeListBuilder) ValueBuilder() Builder {
return b.values
}
// NewArray creates a List array from the memory buffers used by the builder and resets the FixedSizeListBuilder
// so it can be used to build a new array.
func (b *FixedSizeListBuilder) NewArray() Interface {
return b.NewListArray()
}
// NewListArray creates a List array from the memory buffers used by the builder and resets the FixedSizeListBuilder
// so it can be used to build a new array.
func (b *FixedSizeListBuilder) NewListArray() (a *FixedSizeList) {
data := b.newData()
a = NewFixedSizeListData(data)
data.Release()
return
}
func (b *FixedSizeListBuilder) newData() (data *Data) {
values := b.values.NewArray()
defer values.Release()
data = NewData(
arrow.FixedSizeListOf(b.n, b.etype), b.length,
[]*memory.Buffer{b.nullBitmap},
[]*Data{values.Data()},
b.nulls,
0,
)
b.reset()
return
}
var (
_ Interface = (*FixedSizeList)(nil)
_ Builder = (*FixedSizeListBuilder)(nil)
)

95
vendor/github.com/apache/arrow/go/arrow/array/fixedsize_binary.go generated vendored Normal file
View File

@ -0,0 +1,95 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"bytes"
"fmt"
"strings"
"github.com/apache/arrow/go/arrow"
)
// A type which represents an immutable sequence of fixed-length binary strings.
type FixedSizeBinary struct {
array
valueBytes []byte
bytewidth int32
}
// NewFixedSizeBinaryData constructs a new fixed-size binary array from data.
func NewFixedSizeBinaryData(data *Data) *FixedSizeBinary {
a := &FixedSizeBinary{bytewidth: int32(data.DataType().(arrow.FixedWidthDataType).BitWidth() / 8)}
a.refCount = 1
a.setData(data)
return a
}
// Value returns the fixed-size slice at index i. This value should not be mutated.
func (a *FixedSizeBinary) Value(i int) []byte {
i += a.array.data.offset
var (
bw = int(a.bytewidth)
beg = i * bw
end = (i + 1) * bw
)
return a.valueBytes[beg:end]
}
func (a *FixedSizeBinary) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i := 0; i < a.Len(); i++ {
if i > 0 {
o.WriteString(" ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%q", a.Value(i))
}
}
o.WriteString("]")
return o.String()
}
func (a *FixedSizeBinary) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.valueBytes = vals.Bytes()
}
}
func arrayEqualFixedSizeBinary(left, right *FixedSizeBinary) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if bytes.Compare(left.Value(i), right.Value(i)) != 0 {
return false
}
}
return true
}
var (
_ Interface = (*FixedSizeBinary)(nil)
)

154
vendor/github.com/apache/arrow/go/arrow/array/fixedsize_binarybuilder.go generated vendored Normal file
View File

@ -0,0 +1,154 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"fmt"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
// A FixedSizeBinaryBuilder is used to build a FixedSizeBinary array using the Append methods.
type FixedSizeBinaryBuilder struct {
builder
dtype *arrow.FixedSizeBinaryType
values *byteBufferBuilder
}
func NewFixedSizeBinaryBuilder(mem memory.Allocator, dtype *arrow.FixedSizeBinaryType) *FixedSizeBinaryBuilder {
b := &FixedSizeBinaryBuilder{
builder: builder{refCount: 1, mem: mem},
dtype: dtype,
values: newByteBufferBuilder(mem),
}
return b
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (b *FixedSizeBinaryBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
if b.values != nil {
b.values.Release()
b.values = nil
}
}
}
func (b *FixedSizeBinaryBuilder) Append(v []byte) {
if len(v) != b.dtype.ByteWidth {
// TODO(alexandre): should we return an error instead?
panic("len(v) != b.dtype.ByteWidth")
}
b.Reserve(1)
b.values.Append(v)
b.UnsafeAppendBoolToBitmap(true)
}
func (b *FixedSizeBinaryBuilder) AppendNull() {
b.Reserve(1)
b.values.Advance(b.dtype.ByteWidth)
b.UnsafeAppendBoolToBitmap(false)
}
// AppendValues will append the values in the v slice. The valid slice determines which values
// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
// all values in v are appended and considered valid.
func (b *FixedSizeBinaryBuilder) AppendValues(v [][]byte, valid []bool) {
if len(v) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
if len(v) == 0 {
return
}
b.Reserve(len(v))
for _, vv := range v {
switch len(vv) {
case 0:
b.values.Advance(b.dtype.ByteWidth)
case b.dtype.ByteWidth:
b.values.Append(vv)
default:
panic(fmt.Errorf("array: invalid binary length (got=%d, want=%d)", len(vv), b.dtype.ByteWidth))
}
}
b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
}
func (b *FixedSizeBinaryBuilder) init(capacity int) {
b.builder.init(capacity)
b.values.resize(capacity * b.dtype.ByteWidth)
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *FixedSizeBinaryBuilder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *FixedSizeBinaryBuilder) Resize(n int) {
b.builder.resize(n, b.init)
}
// NewArray creates a FixedSizeBinary array from the memory buffers used by the
// builder and resets the FixedSizeBinaryBuilder so it can be used to build a new array.
func (b *FixedSizeBinaryBuilder) NewArray() Interface {
return b.NewFixedSizeBinaryArray()
}
// NewFixedSizeBinaryArray creates a FixedSizeBinary array from the memory buffers used by the builder and resets the FixedSizeBinaryBuilder
// so it can be used to build a new array.
func (b *FixedSizeBinaryBuilder) NewFixedSizeBinaryArray() (a *FixedSizeBinary) {
data := b.newData()
a = NewFixedSizeBinaryData(data)
data.Release()
return
}
func (b *FixedSizeBinaryBuilder) newData() (data *Data) {
values := b.values.Finish()
data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, values}, nil, b.nulls, 0)
if values != nil {
values.Release()
}
b.builder.reset()
return
}
var (
_ Builder = (*FixedSizeBinaryBuilder)(nil)
)

87
vendor/github.com/apache/arrow/go/arrow/array/float16.go generated vendored Normal file
View File

@ -0,0 +1,87 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"fmt"
"strings"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/float16"
)
// A type which represents an immutable sequence of Float16 values.
type Float16 struct {
array
values []float16.Num
}
func NewFloat16Data(data *Data) *Float16 {
a := &Float16{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Float16) Value(i int) float16.Num { return a.values[i] }
func (a *Float16) Values() []float16.Num { return a.values }
func (a *Float16) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i := 0; i < a.Len(); i++ {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", a.values[i].Float32())
}
}
o.WriteString("]")
return o.String()
}
func (a *Float16) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.Float16Traits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualFloat16(left, right *Float16) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
var (
_ Interface = (*Float16)(nil)
)

165
vendor/github.com/apache/arrow/go/arrow/array/float16_builder.go generated vendored Normal file
View File

@ -0,0 +1,165 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/float16"
"github.com/apache/arrow/go/arrow/internal/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
type Float16Builder struct {
builder
data *memory.Buffer
rawData []float16.Num
}
func NewFloat16Builder(mem memory.Allocator) *Float16Builder {
return &Float16Builder{builder: builder{refCount: 1, mem: mem}}
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
func (b *Float16Builder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
}
}
func (b *Float16Builder) Append(v float16.Num) {
b.Reserve(1)
b.UnsafeAppend(v)
}
func (b *Float16Builder) UnsafeAppend(v float16.Num) {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
b.rawData[b.length] = v
b.length++
}
func (b *Float16Builder) AppendNull() {
b.Reserve(1)
b.UnsafeAppendBoolToBitmap(false)
}
func (b *Float16Builder) UnsafeAppendBoolToBitmap(isValid bool) {
if isValid {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
} else {
b.nulls++
}
b.length++
}
// AppendValues will append the values in the v slice. The valid slice determines which values
// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
// all values in v are appended and considered valid.
func (b *Float16Builder) AppendValues(v []float16.Num, valid []bool) {
if len(v) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
if len(v) == 0 {
return
}
b.Reserve(len(v))
if len(v) > 0 {
arrow.Float16Traits.Copy(b.rawData[b.length:], v)
}
b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
}
func (b *Float16Builder) init(capacity int) {
b.builder.init(capacity)
b.data = memory.NewResizableBuffer(b.mem)
bytesN := arrow.Uint16Traits.BytesRequired(capacity)
b.data.Resize(bytesN)
b.rawData = arrow.Float16Traits.CastFromBytes(b.data.Bytes())
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *Float16Builder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *Float16Builder) Resize(n int) {
nBuilder := n
if n < minBuilderCapacity {
n = minBuilderCapacity
}
if b.capacity == 0 {
b.init(n)
} else {
b.builder.resize(nBuilder, b.init)
b.data.Resize(arrow.Float16Traits.BytesRequired(n))
b.rawData = arrow.Float16Traits.CastFromBytes(b.data.Bytes())
}
}
// NewArray creates a Float16 array from the memory buffers used by the builder and resets the Float16Builder
// so it can be used to build a new array.
func (b *Float16Builder) NewArray() Interface {
return b.NewFloat16Array()
}
// NewFloat16Array creates a Float16 array from the memory buffers used by the builder and resets the Float16Builder
// so it can be used to build a new array.
func (b *Float16Builder) NewFloat16Array() (a *Float16) {
data := b.newData()
a = NewFloat16Data(data)
data.Release()
return
}
func (b *Float16Builder) newData() (data *Data) {
bytesRequired := arrow.Float16Traits.BytesRequired(b.length)
if bytesRequired > 0 && bytesRequired < b.data.Len() {
// trim buffers
b.data.Resize(bytesRequired)
}
data = NewData(arrow.FixedWidthTypes.Float16, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0)
b.reset()
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
return
}

434
vendor/github.com/apache/arrow/go/arrow/array/interval.go generated vendored Normal file
View File

@ -0,0 +1,434 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array // import "github.com/apache/arrow/go/arrow/array"
import (
"fmt"
"strings"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
"github.com/pkg/errors"
)
func NewIntervalData(data *Data) Interface {
switch data.dtype.(type) {
case *arrow.MonthIntervalType:
return NewMonthIntervalData(data)
case *arrow.DayTimeIntervalType:
return NewDayTimeIntervalData(data)
default:
panic(errors.Errorf("arrow/array: unknown interval data type %T", data.dtype))
}
}
// A type which represents an immutable sequence of arrow.MonthInterval values.
type MonthInterval struct {
array
values []arrow.MonthInterval
}
func NewMonthIntervalData(data *Data) *MonthInterval {
a := &MonthInterval{}
a.refCount = 1
a.setData(data)
return a
}
func (a *MonthInterval) Value(i int) arrow.MonthInterval { return a.values[i] }
func (a *MonthInterval) MonthIntervalValues() []arrow.MonthInterval { return a.values }
func (a *MonthInterval) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *MonthInterval) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.MonthIntervalTraits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualMonthInterval(left, right *MonthInterval) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
type MonthIntervalBuilder struct {
builder
data *memory.Buffer
rawData []arrow.MonthInterval
}
func NewMonthIntervalBuilder(mem memory.Allocator) *MonthIntervalBuilder {
return &MonthIntervalBuilder{builder: builder{refCount: 1, mem: mem}}
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
func (b *MonthIntervalBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
}
}
func (b *MonthIntervalBuilder) Append(v arrow.MonthInterval) {
b.Reserve(1)
b.UnsafeAppend(v)
}
func (b *MonthIntervalBuilder) AppendNull() {
b.Reserve(1)
b.UnsafeAppendBoolToBitmap(false)
}
func (b *MonthIntervalBuilder) UnsafeAppend(v arrow.MonthInterval) {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
b.rawData[b.length] = v
b.length++
}
func (b *MonthIntervalBuilder) UnsafeAppendBoolToBitmap(isValid bool) {
if isValid {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
} else {
b.nulls++
}
b.length++
}
// AppendValues will append the values in the v slice. The valid slice determines which values
// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
// all values in v are appended and considered valid.
func (b *MonthIntervalBuilder) AppendValues(v []arrow.MonthInterval, valid []bool) {
if len(v) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
if len(v) == 0 {
return
}
b.Reserve(len(v))
arrow.MonthIntervalTraits.Copy(b.rawData[b.length:], v)
b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
}
func (b *MonthIntervalBuilder) init(capacity int) {
b.builder.init(capacity)
b.data = memory.NewResizableBuffer(b.mem)
bytesN := arrow.MonthIntervalTraits.BytesRequired(capacity)
b.data.Resize(bytesN)
b.rawData = arrow.MonthIntervalTraits.CastFromBytes(b.data.Bytes())
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *MonthIntervalBuilder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *MonthIntervalBuilder) Resize(n int) {
nBuilder := n
if n < minBuilderCapacity {
n = minBuilderCapacity
}
if b.capacity == 0 {
b.init(n)
} else {
b.builder.resize(nBuilder, b.init)
b.data.Resize(arrow.MonthIntervalTraits.BytesRequired(n))
b.rawData = arrow.MonthIntervalTraits.CastFromBytes(b.data.Bytes())
}
}
// NewArray creates a MonthInterval array from the memory buffers used by the builder and resets the MonthIntervalBuilder
// so it can be used to build a new array.
func (b *MonthIntervalBuilder) NewArray() Interface {
return b.NewMonthIntervalArray()
}
// NewMonthIntervalArray creates a MonthInterval array from the memory buffers used by the builder and resets the MonthIntervalBuilder
// so it can be used to build a new array.
func (b *MonthIntervalBuilder) NewMonthIntervalArray() (a *MonthInterval) {
data := b.newData()
a = NewMonthIntervalData(data)
data.Release()
return
}
func (b *MonthIntervalBuilder) newData() (data *Data) {
bytesRequired := arrow.MonthIntervalTraits.BytesRequired(b.length)
if bytesRequired > 0 && bytesRequired < b.data.Len() {
// trim buffers
b.data.Resize(bytesRequired)
}
data = NewData(arrow.FixedWidthTypes.MonthInterval, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0)
b.reset()
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
return
}
// A type which represents an immutable sequence of arrow.DayTimeInterval values.
type DayTimeInterval struct {
array
values []arrow.DayTimeInterval
}
func NewDayTimeIntervalData(data *Data) *DayTimeInterval {
a := &DayTimeInterval{}
a.refCount = 1
a.setData(data)
return a
}
func (a *DayTimeInterval) Value(i int) arrow.DayTimeInterval { return a.values[i] }
func (a *DayTimeInterval) DayTimeIntervalValues() []arrow.DayTimeInterval { return a.values }
func (a *DayTimeInterval) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *DayTimeInterval) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.DayTimeIntervalTraits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualDayTimeInterval(left, right *DayTimeInterval) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
type DayTimeIntervalBuilder struct {
builder
data *memory.Buffer
rawData []arrow.DayTimeInterval
}
func NewDayTimeIntervalBuilder(mem memory.Allocator) *DayTimeIntervalBuilder {
return &DayTimeIntervalBuilder{builder: builder{refCount: 1, mem: mem}}
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
func (b *DayTimeIntervalBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
}
}
func (b *DayTimeIntervalBuilder) Append(v arrow.DayTimeInterval) {
b.Reserve(1)
b.UnsafeAppend(v)
}
func (b *DayTimeIntervalBuilder) AppendNull() {
b.Reserve(1)
b.UnsafeAppendBoolToBitmap(false)
}
func (b *DayTimeIntervalBuilder) UnsafeAppend(v arrow.DayTimeInterval) {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
b.rawData[b.length] = v
b.length++
}
func (b *DayTimeIntervalBuilder) UnsafeAppendBoolToBitmap(isValid bool) {
if isValid {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
} else {
b.nulls++
}
b.length++
}
// AppendValues will append the values in the v slice. The valid slice determines which values
// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
// all values in v are appended and considered valid.
func (b *DayTimeIntervalBuilder) AppendValues(v []arrow.DayTimeInterval, valid []bool) {
if len(v) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
if len(v) == 0 {
return
}
b.Reserve(len(v))
arrow.DayTimeIntervalTraits.Copy(b.rawData[b.length:], v)
b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
}
func (b *DayTimeIntervalBuilder) init(capacity int) {
b.builder.init(capacity)
b.data = memory.NewResizableBuffer(b.mem)
bytesN := arrow.DayTimeIntervalTraits.BytesRequired(capacity)
b.data.Resize(bytesN)
b.rawData = arrow.DayTimeIntervalTraits.CastFromBytes(b.data.Bytes())
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *DayTimeIntervalBuilder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *DayTimeIntervalBuilder) Resize(n int) {
nBuilder := n
if n < minBuilderCapacity {
n = minBuilderCapacity
}
if b.capacity == 0 {
b.init(n)
} else {
b.builder.resize(nBuilder, b.init)
b.data.Resize(arrow.DayTimeIntervalTraits.BytesRequired(n))
b.rawData = arrow.DayTimeIntervalTraits.CastFromBytes(b.data.Bytes())
}
}
// NewArray creates a DayTimeInterval array from the memory buffers used by the builder and resets the DayTimeIntervalBuilder
// so it can be used to build a new array.
func (b *DayTimeIntervalBuilder) NewArray() Interface {
return b.NewDayTimeIntervalArray()
}
// NewDayTimeIntervalArray creates a DayTimeInterval array from the memory buffers used by the builder and resets the DayTimeIntervalBuilder
// so it can be used to build a new array.
func (b *DayTimeIntervalBuilder) NewDayTimeIntervalArray() (a *DayTimeInterval) {
data := b.newData()
a = NewDayTimeIntervalData(data)
data.Release()
return
}
func (b *DayTimeIntervalBuilder) newData() (data *Data) {
bytesRequired := arrow.DayTimeIntervalTraits.BytesRequired(b.length)
if bytesRequired > 0 && bytesRequired < b.data.Len() {
// trim buffers
b.data.Resize(bytesRequired)
}
data = NewData(arrow.FixedWidthTypes.DayTimeInterval, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0)
b.reset()
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
return
}
var (
_ Interface = (*MonthInterval)(nil)
_ Interface = (*DayTimeInterval)(nil)
_ Builder = (*MonthIntervalBuilder)(nil)
_ Builder = (*DayTimeIntervalBuilder)(nil)
)

264
vendor/github.com/apache/arrow/go/arrow/array/list.go generated vendored Normal file
View File

@ -0,0 +1,264 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"fmt"
"strings"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
// List represents an immutable sequence of array values.
type List struct {
array
values Interface
offsets []int32
}
// NewListData returns a new List array value, from data.
func NewListData(data *Data) *List {
a := &List{}
a.refCount = 1
a.setData(data)
return a
}
func (a *List) ListValues() Interface { return a.values }
func (a *List) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i := 0; i < a.Len(); i++ {
if i > 0 {
o.WriteString(" ")
}
if !a.IsValid(i) {
o.WriteString("(null)")
continue
}
sub := a.newListValue(i)
fmt.Fprintf(o, "%v", sub)
sub.Release()
}
o.WriteString("]")
return o.String()
}
func (a *List) newListValue(i int) Interface {
j := i + a.array.data.offset
beg := int64(a.offsets[j])
end := int64(a.offsets[j+1])
return NewSlice(a.values, beg, end)
}
func (a *List) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.offsets = arrow.Int32Traits.CastFromBytes(vals.Bytes())
}
a.values = MakeFromData(data.childData[0])
}
func arrayEqualList(left, right *List) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
o := func() bool {
l := left.newListValue(i)
defer l.Release()
r := right.newListValue(i)
defer r.Release()
return ArrayEqual(l, r)
}()
if !o {
return false
}
}
return true
}
// Len returns the number of elements in the array.
func (a *List) Len() int { return a.array.Len() }
func (a *List) Offsets() []int32 { return a.offsets }
func (a *List) Retain() {
a.array.Retain()
a.values.Retain()
}
func (a *List) Release() {
a.array.Release()
a.values.Release()
}
type ListBuilder struct {
builder
etype arrow.DataType // data type of the list's elements.
values Builder // value builder for the list's elements.
offsets *Int32Builder
}
// NewListBuilder returns a builder, using the provided memory allocator.
// The created list builder will create a list whose elements will be of type etype.
func NewListBuilder(mem memory.Allocator, etype arrow.DataType) *ListBuilder {
return &ListBuilder{
builder: builder{refCount: 1, mem: mem},
etype: etype,
values: newBuilder(mem, etype),
offsets: NewInt32Builder(mem),
}
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
func (b *ListBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
}
b.values.Release()
b.offsets.Release()
}
func (b *ListBuilder) appendNextOffset() {
b.offsets.Append(int32(b.values.Len()))
}
func (b *ListBuilder) Append(v bool) {
b.Reserve(1)
b.unsafeAppendBoolToBitmap(v)
b.appendNextOffset()
}
func (b *ListBuilder) AppendNull() {
b.Reserve(1)
b.unsafeAppendBoolToBitmap(false)
b.appendNextOffset()
}
func (b *ListBuilder) AppendValues(offsets []int32, valid []bool) {
b.Reserve(len(valid))
b.offsets.AppendValues(offsets, nil)
b.builder.unsafeAppendBoolsToBitmap(valid, len(valid))
}
func (b *ListBuilder) unsafeAppend(v bool) {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
b.length++
}
func (b *ListBuilder) unsafeAppendBoolToBitmap(isValid bool) {
if isValid {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
} else {
b.nulls++
}
b.length++
}
func (b *ListBuilder) init(capacity int) {
b.builder.init(capacity)
b.offsets.init(capacity + 1)
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *ListBuilder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *ListBuilder) Resize(n int) {
if n < minBuilderCapacity {
n = minBuilderCapacity
}
if b.capacity == 0 {
b.init(n)
} else {
b.builder.resize(n, b.builder.init)
b.offsets.resize(n+1, b.offsets.init)
}
}
func (b *ListBuilder) ValueBuilder() Builder {
return b.values
}
// NewArray creates a List array from the memory buffers used by the builder and resets the ListBuilder
// so it can be used to build a new array.
func (b *ListBuilder) NewArray() Interface {
return b.NewListArray()
}
// NewListArray creates a List array from the memory buffers used by the builder and resets the ListBuilder
// so it can be used to build a new array.
func (b *ListBuilder) NewListArray() (a *List) {
if b.offsets.Len() != b.length+1 {
b.appendNextOffset()
}
data := b.newData()
a = NewListData(data)
data.Release()
return
}
func (b *ListBuilder) newData() (data *Data) {
values := b.values.NewArray()
defer values.Release()
var offsets *memory.Buffer
if b.offsets != nil {
arr := b.offsets.NewInt32Array()
defer arr.Release()
offsets = arr.Data().buffers[1]
}
data = NewData(
arrow.ListOf(b.etype), b.length,
[]*memory.Buffer{
b.nullBitmap,
offsets,
},
[]*Data{values.Data()},
b.nulls,
0,
)
b.reset()
return
}
var (
_ Interface = (*List)(nil)
_ Builder = (*ListBuilder)(nil)
)

126
vendor/github.com/apache/arrow/go/arrow/array/null.go generated vendored Normal file
View File

@ -0,0 +1,126 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
// Null represents an immutable, degenerate array with no physical storage.
type Null struct {
array
}
// NewNull returns a new Null array value of size n.
func NewNull(n int) *Null {
a := &Null{}
a.refCount = 1
data := NewData(
arrow.Null, n,
[]*memory.Buffer{nil},
nil,
n,
0,
)
a.setData(data)
data.Release()
return a
}
// NewNullData returns a new Null array value, from data.
func NewNullData(data *Data) *Null {
a := &Null{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Null) setData(data *Data) {
a.array.setData(data)
a.array.nullBitmapBytes = nil
a.array.data.nulls = a.array.data.length
}
type NullBuilder struct {
builder
}
// NewNullBuilder returns a builder, using the provided memory allocator.
func NewNullBuilder(mem memory.Allocator) *NullBuilder {
return &NullBuilder{builder: builder{refCount: 1, mem: mem}}
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
func (b *NullBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
}
}
func (b *NullBuilder) AppendNull() {
b.builder.length++
b.builder.nulls++
}
func (*NullBuilder) Reserve(size int) {}
func (*NullBuilder) Resize(size int) {}
func (*NullBuilder) init(cap int) {}
func (*NullBuilder) resize(newBits int, init func(int)) {}
// NewArray creates a Null array from the memory buffers used by the builder and resets the NullBuilder
// so it can be used to build a new array.
func (b *NullBuilder) NewArray() Interface {
return b.NewNullArray()
}
// NewNullArray creates a Null array from the memory buffers used by the builder and resets the NullBuilder
// so it can be used to build a new array.
func (b *NullBuilder) NewNullArray() (a *Null) {
data := b.newData()
a = NewNullData(data)
data.Release()
return
}
func (b *NullBuilder) newData() (data *Data) {
data = NewData(
arrow.Null, b.length,
[]*memory.Buffer{nil},
nil,
b.nulls,
0,
)
b.reset()
return
}
var (
_ Interface = (*Null)(nil)
_ Builder = (*NullBuilder)(nil)
)

938
vendor/github.com/apache/arrow/go/arrow/array/numeric.gen.go generated vendored Normal file
View File

@ -0,0 +1,938 @@
// Code generated by array/numeric.gen.go.tmpl. DO NOT EDIT.
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"fmt"
"strings"
"github.com/apache/arrow/go/arrow"
)
// A type which represents an immutable sequence of int64 values.
type Int64 struct {
array
values []int64
}
func NewInt64Data(data *Data) *Int64 {
a := &Int64{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Int64) Value(i int) int64 { return a.values[i] }
func (a *Int64) Int64Values() []int64 { return a.values }
func (a *Int64) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *Int64) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.Int64Traits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualInt64(left, right *Int64) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
// A type which represents an immutable sequence of uint64 values.
type Uint64 struct {
array
values []uint64
}
func NewUint64Data(data *Data) *Uint64 {
a := &Uint64{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Uint64) Value(i int) uint64 { return a.values[i] }
func (a *Uint64) Uint64Values() []uint64 { return a.values }
func (a *Uint64) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *Uint64) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.Uint64Traits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualUint64(left, right *Uint64) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
// A type which represents an immutable sequence of float64 values.
type Float64 struct {
array
values []float64
}
func NewFloat64Data(data *Data) *Float64 {
a := &Float64{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Float64) Value(i int) float64 { return a.values[i] }
func (a *Float64) Float64Values() []float64 { return a.values }
func (a *Float64) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *Float64) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.Float64Traits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualFloat64(left, right *Float64) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
// A type which represents an immutable sequence of int32 values.
type Int32 struct {
array
values []int32
}
func NewInt32Data(data *Data) *Int32 {
a := &Int32{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Int32) Value(i int) int32 { return a.values[i] }
func (a *Int32) Int32Values() []int32 { return a.values }
func (a *Int32) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *Int32) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.Int32Traits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualInt32(left, right *Int32) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
// A type which represents an immutable sequence of uint32 values.
type Uint32 struct {
array
values []uint32
}
func NewUint32Data(data *Data) *Uint32 {
a := &Uint32{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Uint32) Value(i int) uint32 { return a.values[i] }
func (a *Uint32) Uint32Values() []uint32 { return a.values }
func (a *Uint32) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *Uint32) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.Uint32Traits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualUint32(left, right *Uint32) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
// A type which represents an immutable sequence of float32 values.
type Float32 struct {
array
values []float32
}
func NewFloat32Data(data *Data) *Float32 {
a := &Float32{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Float32) Value(i int) float32 { return a.values[i] }
func (a *Float32) Float32Values() []float32 { return a.values }
func (a *Float32) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *Float32) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.Float32Traits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualFloat32(left, right *Float32) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
// A type which represents an immutable sequence of int16 values.
type Int16 struct {
array
values []int16
}
func NewInt16Data(data *Data) *Int16 {
a := &Int16{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Int16) Value(i int) int16 { return a.values[i] }
func (a *Int16) Int16Values() []int16 { return a.values }
func (a *Int16) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *Int16) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.Int16Traits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualInt16(left, right *Int16) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
// A type which represents an immutable sequence of uint16 values.
type Uint16 struct {
array
values []uint16
}
func NewUint16Data(data *Data) *Uint16 {
a := &Uint16{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Uint16) Value(i int) uint16 { return a.values[i] }
func (a *Uint16) Uint16Values() []uint16 { return a.values }
func (a *Uint16) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *Uint16) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.Uint16Traits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualUint16(left, right *Uint16) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
// A type which represents an immutable sequence of int8 values.
type Int8 struct {
array
values []int8
}
func NewInt8Data(data *Data) *Int8 {
a := &Int8{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Int8) Value(i int) int8 { return a.values[i] }
func (a *Int8) Int8Values() []int8 { return a.values }
func (a *Int8) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *Int8) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.Int8Traits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualInt8(left, right *Int8) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
// A type which represents an immutable sequence of uint8 values.
type Uint8 struct {
array
values []uint8
}
func NewUint8Data(data *Data) *Uint8 {
a := &Uint8{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Uint8) Value(i int) uint8 { return a.values[i] }
func (a *Uint8) Uint8Values() []uint8 { return a.values }
func (a *Uint8) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *Uint8) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.Uint8Traits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualUint8(left, right *Uint8) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
// A type which represents an immutable sequence of arrow.Timestamp values.
type Timestamp struct {
array
values []arrow.Timestamp
}
func NewTimestampData(data *Data) *Timestamp {
a := &Timestamp{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Timestamp) Value(i int) arrow.Timestamp { return a.values[i] }
func (a *Timestamp) TimestampValues() []arrow.Timestamp { return a.values }
func (a *Timestamp) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *Timestamp) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.TimestampTraits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualTimestamp(left, right *Timestamp) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
// A type which represents an immutable sequence of arrow.Time32 values.
type Time32 struct {
array
values []arrow.Time32
}
func NewTime32Data(data *Data) *Time32 {
a := &Time32{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Time32) Value(i int) arrow.Time32 { return a.values[i] }
func (a *Time32) Time32Values() []arrow.Time32 { return a.values }
func (a *Time32) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *Time32) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.Time32Traits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualTime32(left, right *Time32) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
// A type which represents an immutable sequence of arrow.Time64 values.
type Time64 struct {
array
values []arrow.Time64
}
func NewTime64Data(data *Data) *Time64 {
a := &Time64{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Time64) Value(i int) arrow.Time64 { return a.values[i] }
func (a *Time64) Time64Values() []arrow.Time64 { return a.values }
func (a *Time64) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *Time64) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.Time64Traits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualTime64(left, right *Time64) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
// A type which represents an immutable sequence of arrow.Date32 values.
type Date32 struct {
array
values []arrow.Date32
}
func NewDate32Data(data *Data) *Date32 {
a := &Date32{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Date32) Value(i int) arrow.Date32 { return a.values[i] }
func (a *Date32) Date32Values() []arrow.Date32 { return a.values }
func (a *Date32) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *Date32) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.Date32Traits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualDate32(left, right *Date32) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
// A type which represents an immutable sequence of arrow.Date64 values.
type Date64 struct {
array
values []arrow.Date64
}
func NewDate64Data(data *Data) *Date64 {
a := &Date64{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Date64) Value(i int) arrow.Date64 { return a.values[i] }
func (a *Date64) Date64Values() []arrow.Date64 { return a.values }
func (a *Date64) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *Date64) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.Date64Traits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualDate64(left, right *Date64) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
// A type which represents an immutable sequence of arrow.Duration values.
type Duration struct {
array
values []arrow.Duration
}
func NewDurationData(data *Data) *Duration {
a := &Duration{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Duration) Value(i int) arrow.Duration { return a.values[i] }
func (a *Duration) DurationValues() []arrow.Duration { return a.values }
func (a *Duration) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *Duration) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.DurationTraits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqualDuration(left, right *Duration) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}

85
vendor/github.com/apache/arrow/go/arrow/array/numeric.gen.go.tmpl generated vendored Normal file
View File

@ -0,0 +1,85 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"fmt"
"strings"
"github.com/apache/arrow/go/arrow"
)
{{range .In}}
// A type which represents an immutable sequence of {{or .QualifiedType .Type}} values.
type {{.Name}} struct {
array
values []{{or .QualifiedType .Type}}
}
func New{{.Name}}Data(data *Data) *{{.Name}} {
a := &{{.Name}}{}
a.refCount = 1
a.setData(data)
return a
}
func (a *{{.Name}}) Value(i int) {{or .QualifiedType .Type}} { return a.values[i] }
func (a *{{.Name}}) {{.Name}}Values() []{{or .QualifiedType .Type}} { return a.values }
func (a *{{.Name}}) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i, v := range a.values {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", v)
}
}
o.WriteString("]")
return o.String()
}
func (a *{{.Name}}) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = arrow.{{.Name}}Traits.CastFromBytes(vals.Bytes())
beg := a.array.data.offset
end := beg + a.array.data.length
a.values = a.values[beg:end]
}
}
func arrayEqual{{.Name}}(left, right *{{.Name}}) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
{{end}}

2227
vendor/github.com/apache/arrow/go/arrow/array/numericbuilder.gen.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

182
vendor/github.com/apache/arrow/go/arrow/array/numericbuilder.gen.go.tmpl generated vendored Normal file
View File

@ -0,0 +1,182 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
{{range .In}}
type {{.Name}}Builder struct {
builder
{{if .Opt.Parametric -}}
dtype *arrow.{{.Name}}Type
{{end -}}
data *memory.Buffer
rawData []{{or .QualifiedType .Type}}
}
{{if .Opt.Parametric}}
func New{{.Name}}Builder(mem memory.Allocator, dtype *arrow.{{.Name}}Type) *{{.Name}}Builder {
return &{{.Name}}Builder{builder: builder{refCount:1, mem: mem}, dtype: dtype}
}
{{else}}
func New{{.Name}}Builder(mem memory.Allocator) *{{.Name}}Builder {
return &{{.Name}}Builder{builder: builder{refCount:1, mem: mem}}
}
{{end}}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
func (b *{{.Name}}Builder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
}
}
func (b *{{.Name}}Builder) Append(v {{or .QualifiedType .Type}}) {
b.Reserve(1)
b.UnsafeAppend(v)
}
func (b *{{.Name}}Builder) AppendNull() {
b.Reserve(1)
b.UnsafeAppendBoolToBitmap(false)
}
func (b *{{.Name}}Builder) UnsafeAppend(v {{or .QualifiedType .Type}}) {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
b.rawData[b.length] = v
b.length++
}
func (b *{{.Name}}Builder) UnsafeAppendBoolToBitmap(isValid bool) {
if isValid {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
} else {
b.nulls++
}
b.length++
}
// AppendValues will append the values in the v slice. The valid slice determines which values
// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
// all values in v are appended and considered valid.
func (b *{{.Name}}Builder) AppendValues(v []{{or .QualifiedType .Type}}, valid []bool) {
if len(v) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
if len(v) == 0 {
return
}
b.Reserve(len(v))
arrow.{{.Name}}Traits.Copy(b.rawData[b.length:], v)
b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
}
func (b *{{.Name}}Builder) init(capacity int) {
b.builder.init(capacity)
b.data = memory.NewResizableBuffer(b.mem)
bytesN := arrow.{{.Name}}Traits.BytesRequired(capacity)
b.data.Resize(bytesN)
b.rawData = arrow.{{.Name}}Traits.CastFromBytes(b.data.Bytes())
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *{{.Name}}Builder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *{{.Name}}Builder) Resize(n int) {
nBuilder := n
if n < minBuilderCapacity {
n = minBuilderCapacity
}
if b.capacity == 0 {
b.init(n)
} else {
b.builder.resize(nBuilder, b.init)
b.data.Resize(arrow.{{.Name}}Traits.BytesRequired(n))
b.rawData = arrow.{{.Name}}Traits.CastFromBytes(b.data.Bytes())
}
}
// NewArray creates a {{.Name}} array from the memory buffers used by the builder and resets the {{.Name}}Builder
// so it can be used to build a new array.
func (b *{{.Name}}Builder) NewArray() Interface {
return b.New{{.Name}}Array()
}
// New{{.Name}}Array creates a {{.Name}} array from the memory buffers used by the builder and resets the {{.Name}}Builder
// so it can be used to build a new array.
func (b *{{.Name}}Builder) New{{.Name}}Array() (a *{{.Name}}) {
data := b.newData()
a = New{{.Name}}Data(data)
data.Release()
return
}
func (b *{{.Name}}Builder) newData() (data *Data) {
bytesRequired := arrow.{{.Name}}Traits.BytesRequired(b.length)
if bytesRequired > 0 && bytesRequired < b.data.Len() {
// trim buffers
b.data.Resize(bytesRequired)
}
{{if .Opt.Parametric -}}
data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0)
{{else -}}
data = NewData(arrow.PrimitiveTypes.{{.Name}}, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0)
{{end -}}
b.reset()
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
return
}
{{end}}
var (
{{- range .In}}
_ Builder = (*{{.Name}}Builder)(nil)
{{- end}}
)

216
vendor/github.com/apache/arrow/go/arrow/array/numericbuilder.gen_test.go.tmpl generated vendored Normal file
View File

@ -0,0 +1,216 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array_test
import (
"testing"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/array"
"github.com/apache/arrow/go/arrow/memory"
"github.com/stretchr/testify/assert"
)
{{range .In}}
func TestNew{{.Name}}Builder(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
{{if .Opt.Parametric -}}
dtype := &arrow.{{.Name}}Type{Unit: arrow.Second}
ab := array.New{{.Name}}Builder(mem, dtype)
{{else}}
ab := array.New{{.Name}}Builder(mem)
{{end -}}
defer ab.Release()
ab.Retain()
ab.Release()
ab.Append(1)
ab.Append(2)
ab.Append(3)
ab.AppendNull()
ab.Append(5)
ab.Append(6)
ab.AppendNull()
ab.Append(8)
ab.Append(9)
ab.Append(10)
// check state of builder before New{{.Name}}Array
assert.Equal(t, 10, ab.Len(), "unexpected Len()")
assert.Equal(t, 2, ab.NullN(), "unexpected NullN()")
a := ab.New{{.Name}}Array()
// check state of builder after New{{.Name}}Array
assert.Zero(t, ab.Len(), "unexpected ArrayBuilder.Len(), New{{.Name}}Array did not reset state")
assert.Zero(t, ab.Cap(), "unexpected ArrayBuilder.Cap(), New{{.Name}}Array did not reset state")
assert.Zero(t, ab.NullN(), "unexpected ArrayBuilder.NullN(), New{{.Name}}Array did not reset state")
// check state of array
assert.Equal(t, 2, a.NullN(), "unexpected null count")
assert.Equal(t, []{{or .QualifiedType .Type}}{1, 2, 3, 0, 5, 6, 0, 8, 9, 10}, a.{{.Name}}Values(), "unexpected {{.Name}}Values")
assert.Equal(t, []byte{0xb7}, a.NullBitmapBytes()[:1]) // 4 bytes due to minBuilderCapacity
assert.Len(t, a.{{.Name}}Values(), 10, "unexpected length of {{.Name}}Values")
a.Release()
ab.Append(7)
ab.Append(8)
a = ab.New{{.Name}}Array()
assert.Equal(t, 0, a.NullN())
assert.Equal(t, []{{or .QualifiedType .Type}}{7, 8}, a.{{.Name}}Values())
assert.Len(t, a.{{.Name}}Values(), 2)
a.Release()
var (
want = []{{or .QualifiedType .Type}}{1, 2, 3, 4}
valids = []bool{true, true, false, true}
)
ab.AppendValues(want, valids)
a = ab.New{{.Name}}Array()
sub := array.MakeFromData(a.Data())
defer sub.Release()
if got, want := sub.DataType().ID(), a.DataType().ID(); got != want {
t.Fatalf("invalid type: got=%q, want=%q", got, want)
}
if _, ok := sub.(*array.{{.Name}}); !ok {
t.Fatalf("could not type-assert to array.{{.Name}}")
}
if got, want := a.String(), `[1 2 (null) 4]`; got != want {
t.Fatalf("got=%q, want=%q", got, want)
}
slice := array.NewSliceData(a.Data(), 2, 4)
defer slice.Release()
sub1 := array.MakeFromData(slice)
defer sub1.Release()
v, ok := sub1.(*array.{{.Name}})
if !ok {
t.Fatalf("could not type-assert to array.{{.Name}}")
}
if got, want := v.String(), `[(null) 4]`; got != want {
t.Fatalf("got=%q, want=%q", got, want)
}
a.Release()
}
func Test{{.Name}}Builder_AppendValues(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
{{if .Opt.Parametric -}}
dtype := &arrow.{{.Name}}Type{Unit: arrow.Second}
ab := array.New{{.Name}}Builder(mem, dtype)
{{else}}
ab := array.New{{.Name}}Builder(mem)
{{end -}}
defer ab.Release()
exp := []{{or .QualifiedType .Type}}{0, 1, 2, 3}
ab.AppendValues(exp, nil)
a := ab.New{{.Name}}Array()
assert.Equal(t, exp, a.{{.Name}}Values())
a.Release()
}
func Test{{.Name}}Builder_Empty(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
{{if .Opt.Parametric -}}
dtype := &arrow.{{.Name}}Type{Unit: arrow.Second}
ab := array.New{{.Name}}Builder(mem, dtype)
{{else}}
ab := array.New{{.Name}}Builder(mem)
{{end -}}
defer ab.Release()
exp := []{{or .QualifiedType .Type}}{0, 1, 2, 3}
ab.AppendValues([]{{or .QualifiedType .Type}}{}, nil)
a := ab.New{{.Name}}Array()
assert.Zero(t, a.Len())
a.Release()
ab.AppendValues(nil, nil)
a = ab.New{{.Name}}Array()
assert.Zero(t, a.Len())
a.Release()
ab.AppendValues([]{{or .QualifiedType .Type}}{}, nil)
ab.AppendValues(exp, nil)
a = ab.New{{.Name}}Array()
assert.Equal(t, exp, a.{{.Name}}Values())
a.Release()
ab.AppendValues(exp, nil)
ab.AppendValues([]{{or .QualifiedType .Type}}{}, nil)
a = ab.New{{.Name}}Array()
assert.Equal(t, exp, a.{{.Name}}Values())
a.Release()
}
func Test{{.Name}}Builder_Resize(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
defer mem.AssertSize(t, 0)
{{if .Opt.Parametric -}}
dtype := &arrow.{{.Name}}Type{Unit: arrow.Second}
ab := array.New{{.Name}}Builder(mem, dtype)
{{else}}
ab := array.New{{.Name}}Builder(mem)
{{end -}}
defer ab.Release()
assert.Equal(t, 0, ab.Cap())
assert.Equal(t, 0, ab.Len())
ab.Reserve(63)
assert.Equal(t, 64, ab.Cap())
assert.Equal(t, 0, ab.Len())
for i := 0; i < 63; i++ {
ab.Append(0)
}
assert.Equal(t, 64, ab.Cap())
assert.Equal(t, 63, ab.Len())
ab.Resize(5)
assert.Equal(t, 5, ab.Len())
ab.Resize(32)
assert.Equal(t, 5, ab.Len())
}
{{end}}

345
vendor/github.com/apache/arrow/go/arrow/array/record.go generated vendored Normal file
View File

@ -0,0 +1,345 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"fmt"
"strings"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
// RecordReader reads a stream of records.
type RecordReader interface {
Retain()
Release()
Schema() *arrow.Schema
Next() bool
Record() Record
}
// simpleRecords is a simple iterator over a collection of records.
type simpleRecords struct {
refCount int64
schema *arrow.Schema
recs []Record
cur Record
}
// NewRecordReader returns a simple iterator over the given slice of records.
func NewRecordReader(schema *arrow.Schema, recs []Record) (*simpleRecords, error) {
rs := &simpleRecords{
refCount: 1,
schema: schema,
recs: recs,
cur: nil,
}
for _, rec := range rs.recs {
rec.Retain()
}
for _, rec := range recs {
if !rec.Schema().Equal(rs.schema) {
rs.Release()
return nil, fmt.Errorf("arrow/array: mismatch schema")
}
}
return rs, nil
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (rs *simpleRecords) Retain() {
atomic.AddInt64(&rs.refCount, 1)
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (rs *simpleRecords) Release() {
debug.Assert(atomic.LoadInt64(&rs.refCount) > 0, "too many releases")
if atomic.AddInt64(&rs.refCount, -1) == 0 {
if rs.cur != nil {
rs.cur.Release()
}
for _, rec := range rs.recs {
rec.Release()
}
rs.recs = nil
}
}
func (rs *simpleRecords) Schema() *arrow.Schema { return rs.schema }
func (rs *simpleRecords) Record() Record { return rs.cur }
func (rs *simpleRecords) Next() bool {
if len(rs.recs) == 0 {
return false
}
if rs.cur != nil {
rs.cur.Release()
}
rs.cur = rs.recs[0]
rs.recs = rs.recs[1:]
return true
}
// Record is a collection of equal-length arrays
// matching a particular Schema.
type Record interface {
Release()
Retain()
Schema() *arrow.Schema
NumRows() int64
NumCols() int64
Columns() []Interface
Column(i int) Interface
ColumnName(i int) string
// NewSlice constructs a zero-copy slice of the record with the indicated
// indices i and j, corresponding to array[i:j].
// The returned record must be Release()'d after use.
//
// NewSlice panics if the slice is outside the valid range of the record array.
// NewSlice panics if j < i.
NewSlice(i, j int64) Record
}
// simpleRecord is a basic, non-lazy in-memory record batch.
type simpleRecord struct {
refCount int64
schema *arrow.Schema
rows int64
arrs []Interface
}
// NewRecord returns a basic, non-lazy in-memory record batch.
//
// NewRecord panics if the columns and schema are inconsistent.
// NewRecord panics if rows is larger than the height of the columns.
func NewRecord(schema *arrow.Schema, cols []Interface, nrows int64) *simpleRecord {
rec := &simpleRecord{
refCount: 1,
schema: schema,
rows: nrows,
arrs: make([]Interface, len(cols)),
}
copy(rec.arrs, cols)
for _, arr := range rec.arrs {
arr.Retain()
}
if rec.rows < 0 {
switch len(rec.arrs) {
case 0:
rec.rows = 0
default:
rec.rows = int64(rec.arrs[0].Len())
}
}
err := rec.validate()
if err != nil {
rec.Release()
panic(err)
}
return rec
}
func (rec *simpleRecord) validate() error {
if len(rec.arrs) != len(rec.schema.Fields()) {
return fmt.Errorf("arrow/array: number of columns/fields mismatch")
}
for i, arr := range rec.arrs {
f := rec.schema.Field(i)
if int64(arr.Len()) < rec.rows {
return fmt.Errorf("arrow/array: mismatch number of rows in column %q: got=%d, want=%d",
f.Name,
arr.Len(), rec.rows,
)
}
if !arrow.TypeEquals(f.Type, arr.DataType()) {
return fmt.Errorf("arrow/array: column %q type mismatch: got=%v, want=%v",
f.Name,
arr.DataType(), f.Type,
)
}
}
return nil
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (rec *simpleRecord) Retain() {
atomic.AddInt64(&rec.refCount, 1)
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (rec *simpleRecord) Release() {
debug.Assert(atomic.LoadInt64(&rec.refCount) > 0, "too many releases")
if atomic.AddInt64(&rec.refCount, -1) == 0 {
for _, arr := range rec.arrs {
arr.Release()
}
rec.arrs = nil
}
}
func (rec *simpleRecord) Schema() *arrow.Schema { return rec.schema }
func (rec *simpleRecord) NumRows() int64 { return rec.rows }
func (rec *simpleRecord) NumCols() int64 { return int64(len(rec.arrs)) }
func (rec *simpleRecord) Columns() []Interface { return rec.arrs }
func (rec *simpleRecord) Column(i int) Interface { return rec.arrs[i] }
func (rec *simpleRecord) ColumnName(i int) string { return rec.schema.Field(i).Name }
// NewSlice constructs a zero-copy slice of the record with the indicated
// indices i and j, corresponding to array[i:j].
// The returned record must be Release()'d after use.
//
// NewSlice panics if the slice is outside the valid range of the record array.
// NewSlice panics if j < i.
func (rec *simpleRecord) NewSlice(i, j int64) Record {
arrs := make([]Interface, len(rec.arrs))
for ii, arr := range rec.arrs {
arrs[ii] = NewSlice(arr, i, j)
}
defer func() {
for _, arr := range arrs {
arr.Release()
}
}()
return NewRecord(rec.schema, arrs, j-i)
}
func (rec *simpleRecord) String() string {
o := new(strings.Builder)
fmt.Fprintf(o, "record:\n %v\n", rec.schema)
fmt.Fprintf(o, " rows: %d\n", rec.rows)
for i, col := range rec.arrs {
fmt.Fprintf(o, " col[%d][%s]: %v\n", i, rec.schema.Field(i).Name, col)
}
return o.String()
}
// RecordBuilder eases the process of building a Record, iteratively, from
// a known Schema.
type RecordBuilder struct {
refCount int64
mem memory.Allocator
schema *arrow.Schema
fields []Builder
}
// NewRecordBuilder returns a builder, using the provided memory allocator and a schema.
func NewRecordBuilder(mem memory.Allocator, schema *arrow.Schema) *RecordBuilder {
b := &RecordBuilder{
refCount: 1,
mem: mem,
schema: schema,
fields: make([]Builder, len(schema.Fields())),
}
for i, f := range schema.Fields() {
b.fields[i] = newBuilder(b.mem, f.Type)
}
return b
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (b *RecordBuilder) Retain() {
atomic.AddInt64(&b.refCount, 1)
}
// Release decreases the reference count by 1.
func (b *RecordBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
for _, f := range b.fields {
f.Release()
}
if atomic.AddInt64(&b.refCount, -1) == 0 {
b.fields = nil
}
}
func (b *RecordBuilder) Schema() *arrow.Schema { return b.schema }
func (b *RecordBuilder) Fields() []Builder { return b.fields }
func (b *RecordBuilder) Field(i int) Builder { return b.fields[i] }
func (b *RecordBuilder) Reserve(size int) {
for _, f := range b.fields {
f.Reserve(size)
}
}
// NewRecord creates a new record from the memory buffers and resets the
// RecordBuilder so it can be used to build a new record.
//
// The returned Record must be Release()'d after use.
//
// NewRecord panics if the fields' builder do not have the same length.
func (b *RecordBuilder) NewRecord() Record {
cols := make([]Interface, len(b.fields))
rows := int64(0)
defer func(cols []Interface) {
for _, col := range cols {
if col == nil {
continue
}
col.Release()
}
}(cols)
for i, f := range b.fields {
cols[i] = f.NewArray()
irow := int64(cols[i].Len())
if i > 0 && irow != rows {
panic(fmt.Errorf("arrow/array: field %d has %d rows. want=%d", i, irow, rows))
}
rows = irow
}
return NewRecord(b.schema, cols, rows)
}
var (
_ Record = (*simpleRecord)(nil)
_ RecordReader = (*simpleRecords)(nil)
)

194
vendor/github.com/apache/arrow/go/arrow/array/string.go generated vendored Normal file
View File

@ -0,0 +1,194 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"fmt"
"math"
"strings"
"unsafe"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/memory"
)
const (
stringArrayMaximumCapacity = math.MaxInt32
)
// A type which represents an immutable sequence of variable-length UTF-8 strings.
type String struct {
array
offsets []int32
values string
}
// NewStringData constructs a new String array from data.
func NewStringData(data *Data) *String {
a := &String{}
a.refCount = 1
a.setData(data)
return a
}
// Value returns the slice at index i. This value should not be mutated.
func (a *String) Value(i int) string {
i = i + a.array.data.offset
return a.values[a.offsets[i]:a.offsets[i+1]]
}
func (a *String) ValueOffset(i int) int { return int(a.offsets[i]) }
func (a *String) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i := 0; i < a.Len(); i++ {
if i > 0 {
o.WriteString(" ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%q", a.Value(i))
}
}
o.WriteString("]")
return o.String()
}
func (a *String) setData(data *Data) {
if len(data.buffers) != 3 {
panic("arrow/array: len(data.buffers) != 3")
}
a.array.setData(data)
if vdata := data.buffers[2]; vdata != nil {
b := vdata.Bytes()
a.values = *(*string)(unsafe.Pointer(&b))
}
if offsets := data.buffers[1]; offsets != nil {
a.offsets = arrow.Int32Traits.CastFromBytes(offsets.Bytes())
}
}
func arrayEqualString(left, right *String) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
// A StringBuilder is used to build a String array using the Append methods.
type StringBuilder struct {
builder *BinaryBuilder
}
func NewStringBuilder(mem memory.Allocator) *StringBuilder {
b := &StringBuilder{
builder: NewBinaryBuilder(mem, arrow.BinaryTypes.String),
}
return b
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (b *StringBuilder) Release() {
b.builder.Release()
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (b *StringBuilder) Retain() {
b.builder.Retain()
}
//
// Len returns the number of elements in the array builder.
func (b *StringBuilder) Len() int { return b.builder.Len() }
// Cap returns the total number of elements that can be stored without allocating additional memory.
func (b *StringBuilder) Cap() int { return b.builder.Cap() }
// NullN returns the number of null values in the array builder.
func (b *StringBuilder) NullN() int { return b.builder.NullN() }
func (b *StringBuilder) Append(v string) {
b.builder.Append([]byte(v))
}
func (b *StringBuilder) AppendNull() {
b.builder.AppendNull()
}
// AppendValues will append the values in the v slice. The valid slice determines which values
// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
// all values in v are appended and considered valid.
func (b *StringBuilder) AppendValues(v []string, valid []bool) {
b.builder.AppendStringValues(v, valid)
}
func (b *StringBuilder) Value(i int) string {
return string(b.builder.Value(i))
}
func (b *StringBuilder) init(capacity int) {
b.builder.init(capacity)
}
func (b *StringBuilder) resize(newBits int, init func(int)) {
b.builder.resize(newBits, init)
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *StringBuilder) Reserve(n int) {
b.builder.Reserve(n)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *StringBuilder) Resize(n int) {
b.builder.Resize(n)
}
// NewArray creates a String array from the memory buffers used by the builder and resets the StringBuilder
// so it can be used to build a new array.
func (b *StringBuilder) NewArray() Interface {
return b.NewStringArray()
}
// NewStringArray creates a String array from the memory buffers used by the builder and resets the StringBuilder
// so it can be used to build a new array.
func (b *StringBuilder) NewStringArray() (a *String) {
data := b.builder.newData()
a = NewStringData(data)
data.Release()
return
}
var (
_ Interface = (*String)(nil)
_ Builder = (*StringBuilder)(nil)
)

234
vendor/github.com/apache/arrow/go/arrow/array/struct.go generated vendored Normal file
View File

@ -0,0 +1,234 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"fmt"
"strings"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
// Struct represents an ordered sequence of relative types.
type Struct struct {
array
fields []Interface
}
// NewStructData returns a new Struct array value from data.
func NewStructData(data *Data) *Struct {
a := &Struct{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Struct) NumField() int { return len(a.fields) }
func (a *Struct) Field(i int) Interface { return a.fields[i] }
func (a *Struct) String() string {
o := new(strings.Builder)
o.WriteString("{")
for i, v := range a.fields {
if i > 0 {
o.WriteString(" ")
}
fmt.Fprintf(o, "%v", v)
}
o.WriteString("}")
return o.String()
}
func (a *Struct) setData(data *Data) {
a.array.setData(data)
a.fields = make([]Interface, len(data.childData))
for i, child := range data.childData {
a.fields[i] = MakeFromData(child)
}
}
func arrayEqualStruct(left, right *Struct) bool {
for i, lf := range left.fields {
rf := right.fields[i]
if !ArrayEqual(lf, rf) {
return false
}
}
return true
}
func (a *Struct) Retain() {
a.array.Retain()
for _, f := range a.fields {
f.Retain()
}
}
func (a *Struct) Release() {
a.array.Release()
for _, f := range a.fields {
f.Release()
}
}
type StructBuilder struct {
builder
dtype arrow.DataType
fields []Builder
}
// NewStructBuilder returns a builder, using the provided memory allocator.
func NewStructBuilder(mem memory.Allocator, dtype *arrow.StructType) *StructBuilder {
b := &StructBuilder{
builder: builder{refCount: 1, mem: mem},
dtype: dtype,
fields: make([]Builder, len(dtype.Fields())),
}
for i, f := range dtype.Fields() {
b.fields[i] = newBuilder(b.mem, f.Type)
}
return b
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
func (b *StructBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
}
for _, f := range b.fields {
f.Release()
}
}
func (b *StructBuilder) Append(v bool) {
b.Reserve(1)
b.unsafeAppendBoolToBitmap(v)
if !v {
for _, f := range b.fields {
f.AppendNull()
}
}
}
func (b *StructBuilder) AppendValues(valids []bool) {
b.Reserve(len(valids))
b.builder.unsafeAppendBoolsToBitmap(valids, len(valids))
}
func (b *StructBuilder) AppendNull() { b.Append(false) }
func (b *StructBuilder) unsafeAppend(v bool) {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
b.length++
}
func (b *StructBuilder) unsafeAppendBoolToBitmap(isValid bool) {
if isValid {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
} else {
b.nulls++
}
b.length++
}
func (b *StructBuilder) init(capacity int) {
b.builder.init(capacity)
for _, f := range b.fields {
f.init(capacity)
}
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *StructBuilder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *StructBuilder) Resize(n int) {
if n < minBuilderCapacity {
n = minBuilderCapacity
}
if b.capacity == 0 {
b.init(n)
} else {
b.builder.resize(n, b.builder.init)
for _, f := range b.fields {
f.resize(n, f.init)
}
}
}
func (b *StructBuilder) NumField() int { return len(b.fields) }
func (b *StructBuilder) FieldBuilder(i int) Builder { return b.fields[i] }
// NewArray creates a Struct array from the memory buffers used by the builder and resets the StructBuilder
// so it can be used to build a new array.
func (b *StructBuilder) NewArray() Interface {
return b.NewStructArray()
}
// NewStructArray creates a Struct array from the memory buffers used by the builder and resets the StructBuilder
// so it can be used to build a new array.
func (b *StructBuilder) NewStructArray() (a *Struct) {
data := b.newData()
a = NewStructData(data)
data.Release()
return
}
func (b *StructBuilder) newData() (data *Data) {
fields := make([]*Data, len(b.fields))
for i, f := range b.fields {
arr := f.NewArray()
defer arr.Release()
fields[i] = arr.Data()
}
data = NewData(
b.dtype, b.length,
[]*memory.Buffer{
b.nullBitmap,
nil, // FIXME(sbinet)
},
fields,
b.nulls,
0,
)
b.reset()
return
}
var (
_ Interface = (*Struct)(nil)
_ Builder = (*StructBuilder)(nil)
)

455
vendor/github.com/apache/arrow/go/arrow/array/table.go generated vendored Normal file
View File

@ -0,0 +1,455 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"errors"
"fmt"
"math"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/debug"
)
// Table represents a logical sequence of chunked arrays.
type Table interface {
Schema() *arrow.Schema
NumRows() int64
NumCols() int64
Column(i int) *Column
Retain()
Release()
}
// Column is an immutable column data structure consisting of
// a field (type metadata) and a chunked data array.
type Column struct {
field arrow.Field
data *Chunked
}
// NewColumn returns a column from a field and a chunked data array.
//
// NewColumn panics if the field's data type is inconsistent with the data type
// of the chunked data array.
func NewColumn(field arrow.Field, chunks *Chunked) *Column {
col := Column{
field: field,
data: chunks,
}
col.data.Retain()
if !arrow.TypeEquals(col.data.DataType(), col.field.Type) {
col.data.Release()
panic("arrow/array: inconsistent data type")
}
return &col
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (col *Column) Retain() {
col.data.Retain()
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (col *Column) Release() {
col.data.Release()
}
func (col *Column) Len() int { return col.data.Len() }
func (col *Column) NullN() int { return col.data.NullN() }
func (col *Column) Data() *Chunked { return col.data }
func (col *Column) Field() arrow.Field { return col.field }
func (col *Column) Name() string { return col.field.Name }
func (col *Column) DataType() arrow.DataType { return col.field.Type }
// NewSlice returns a new zero-copy slice of the column with the indicated
// indices i and j, corresponding to the column's array[i:j].
// The returned column must be Release()'d after use.
//
// NewSlice panics if the slice is outside the valid range of the column's array.
// NewSlice panics if j < i.
func (col *Column) NewSlice(i, j int64) *Column {
return &Column{
field: col.field,
data: col.data.NewSlice(i, j),
}
}
// Chunked manages a collection of primitives arrays as one logical large array.
type Chunked struct {
chunks []Interface
refCount int64
length int
nulls int
dtype arrow.DataType
}
// NewChunked returns a new chunked array from the slice of arrays.
//
// NewChunked panics if the chunks do not have the same data type.
func NewChunked(dtype arrow.DataType, chunks []Interface) *Chunked {
arr := &Chunked{
chunks: make([]Interface, len(chunks)),
refCount: 1,
dtype: dtype,
}
for i, chunk := range chunks {
if !arrow.TypeEquals(chunk.DataType(), dtype) {
panic("arrow/array: mismatch data type")
}
chunk.Retain()
arr.chunks[i] = chunk
arr.length += chunk.Len()
arr.nulls += chunk.NullN()
}
return arr
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (a *Chunked) Retain() {
atomic.AddInt64(&a.refCount, 1)
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (a *Chunked) Release() {
debug.Assert(atomic.LoadInt64(&a.refCount) > 0, "too many releases")
if atomic.AddInt64(&a.refCount, -1) == 0 {
for _, arr := range a.chunks {
arr.Release()
}
a.chunks = nil
a.length = 0
a.nulls = 0
}
}
func (a *Chunked) Len() int { return a.length }
func (a *Chunked) NullN() int { return a.nulls }
func (a *Chunked) DataType() arrow.DataType { return a.dtype }
func (a *Chunked) Chunks() []Interface { return a.chunks }
func (a *Chunked) Chunk(i int) Interface { return a.chunks[i] }
// NewSlice constructs a zero-copy slice of the chunked array with the indicated
// indices i and j, corresponding to array[i:j].
// The returned chunked array must be Release()'d after use.
//
// NewSlice panics if the slice is outside the valid range of the input array.
// NewSlice panics if j < i.
func (a *Chunked) NewSlice(i, j int64) *Chunked {
if j > int64(a.length) || i > j || i > int64(a.length) {
panic("arrow/array: index out of range")
}
var (
cur = 0
beg = i
sz = j - i
chunks = make([]Interface, 0, len(a.chunks))
)
for cur < len(a.chunks) && beg >= int64(a.chunks[cur].Len()) {
beg -= int64(a.chunks[cur].Len())
cur++
}
for cur < len(a.chunks) && sz > 0 {
arr := a.chunks[cur]
end := beg + sz
if end > int64(arr.Len()) {
end = int64(arr.Len())
}
chunks = append(chunks, NewSlice(arr, beg, end))
sz -= int64(arr.Len()) - beg
beg = 0
cur++
}
chunks = chunks[:len(chunks):len(chunks)]
defer func() {
for _, chunk := range chunks {
chunk.Release()
}
}()
return NewChunked(a.dtype, chunks)
}
// simpleTable is a basic, non-lazy in-memory table.
type simpleTable struct {
refCount int64
rows int64
cols []Column
schema *arrow.Schema
}
// NewTable returns a new basic, non-lazy in-memory table.
// If rows is negative, the number of rows will be inferred from the height
// of the columns.
//
// NewTable panics if the columns and schema are inconsistent.
// NewTable panics if rows is larger than the height of the columns.
func NewTable(schema *arrow.Schema, cols []Column, rows int64) *simpleTable {
tbl := simpleTable{
refCount: 1,
rows: rows,
cols: cols,
schema: schema,
}
if tbl.rows < 0 {
switch len(tbl.cols) {
case 0:
tbl.rows = 0
default:
tbl.rows = int64(tbl.cols[0].Len())
}
}
// validate the table and its constituents.
// note we retain the columns after having validated the table
// in case the validation fails and panics (and would otherwise leak
// a ref-count on the columns.)
tbl.validate()
for i := range tbl.cols {
tbl.cols[i].Retain()
}
return &tbl
}
// NewTableFromRecords returns a new basic, non-lazy in-memory table.
//
// NewTableFromRecords panics if the records and schema are inconsistent.
func NewTableFromRecords(schema *arrow.Schema, recs []Record) *simpleTable {
arrs := make([]Interface, len(recs))
cols := make([]Column, len(schema.Fields()))
defer func(cols []Column) {
for i := range cols {
cols[i].Release()
}
}(cols)
for i := range cols {
field := schema.Field(i)
for j, rec := range recs {
arrs[j] = rec.Column(i)
}
chunk := NewChunked(field.Type, arrs)
cols[i] = *NewColumn(field, chunk)
chunk.Release()
}
return NewTable(schema, cols, -1)
}
func (tbl *simpleTable) Schema() *arrow.Schema { return tbl.schema }
func (tbl *simpleTable) NumRows() int64 { return tbl.rows }
func (tbl *simpleTable) NumCols() int64 { return int64(len(tbl.cols)) }
func (tbl *simpleTable) Column(i int) *Column { return &tbl.cols[i] }
func (tbl *simpleTable) validate() {
if len(tbl.cols) != len(tbl.schema.Fields()) {
panic(errors.New("arrow/array: table schema mismatch"))
}
for i, col := range tbl.cols {
if !col.field.Equal(tbl.schema.Field(i)) {
panic(fmt.Errorf("arrow/array: column field %q is inconsistent with schema", col.Name()))
}
if int64(col.Len()) < tbl.rows {
panic(fmt.Errorf("arrow/array: column %q expected length >= %d but got length %d", col.Name(), tbl.rows, col.Len()))
}
}
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (tbl *simpleTable) Retain() {
atomic.AddInt64(&tbl.refCount, 1)
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (tbl *simpleTable) Release() {
debug.Assert(atomic.LoadInt64(&tbl.refCount) > 0, "too many releases")
if atomic.AddInt64(&tbl.refCount, -1) == 0 {
for i := range tbl.cols {
tbl.cols[i].Release()
}
tbl.cols = nil
}
}
// TableReader is a Record iterator over a (possibly chunked) Table
type TableReader struct {
refCount int64
tbl Table
cur int64 // current row
max int64 // total number of rows
rec Record // current Record
chksz int64 // chunk size
chunks []*Chunked
slots []int // chunk indices
offsets []int64 // chunk offsets
}
// NewTableReader returns a new TableReader to iterate over the (possibly chunked) Table.
// if chunkSize is <= 0, the biggest possible chunk will be selected.
func NewTableReader(tbl Table, chunkSize int64) *TableReader {
ncols := tbl.NumCols()
tr := &TableReader{
refCount: 1,
tbl: tbl,
cur: 0,
max: int64(tbl.NumRows()),
chksz: chunkSize,
chunks: make([]*Chunked, ncols),
slots: make([]int, ncols),
offsets: make([]int64, ncols),
}
tr.tbl.Retain()
if tr.chksz <= 0 {
tr.chksz = math.MaxInt64
}
for i := range tr.chunks {
col := tr.tbl.Column(i)
tr.chunks[i] = col.Data()
tr.chunks[i].Retain()
}
return tr
}
func (tr *TableReader) Schema() *arrow.Schema { return tr.tbl.Schema() }
func (tr *TableReader) Record() Record { return tr.rec }
func (tr *TableReader) Next() bool {
if tr.cur >= tr.max {
return false
}
if tr.rec != nil {
tr.rec.Release()
}
// determine the minimum contiguous slice across all columns
chunksz := imin64(tr.max, tr.chksz)
chunks := make([]Interface, len(tr.chunks))
for i := range chunks {
j := tr.slots[i]
chunk := tr.chunks[i].Chunk(j)
remain := int64(chunk.Len()) - tr.offsets[i]
if remain < chunksz {
chunksz = remain
}
chunks[i] = chunk
}
// slice the chunks, advance each chunk slot as appropriate.
batch := make([]Interface, len(tr.chunks))
for i, chunk := range chunks {
var slice Interface
offset := tr.offsets[i]
switch int64(chunk.Len()) - offset {
case chunksz:
tr.slots[i]++
tr.offsets[i] = 0
if offset > 0 {
// need to slice
slice = NewSlice(chunk, offset, offset+chunksz)
} else {
// no need to slice
slice = chunk
slice.Retain()
}
default:
tr.offsets[i] += chunksz
slice = NewSlice(chunk, offset, offset+chunksz)
}
batch[i] = slice
}
tr.cur += chunksz
tr.rec = NewRecord(tr.tbl.Schema(), batch, chunksz)
for _, arr := range batch {
arr.Release()
}
return true
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (tr *TableReader) Retain() {
atomic.AddInt64(&tr.refCount, 1)
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (tr *TableReader) Release() {
debug.Assert(atomic.LoadInt64(&tr.refCount) > 0, "too many releases")
if atomic.AddInt64(&tr.refCount, -1) == 0 {
tr.tbl.Release()
for _, chk := range tr.chunks {
chk.Release()
}
if tr.rec != nil {
tr.rec.Release()
}
tr.tbl = nil
tr.chunks = nil
tr.slots = nil
tr.offsets = nil
}
}
func imin64(a, b int64) int64 {
if a < b {
return a
}
return b
}
var (
_ Table = (*simpleTable)(nil)
_ RecordReader = (*TableReader)(nil)
)

24
vendor/github.com/apache/arrow/go/arrow/array/util.go generated vendored Normal file
View File

@ -0,0 +1,24 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
func min(a, b int) int {
if a < b {
return a
}
return b
}

91
vendor/github.com/apache/arrow/go/arrow/arrio/arrio.go generated vendored Normal file
View File

@ -0,0 +1,91 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package arrio exposes functions to manipulate records, exposing and using
// interfaces not unlike the ones defined in the stdlib io package.
package arrio // import "github.com/apache/arrow/go/arrow/arrio"
import (
"io"
"github.com/apache/arrow/go/arrow/array"
)
// Reader is the interface that wraps the Read method.
type Reader interface {
// Read reads the current record from the underlying stream and an error, if any.
// When the Reader reaches the end of the underlying stream, it returns (nil, io.EOF).
Read() (array.Record, error)
}
// ReaderAt is the interface that wraps the ReadAt method.
type ReaderAt interface {
// ReadAt reads the i-th record from the underlying stream and an error, if any.
ReadAt(i int64) (array.Record, error)
}
// Writer is the interface that wraps the Write method.
type Writer interface {
Write(rec array.Record) error
}
// Copy copies all the records available from src to dst.
// Copy returns the number of records copied and the first error
// encountered while copying, if any.
//
// A successful Copy returns err == nil, not err == EOF. Because Copy is
// defined to read from src until EOF, it does not treat an EOF from Read as an
// error to be reported.
func Copy(dst Writer, src Reader) (n int64, err error) {
for {
rec, err := src.Read()
if err != nil {
if err == io.EOF {
return n, nil
}
return n, err
}
err = dst.Write(rec)
if err != nil {
return n, err
}
n++
}
}
// CopyN copies n records (or until an error) from src to dst. It returns the
// number of records copied and the earliest error encountered while copying. On
// return, written == n if and only if err == nil.
func CopyN(dst Writer, src Reader, n int64) (written int64, err error) {
for ; written < n; written++ {
rec, err := src.Read()
if err != nil {
if err == io.EOF && written == n {
return written, nil
}
return written, err
}
err = dst.Write(rec)
if err != nil {
return written, err
}
}
if written != n && err == nil {
err = io.EOF
}
return written, err
}

79
vendor/github.com/apache/arrow/go/arrow/compare.go generated vendored Normal file
View File

@ -0,0 +1,79 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package arrow
import (
"reflect"
)
type typeEqualsConfig struct {
metadata bool
}
// TypeEqualsOption is a functional option type used for configuring type
// equality checks.
type TypeEqualsOption func(*typeEqualsConfig)
// CheckMetadata is an option for TypeEquals that allows checking for metadata
// equality besides type equality. It only makes sense for STRUCT type.
func CheckMetadata() TypeEqualsOption {
return func(cfg *typeEqualsConfig) {
cfg.metadata = true
}
}
// TypeEquals checks if two DataType are the same, optionally checking metadata
// equality for STRUCT types.
func TypeEquals(left, right DataType, opts ...TypeEqualsOption) bool {
var cfg typeEqualsConfig
for _, opt := range opts {
opt(&cfg)
}
switch {
case left == nil || right == nil:
return false
case left.ID() != right.ID():
return false
}
// StructType is the only type that has metadata.
l, ok := left.(*StructType)
if !ok || cfg.metadata {
return reflect.DeepEqual(left, right)
}
r := right.(*StructType)
switch {
case len(l.fields) != len(r.fields):
return false
case !reflect.DeepEqual(l.index, r.index):
return false
}
for i := range l.fields {
leftField, rightField := l.fields[i], r.fields[i]
switch {
case leftField.Name != rightField.Name:
return false
case leftField.Nullable != rightField.Nullable:
return false
case !TypeEquals(leftField.Type, rightField.Type, opts...):
return false
}
}
return true
}

143
vendor/github.com/apache/arrow/go/arrow/datatype.go generated vendored Normal file
View File

@ -0,0 +1,143 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package arrow
// Type is a logical type. They can be expressed as
// either a primitive physical type (bytes or bits of some fixed size), a
// nested type consisting of other data types, or another data type (e.g. a
// timestamp encoded as an int64)
type Type int
const (
// NULL type having no physical storage
NULL Type = iota
// BOOL is a 1 bit, LSB bit-packed ordering
BOOL
// UINT8 is an Unsigned 8-bit little-endian integer
UINT8
// INT8 is a Signed 8-bit little-endian integer
INT8
// UINT16 is an Unsigned 16-bit little-endian integer
UINT16
// INT16 is a Signed 16-bit little-endian integer
INT16
// UINT32 is an Unsigned 32-bit little-endian integer
UINT32
// INT32 is a Signed 32-bit little-endian integer
INT32
// UINT64 is an Unsigned 64-bit little-endian integer
UINT64
// INT64 is a Signed 64-bit little-endian integer
INT64
// FLOAT16 is a 2-byte floating point value
FLOAT16
// FLOAT32 is a 4-byte floating point value
FLOAT32
// FLOAT64 is an 8-byte floating point value
FLOAT64
// STRING is a UTF8 variable-length string
STRING
// BINARY is a Variable-length byte type (no guarantee of UTF8-ness)
BINARY
// FIXED_SIZE_BINARY is a binary where each value occupies the same number of bytes
FIXED_SIZE_BINARY
// DATE32 is int32 days since the UNIX epoch
DATE32
// DATE64 is int64 milliseconds since the UNIX epoch
DATE64
// TIMESTAMP is an exact timestamp encoded with int64 since UNIX epoch
// Default unit millisecond
TIMESTAMP
// TIME32 is a signed 32-bit integer, representing either seconds or
// milliseconds since midnight
TIME32
// TIME64 is a signed 64-bit integer, representing either microseconds or
// nanoseconds since midnight
TIME64
// INTERVAL is YEAR_MONTH or DAY_TIME interval in SQL style
INTERVAL
// DECIMAL is a precision- and scale-based decimal type. Storage type depends on the
// parameters.
DECIMAL
// LIST is a list of some logical data type
LIST
// STRUCT of logical types
STRUCT
// UNION of logical types
UNION
// DICTIONARY aka Category type
DICTIONARY
// MAP is a repeated struct logical type
MAP
// Custom data type, implemented by user
EXTENSION
// Fixed size list of some logical type
FIXED_SIZE_LIST
// Measure of elapsed time in either seconds, milliseconds, microseconds
// or nanoseconds.
DURATION
)
// DataType is the representation of an Arrow type.
type DataType interface {
ID() Type
// Name is name of the data type.
Name() string
}
// FixedWidthDataType is the representation of an Arrow type that
// requires a fixed number of bits in memory for each element.
type FixedWidthDataType interface {
DataType
// BitWidth returns the number of bits required to store a single element of this data type in memory.
BitWidth() int
}
type BinaryDataType interface {
DataType
binary()
}

41
vendor/github.com/apache/arrow/go/arrow/datatype_binary.go generated vendored Normal file
View File

@ -0,0 +1,41 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package arrow
type BinaryType struct{}
func (t *BinaryType) ID() Type { return BINARY }
func (t *BinaryType) Name() string { return "binary" }
func (t *BinaryType) String() string { return "binary" }
func (t *BinaryType) binary() {}
type StringType struct{}
func (t *StringType) ID() Type { return STRING }
func (t *StringType) Name() string { return "utf8" }
func (t *StringType) String() string { return "utf8" }
func (t *StringType) binary() {}
var (
BinaryTypes = struct {
Binary BinaryDataType
String BinaryDataType
}{
Binary: &BinaryType{},
String: &StringType{},
}
)

213
vendor/github.com/apache/arrow/go/arrow/datatype_fixedwidth.go generated vendored Normal file
View File

@ -0,0 +1,213 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package arrow
import (
"fmt"
"strconv"
)
type BooleanType struct{}
func (t *BooleanType) ID() Type { return BOOL }
func (t *BooleanType) Name() string { return "bool" }
func (t *BooleanType) String() string { return "bool" }
// BitWidth returns the number of bits required to store a single element of this data type in memory.
func (t *BooleanType) BitWidth() int { return 1 }
type FixedSizeBinaryType struct {
ByteWidth int
}
func (*FixedSizeBinaryType) ID() Type { return FIXED_SIZE_BINARY }
func (*FixedSizeBinaryType) Name() string { return "fixed_size_binary" }
func (t *FixedSizeBinaryType) BitWidth() int { return 8 * t.ByteWidth }
func (t *FixedSizeBinaryType) String() string {
return "fixed_size_binary[" + strconv.Itoa(t.ByteWidth) + "]"
}
type (
Timestamp int64
Time32 int32
Time64 int64
TimeUnit int
Date32 int32
Date64 int64
Duration int64
)
const (
Nanosecond TimeUnit = iota
Microsecond
Millisecond
Second
)
func (u TimeUnit) String() string { return [...]string{"ns", "us", "ms", "s"}[uint(u)&3] }
// TimestampType is encoded as a 64-bit signed integer since the UNIX epoch (2017-01-01T00:00:00Z).
// The zero-value is a nanosecond and time zone neutral. Time zone neutral can be
// considered UTC without having "UTC" as a time zone.
type TimestampType struct {
Unit TimeUnit
TimeZone string
}
func (*TimestampType) ID() Type { return TIMESTAMP }
func (*TimestampType) Name() string { return "timestamp" }
func (t *TimestampType) String() string {
switch len(t.TimeZone) {
case 0:
return "timestamp[" + t.Unit.String() + "]"
default:
return "timestamp[" + t.Unit.String() + ", tz=" + t.TimeZone + "]"
}
}
// BitWidth returns the number of bits required to store a single element of this data type in memory.
func (*TimestampType) BitWidth() int { return 64 }
// Time32Type is encoded as a 32-bit signed integer, representing either seconds or milliseconds since midnight.
type Time32Type struct {
Unit TimeUnit
}
func (*Time32Type) ID() Type { return TIME32 }
func (*Time32Type) Name() string { return "time32" }
func (*Time32Type) BitWidth() int { return 32 }
func (t *Time32Type) String() string { return "time32[" + t.Unit.String() + "]" }
// Time64Type is encoded as a 64-bit signed integer, representing either microseconds or nanoseconds since midnight.
type Time64Type struct {
Unit TimeUnit
}
func (*Time64Type) ID() Type { return TIME64 }
func (*Time64Type) Name() string { return "time64" }
func (*Time64Type) BitWidth() int { return 64 }
func (t *Time64Type) String() string { return "time64[" + t.Unit.String() + "]" }
// DurationType is encoded as a 64-bit signed integer, representing an amount
// of elapsed time without any relation to a calendar artifact.
type DurationType struct {
Unit TimeUnit
}
func (*DurationType) ID() Type { return DURATION }
func (*DurationType) Name() string { return "duration" }
func (*DurationType) BitWidth() int { return 64 }
func (t *DurationType) String() string { return "duration[" + t.Unit.String() + "]" }
// Float16Type represents a floating point value encoded with a 16-bit precision.
type Float16Type struct{}
func (t *Float16Type) ID() Type { return FLOAT16 }
func (t *Float16Type) Name() string { return "float16" }
func (t *Float16Type) String() string { return "float16" }
// BitWidth returns the number of bits required to store a single element of this data type in memory.
func (t *Float16Type) BitWidth() int { return 16 }
// Decimal128Type represents a fixed-size 128-bit decimal type.
type Decimal128Type struct {
Precision int32
Scale int32
}
func (*Decimal128Type) ID() Type { return DECIMAL }
func (*Decimal128Type) Name() string { return "decimal" }
func (*Decimal128Type) BitWidth() int { return 16 }
func (t *Decimal128Type) String() string {
return fmt.Sprintf("%s(%d, %d)", t.Name(), t.Precision, t.Scale)
}
// MonthInterval represents a number of months.
type MonthInterval int32
// MonthIntervalType is encoded as a 32-bit signed integer,
// representing a number of months.
type MonthIntervalType struct{}
func (*MonthIntervalType) ID() Type { return INTERVAL }
func (*MonthIntervalType) Name() string { return "month_interval" }
func (*MonthIntervalType) String() string { return "month_interval" }
// BitWidth returns the number of bits required to store a single element of this data type in memory.
func (t *MonthIntervalType) BitWidth() int { return 32 }
// DayTimeInterval represents a number of days and milliseconds (fraction of day).
type DayTimeInterval struct {
Days int32 `json:"days"`
Milliseconds int32 `json:"milliseconds"`
}
// DayTimeIntervalType is encoded as a pair of 32-bit signed integer,
// representing a number of days and milliseconds (fraction of day).
type DayTimeIntervalType struct{}
func (*DayTimeIntervalType) ID() Type { return INTERVAL }
func (*DayTimeIntervalType) Name() string { return "day_time_interval" }
func (*DayTimeIntervalType) String() string { return "day_time_interval" }
// BitWidth returns the number of bits required to store a single element of this data type in memory.
func (t *DayTimeIntervalType) BitWidth() int { return 64 }
var (
FixedWidthTypes = struct {
Boolean FixedWidthDataType
Date32 FixedWidthDataType
Date64 FixedWidthDataType
DayTimeInterval FixedWidthDataType
Duration_s FixedWidthDataType
Duration_ms FixedWidthDataType
Duration_us FixedWidthDataType
Duration_ns FixedWidthDataType
Float16 FixedWidthDataType
MonthInterval FixedWidthDataType
Time32s FixedWidthDataType
Time32ms FixedWidthDataType
Time64us FixedWidthDataType
Time64ns FixedWidthDataType
Timestamp_s FixedWidthDataType
Timestamp_ms FixedWidthDataType
Timestamp_us FixedWidthDataType
Timestamp_ns FixedWidthDataType
}{
Boolean: &BooleanType{},
Date32: &Date32Type{},
Date64: &Date64Type{},
DayTimeInterval: &DayTimeIntervalType{},
Duration_s: &DurationType{Unit: Second},
Duration_ms: &DurationType{Unit: Millisecond},
Duration_us: &DurationType{Unit: Microsecond},
Duration_ns: &DurationType{Unit: Nanosecond},
Float16: &Float16Type{},
MonthInterval: &MonthIntervalType{},
Time32s: &Time32Type{Unit: Second},
Time32ms: &Time32Type{Unit: Millisecond},
Time64us: &Time64Type{Unit: Microsecond},
Time64ns: &Time64Type{Unit: Nanosecond},
Timestamp_s: &TimestampType{Unit: Second, TimeZone: "UTC"},
Timestamp_ms: &TimestampType{Unit: Millisecond, TimeZone: "UTC"},
Timestamp_us: &TimestampType{Unit: Microsecond, TimeZone: "UTC"},
Timestamp_ns: &TimestampType{Unit: Nanosecond, TimeZone: "UTC"},
}
_ FixedWidthDataType = (*FixedSizeBinaryType)(nil)
)

180
vendor/github.com/apache/arrow/go/arrow/datatype_nested.go generated vendored Normal file
View File

@ -0,0 +1,180 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package arrow
import (
"fmt"
"reflect"
"strings"
)
// ListType describes a nested type in which each array slot contains
// a variable-size sequence of values, all having the same relative type.
type ListType struct {
elem DataType // DataType of the list's elements
}
// ListOf returns the list type with element type t.
// For example, if t represents int32, ListOf(t) represents []int32.
//
// ListOf panics if t is nil or invalid.
func ListOf(t DataType) *ListType {
if t == nil {
panic("arrow: nil DataType")
}
return &ListType{elem: t}
}
func (*ListType) ID() Type { return LIST }
func (*ListType) Name() string { return "list" }
func (t *ListType) String() string { return fmt.Sprintf("list<item: %v>", t.elem) }
// Elem returns the ListType's element type.
func (t *ListType) Elem() DataType { return t.elem }
// FixedSizeListType describes a nested type in which each array slot contains
// a fixed-size sequence of values, all having the same relative type.
type FixedSizeListType struct {
n int32 // number of elements in the list
elem DataType // DataType of the list's elements
}
// FixedSizeListOf returns the list type with element type t.
// For example, if t represents int32, FixedSizeListOf(10, t) represents [10]int32.
//
// FixedSizeListOf panics if t is nil or invalid.
// FixedSizeListOf panics if n is <= 0.
func FixedSizeListOf(n int32, t DataType) *FixedSizeListType {
if t == nil {
panic("arrow: nil DataType")
}
if n <= 0 {
panic("arrow: invalid size")
}
return &FixedSizeListType{elem: t, n: n}
}
func (*FixedSizeListType) ID() Type { return FIXED_SIZE_LIST }
func (*FixedSizeListType) Name() string { return "fixed_size_list" }
func (t *FixedSizeListType) String() string {
return fmt.Sprintf("fixed_size_list<item: %v>[%d]", t.elem, t.n)
}
// Elem returns the FixedSizeListType's element type.
func (t *FixedSizeListType) Elem() DataType { return t.elem }
// Len returns the FixedSizeListType's size.
func (t *FixedSizeListType) Len() int32 { return t.n }
// StructType describes a nested type parameterized by an ordered sequence
// of relative types, called its fields.
type StructType struct {
fields []Field
index map[string]int
meta Metadata
}
// StructOf returns the struct type with fields fs.
//
// StructOf panics if there are duplicated fields.
// StructOf panics if there is a field with an invalid DataType.
func StructOf(fs ...Field) *StructType {
n := len(fs)
if n == 0 {
return &StructType{}
}
t := &StructType{
fields: make([]Field, n),
index: make(map[string]int, n),
}
for i, f := range fs {
if f.Type == nil {
panic("arrow: field with nil DataType")
}
t.fields[i] = Field{
Name: f.Name,
Type: f.Type,
Nullable: f.Nullable,
Metadata: f.Metadata.clone(),
}
if _, dup := t.index[f.Name]; dup {
panic(fmt.Errorf("arrow: duplicate field with name %q", f.Name))
}
t.index[f.Name] = i
}
return t
}
func (*StructType) ID() Type { return STRUCT }
func (*StructType) Name() string { return "struct" }
func (t *StructType) String() string {
o := new(strings.Builder)
o.WriteString("struct<")
for i, f := range t.fields {
if i > 0 {
o.WriteString(", ")
}
o.WriteString(fmt.Sprintf("%s: %v", f.Name, f.Type))
}
o.WriteString(">")
return o.String()
}
func (t *StructType) Fields() []Field { return t.fields }
func (t *StructType) Field(i int) Field { return t.fields[i] }
func (t *StructType) FieldByName(name string) (Field, bool) {
i, ok := t.index[name]
if !ok {
return Field{}, false
}
return t.fields[i], true
}
type Field struct {
Name string // Field name
Type DataType // The field's data type
Nullable bool // Fields can be nullable
Metadata Metadata // The field's metadata, if any
}
func (f Field) HasMetadata() bool { return f.Metadata.Len() != 0 }
func (f Field) Equal(o Field) bool {
return reflect.DeepEqual(f, o)
}
func (f Field) String() string {
o := new(strings.Builder)
nullable := ""
if f.Nullable {
nullable = ", nullable"
}
fmt.Fprintf(o, "%s: type=%v%v", f.Name, f.Type, nullable)
if f.HasMetadata() {
fmt.Fprintf(o, "\n%*.smetadata: %v", len(f.Name)+2, "", f.Metadata)
}
return o.String()
}
var (
_ DataType = (*ListType)(nil)
_ DataType = (*StructType)(nil)
)

29
vendor/github.com/apache/arrow/go/arrow/datatype_null.go generated vendored Normal file
View File

@ -0,0 +1,29 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package arrow
// NullType describes a degenerate array, with zero physical storage.
type NullType struct{}
func (*NullType) ID() Type { return NULL }
func (*NullType) Name() string { return "null" }
func (*NullType) String() string { return "null" }
var (
Null *NullType
_ DataType = Null
)

134
vendor/github.com/apache/arrow/go/arrow/datatype_numeric.gen.go generated vendored Normal file
View File

@ -0,0 +1,134 @@
// Code generated by datatype_numeric.gen.go.tmpl. DO NOT EDIT.
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package arrow
type Int8Type struct{}
func (t *Int8Type) ID() Type { return INT8 }
func (t *Int8Type) Name() string { return "int8" }
func (t *Int8Type) String() string { return "int8" }
func (t *Int8Type) BitWidth() int { return 8 }
type Int16Type struct{}
func (t *Int16Type) ID() Type { return INT16 }
func (t *Int16Type) Name() string { return "int16" }
func (t *Int16Type) String() string { return "int16" }
func (t *Int16Type) BitWidth() int { return 16 }
type Int32Type struct{}
func (t *Int32Type) ID() Type { return INT32 }
func (t *Int32Type) Name() string { return "int32" }
func (t *Int32Type) String() string { return "int32" }
func (t *Int32Type) BitWidth() int { return 32 }
type Int64Type struct{}
func (t *Int64Type) ID() Type { return INT64 }
func (t *Int64Type) Name() string { return "int64" }
func (t *Int64Type) String() string { return "int64" }
func (t *Int64Type) BitWidth() int { return 64 }
type Uint8Type struct{}
func (t *Uint8Type) ID() Type { return UINT8 }
func (t *Uint8Type) Name() string { return "uint8" }
func (t *Uint8Type) String() string { return "uint8" }
func (t *Uint8Type) BitWidth() int { return 8 }
type Uint16Type struct{}
func (t *Uint16Type) ID() Type { return UINT16 }
func (t *Uint16Type) Name() string { return "uint16" }
func (t *Uint16Type) String() string { return "uint16" }
func (t *Uint16Type) BitWidth() int { return 16 }
type Uint32Type struct{}
func (t *Uint32Type) ID() Type { return UINT32 }
func (t *Uint32Type) Name() string { return "uint32" }
func (t *Uint32Type) String() string { return "uint32" }
func (t *Uint32Type) BitWidth() int { return 32 }
type Uint64Type struct{}
func (t *Uint64Type) ID() Type { return UINT64 }
func (t *Uint64Type) Name() string { return "uint64" }
func (t *Uint64Type) String() string { return "uint64" }
func (t *Uint64Type) BitWidth() int { return 64 }
type Float32Type struct{}
func (t *Float32Type) ID() Type { return FLOAT32 }
func (t *Float32Type) Name() string { return "float32" }
func (t *Float32Type) String() string { return "float32" }
func (t *Float32Type) BitWidth() int { return 32 }
type Float64Type struct{}
func (t *Float64Type) ID() Type { return FLOAT64 }
func (t *Float64Type) Name() string { return "float64" }
func (t *Float64Type) String() string { return "float64" }
func (t *Float64Type) BitWidth() int { return 64 }
type Date32Type struct{}
func (t *Date32Type) ID() Type { return DATE32 }
func (t *Date32Type) Name() string { return "date32" }
func (t *Date32Type) String() string { return "date32" }
func (t *Date32Type) BitWidth() int { return 32 }
type Date64Type struct{}
func (t *Date64Type) ID() Type { return DATE64 }
func (t *Date64Type) Name() string { return "date64" }
func (t *Date64Type) String() string { return "date64" }
func (t *Date64Type) BitWidth() int { return 64 }
var (
PrimitiveTypes = struct {
Int8 DataType
Int16 DataType
Int32 DataType
Int64 DataType
Uint8 DataType
Uint16 DataType
Uint32 DataType
Uint64 DataType
Float32 DataType
Float64 DataType
Date32 DataType
Date64 DataType
}{
Int8: &Int8Type{},
Int16: &Int16Type{},
Int32: &Int32Type{},
Int64: &Int64Type{},
Uint8: &Uint8Type{},
Uint16: &Uint16Type{},
Uint32: &Uint32Type{},
Uint64: &Uint64Type{},
Float32: &Float32Type{},
Float64: &Float64Type{},
Date32: &Date32Type{},
Date64: &Date64Type{},
}
)

40
vendor/github.com/apache/arrow/go/arrow/datatype_numeric.gen.go.tmpl generated vendored Normal file
View File

@ -0,0 +1,40 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package arrow
{{range .In}}
type {{.Name}}Type struct {}
func (t *{{.Name}}Type) ID() Type { return {{.Name|upper}} }
func (t *{{.Name}}Type) Name() string { return "{{.Name|lower}}" }
func (t *{{.Name}}Type) String() string { return "{{.Name|lower}}" }
func (t *{{.Name}}Type) BitWidth() int { return {{.Size}} }
{{end}}
var (
PrimitiveTypes = struct {
{{range .In}}
{{.Name}} DataType
{{- end}}
}{
{{range .In}}
{{.Name}}: &{{.Name}}Type{},
{{- end}}
}
)

62
vendor/github.com/apache/arrow/go/arrow/datatype_numeric.gen.go.tmpldata generated vendored Normal file
View File

@ -0,0 +1,62 @@
[
{
"Name": "Int8",
"Type": "int8",
"Size": 8
},
{
"Name": "Int16",
"Type": "int16",
"Size": 16
},
{
"Name": "Int32",
"Type": "int32",
"Size": 32
},
{
"Name": "Int64",
"Type": "int64",
"Size": 64
},
{
"Name": "Uint8",
"Type": "uint8",
"Size": 8
},
{
"Name": "Uint16",
"Type": "uint16",
"Size": 16
},
{
"Name": "Uint32",
"Type": "uint32",
"Size": 32
},
{
"Name": "Uint64",
"Type": "uint64",
"Size": 64
},
{
"Name": "Float32",
"Type": "float32",
"Size": 32
},
{
"Name": "Float64",
"Type": "float64",
"Size": 64
},
{
"Name": "Date32",
"Type": "date32",
"Size": 32
},
{
"Name": "Date64",
"Type": "date64",
"Size": 64
}
]

73
vendor/github.com/apache/arrow/go/arrow/decimal128/decimal128.go generated vendored Normal file
View File

@ -0,0 +1,73 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package decimal128 // import "github.com/apache/arrow/go/arrow/decimal128"
var (
MaxDecimal128 = New(542101086242752217, 687399551400673280-1)
)
// Num represents a signed 128-bit integer in two's complement.
// Calculations wrap around and overflow is ignored.
//
// For a discussion of the algorithms, look at Knuth's volume 2,
// Semi-numerical Algorithms section 4.3.1.
//
// Adapted from the Apache ORC C++ implementation
type Num struct {
lo uint64 // low bits
hi int64 // high bits
}
// New returns a new signed 128-bit integer value.
func New(hi int64, lo uint64) Num {
return Num{lo: lo, hi: hi}
}
// FromU64 returns a new signed 128-bit integer value from the provided uint64 one.
func FromU64(v uint64) Num {
return New(0, v)
}
// FromI64 returns a new signed 128-bit integer value from the provided int64 one.
func FromI64(v int64) Num {
switch {
case v > 0:
return New(0, uint64(v))
case v < 0:
return New(-1, uint64(v))
default:
return Num{}
}
}
// LowBits returns the low bits of the two's complement representation of the number.
func (n Num) LowBits() uint64 { return n.lo }
// HighBits returns the high bits of the two's complement representation of the number.
func (n Num) HighBits() int64 { return n.hi }
// Sign returns:
//
// -1 if x < 0
// 0 if x == 0
// +1 if x > 0
func (n Num) Sign() int {
if n == (Num{}) {
return 0
}
return int(1 | (n.hi >> 63))
}

39
vendor/github.com/apache/arrow/go/arrow/doc.go generated vendored Normal file
View File

@ -0,0 +1,39 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
Package arrow provides an implementation of Apache Arrow.
Apache Arrow is a cross-language development platform for in-memory data. It specifies a standardized
language-independent columnar memory format for flat and hierarchical data, organized for efficient analytic
operations on modern hardware. It also provides computational libraries and zero-copy streaming
messaging and inter-process communication.
Basics
The fundamental data structure in Arrow is an Array, which holds a sequence of values of the same type. An array
consists of memory holding the data and an additional validity bitmap that indicates if the corresponding entry in the
array is valid (not null). If the array has no null entries, it is possible to omit this bitmap.
*/
package arrow
//go:generate go run _tools/tmpl/main.go -i -data=numeric.tmpldata type_traits_numeric.gen.go.tmpl type_traits_numeric.gen_test.go.tmpl array/numeric.gen.go.tmpl array/numericbuilder.gen.go.tmpl array/bufferbuilder_numeric.gen.go.tmpl
//go:generate go run _tools/tmpl/main.go -i -data=datatype_numeric.gen.go.tmpldata datatype_numeric.gen.go.tmpl tensor/numeric.gen.go.tmpl tensor/numeric.gen_test.go.tmpl
//go:generate go run ./gen-flatbuffers.go
// stringer
//go:generate stringer -type=Type

70
vendor/github.com/apache/arrow/go/arrow/float16/float16.go generated vendored Normal file
View File

@ -0,0 +1,70 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package float16 // import "github.com/apache/arrow/go/arrow/float16"
import (
"math"
"strconv"
)
// Num represents a half-precision floating point value (float16)
// stored on 16 bits.
//
// See https://en.wikipedia.org/wiki/Half-precision_floating-point_format for more informations.
type Num struct {
bits uint16
}
// New creates a new half-precision floating point value from the provided
// float32 value.
func New(f float32) Num {
b := math.Float32bits(f)
sn := uint16((b >> 31) & 0x1)
exp := (b >> 23) & 0xff
res := int16(exp) - 127 + 15
fc := uint16(b>>13) & 0x3ff
switch {
case exp == 0:
res = 0
case exp == 0xff:
res = 0x1f
case res > 0x1e:
res = 0x1f
fc = 0
case res < 0x01:
res = 0
fc = 0
}
return Num{bits: (sn << 15) | uint16(res<<10) | fc}
}
func (f Num) Float32() float32 {
sn := uint32((f.bits >> 15) & 0x1)
exp := (f.bits >> 10) & 0x1f
res := uint32(exp) + 127 - 15
fc := uint32(f.bits & 0x3ff)
switch {
case exp == 0:
res = 0
case exp == 0x1f:
res = 0xff
}
return math.Float32frombits((sn << 31) | (res << 23) | (fc << 13))
}
func (f Num) Uint16() uint16 { return f.bits }
func (f Num) String() string { return strconv.FormatFloat(float64(f.Float32()), 'g', -1, 32) }

27
vendor/github.com/apache/arrow/go/arrow/go.mod generated vendored Normal file
View File

@ -0,0 +1,27 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
module github.com/apache/arrow/go/arrow
go 1.12
require (
github.com/davecgh/go-spew v1.1.0 // indirect
github.com/google/flatbuffers v1.11.0
github.com/pkg/errors v0.8.1
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/stretchr/testify v1.2.0
)

10
vendor/github.com/apache/arrow/go/arrow/go.sum generated vendored Normal file
View File

@ -0,0 +1,10 @@
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/flatbuffers v1.11.0 h1:O7CEyB8Cb3/DmtxODGtLHcEvpr81Jm5qLg/hsHnxA2A=
github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.2.0 h1:LThGCOvhuJic9Gyd1VBCkhyUXmO8vKaBFvBsJ2k03rg=
github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=

156
vendor/github.com/apache/arrow/go/arrow/internal/bitutil/bitutil.go generated vendored Normal file
View File

@ -0,0 +1,156 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bitutil
import (
"math/bits"
"reflect"
"unsafe"
)
var (
BitMask = [8]byte{1, 2, 4, 8, 16, 32, 64, 128}
FlippedBitMask = [8]byte{254, 253, 251, 247, 239, 223, 191, 127}
)
// IsMultipleOf8 returns whether v is a multiple of 8.
func IsMultipleOf8(v int64) bool { return v&7 == 0 }
func BytesForBits(bits int64) int64 { return (bits + 7) >> 3 }
// NextPowerOf2 rounds x to the next power of two.
func NextPowerOf2(x int) int { return 1 << uint(bits.Len(uint(x))) }
// CeilByte rounds size to the next multiple of 8.
func CeilByte(size int) int { return (size + 7) &^ 7 }
// CeilByte64 rounds size to the next multiple of 8.
func CeilByte64(size int64) int64 { return (size + 7) &^ 7 }
// BitIsSet returns true if the bit at index i in buf is set (1).
func BitIsSet(buf []byte, i int) bool { return (buf[uint(i)/8] & BitMask[byte(i)%8]) != 0 }
// BitIsNotSet returns true if the bit at index i in buf is not set (0).
func BitIsNotSet(buf []byte, i int) bool { return (buf[uint(i)/8] & BitMask[byte(i)%8]) == 0 }
// SetBit sets the bit at index i in buf to 1.
func SetBit(buf []byte, i int) { buf[uint(i)/8] |= BitMask[byte(i)%8] }
// ClearBit sets the bit at index i in buf to 0.
func ClearBit(buf []byte, i int) { buf[uint(i)/8] &= FlippedBitMask[byte(i)%8] }
// SetBitTo sets the bit at index i in buf to val.
func SetBitTo(buf []byte, i int, val bool) {
if val {
SetBit(buf, i)
} else {
ClearBit(buf, i)
}
}
// CountSetBits counts the number of 1's in buf up to n bits.
func CountSetBits(buf []byte, offset, n int) int {
if offset > 0 {
return countSetBitsWithOffset(buf, offset, n)
}
count := 0
uint64Bytes := n / uint64SizeBits * 8
for _, v := range bytesToUint64(buf[:uint64Bytes]) {
count += bits.OnesCount64(v)
}
for _, v := range buf[uint64Bytes : n/8] {
count += bits.OnesCount8(v)
}
// tail bits
for i := n &^ 0x7; i < n; i++ {
if BitIsSet(buf, i) {
count++
}
}
return count
}
func countSetBitsWithOffset(buf []byte, offset, n int) int {
count := 0
beg := offset
end := offset + n
begU8 := roundUp(beg, uint64SizeBits)
init := min(n, begU8-beg)
for i := offset; i < beg+init; i++ {
if BitIsSet(buf, i) {
count++
}
}
nU64 := (n - init) / uint64SizeBits
begU64 := begU8 / uint64SizeBits
endU64 := begU64 + nU64
bufU64 := bytesToUint64(buf)
if begU64 < len(bufU64) {
for _, v := range bufU64[begU64:endU64] {
count += bits.OnesCount64(v)
}
}
// FIXME: use a fallback to bits.OnesCount8
// before counting the tail bits.
tail := beg + init + nU64*uint64SizeBits
for i := tail; i < end; i++ {
if BitIsSet(buf, i) {
count++
}
}
return count
}
func roundUp(v, f int) int {
return (v + (f - 1)) / f * f
}
func min(a, b int) int {
if a < b {
return a
}
return b
}
const (
uint64SizeBytes = int(unsafe.Sizeof(uint64(0)))
uint64SizeBits = uint64SizeBytes * 8
)
func bytesToUint64(b []byte) []uint64 {
h := (*reflect.SliceHeader)(unsafe.Pointer(&b))
var res []uint64
s := (*reflect.SliceHeader)(unsafe.Pointer(&res))
s.Data = h.Data
s.Len = h.Len / uint64SizeBytes
s.Cap = h.Cap / uint64SizeBytes
return res
}

42
vendor/github.com/apache/arrow/go/arrow/internal/cpu/README.md generated vendored Normal file
View File

@ -0,0 +1,42 @@
<!---
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Package cpu
Copied from Go src/internal/cpu
## Extras
### Intel
The `INTEL_DISABLE_EXT` environment variable can control which CPU extensions are available for
the running process. It should be a comma-separate list of upper-case strings as follows
| Flag | Description |
| -------- | ----------- |
| `ALL` | Disable all CPU extensions and fall back to Go implementation |
| `AVX2` | Disable AVX2 optimizations |
| `AVX` | Disable AVX optimizations |
| `SSE` | Disable all SSE optimizations |
| `SSE4` | Disable SSE42, SSE41 optimizations |
| `SSSE3` | Disable supplemental SSE3 optimizations |
| `SSE3` | Disable SSE3 optimizations |
| `SSE2` | Disable SSE2 optimizations |
Any unrecognized flags will be ignored and therefore it is possible to leave the environment variable with a bogus value such as `NONE` when experimenting.

77
vendor/github.com/apache/arrow/go/arrow/internal/cpu/cpu.go generated vendored Normal file
View File

@ -0,0 +1,77 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package cpu implements processor feature detection
// used by the Go standard library.
package cpu
var X86 x86
// The booleans in x86 contain the correspondingly named cpuid feature bit.
// HasAVX and HasAVX2 are only set if the OS does support XMM and YMM registers
// in addition to the cpuid feature bit being set.
// The struct is padded to avoid false sharing.
type x86 struct {
_ [CacheLineSize]byte
HasAES bool
HasADX bool
HasAVX bool
HasAVX2 bool
HasBMI1 bool
HasBMI2 bool
HasERMS bool
HasFMA bool
HasOSXSAVE bool
HasPCLMULQDQ bool
HasPOPCNT bool
HasSSE2 bool
HasSSE3 bool
HasSSSE3 bool
HasSSE41 bool
HasSSE42 bool
_ [CacheLineSize]byte
}
var PPC64 ppc64
// For ppc64x, it is safe to check only for ISA level starting on ISA v3.00,
// since there are no optional categories. There are some exceptions that also
// require kernel support to work (darn, scv), so there are capability bits for
// those as well. The minimum processor requirement is POWER8 (ISA 2.07), so we
// maintain some of the old capability checks for optional categories for
// safety.
// The struct is padded to avoid false sharing.
type ppc64 struct {
_ [CacheLineSize]byte
HasVMX bool // Vector unit (Altivec)
HasDFP bool // Decimal Floating Point unit
HasVSX bool // Vector-scalar unit
HasHTM bool // Hardware Transactional Memory
HasISEL bool // Integer select
HasVCRYPTO bool // Vector cryptography
HasHTMNOSC bool // HTM: kernel-aborted transaction in syscalls
HasDARN bool // Hardware random number generator (requires kernel enablement)
HasSCV bool // Syscall vectored (requires kernel enablement)
IsPOWER8 bool // ISA v2.07 (POWER8)
IsPOWER9 bool // ISA v3.00 (POWER9)
_ [CacheLineSize]byte
}
var ARM64 arm64
// The booleans in arm64 contain the correspondingly named cpu feature bit.
// The struct is padded to avoid false sharing.
type arm64 struct {
_ [CacheLineSize]byte
HasFP bool
HasASIMD bool
HasEVTSTRM bool
HasAES bool
HasPMULL bool
HasSHA1 bool
HasSHA2 bool
HasCRC32 bool
HasATOMICS bool
_ [CacheLineSize]byte
}

107
vendor/github.com/apache/arrow/go/arrow/internal/cpu/cpu_x86.go generated vendored Normal file
View File

@ -0,0 +1,107 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build 386 amd64 amd64p32
package cpu
import (
"os"
"strings"
)
const CacheLineSize = 64
// cpuid is implemented in cpu_x86.s.
func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32)
// xgetbv with ecx = 0 is implemented in cpu_x86.s.
func xgetbv() (eax, edx uint32)
func init() {
maxID, _, _, _ := cpuid(0, 0)
if maxID < 1 {
return
}
_, _, ecx1, edx1 := cpuid(1, 0)
X86.HasSSE2 = isSet(26, edx1)
X86.HasSSE3 = isSet(0, ecx1)
X86.HasPCLMULQDQ = isSet(1, ecx1)
X86.HasSSSE3 = isSet(9, ecx1)
X86.HasFMA = isSet(12, ecx1)
X86.HasSSE41 = isSet(19, ecx1)
X86.HasSSE42 = isSet(20, ecx1)
X86.HasPOPCNT = isSet(23, ecx1)
X86.HasAES = isSet(25, ecx1)
X86.HasOSXSAVE = isSet(27, ecx1)
osSupportsAVX := false
// For XGETBV, OSXSAVE bit is required and sufficient.
if X86.HasOSXSAVE {
eax, _ := xgetbv()
// Check if XMM and YMM registers have OS support.
osSupportsAVX = isSet(1, eax) && isSet(2, eax)
}
X86.HasAVX = isSet(28, ecx1) && osSupportsAVX
if maxID < 7 {
return
}
_, ebx7, _, _ := cpuid(7, 0)
X86.HasBMI1 = isSet(3, ebx7)
X86.HasAVX2 = isSet(5, ebx7) && osSupportsAVX
X86.HasBMI2 = isSet(8, ebx7)
X86.HasERMS = isSet(9, ebx7)
X86.HasADX = isSet(19, ebx7)
// NOTE(sgc): added ability to disable extension via environment
checkEnvironment()
}
func checkEnvironment() {
if ext, ok := os.LookupEnv("INTEL_DISABLE_EXT"); ok {
exts := strings.Split(ext, ",")
for _, x := range exts {
switch x {
case "ALL":
X86.HasAVX2 = false
X86.HasAVX = false
X86.HasSSE42 = false
X86.HasSSE41 = false
X86.HasSSSE3 = false
X86.HasSSE3 = false
X86.HasSSE2 = false
case "AVX2":
X86.HasAVX2 = false
case "AVX":
X86.HasAVX = false
case "SSE":
X86.HasSSE42 = false
X86.HasSSE41 = false
X86.HasSSSE3 = false
X86.HasSSE3 = false
X86.HasSSE2 = false
case "SSE4":
X86.HasSSE42 = false
X86.HasSSE41 = false
case "SSSE3":
X86.HasSSSE3 = false
case "SSE3":
X86.HasSSE3 = false
case "SSE2":
X86.HasSSE2 = false
}
}
}
}
func isSet(bitpos uint, value uint32) bool {
return value&(1<<bitpos) != 0
}

32
vendor/github.com/apache/arrow/go/arrow/internal/cpu/cpu_x86.s generated vendored Normal file
View File

@ -0,0 +1,32 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build 386 amd64 amd64p32
#include "textflag.h"
// func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32)
TEXT ·cpuid(SB), NOSPLIT, $0-24
MOVL eaxArg+0(FP), AX
MOVL ecxArg+4(FP), CX
CPUID
MOVL AX, eax+8(FP)
MOVL BX, ebx+12(FP)
MOVL CX, ecx+16(FP)
MOVL DX, edx+20(FP)
RET
// func xgetbv() (eax, edx uint32)
TEXT ·xgetbv(SB),NOSPLIT,$0-8
#ifdef GOOS_nacl
// nacl does not support XGETBV.
MOVL $0, eax+0(FP)
MOVL $0, edx+4(FP)
#else
MOVL $0, CX
XGETBV
MOVL AX, eax+0(FP)
MOVL DX, edx+4(FP)
#endif
RET

24
vendor/github.com/apache/arrow/go/arrow/internal/debug/assert_off.go generated vendored Normal file
View File

@ -0,0 +1,24 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !assert
package debug
// Assert will panic with msg if cond is false.
//
// msg must be a string, func() string or fmt.Stringer.
func Assert(cond bool, msg interface{}) {}

28
vendor/github.com/apache/arrow/go/arrow/internal/debug/assert_on.go generated vendored Normal file
View File

@ -0,0 +1,28 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build assert
package debug
// Assert will panic with msg if cond is false.
//
// msg must be a string, func() string or fmt.Stringer.
func Assert(cond bool, msg interface{}) {
if !cond {
panic(getStringValue(msg))
}
}

32
vendor/github.com/apache/arrow/go/arrow/internal/debug/doc.go generated vendored Normal file
View File

@ -0,0 +1,32 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
Package debug provides APIs for conditional runtime assertions and debug logging.
Using Assert
To enable runtime assertions, build with the assert tag. When the assert tag is omitted,
the code for the assertion will be omitted from the binary.
Using Log
To enable runtime debug logs, build with the debug tag. When the debug tag is omitted,
the code for logging will be omitted from the binary.
*/
package debug

21
vendor/github.com/apache/arrow/go/arrow/internal/debug/log_off.go generated vendored Normal file
View File

@ -0,0 +1,21 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !debug
package debug
func Log(interface{}) {}

32
vendor/github.com/apache/arrow/go/arrow/internal/debug/log_on.go generated vendored Normal file
View File

@ -0,0 +1,32 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build debug
package debug
import (
"log"
"os"
)
var (
debug = log.New(os.Stderr, "[D] ", log.LstdFlags)
)
func Log(msg interface{}) {
debug.Output(1, getStringValue(msg))
}

37
vendor/github.com/apache/arrow/go/arrow/internal/debug/util.go generated vendored Normal file
View File

@ -0,0 +1,37 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build debug assert
package debug
import "fmt"
func getStringValue(v interface{}) string {
switch a := v.(type) {
case func() string:
return a()
case string:
return a
case fmt.Stringer:
return a.String()
default:
panic(fmt.Sprintf("unexpected type, %t", v))
}
}

50
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Binary.go generated vendored Normal file
View File

@ -0,0 +1,50 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
type Binary struct {
_tab flatbuffers.Table
}
func GetRootAsBinary(buf []byte, offset flatbuffers.UOffsetT) *Binary {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Binary{}
x.Init(buf, n+offset)
return x
}
func (rcv *Binary) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Binary) Table() flatbuffers.Table {
return rcv._tab
}
func BinaryStart(builder *flatbuffers.Builder) {
builder.StartObject(0)
}
func BinaryEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

74
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Block.go generated vendored Normal file
View File

@ -0,0 +1,74 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
type Block struct {
_tab flatbuffers.Struct
}
func (rcv *Block) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Block) Table() flatbuffers.Table {
return rcv._tab.Table
}
/// Index to the start of the RecordBlock (note this is past the Message header)
func (rcv *Block) Offset() int64 {
return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(0))
}
/// Index to the start of the RecordBlock (note this is past the Message header)
func (rcv *Block) MutateOffset(n int64) bool {
return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(0), n)
}
/// Length of the metadata
func (rcv *Block) MetaDataLength() int32 {
return rcv._tab.GetInt32(rcv._tab.Pos + flatbuffers.UOffsetT(8))
}
/// Length of the metadata
func (rcv *Block) MutateMetaDataLength(n int32) bool {
return rcv._tab.MutateInt32(rcv._tab.Pos+flatbuffers.UOffsetT(8), n)
}
/// Length of the data (this is aligned so there can be a gap between this and
/// the metatdata).
func (rcv *Block) BodyLength() int64 {
return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(16))
}
/// Length of the data (this is aligned so there can be a gap between this and
/// the metatdata).
func (rcv *Block) MutateBodyLength(n int64) bool {
return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(16), n)
}
func CreateBlock(builder *flatbuffers.Builder, offset int64, metaDataLength int32, bodyLength int64) flatbuffers.UOffsetT {
builder.Prep(8, 24)
builder.PrependInt64(bodyLength)
builder.Pad(4)
builder.PrependInt32(metaDataLength)
builder.PrependInt64(offset)
return builder.Offset()
}

50
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Bool.go generated vendored Normal file
View File

@ -0,0 +1,50 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
type Bool struct {
_tab flatbuffers.Table
}
func GetRootAsBool(buf []byte, offset flatbuffers.UOffsetT) *Bool {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Bool{}
x.Init(buf, n+offset)
return x
}
func (rcv *Bool) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Bool) Table() flatbuffers.Table {
return rcv._tab
}
func BoolStart(builder *flatbuffers.Builder) {
builder.StartObject(0)
}
func BoolEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

67
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Buffer.go generated vendored Normal file
View File

@ -0,0 +1,67 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
/// ----------------------------------------------------------------------
/// A Buffer represents a single contiguous memory segment
type Buffer struct {
_tab flatbuffers.Struct
}
func (rcv *Buffer) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Buffer) Table() flatbuffers.Table {
return rcv._tab.Table
}
/// The relative offset into the shared memory page where the bytes for this
/// buffer starts
func (rcv *Buffer) Offset() int64 {
return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(0))
}
/// The relative offset into the shared memory page where the bytes for this
/// buffer starts
func (rcv *Buffer) MutateOffset(n int64) bool {
return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(0), n)
}
/// The absolute length (in bytes) of the memory buffer. The memory is found
/// from offset (inclusive) to offset + length (non-inclusive).
func (rcv *Buffer) Length() int64 {
return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(8))
}
/// The absolute length (in bytes) of the memory buffer. The memory is found
/// from offset (inclusive) to offset + length (non-inclusive).
func (rcv *Buffer) MutateLength(n int64) bool {
return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(8), n)
}
func CreateBuffer(builder *flatbuffers.Builder, offset int64, length int64) flatbuffers.UOffsetT {
builder.Prep(8, 16)
builder.PrependInt64(length)
builder.PrependInt64(offset)
return builder.Offset()
}

71
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Date.go generated vendored Normal file
View File

@ -0,0 +1,71 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
/// Date is either a 32-bit or 64-bit type representing elapsed time since UNIX
/// epoch (1970-01-01), stored in either of two units:
///
/// * Milliseconds (64 bits) indicating UNIX time elapsed since the epoch (no
/// leap seconds), where the values are evenly divisible by 86400000
/// * Days (32 bits) since the UNIX epoch
type Date struct {
_tab flatbuffers.Table
}
func GetRootAsDate(buf []byte, offset flatbuffers.UOffsetT) *Date {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Date{}
x.Init(buf, n+offset)
return x
}
func (rcv *Date) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Date) Table() flatbuffers.Table {
return rcv._tab
}
func (rcv *Date) Unit() DateUnit {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.GetInt16(o + rcv._tab.Pos)
}
return 1
}
func (rcv *Date) MutateUnit(n DateUnit) bool {
return rcv._tab.MutateInt16Slot(4, n)
}
func DateStart(builder *flatbuffers.Builder) {
builder.StartObject(1)
}
func DateAddUnit(builder *flatbuffers.Builder, unit int16) {
builder.PrependInt16Slot(0, unit, 1)
}
func DateEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

31
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/DateUnit.go generated vendored Normal file
View File

@ -0,0 +1,31 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
type DateUnit = int16
const (
DateUnitDAY DateUnit = 0
DateUnitMILLISECOND DateUnit = 1
)
var EnumNamesDateUnit = map[DateUnit]string{
DateUnitDAY:"DAY",
DateUnitMILLISECOND:"MILLISECOND",
}

84
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Decimal.go generated vendored Normal file
View File

@ -0,0 +1,84 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
type Decimal struct {
_tab flatbuffers.Table
}
func GetRootAsDecimal(buf []byte, offset flatbuffers.UOffsetT) *Decimal {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Decimal{}
x.Init(buf, n+offset)
return x
}
func (rcv *Decimal) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Decimal) Table() flatbuffers.Table {
return rcv._tab
}
/// Total number of decimal digits
func (rcv *Decimal) Precision() int32 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.GetInt32(o + rcv._tab.Pos)
}
return 0
}
/// Total number of decimal digits
func (rcv *Decimal) MutatePrecision(n int32) bool {
return rcv._tab.MutateInt32Slot(4, n)
}
/// Number of digits after the decimal point "."
func (rcv *Decimal) Scale() int32 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
return rcv._tab.GetInt32(o + rcv._tab.Pos)
}
return 0
}
/// Number of digits after the decimal point "."
func (rcv *Decimal) MutateScale(n int32) bool {
return rcv._tab.MutateInt32Slot(6, n)
}
func DecimalStart(builder *flatbuffers.Builder) {
builder.StartObject(2)
}
func DecimalAddPrecision(builder *flatbuffers.Builder, precision int32) {
builder.PrependInt32Slot(0, precision, 0)
}
func DecimalAddScale(builder *flatbuffers.Builder, scale int32) {
builder.PrependInt32Slot(1, scale, 0)
}
func DecimalEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

106
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/DictionaryBatch.go generated vendored Normal file
View File

@ -0,0 +1,106 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
/// For sending dictionary encoding information. Any Field can be
/// dictionary-encoded, but in this case none of its children may be
/// dictionary-encoded.
/// There is one vector / column per dictionary, but that vector / column
/// may be spread across multiple dictionary batches by using the isDelta
/// flag
type DictionaryBatch struct {
_tab flatbuffers.Table
}
func GetRootAsDictionaryBatch(buf []byte, offset flatbuffers.UOffsetT) *DictionaryBatch {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &DictionaryBatch{}
x.Init(buf, n+offset)
return x
}
func (rcv *DictionaryBatch) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *DictionaryBatch) Table() flatbuffers.Table {
return rcv._tab
}
func (rcv *DictionaryBatch) Id() int64 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.GetInt64(o + rcv._tab.Pos)
}
return 0
}
func (rcv *DictionaryBatch) MutateId(n int64) bool {
return rcv._tab.MutateInt64Slot(4, n)
}
func (rcv *DictionaryBatch) Data(obj *RecordBatch) *RecordBatch {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
x := rcv._tab.Indirect(o + rcv._tab.Pos)
if obj == nil {
obj = new(RecordBatch)
}
obj.Init(rcv._tab.Bytes, x)
return obj
}
return nil
}
/// If isDelta is true the values in the dictionary are to be appended to a
/// dictionary with the indicated id
func (rcv *DictionaryBatch) IsDelta() bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
return rcv._tab.GetBool(o + rcv._tab.Pos)
}
return false
}
/// If isDelta is true the values in the dictionary are to be appended to a
/// dictionary with the indicated id
func (rcv *DictionaryBatch) MutateIsDelta(n bool) bool {
return rcv._tab.MutateBoolSlot(8, n)
}
func DictionaryBatchStart(builder *flatbuffers.Builder) {
builder.StartObject(3)
}
func DictionaryBatchAddId(builder *flatbuffers.Builder, id int64) {
builder.PrependInt64Slot(0, id, 0)
}
func DictionaryBatchAddData(builder *flatbuffers.Builder, data flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(data), 0)
}
func DictionaryBatchAddIsDelta(builder *flatbuffers.Builder, isDelta bool) {
builder.PrependBoolSlot(2, isDelta, false)
}
func DictionaryBatchEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

116
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/DictionaryEncoding.go generated vendored Normal file
View File

@ -0,0 +1,116 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
/// ----------------------------------------------------------------------
/// Dictionary encoding metadata
type DictionaryEncoding struct {
_tab flatbuffers.Table
}
func GetRootAsDictionaryEncoding(buf []byte, offset flatbuffers.UOffsetT) *DictionaryEncoding {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &DictionaryEncoding{}
x.Init(buf, n+offset)
return x
}
func (rcv *DictionaryEncoding) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *DictionaryEncoding) Table() flatbuffers.Table {
return rcv._tab
}
/// The known dictionary id in the application where this data is used. In
/// the file or streaming formats, the dictionary ids are found in the
/// DictionaryBatch messages
func (rcv *DictionaryEncoding) Id() int64 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.GetInt64(o + rcv._tab.Pos)
}
return 0
}
/// The known dictionary id in the application where this data is used. In
/// the file or streaming formats, the dictionary ids are found in the
/// DictionaryBatch messages
func (rcv *DictionaryEncoding) MutateId(n int64) bool {
return rcv._tab.MutateInt64Slot(4, n)
}
/// The dictionary indices are constrained to be positive integers. If this
/// field is null, the indices must be signed int32
func (rcv *DictionaryEncoding) IndexType(obj *Int) *Int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
x := rcv._tab.Indirect(o + rcv._tab.Pos)
if obj == nil {
obj = new(Int)
}
obj.Init(rcv._tab.Bytes, x)
return obj
}
return nil
}
/// The dictionary indices are constrained to be positive integers. If this
/// field is null, the indices must be signed int32
/// By default, dictionaries are not ordered, or the order does not have
/// semantic meaning. In some statistical, applications, dictionary-encoding
/// is used to represent ordered categorical data, and we provide a way to
/// preserve that metadata here
func (rcv *DictionaryEncoding) IsOrdered() bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
return rcv._tab.GetBool(o + rcv._tab.Pos)
}
return false
}
/// By default, dictionaries are not ordered, or the order does not have
/// semantic meaning. In some statistical, applications, dictionary-encoding
/// is used to represent ordered categorical data, and we provide a way to
/// preserve that metadata here
func (rcv *DictionaryEncoding) MutateIsOrdered(n bool) bool {
return rcv._tab.MutateBoolSlot(8, n)
}
func DictionaryEncodingStart(builder *flatbuffers.Builder) {
builder.StartObject(3)
}
func DictionaryEncodingAddId(builder *flatbuffers.Builder, id int64) {
builder.PrependInt64Slot(0, id, 0)
}
func DictionaryEncodingAddIndexType(builder *flatbuffers.Builder, indexType flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(indexType), 0)
}
func DictionaryEncodingAddIsOrdered(builder *flatbuffers.Builder, isOrdered bool) {
builder.PrependBoolSlot(2, isOrdered, false)
}
func DictionaryEncodingEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

65
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Duration.go generated vendored Normal file
View File

@ -0,0 +1,65 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
type Duration struct {
_tab flatbuffers.Table
}
func GetRootAsDuration(buf []byte, offset flatbuffers.UOffsetT) *Duration {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Duration{}
x.Init(buf, n+offset)
return x
}
func (rcv *Duration) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Duration) Table() flatbuffers.Table {
return rcv._tab
}
func (rcv *Duration) Unit() TimeUnit {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.GetInt16(o + rcv._tab.Pos)
}
return 1
}
func (rcv *Duration) MutateUnit(n TimeUnit) bool {
return rcv._tab.MutateInt16Slot(4, n)
}
func DurationStart(builder *flatbuffers.Builder) {
builder.StartObject(1)
}
func DurationAddUnit(builder *flatbuffers.Builder, unit int16) {
builder.PrependInt16Slot(0, unit, 1)
}
func DurationEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

33
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Endianness.go generated vendored Normal file
View File

@ -0,0 +1,33 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
/// ----------------------------------------------------------------------
/// Endianness of the platform producing the data
type Endianness = int16
const (
EndiannessLittle Endianness = 0
EndiannessBig Endianness = 1
)
var EnumNamesEndianness = map[Endianness]string{
EndiannessLittle:"Little",
EndiannessBig:"Big",
}

188
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Field.go generated vendored Normal file
View File

@ -0,0 +1,188 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
/// ----------------------------------------------------------------------
/// A field represents a named column in a record / row batch or child of a
/// nested type.
type Field struct {
_tab flatbuffers.Table
}
func GetRootAsField(buf []byte, offset flatbuffers.UOffsetT) *Field {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Field{}
x.Init(buf, n+offset)
return x
}
func (rcv *Field) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Field) Table() flatbuffers.Table {
return rcv._tab
}
/// Name is not required, in i.e. a List
func (rcv *Field) Name() []byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.ByteVector(o + rcv._tab.Pos)
}
return nil
}
/// Name is not required, in i.e. a List
/// Whether or not this field can contain nulls. Should be true in general.
func (rcv *Field) Nullable() bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
return rcv._tab.GetBool(o + rcv._tab.Pos)
}
return false
}
/// Whether or not this field can contain nulls. Should be true in general.
func (rcv *Field) MutateNullable(n bool) bool {
return rcv._tab.MutateBoolSlot(6, n)
}
func (rcv *Field) TypeType() byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
return rcv._tab.GetByte(o + rcv._tab.Pos)
}
return 0
}
func (rcv *Field) MutateTypeType(n byte) bool {
return rcv._tab.MutateByteSlot(8, n)
}
/// This is the type of the decoded value if the field is dictionary encoded.
func (rcv *Field) Type(obj *flatbuffers.Table) bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
if o != 0 {
rcv._tab.Union(obj, o)
return true
}
return false
}
/// This is the type of the decoded value if the field is dictionary encoded.
/// Present only if the field is dictionary encoded.
func (rcv *Field) Dictionary(obj *DictionaryEncoding) *DictionaryEncoding {
o := flatbuffers.UOffsetT(rcv._tab.Offset(12))
if o != 0 {
x := rcv._tab.Indirect(o + rcv._tab.Pos)
if obj == nil {
obj = new(DictionaryEncoding)
}
obj.Init(rcv._tab.Bytes, x)
return obj
}
return nil
}
/// Present only if the field is dictionary encoded.
/// children apply only to nested data types like Struct, List and Union. For
/// primitive types children will have length 0.
func (rcv *Field) Children(obj *Field, j int) bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(14))
if o != 0 {
x := rcv._tab.Vector(o)
x += flatbuffers.UOffsetT(j) * 4
x = rcv._tab.Indirect(x)
obj.Init(rcv._tab.Bytes, x)
return true
}
return false
}
func (rcv *Field) ChildrenLength() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(14))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}
/// children apply only to nested data types like Struct, List and Union. For
/// primitive types children will have length 0.
/// User-defined metadata
func (rcv *Field) CustomMetadata(obj *KeyValue, j int) bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(16))
if o != 0 {
x := rcv._tab.Vector(o)
x += flatbuffers.UOffsetT(j) * 4
x = rcv._tab.Indirect(x)
obj.Init(rcv._tab.Bytes, x)
return true
}
return false
}
func (rcv *Field) CustomMetadataLength() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(16))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}
/// User-defined metadata
func FieldStart(builder *flatbuffers.Builder) {
builder.StartObject(7)
}
func FieldAddName(builder *flatbuffers.Builder, name flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(name), 0)
}
func FieldAddNullable(builder *flatbuffers.Builder, nullable bool) {
builder.PrependBoolSlot(1, nullable, false)
}
func FieldAddTypeType(builder *flatbuffers.Builder, typeType byte) {
builder.PrependByteSlot(2, typeType, 0)
}
func FieldAddType(builder *flatbuffers.Builder, type_ flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(type_), 0)
}
func FieldAddDictionary(builder *flatbuffers.Builder, dictionary flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(4, flatbuffers.UOffsetT(dictionary), 0)
}
func FieldAddChildren(builder *flatbuffers.Builder, children flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(5, flatbuffers.UOffsetT(children), 0)
}
func FieldStartChildrenVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(4, numElems, 4)
}
func FieldAddCustomMetadata(builder *flatbuffers.Builder, customMetadata flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(6, flatbuffers.UOffsetT(customMetadata), 0)
}
func FieldStartCustomMetadataVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(4, numElems, 4)
}
func FieldEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

76
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/FieldNode.go generated vendored Normal file
View File

@ -0,0 +1,76 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
/// ----------------------------------------------------------------------
/// Data structures for describing a table row batch (a collection of
/// equal-length Arrow arrays)
/// Metadata about a field at some level of a nested type tree (but not
/// its children).
///
/// For example, a List<Int16> with values [[1, 2, 3], null, [4], [5, 6], null]
/// would have {length: 5, null_count: 2} for its List node, and {length: 6,
/// null_count: 0} for its Int16 node, as separate FieldNode structs
type FieldNode struct {
_tab flatbuffers.Struct
}
func (rcv *FieldNode) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *FieldNode) Table() flatbuffers.Table {
return rcv._tab.Table
}
/// The number of value slots in the Arrow array at this level of a nested
/// tree
func (rcv *FieldNode) Length() int64 {
return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(0))
}
/// The number of value slots in the Arrow array at this level of a nested
/// tree
func (rcv *FieldNode) MutateLength(n int64) bool {
return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(0), n)
}
/// The number of observed nulls. Fields with null_count == 0 may choose not
/// to write their physical validity bitmap out as a materialized buffer,
/// instead setting the length of the bitmap buffer to 0.
func (rcv *FieldNode) NullCount() int64 {
return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(8))
}
/// The number of observed nulls. Fields with null_count == 0 may choose not
/// to write their physical validity bitmap out as a materialized buffer,
/// instead setting the length of the bitmap buffer to 0.
func (rcv *FieldNode) MutateNullCount(n int64) bool {
return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(8), n)
}
func CreateFieldNode(builder *flatbuffers.Builder, length int64, nullCount int64) flatbuffers.UOffsetT {
builder.Prep(8, 16)
builder.PrependInt64(nullCount)
builder.PrependInt64(length)
return builder.Offset()
}

67
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/FixedSizeBinary.go generated vendored Normal file
View File

@ -0,0 +1,67 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
type FixedSizeBinary struct {
_tab flatbuffers.Table
}
func GetRootAsFixedSizeBinary(buf []byte, offset flatbuffers.UOffsetT) *FixedSizeBinary {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &FixedSizeBinary{}
x.Init(buf, n+offset)
return x
}
func (rcv *FixedSizeBinary) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *FixedSizeBinary) Table() flatbuffers.Table {
return rcv._tab
}
/// Number of bytes per value
func (rcv *FixedSizeBinary) ByteWidth() int32 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.GetInt32(o + rcv._tab.Pos)
}
return 0
}
/// Number of bytes per value
func (rcv *FixedSizeBinary) MutateByteWidth(n int32) bool {
return rcv._tab.MutateInt32Slot(4, n)
}
func FixedSizeBinaryStart(builder *flatbuffers.Builder) {
builder.StartObject(1)
}
func FixedSizeBinaryAddByteWidth(builder *flatbuffers.Builder, byteWidth int32) {
builder.PrependInt32Slot(0, byteWidth, 0)
}
func FixedSizeBinaryEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

67
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/FixedSizeList.go generated vendored Normal file
View File

@ -0,0 +1,67 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
type FixedSizeList struct {
_tab flatbuffers.Table
}
func GetRootAsFixedSizeList(buf []byte, offset flatbuffers.UOffsetT) *FixedSizeList {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &FixedSizeList{}
x.Init(buf, n+offset)
return x
}
func (rcv *FixedSizeList) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *FixedSizeList) Table() flatbuffers.Table {
return rcv._tab
}
/// Number of list items per value
func (rcv *FixedSizeList) ListSize() int32 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.GetInt32(o + rcv._tab.Pos)
}
return 0
}
/// Number of list items per value
func (rcv *FixedSizeList) MutateListSize(n int32) bool {
return rcv._tab.MutateInt32Slot(4, n)
}
func FixedSizeListStart(builder *flatbuffers.Builder) {
builder.StartObject(1)
}
func FixedSizeListAddListSize(builder *flatbuffers.Builder, listSize int32) {
builder.PrependInt32Slot(0, listSize, 0)
}
func FixedSizeListEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

65
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/FloatingPoint.go generated vendored Normal file
View File

@ -0,0 +1,65 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
type FloatingPoint struct {
_tab flatbuffers.Table
}
func GetRootAsFloatingPoint(buf []byte, offset flatbuffers.UOffsetT) *FloatingPoint {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &FloatingPoint{}
x.Init(buf, n+offset)
return x
}
func (rcv *FloatingPoint) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *FloatingPoint) Table() flatbuffers.Table {
return rcv._tab
}
func (rcv *FloatingPoint) Precision() Precision {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.GetInt16(o + rcv._tab.Pos)
}
return 0
}
func (rcv *FloatingPoint) MutatePrecision(n Precision) bool {
return rcv._tab.MutateInt16Slot(4, n)
}
func FloatingPointStart(builder *flatbuffers.Builder) {
builder.StartObject(1)
}
func FloatingPointAddPrecision(builder *flatbuffers.Builder, precision int16) {
builder.PrependInt16Slot(0, precision, 0)
}
func FloatingPointEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

134
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Footer.go generated vendored Normal file
View File

@ -0,0 +1,134 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
/// ----------------------------------------------------------------------
/// Arrow File metadata
///
type Footer struct {
_tab flatbuffers.Table
}
func GetRootAsFooter(buf []byte, offset flatbuffers.UOffsetT) *Footer {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Footer{}
x.Init(buf, n+offset)
return x
}
func (rcv *Footer) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Footer) Table() flatbuffers.Table {
return rcv._tab
}
func (rcv *Footer) Version() MetadataVersion {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.GetInt16(o + rcv._tab.Pos)
}
return 0
}
func (rcv *Footer) MutateVersion(n MetadataVersion) bool {
return rcv._tab.MutateInt16Slot(4, n)
}
func (rcv *Footer) Schema(obj *Schema) *Schema {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
x := rcv._tab.Indirect(o + rcv._tab.Pos)
if obj == nil {
obj = new(Schema)
}
obj.Init(rcv._tab.Bytes, x)
return obj
}
return nil
}
func (rcv *Footer) Dictionaries(obj *Block, j int) bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
x := rcv._tab.Vector(o)
x += flatbuffers.UOffsetT(j) * 24
obj.Init(rcv._tab.Bytes, x)
return true
}
return false
}
func (rcv *Footer) DictionariesLength() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}
func (rcv *Footer) RecordBatches(obj *Block, j int) bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
if o != 0 {
x := rcv._tab.Vector(o)
x += flatbuffers.UOffsetT(j) * 24
obj.Init(rcv._tab.Bytes, x)
return true
}
return false
}
func (rcv *Footer) RecordBatchesLength() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}
func FooterStart(builder *flatbuffers.Builder) {
builder.StartObject(4)
}
func FooterAddVersion(builder *flatbuffers.Builder, version int16) {
builder.PrependInt16Slot(0, version, 0)
}
func FooterAddSchema(builder *flatbuffers.Builder, schema flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(schema), 0)
}
func FooterAddDictionaries(builder *flatbuffers.Builder, dictionaries flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(dictionaries), 0)
}
func FooterStartDictionariesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(24, numElems, 8)
}
func FooterAddRecordBatches(builder *flatbuffers.Builder, recordBatches flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(recordBatches), 0)
}
func FooterStartRecordBatchesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(24, numElems, 8)
}
func FooterEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

80
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Int.go generated vendored Normal file
View File

@ -0,0 +1,80 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
type Int struct {
_tab flatbuffers.Table
}
func GetRootAsInt(buf []byte, offset flatbuffers.UOffsetT) *Int {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Int{}
x.Init(buf, n+offset)
return x
}
func (rcv *Int) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Int) Table() flatbuffers.Table {
return rcv._tab
}
func (rcv *Int) BitWidth() int32 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.GetInt32(o + rcv._tab.Pos)
}
return 0
}
func (rcv *Int) MutateBitWidth(n int32) bool {
return rcv._tab.MutateInt32Slot(4, n)
}
func (rcv *Int) IsSigned() bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
return rcv._tab.GetBool(o + rcv._tab.Pos)
}
return false
}
func (rcv *Int) MutateIsSigned(n bool) bool {
return rcv._tab.MutateBoolSlot(6, n)
}
func IntStart(builder *flatbuffers.Builder) {
builder.StartObject(2)
}
func IntAddBitWidth(builder *flatbuffers.Builder, bitWidth int32) {
builder.PrependInt32Slot(0, bitWidth, 0)
}
func IntAddIsSigned(builder *flatbuffers.Builder, isSigned bool) {
builder.PrependBoolSlot(1, isSigned, false)
}
func IntEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

65
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Interval.go generated vendored Normal file
View File

@ -0,0 +1,65 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
type Interval struct {
_tab flatbuffers.Table
}
func GetRootAsInterval(buf []byte, offset flatbuffers.UOffsetT) *Interval {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Interval{}
x.Init(buf, n+offset)
return x
}
func (rcv *Interval) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Interval) Table() flatbuffers.Table {
return rcv._tab
}
func (rcv *Interval) Unit() IntervalUnit {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.GetInt16(o + rcv._tab.Pos)
}
return 0
}
func (rcv *Interval) MutateUnit(n IntervalUnit) bool {
return rcv._tab.MutateInt16Slot(4, n)
}
func IntervalStart(builder *flatbuffers.Builder) {
builder.StartObject(1)
}
func IntervalAddUnit(builder *flatbuffers.Builder, unit int16) {
builder.PrependInt16Slot(0, unit, 0)
}
func IntervalEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

31
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/IntervalUnit.go generated vendored Normal file
View File

@ -0,0 +1,31 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
type IntervalUnit = int16
const (
IntervalUnitYEAR_MONTH IntervalUnit = 0
IntervalUnitDAY_TIME IntervalUnit = 1
)
var EnumNamesIntervalUnit = map[IntervalUnit]string{
IntervalUnitYEAR_MONTH:"YEAR_MONTH",
IntervalUnitDAY_TIME:"DAY_TIME",
}

75
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/KeyValue.go generated vendored Normal file
View File

@ -0,0 +1,75 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
/// ----------------------------------------------------------------------
/// user defined key value pairs to add custom metadata to arrow
/// key namespacing is the responsibility of the user
type KeyValue struct {
_tab flatbuffers.Table
}
func GetRootAsKeyValue(buf []byte, offset flatbuffers.UOffsetT) *KeyValue {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &KeyValue{}
x.Init(buf, n+offset)
return x
}
func (rcv *KeyValue) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *KeyValue) Table() flatbuffers.Table {
return rcv._tab
}
func (rcv *KeyValue) Key() []byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.ByteVector(o + rcv._tab.Pos)
}
return nil
}
func (rcv *KeyValue) Value() []byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
return rcv._tab.ByteVector(o + rcv._tab.Pos)
}
return nil
}
func KeyValueStart(builder *flatbuffers.Builder) {
builder.StartObject(2)
}
func KeyValueAddKey(builder *flatbuffers.Builder, key flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(key), 0)
}
func KeyValueAddValue(builder *flatbuffers.Builder, value flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(value), 0)
}
func KeyValueEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

50
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/List.go generated vendored Normal file
View File

@ -0,0 +1,50 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
type List struct {
_tab flatbuffers.Table
}
func GetRootAsList(buf []byte, offset flatbuffers.UOffsetT) *List {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &List{}
x.Init(buf, n+offset)
return x
}
func (rcv *List) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *List) Table() flatbuffers.Table {
return rcv._tab
}
func ListStart(builder *flatbuffers.Builder) {
builder.StartObject(0)
}
func ListEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

91
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Map.go generated vendored Normal file
View File

@ -0,0 +1,91 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
/// A Map is a logical nested type that is represented as
///
/// List<entry: Struct<key: K, value: V>>
///
/// In this layout, the keys and values are each respectively contiguous. We do
/// not constrain the key and value types, so the application is responsible
/// for ensuring that the keys are hashable and unique. Whether the keys are sorted
/// may be set in the metadata for this field
///
/// In a Field with Map type, the Field has a child Struct field, which then
/// has two children: key type and the second the value type. The names of the
/// child fields may be respectively "entry", "key", and "value", but this is
/// not enforced
///
/// Map
/// - child[0] entry: Struct
/// - child[0] key: K
/// - child[1] value: V
///
/// Neither the "entry" field nor the "key" field may be nullable.
///
/// The metadata is structured so that Arrow systems without special handling
/// for Map can make Map an alias for List. The "layout" attribute for the Map
/// field must have the same contents as a List.
type Map struct {
_tab flatbuffers.Table
}
func GetRootAsMap(buf []byte, offset flatbuffers.UOffsetT) *Map {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Map{}
x.Init(buf, n+offset)
return x
}
func (rcv *Map) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Map) Table() flatbuffers.Table {
return rcv._tab
}
/// Set to true if the keys within each value are sorted
func (rcv *Map) KeysSorted() bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.GetBool(o + rcv._tab.Pos)
}
return false
}
/// Set to true if the keys within each value are sorted
func (rcv *Map) MutateKeysSorted(n bool) bool {
return rcv._tab.MutateBoolSlot(4, n)
}
func MapStart(builder *flatbuffers.Builder) {
builder.StartObject(1)
}
func MapAddKeysSorted(builder *flatbuffers.Builder, keysSorted bool) {
builder.PrependBoolSlot(0, keysSorted, false)
}
func MapEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

133
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Message.go generated vendored Normal file
View File

@ -0,0 +1,133 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
type Message struct {
_tab flatbuffers.Table
}
func GetRootAsMessage(buf []byte, offset flatbuffers.UOffsetT) *Message {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Message{}
x.Init(buf, n+offset)
return x
}
func (rcv *Message) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Message) Table() flatbuffers.Table {
return rcv._tab
}
func (rcv *Message) Version() MetadataVersion {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.GetInt16(o + rcv._tab.Pos)
}
return 0
}
func (rcv *Message) MutateVersion(n MetadataVersion) bool {
return rcv._tab.MutateInt16Slot(4, n)
}
func (rcv *Message) HeaderType() byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
return rcv._tab.GetByte(o + rcv._tab.Pos)
}
return 0
}
func (rcv *Message) MutateHeaderType(n byte) bool {
return rcv._tab.MutateByteSlot(6, n)
}
func (rcv *Message) Header(obj *flatbuffers.Table) bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
rcv._tab.Union(obj, o)
return true
}
return false
}
func (rcv *Message) BodyLength() int64 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
if o != 0 {
return rcv._tab.GetInt64(o + rcv._tab.Pos)
}
return 0
}
func (rcv *Message) MutateBodyLength(n int64) bool {
return rcv._tab.MutateInt64Slot(10, n)
}
func (rcv *Message) CustomMetadata(obj *KeyValue, j int) bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(12))
if o != 0 {
x := rcv._tab.Vector(o)
x += flatbuffers.UOffsetT(j) * 4
x = rcv._tab.Indirect(x)
obj.Init(rcv._tab.Bytes, x)
return true
}
return false
}
func (rcv *Message) CustomMetadataLength() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(12))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}
func MessageStart(builder *flatbuffers.Builder) {
builder.StartObject(5)
}
func MessageAddVersion(builder *flatbuffers.Builder, version int16) {
builder.PrependInt16Slot(0, version, 0)
}
func MessageAddHeaderType(builder *flatbuffers.Builder, headerType byte) {
builder.PrependByteSlot(1, headerType, 0)
}
func MessageAddHeader(builder *flatbuffers.Builder, header flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(header), 0)
}
func MessageAddBodyLength(builder *flatbuffers.Builder, bodyLength int64) {
builder.PrependInt64Slot(3, bodyLength, 0)
}
func MessageAddCustomMetadata(builder *flatbuffers.Builder, customMetadata flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(4, flatbuffers.UOffsetT(customMetadata), 0)
}
func MessageStartCustomMetadataVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(4, numElems, 4)
}
func MessageEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

47
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/MessageHeader.go generated vendored Normal file
View File

@ -0,0 +1,47 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
/// ----------------------------------------------------------------------
/// The root Message type
/// This union enables us to easily send different message types without
/// redundant storage, and in the future we can easily add new message types.
///
/// Arrow implementations do not need to implement all of the message types,
/// which may include experimental metadata types. For maximum compatibility,
/// it is best to send data using RecordBatch
type MessageHeader = byte
const (
MessageHeaderNONE MessageHeader = 0
MessageHeaderSchema MessageHeader = 1
MessageHeaderDictionaryBatch MessageHeader = 2
MessageHeaderRecordBatch MessageHeader = 3
MessageHeaderTensor MessageHeader = 4
MessageHeaderSparseTensor MessageHeader = 5
)
var EnumNamesMessageHeader = map[MessageHeader]string{
MessageHeaderNONE:"NONE",
MessageHeaderSchema:"Schema",
MessageHeaderDictionaryBatch:"DictionaryBatch",
MessageHeaderRecordBatch:"RecordBatch",
MessageHeaderTensor:"Tensor",
MessageHeaderSparseTensor:"SparseTensor",
}

39
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/MetadataVersion.go generated vendored Normal file
View File

@ -0,0 +1,39 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
type MetadataVersion = int16
const (
/// 0.1.0
MetadataVersionV1 MetadataVersion = 0
/// 0.2.0
MetadataVersionV2 MetadataVersion = 1
/// 0.3.0 -> 0.7.1
MetadataVersionV3 MetadataVersion = 2
/// >= 0.8.0
MetadataVersionV4 MetadataVersion = 3
)
var EnumNamesMetadataVersion = map[MetadataVersion]string{
MetadataVersionV1:"V1",
MetadataVersionV2:"V2",
MetadataVersionV3:"V3",
MetadataVersionV4:"V4",
}

51
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Null.go generated vendored Normal file
View File

@ -0,0 +1,51 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
/// These are stored in the flatbuffer in the Type union below
type Null struct {
_tab flatbuffers.Table
}
func GetRootAsNull(buf []byte, offset flatbuffers.UOffsetT) *Null {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Null{}
x.Init(buf, n+offset)
return x
}
func (rcv *Null) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Null) Table() flatbuffers.Table {
return rcv._tab
}
func NullStart(builder *flatbuffers.Builder) {
builder.StartObject(0)
}
func NullEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

33
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Precision.go generated vendored Normal file
View File

@ -0,0 +1,33 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
type Precision = int16
const (
PrecisionHALF Precision = 0
PrecisionSINGLE Precision = 1
PrecisionDOUBLE Precision = 2
)
var EnumNamesPrecision = map[Precision]string{
PrecisionHALF:"HALF",
PrecisionSINGLE:"SINGLE",
PrecisionDOUBLE:"DOUBLE",
}

136
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/RecordBatch.go generated vendored Normal file
View File

@ -0,0 +1,136 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
/// A data header describing the shared memory layout of a "record" or "row"
/// batch. Some systems call this a "row batch" internally and others a "record
/// batch".
type RecordBatch struct {
_tab flatbuffers.Table
}
func GetRootAsRecordBatch(buf []byte, offset flatbuffers.UOffsetT) *RecordBatch {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &RecordBatch{}
x.Init(buf, n+offset)
return x
}
func (rcv *RecordBatch) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *RecordBatch) Table() flatbuffers.Table {
return rcv._tab
}
/// number of records / rows. The arrays in the batch should all have this
/// length
func (rcv *RecordBatch) Length() int64 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.GetInt64(o + rcv._tab.Pos)
}
return 0
}
/// number of records / rows. The arrays in the batch should all have this
/// length
func (rcv *RecordBatch) MutateLength(n int64) bool {
return rcv._tab.MutateInt64Slot(4, n)
}
/// Nodes correspond to the pre-ordered flattened logical schema
func (rcv *RecordBatch) Nodes(obj *FieldNode, j int) bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
x := rcv._tab.Vector(o)
x += flatbuffers.UOffsetT(j) * 16
obj.Init(rcv._tab.Bytes, x)
return true
}
return false
}
func (rcv *RecordBatch) NodesLength() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}
/// Nodes correspond to the pre-ordered flattened logical schema
/// Buffers correspond to the pre-ordered flattened buffer tree
///
/// The number of buffers appended to this list depends on the schema. For
/// example, most primitive arrays will have 2 buffers, 1 for the validity
/// bitmap and 1 for the values. For struct arrays, there will only be a
/// single buffer for the validity (nulls) bitmap
func (rcv *RecordBatch) Buffers(obj *Buffer, j int) bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
x := rcv._tab.Vector(o)
x += flatbuffers.UOffsetT(j) * 16
obj.Init(rcv._tab.Bytes, x)
return true
}
return false
}
func (rcv *RecordBatch) BuffersLength() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}
/// Buffers correspond to the pre-ordered flattened buffer tree
///
/// The number of buffers appended to this list depends on the schema. For
/// example, most primitive arrays will have 2 buffers, 1 for the validity
/// bitmap and 1 for the values. For struct arrays, there will only be a
/// single buffer for the validity (nulls) bitmap
func RecordBatchStart(builder *flatbuffers.Builder) {
builder.StartObject(3)
}
func RecordBatchAddLength(builder *flatbuffers.Builder, length int64) {
builder.PrependInt64Slot(0, length, 0)
}
func RecordBatchAddNodes(builder *flatbuffers.Builder, nodes flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(nodes), 0)
}
func RecordBatchStartNodesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(16, numElems, 8)
}
func RecordBatchAddBuffers(builder *flatbuffers.Builder, buffers flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(buffers), 0)
}
func RecordBatchStartBuffersVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(16, numElems, 8)
}
func RecordBatchEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

125
vendor/github.com/apache/arrow/go/arrow/internal/flatbuf/Schema.go generated vendored Normal file
View File

@ -0,0 +1,125 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by the FlatBuffers compiler. DO NOT EDIT.
package flatbuf
import (
flatbuffers "github.com/google/flatbuffers/go"
)
/// ----------------------------------------------------------------------
/// A Schema describes the columns in a row batch
type Schema struct {
_tab flatbuffers.Table
}
func GetRootAsSchema(buf []byte, offset flatbuffers.UOffsetT) *Schema {
n := flatbuffers.GetUOffsetT(buf[offset:])
x := &Schema{}
x.Init(buf, n+offset)
return x
}
func (rcv *Schema) Init(buf []byte, i flatbuffers.UOffsetT) {
rcv._tab.Bytes = buf
rcv._tab.Pos = i
}
func (rcv *Schema) Table() flatbuffers.Table {
return rcv._tab
}
/// endianness of the buffer
/// it is Little Endian by default
/// if endianness doesn't match the underlying system then the vectors need to be converted
func (rcv *Schema) Endianness() Endianness {
o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
if o != 0 {
return rcv._tab.GetInt16(o + rcv._tab.Pos)
}
return 0
}
/// endianness of the buffer
/// it is Little Endian by default
/// if endianness doesn't match the underlying system then the vectors need to be converted
func (rcv *Schema) MutateEndianness(n Endianness) bool {
return rcv._tab.MutateInt16Slot(4, n)
}
func (rcv *Schema) Fields(obj *Field, j int) bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
x := rcv._tab.Vector(o)
x += flatbuffers.UOffsetT(j) * 4
x = rcv._tab.Indirect(x)
obj.Init(rcv._tab.Bytes, x)
return true
}
return false
}
func (rcv *Schema) FieldsLength() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}
func (rcv *Schema) CustomMetadata(obj *KeyValue, j int) bool {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
x := rcv._tab.Vector(o)
x += flatbuffers.UOffsetT(j) * 4
x = rcv._tab.Indirect(x)
obj.Init(rcv._tab.Bytes, x)
return true
}
return false
}
func (rcv *Schema) CustomMetadataLength() int {
o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
if o != 0 {
return rcv._tab.VectorLen(o)
}
return 0
}
func SchemaStart(builder *flatbuffers.Builder) {
builder.StartObject(3)
}
func SchemaAddEndianness(builder *flatbuffers.Builder, endianness int16) {
builder.PrependInt16Slot(0, endianness, 0)
}
func SchemaAddFields(builder *flatbuffers.Builder, fields flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(fields), 0)
}
func SchemaStartFieldsVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(4, numElems, 4)
}
func SchemaAddCustomMetadata(builder *flatbuffers.Builder, customMetadata flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(customMetadata), 0)
}
func SchemaStartCustomMetadataVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
return builder.StartVector(4, numElems, 4)
}
func SchemaEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

Some files were not shown because too many files have changed in this diff Show More