Skip to content
GitLab
Unverified Commit 636d0371 authored by Lauren Voswinkel's avatar Lauren Voswinkel Committed by GitHub
Browse files

Adding snowflake as a bundled database secrets plugin (#10603) 

* Adding snowflake as a bundled database secrets plugin

* Add snowflake-database-plugin to expected bundled plugins

* Add snowflake plugin name to the mockBuiltinRegistry
parent 0ddc32f2
Hide whitespace changes
Inline Side-by-side
Showing
with 1314 additions and 23 deletions
+1314 -23
command/base_predict_test.go
+ 1
- 0
View file @ 636d0371
......@@ -390,6 +390,7 @@ func TestPredict_Plugins(t *testing.T) {
"rabbitmq",
"radius",
"redshift-database-plugin",
"snowflake-database-plugin",
"ssh",
"totp",
"transform",
......
go.sum
+ 16
- 0
View file @ 636d0371
......@@ -145,6 +145,8 @@ github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190620160927-9418d7b0cd0f h1:oRD
github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190620160927-9418d7b0cd0f/go.mod h1:myCDvQSzCW+wB1WAlocEru4wMGJxy+vlxHdhegi1CDQ=
github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5 h1:nWDRPCyCltiTsANwC/n3QZH7Vww33Npq9MKqlwRzI/c=
github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/apache/arrow/go/arrow v0.0.0-20200601151325-b2287a20f230 h1:5ultmol0yeX75oh1hY78uAFn3dupBQ/QUNxERCkiaUQ=
github.com/apache/arrow/go/arrow v0.0.0-20200601151325-b2287a20f230/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/apple/foundationdb/bindings/go v0.0.0-20190411004307-cd5c9d91fad2 h1:VoHKYIXEQU5LWoambPBOvYxyLqZYHuj+rj5DVnMUc3k=
......@@ -275,6 +277,8 @@ github.com/denverdino/aliyungo v0.0.0-20170926055100-d3308649c661 h1:lrWnAyy/F72
github.com/denverdino/aliyungo v0.0.0-20170926055100-d3308649c661/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1 h1:CaO/zOnF8VvUfEbhRatPcwKVWamvbYd8tQGRWacE9kU=
github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/digitalocean/godo v1.7.5 h1:JOQbAO6QT1GGjor0doT0mXefX2FgUDPOpYh2RaXA+ko=
github.com/digitalocean/godo v1.7.5/go.mod h1:h6faOIcZ8lWIwNQ+DN7b3CgX4Kwby5T+nbpNqkUIozU=
......@@ -453,6 +457,8 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/flatbuffers v1.11.0 h1:O7CEyB8Cb3/DmtxODGtLHcEvpr81Jm5qLg/hsHnxA2A=
github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
......@@ -648,6 +654,8 @@ github.com/hashicorp/vault-plugin-database-elasticsearch v0.6.1 h1:C3NF3pVF7/Emx
github.com/hashicorp/vault-plugin-database-elasticsearch v0.6.1/go.mod h1:813Nvr1IQqAKdlk3yIY97M5WyxMhWOrXtYioPf9PqJg=
github.com/hashicorp/vault-plugin-database-mongodbatlas v0.2.1 h1:Yc8ZJJINvCH6JcJ8uvNkZ6W33KYzVdG4zI98dvbQ8lE=
github.com/hashicorp/vault-plugin-database-mongodbatlas v0.2.1/go.mod h1:2So20ndRRsDAMDyG52az6nd7NwFOZTQER9EsrgPCgVg=
github.com/hashicorp/vault-plugin-database-snowflake v0.1.0 h1:SfzEIHgPqaoDfSN8Key2yLXZfUI3AhI45OEn5jXN8Eo=
github.com/hashicorp/vault-plugin-database-snowflake v0.1.0/go.mod h1:EUOuoWLg+RwiIjxydK+hH7Eeb4x66cESYNmTHLqqbuM=
github.com/hashicorp/vault-plugin-mock v0.16.1 h1:5QQvSUHxDjEEbrd2REOeacqyJnCLPD51IQzy71hx8P0=
github.com/hashicorp/vault-plugin-mock v0.16.1/go.mod h1:83G4JKlOwUtxVourn5euQfze3ZWyXcUiLj2wqrKSDIM=
github.com/hashicorp/vault-plugin-secrets-ad v0.8.0 h1:SBOxEjYtxkipmp8AC7Yy3vjBVM5H24YM48WKCSCV+zA=
......@@ -953,6 +961,8 @@ github.com/pierrec/lz4 v2.2.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi
github.com/pierrec/lz4 v2.2.6+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI=
github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 h1:49lOXmGaUpV9Fz3gd7TFZY106KVlPVa5jcYD1gaQf98=
github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
......@@ -1053,6 +1063,10 @@ github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:X
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/snowflakedb/glog v0.0.0-20180824191149-f5055e6f21ce h1:CGR1hXCOeoZ1aJhCs8qdKJuEu3xoZnxsLcYoh5Bnr+4=
github.com/snowflakedb/glog v0.0.0-20180824191149-f5055e6f21ce/go.mod h1:EB/w24pR5VKI60ecFnKqXzxX3dOorz1rnVicQTQrGM0=
github.com/snowflakedb/gosnowflake v1.3.11 h1:4VATaWPZv2HEh9bkZG5LaMux4WRiZJDu/PvvMCzrpUg=
github.com/snowflakedb/gosnowflake v1.3.11/go.mod h1:+BMe9ivHWpzcXbM1qSIxWZ8qpWGBBaA46o9Z1qSfrNg=
github.com/softlayer/softlayer-go v0.0.0-20180806151055-260589d94c7d h1:bVQRCxQvfjNUeRqaY/uT0tFuvuFY0ulgnczuR684Xic=
github.com/softlayer/softlayer-go v0.0.0-20180806151055-260589d94c7d/go.mod h1:Cw4GTlQccdRGSEf6KiMju767x0NEHE0YIVPJSaXjlsw=
github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E=
......@@ -1085,6 +1099,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
......@@ -1183,6 +1198,7 @@ golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
......
helper/builtinplugins/registry.go
+ 21
- 21
View file @ 636d0371
......@@ -10,11 +10,18 @@ import (
credKerb "github.com/hashicorp/vault-plugin-auth-kerberos"
credKube "github.com/hashicorp/vault-plugin-auth-kubernetes"
credOCI "github.com/hashicorp/vault-plugin-auth-oci"
"github.com/hashicorp/vault/sdk/database/helper/credsutil"
dbCouchbase "github.com/hashicorp/vault-plugin-database-couchbase"
dbElastic "github.com/hashicorp/vault-plugin-database-elasticsearch"
dbMongoAtlas "github.com/hashicorp/vault-plugin-database-mongodbatlas"
dbSnowflake "github.com/hashicorp/vault-plugin-database-snowflake"
logicalAd "github.com/hashicorp/vault-plugin-secrets-ad/plugin"
logicalAlicloud "github.com/hashicorp/vault-plugin-secrets-alicloud"
logicalAzure "github.com/hashicorp/vault-plugin-secrets-azure"
logicalGcp "github.com/hashicorp/vault-plugin-secrets-gcp/plugin"
logicalGcpKms "github.com/hashicorp/vault-plugin-secrets-gcpkms"
logicalKv "github.com/hashicorp/vault-plugin-secrets-kv"
logicalMongoAtlas "github.com/hashicorp/vault-plugin-secrets-mongodbatlas"
logicalOpenLDAP "github.com/hashicorp/vault-plugin-secrets-openldap"
credAppId "github.com/hashicorp/vault/builtin/credential/app-id"
credAppRole "github.com/hashicorp/vault/builtin/credential/approle"
credAws "github.com/hashicorp/vault/builtin/credential/aws"
......@@ -24,25 +31,6 @@ import (
credOkta "github.com/hashicorp/vault/builtin/credential/okta"
credRadius "github.com/hashicorp/vault/builtin/credential/radius"
credUserpass "github.com/hashicorp/vault/builtin/credential/userpass"
dbCass "github.com/hashicorp/vault/plugins/database/cassandra"
dbHana "github.com/hashicorp/vault/plugins/database/hana"
dbInflux "github.com/hashicorp/vault/plugins/database/influxdb"
dbMongo "github.com/hashicorp/vault/plugins/database/mongodb"
dbMssql "github.com/hashicorp/vault/plugins/database/mssql"
dbMysql "github.com/hashicorp/vault/plugins/database/mysql"
dbPostgres "github.com/hashicorp/vault/plugins/database/postgresql"
dbRedshift "github.com/hashicorp/vault/plugins/database/redshift"
"github.com/hashicorp/vault/sdk/helper/consts"
"github.com/hashicorp/vault/sdk/logical"
logicalAd "github.com/hashicorp/vault-plugin-secrets-ad/plugin"
logicalAlicloud "github.com/hashicorp/vault-plugin-secrets-alicloud"
logicalAzure "github.com/hashicorp/vault-plugin-secrets-azure"
logicalGcp "github.com/hashicorp/vault-plugin-secrets-gcp/plugin"
logicalGcpKms "github.com/hashicorp/vault-plugin-secrets-gcpkms"
logicalKv "github.com/hashicorp/vault-plugin-secrets-kv"
logicalMongoAtlas "github.com/hashicorp/vault-plugin-secrets-mongodbatlas"
logicalOpenLDAP "github.com/hashicorp/vault-plugin-secrets-openldap"
logicalAws "github.com/hashicorp/vault/builtin/logical/aws"
logicalCass "github.com/hashicorp/vault/builtin/logical/cassandra"
logicalConsul "github.com/hashicorp/vault/builtin/logical/consul"
......@@ -56,6 +44,17 @@ import (
logicalSsh "github.com/hashicorp/vault/builtin/logical/ssh"
logicalTotp "github.com/hashicorp/vault/builtin/logical/totp"
logicalTransit "github.com/hashicorp/vault/builtin/logical/transit"
dbCass "github.com/hashicorp/vault/plugins/database/cassandra"
dbHana "github.com/hashicorp/vault/plugins/database/hana"
dbInflux "github.com/hashicorp/vault/plugins/database/influxdb"
dbMongo "github.com/hashicorp/vault/plugins/database/mongodb"
dbMssql "github.com/hashicorp/vault/plugins/database/mssql"
dbMysql "github.com/hashicorp/vault/plugins/database/mysql"
dbPostgres "github.com/hashicorp/vault/plugins/database/postgresql"
dbRedshift "github.com/hashicorp/vault/plugins/database/redshift"
"github.com/hashicorp/vault/sdk/database/helper/credsutil"
"github.com/hashicorp/vault/sdk/helper/consts"
"github.com/hashicorp/vault/sdk/logical"
)
// Registry is inherently thread-safe because it's immutable.
......@@ -110,6 +109,7 @@ func newRegistry() *registry {
"mssql-database-plugin": dbMssql.New,
"postgresql-database-plugin": dbPostgres.New,
"redshift-database-plugin": dbRedshift.New,
"snowflake-database-plugin": dbSnowflake.New,
},
logicalBackends: map[string]logical.Factory{
"ad": logicalAd.Factory,
......
vault/plugin_catalog_test.go
+ 2
- 2
View file @ 636d0371
......@@ -3,6 +3,8 @@ package vault
import (
"context"
"fmt"
"github.com/hashicorp/vault/sdk/helper/consts"
"github.com/hashicorp/vault/sdk/helper/pluginutil"
"io/ioutil"
"os"
"path/filepath"
......@@ -11,8 +13,6 @@ import (
"testing"
"github.com/hashicorp/vault/helper/builtinplugins"
"github.com/hashicorp/vault/sdk/helper/consts"
"github.com/hashicorp/vault/sdk/helper/pluginutil"
)
func TestPluginCatalog_CRUD(t *testing.T) {
......
vault/testing.go
+ 1
- 0
View file @ 636d0371
......@@ -2089,6 +2089,7 @@ func (m *mockBuiltinRegistry) Keys(pluginType consts.PluginType) []string {
"mssql-database-plugin",
"postgresql-database-plugin",
"redshift-database-plugin",
"snowflake-database-plugin",
}
}
......
vendor/github.com/apache/arrow/go/arrow/.editorconfig 0 → 100644
+ 21
- 0
View file @ 636d0371
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
root = true
[*.tmpl]
indent_style = tab
indent_size = 4
\ No newline at end of file
vendor/github.com/apache/arrow/go/arrow/.gitignore 0 → 100644
+ 35
- 0
View file @ 636d0371
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
### Go template
# Binaries for programs and plugins
*.exe
*.dll
*.so
*.dylib
*.o
# Test binary, build with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
.glide/
bin/
vendor/
\ No newline at end of file
vendor/github.com/apache/arrow/go/arrow/Gopkg.lock 0 → 100644
+ 44
- 0
View file @ 636d0371
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
digest = "1:56c130d885a4aacae1dd9c7b71cfe39912c7ebc1ff7d2b46083c8812996dc43b"
name = "github.com/davecgh/go-spew"
packages = ["spew"]
pruneopts = ""
revision = "346938d642f2ec3594ed81d874461961cd0faa76"
version = "v1.1.0"
[[projects]]
digest = "1:1d7e1867c49a6dd9856598ef7c3123604ea3daabf5b83f303ff457bcbc410b1d"
name = "github.com/pkg/errors"
packages = ["."]
pruneopts = ""
revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4"
version = "v0.8.1"
[[projects]]
digest = "1:256484dbbcd271f9ecebc6795b2df8cad4c458dd0f5fd82a8c2fa0c29f233411"
name = "github.com/pmezard/go-difflib"
packages = ["difflib"]
pruneopts = ""
revision = "792786c7400a136282c1664665ae0a8db921c6c2"
version = "v1.0.0"
[[projects]]
digest = "1:2d0dc026c4aef5e2f3a0e06a4dabe268b840d8f63190cf6894e02134a03f52c5"
name = "github.com/stretchr/testify"
packages = ["assert"]
pruneopts = ""
revision = "b91bfb9ebec76498946beb6af7c0230c7cc7ba6c"
version = "v1.2.0"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
input-imports = [
"github.com/pkg/errors",
"github.com/stretchr/testify/assert",
]
solver-name = "gps-cdcl"
solver-version = 1
vendor/github.com/apache/arrow/go/arrow/Gopkg.toml 0 → 100644
+ 23
- 0
View file @ 636d0371
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[[constraint]]
name = "github.com/stretchr/testify"
version = "1.2.0"
[[constraint]]
name = "github.com/pkg/errors"
version = "0.8.1"
\ No newline at end of file
vendor/github.com/apache/arrow/go/arrow/Makefile 0 → 100644
+ 54
- 0
View file @ 636d0371
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
GO_BUILD=go build
GO_GEN=go generate
GO_TEST?=go test
GOPATH=$(realpath ../../../../../..)
GO_SOURCES := $(shell find . -path ./_lib -prune -o -name '*.go' -not -name '*_test.go')
ALL_SOURCES := $(shell find . -path ./_lib -prune -o -name '*.go' -name '*.s' -not -name '*_test.go')
SOURCES_NO_VENDOR := $(shell find . -path ./vendor -prune -o -name "*.go" -not -name '*_test.go' -print)
.PHONEY: test bench assembly generate
assembly:
@$(MAKE) -C memory assembly
@$(MAKE) -C math assembly
generate: bin/tmpl
bin/tmpl -i -data=numeric.tmpldata type_traits_numeric.gen.go.tmpl type_traits_numeric.gen_test.go.tmpl array/numeric.gen.go.tmpl array/numericbuilder.gen_test.go.tmpl array/numericbuilder.gen.go.tmpl array/bufferbuilder_numeric.gen.go.tmpl
bin/tmpl -i -data=datatype_numeric.gen.go.tmpldata datatype_numeric.gen.go.tmpl
@$(MAKE) -C math generate
fmt: $(SOURCES_NO_VENDOR)
goimports -w $^
bench: $(GO_SOURCES) | assembly
$(GO_TEST) $(GO_TEST_ARGS) -bench=. -run=- ./...
bench-noasm: $(GO_SOURCES)
$(GO_TEST) $(GO_TEST_ARGS) -tags='noasm' -bench=. -run=- ./...
test: $(GO_SOURCES) | assembly
$(GO_TEST) $(GO_TEST_ARGS) ./...
test-noasm: $(GO_SOURCES)
$(GO_TEST) $(GO_TEST_ARGS) -tags='noasm' ./...
bin/tmpl: _tools/tmpl/main.go
$(GO_BUILD) -o $@ ./_tools/tmpl
vendor/github.com/apache/arrow/go/arrow/array/array.go 0 → 100644
+ 208
- 0
View file @ 636d0371
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array // import "github.com/apache/arrow/go/arrow/array"
import (
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
)
// A type which satisfies array.Interface represents an immutable sequence of values.
type Interface interface {
// DataType returns the type metadata for this instance.
DataType() arrow.DataType
// NullN returns the number of null values in the array.
NullN() int
// NullBitmapBytes returns a byte slice of the validity bitmap.
NullBitmapBytes() []byte
// IsNull returns true if value at index is null.
// NOTE: IsNull will panic if NullBitmapBytes is not empty and 0 > i ≥ Len.
IsNull(i int) bool
// IsValid returns true if value at index is not null.
// NOTE: IsValid will panic if NullBitmapBytes is not empty and 0 > i ≥ Len.
IsValid(i int) bool
Data() *Data
// Len returns the number of elements in the array.
Len() int
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
Retain()
// Release decreases the reference count by 1.
// Release may be called simultaneously from multiple goroutines.
// When the reference count goes to zero, the memory is freed.
Release()
}
const (
// UnknownNullCount specifies the NullN should be calculated from the null bitmap buffer.
UnknownNullCount = -1
)
type array struct {
refCount int64
data *Data
nullBitmapBytes []byte
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (a *array) Retain() {
atomic.AddInt64(&a.refCount, 1)
}
// Release decreases the reference count by 1.
// Release may be called simultaneously from multiple goroutines.
// When the reference count goes to zero, the memory is freed.
func (a *array) Release() {
debug.Assert(atomic.LoadInt64(&a.refCount) > 0, "too many releases")
if atomic.AddInt64(&a.refCount, -1) == 0 {
a.data.Release()
a.data, a.nullBitmapBytes = nil, nil
}
}
// DataType returns the type metadata for this instance.
func (a *array) DataType() arrow.DataType { return a.data.dtype }
// NullN returns the number of null values in the array.
func (a *array) NullN() int {
if a.data.nulls < 0 {
a.data.nulls = a.data.length - bitutil.CountSetBits(a.nullBitmapBytes, a.data.offset, a.data.length)
}
return a.data.nulls
}
// NullBitmapBytes returns a byte slice of the validity bitmap.
func (a *array) NullBitmapBytes() []byte { return a.nullBitmapBytes }
func (a *array) Data() *Data { return a.data }
// Len returns the number of elements in the array.
func (a *array) Len() int { return a.data.length }
// IsNull returns true if value at index is null.
// NOTE: IsNull will panic if NullBitmapBytes is not empty and 0 > i ≥ Len.
func (a *array) IsNull(i int) bool {
return len(a.nullBitmapBytes) != 0 && bitutil.BitIsNotSet(a.nullBitmapBytes, a.data.offset+i)
}
// IsValid returns true if value at index is not null.
// NOTE: IsValid will panic if NullBitmapBytes is not empty and 0 > i ≥ Len.
func (a *array) IsValid(i int) bool {
return len(a.nullBitmapBytes) == 0 || bitutil.BitIsSet(a.nullBitmapBytes, a.data.offset+i)
}
func (a *array) setData(data *Data) {
// Retain before releasing in case a.data is the same as data.
data.Retain()
if a.data != nil {
a.data.Release()
}
if len(data.buffers) > 0 && data.buffers[0] != nil {
a.nullBitmapBytes = data.buffers[0].Bytes()
}
a.data = data
}
func (a *array) Offset() int {
return a.data.Offset()
}
type arrayConstructorFn func(*Data) Interface
var (
makeArrayFn [32]arrayConstructorFn
)
func unsupportedArrayType(data *Data) Interface {
panic("unsupported data type: " + data.dtype.ID().String())
}
func invalidDataType(data *Data) Interface {
panic("invalid data type: " + data.dtype.ID().String())
}
// MakeFromData constructs a strongly-typed array instance from generic Data.
func MakeFromData(data *Data) Interface {
return makeArrayFn[byte(data.dtype.ID()&0x1f)](data)
}
// NewSlice constructs a zero-copy slice of the array with the indicated
// indices i and j, corresponding to array[i:j].
// The returned array must be Release()'d after use.
//
// NewSlice panics if the slice is outside the valid range of the input array.
// NewSlice panics if j < i.
func NewSlice(arr Interface, i, j int64) Interface {
data := NewSliceData(arr.Data(), i, j)
slice := MakeFromData(data)
data.Release()
return slice
}
func init() {
makeArrayFn = [...]arrayConstructorFn{
arrow.NULL: func(data *Data) Interface { return NewNullData(data) },
arrow.BOOL: func(data *Data) Interface { return NewBooleanData(data) },
arrow.UINT8: func(data *Data) Interface { return NewUint8Data(data) },
arrow.INT8: func(data *Data) Interface { return NewInt8Data(data) },
arrow.UINT16: func(data *Data) Interface { return NewUint16Data(data) },
arrow.INT16: func(data *Data) Interface { return NewInt16Data(data) },
arrow.UINT32: func(data *Data) Interface { return NewUint32Data(data) },
arrow.INT32: func(data *Data) Interface { return NewInt32Data(data) },
arrow.UINT64: func(data *Data) Interface { return NewUint64Data(data) },
arrow.INT64: func(data *Data) Interface { return NewInt64Data(data) },
arrow.FLOAT16: func(data *Data) Interface { return NewFloat16Data(data) },
arrow.FLOAT32: func(data *Data) Interface { return NewFloat32Data(data) },
arrow.FLOAT64: func(data *Data) Interface { return NewFloat64Data(data) },
arrow.STRING: func(data *Data) Interface { return NewStringData(data) },
arrow.BINARY: func(data *Data) Interface { return NewBinaryData(data) },
arrow.FIXED_SIZE_BINARY: func(data *Data) Interface { return NewFixedSizeBinaryData(data) },
arrow.DATE32: func(data *Data) Interface { return NewDate32Data(data) },
arrow.DATE64: func(data *Data) Interface { return NewDate64Data(data) },
arrow.TIMESTAMP: func(data *Data) Interface { return NewTimestampData(data) },
arrow.TIME32: func(data *Data) Interface { return NewTime32Data(data) },
arrow.TIME64: func(data *Data) Interface { return NewTime64Data(data) },
arrow.INTERVAL: func(data *Data) Interface { return NewIntervalData(data) },
arrow.DECIMAL: func(data *Data) Interface { return NewDecimal128Data(data) },
arrow.LIST: func(data *Data) Interface { return NewListData(data) },
arrow.STRUCT: func(data *Data) Interface { return NewStructData(data) },
arrow.UNION: unsupportedArrayType,
arrow.DICTIONARY: unsupportedArrayType,
arrow.MAP: unsupportedArrayType,
arrow.EXTENSION: unsupportedArrayType,
arrow.FIXED_SIZE_LIST: func(data *Data) Interface { return NewFixedSizeListData(data) },
arrow.DURATION: func(data *Data) Interface { return NewDurationData(data) },
// invalid data types to fill out array size 2⁵-1
31: invalidDataType,
}
}
vendor/github.com/apache/arrow/go/arrow/array/binary.go 0 → 100644
+ 134
- 0
View file @ 636d0371
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"bytes"
"fmt"
"strings"
"unsafe"
"github.com/apache/arrow/go/arrow"
)
// A type which represents an immutable sequence of variable-length binary strings.
type Binary struct {
array
valueOffsets []int32
valueBytes []byte
}
// NewBinaryData constructs a new Binary array from data.
func NewBinaryData(data *Data) *Binary {
a := &Binary{}
a.refCount = 1
a.setData(data)
return a
}
// Value returns the slice at index i. This value should not be mutated.
func (a *Binary) Value(i int) []byte {
if i < 0 || i >= a.array.data.length {
panic("arrow/array: index out of range")
}
idx := a.array.data.offset + i
return a.valueBytes[a.valueOffsets[idx]:a.valueOffsets[idx+1]]
}
// ValueString returns the string at index i without performing additional allocations.
// The string is only valid for the lifetime of the Binary array.
func (a *Binary) ValueString(i int) string {
b := a.Value(i)
return *(*string)(unsafe.Pointer(&b))
}
func (a *Binary) ValueOffset(i int) int {
if i < 0 || i >= a.array.data.length {
panic("arrow/array: index out of range")
}
return int(a.valueOffsets[a.array.data.offset+i])
}
func (a *Binary) ValueLen(i int) int {
if i < 0 || i >= a.array.data.length {
panic("arrow/array: index out of range")
}
beg := a.array.data.offset + i
return int(a.valueOffsets[beg+1] - a.valueOffsets[beg])
}
func (a *Binary) ValueOffsets() []int32 {
beg := a.array.data.offset
end := beg + a.array.data.length + 1
return a.valueOffsets[beg:end]
}
func (a *Binary) ValueBytes() []byte {
beg := a.array.data.offset
end := beg + a.array.data.length
return a.valueBytes[a.valueOffsets[beg]:a.valueOffsets[end]]
}
func (a *Binary) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i := 0; i < a.Len(); i++ {
if i > 0 {
o.WriteString(" ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%q", a.ValueString(i))
}
}
o.WriteString("]")
return o.String()
}
func (a *Binary) setData(data *Data) {
if len(data.buffers) != 3 {
panic("len(data.buffers) != 3")
}
a.array.setData(data)
if valueData := data.buffers[2]; valueData != nil {
a.valueBytes = valueData.Bytes()
}
if valueOffsets := data.buffers[1]; valueOffsets != nil {
a.valueOffsets = arrow.Int32Traits.CastFromBytes(valueOffsets.Bytes())
}
}
func arrayEqualBinary(left, right *Binary) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if bytes.Compare(left.Value(i), right.Value(i)) != 0 {
return false
}
}
return true
}
var (
_ Interface = (*Binary)(nil)
)
vendor/github.com/apache/arrow/go/arrow/array/binarybuilder.go 0 → 100644
+ 217
- 0
View file @ 636d0371
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"math"
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
const (
binaryArrayMaximumCapacity = math.MaxInt32
)
// A BinaryBuilder is used to build a Binary array using the Append methods.
type BinaryBuilder struct {
builder
dtype arrow.BinaryDataType
offsets *int32BufferBuilder
values *byteBufferBuilder
}
func NewBinaryBuilder(mem memory.Allocator, dtype arrow.BinaryDataType) *BinaryBuilder {
b := &BinaryBuilder{
builder: builder{refCount: 1, mem: mem},
dtype: dtype,
offsets: newInt32BufferBuilder(mem),
values: newByteBufferBuilder(mem),
}
return b
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (b *BinaryBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
if b.offsets != nil {
b.offsets.Release()
b.offsets = nil
}
if b.values != nil {
b.values.Release()
b.values = nil
}
}
}
func (b *BinaryBuilder) Append(v []byte) {
b.Reserve(1)
b.appendNextOffset()
b.values.Append(v)
b.UnsafeAppendBoolToBitmap(true)
}
func (b *BinaryBuilder) AppendString(v string) {
b.Append([]byte(v))
}
func (b *BinaryBuilder) AppendNull() {
b.Reserve(1)
b.appendNextOffset()
b.UnsafeAppendBoolToBitmap(false)
}
// AppendValues will append the values in the v slice. The valid slice determines which values
// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
// all values in v are appended and considered valid.
func (b *BinaryBuilder) AppendValues(v [][]byte, valid []bool) {
if len(v) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
if len(v) == 0 {
return
}
b.Reserve(len(v))
for _, vv := range v {
b.appendNextOffset()
b.values.Append(vv)
}
b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
}
// AppendStringValues will append the values in the v slice. The valid slice determines which values
// in v are valid (not null). The valid slice must either be empty or be equal in length to v. If empty,
// all values in v are appended and considered valid.
func (b *BinaryBuilder) AppendStringValues(v []string, valid []bool) {
if len(v) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
if len(v) == 0 {
return
}
b.Reserve(len(v))
for _, vv := range v {
b.appendNextOffset()
b.values.Append([]byte(vv))
}
b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
}
func (b *BinaryBuilder) Value(i int) []byte {
offsets := b.offsets.Values()
start := int(offsets[i])
var end int
if i == (b.length - 1) {
end = b.values.Len()
} else {
end = int(offsets[i+1])
}
return b.values.Bytes()[start:end]
}
func (b *BinaryBuilder) init(capacity int) {
b.builder.init(capacity)
b.offsets.resize((capacity + 1) * arrow.Int32SizeBytes)
}
// DataLen returns the number of bytes in the data array.
func (b *BinaryBuilder) DataLen() int { return b.values.length }
// DataCap returns the total number of bytes that can be stored
// without allocating additional memory.
func (b *BinaryBuilder) DataCap() int { return b.values.capacity }
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *BinaryBuilder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// ReserveData ensures there is enough space for appending n bytes
// by checking the capacity and resizing the data buffer if necessary.
func (b *BinaryBuilder) ReserveData(n int) {
if b.values.capacity < b.values.length+n {
b.values.resize(b.values.Len() + n)
}
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may be reduced.
func (b *BinaryBuilder) Resize(n int) {
b.offsets.resize((n + 1) * arrow.Int32SizeBytes)
b.builder.resize(n, b.init)
}
// NewArray creates a Binary array from the memory buffers used by the builder and resets the BinaryBuilder
// so it can be used to build a new array.
func (b *BinaryBuilder) NewArray() Interface {
return b.NewBinaryArray()
}
// NewBinaryArray creates a Binary array from the memory buffers used by the builder and resets the BinaryBuilder
// so it can be used to build a new array.
func (b *BinaryBuilder) NewBinaryArray() (a *Binary) {
data := b.newData()
a = NewBinaryData(data)
data.Release()
return
}
func (b *BinaryBuilder) newData() (data *Data) {
b.appendNextOffset()
offsets, values := b.offsets.Finish(), b.values.Finish()
data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, offsets, values}, nil, b.nulls, 0)
if offsets != nil {
offsets.Release()
}
if values != nil {
values.Release()
}
b.builder.reset()
return
}
func (b *BinaryBuilder) appendNextOffset() {
numBytes := b.values.Len()
// TODO(sgc): check binaryArrayMaximumCapacity?
b.offsets.AppendValue(int32(numBytes))
}
var (
_ Builder = (*BinaryBuilder)(nil)
)
vendor/github.com/apache/arrow/go/arrow/array/boolean.go 0 → 100644
+ 95
- 0
View file @ 636d0371
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"fmt"
"strings"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/bitutil"
"github.com/apache/arrow/go/arrow/memory"
)
// A type which represents an immutable sequence of boolean values.
type Boolean struct {
array
values []byte
}
// NewBoolean creates a boolean array from the data memory.Buffer and contains length elements.
// The nullBitmap buffer can be nil of there are no null values.
// If nulls is not known, use UnknownNullCount to calculate the value of NullN at runtime from the nullBitmap buffer.
func NewBoolean(length int, data *memory.Buffer, nullBitmap *memory.Buffer, nulls int) *Boolean {
return NewBooleanData(NewData(arrow.FixedWidthTypes.Boolean, length, []*memory.Buffer{nullBitmap, data}, nil, nulls, 0))
}
func NewBooleanData(data *Data) *Boolean {
a := &Boolean{}
a.refCount = 1
a.setData(data)
return a
}
func (a *Boolean) Value(i int) bool {
if i < 0 || i >= a.array.data.length {
panic("arrow/array: index out of range")
}
return bitutil.BitIsSet(a.values, a.array.data.offset+i)
}
func (a *Boolean) String() string {
o := new(strings.Builder)
o.WriteString("[")
for i := 0; i < a.Len(); i++ {
if i > 0 {
fmt.Fprintf(o, " ")
}
switch {
case a.IsNull(i):
o.WriteString("(null)")
default:
fmt.Fprintf(o, "%v", a.Value(i))
}
}
o.WriteString("]")
return o.String()
}
func (a *Boolean) setData(data *Data) {
a.array.setData(data)
vals := data.buffers[1]
if vals != nil {
a.values = vals.Bytes()
}
}
func arrayEqualBoolean(left, right *Boolean) bool {
for i := 0; i < left.Len(); i++ {
if left.IsNull(i) {
continue
}
if left.Value(i) != right.Value(i) {
return false
}
}
return true
}
var (
_ Interface = (*Boolean)(nil)
)
vendor/github.com/apache/arrow/go/arrow/array/booleanbuilder.go 0 → 100644
+ 165
- 0
View file @ 636d0371
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"sync/atomic"
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
type BooleanBuilder struct {
builder
data *memory.Buffer
rawData []byte
}
func NewBooleanBuilder(mem memory.Allocator) *BooleanBuilder {
return &BooleanBuilder{builder: builder{refCount: 1, mem: mem}}
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (b *BooleanBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.nullBitmap != nil {
b.nullBitmap.Release()
b.nullBitmap = nil
}
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
}
}
func (b *BooleanBuilder) Append(v bool) {
b.Reserve(1)
b.UnsafeAppend(v)
}
func (b *BooleanBuilder) AppendByte(v byte) {
b.Reserve(1)
b.UnsafeAppend(v != 0)
}
func (b *BooleanBuilder) AppendNull() {
b.Reserve(1)
b.UnsafeAppendBoolToBitmap(false)
}
func (b *BooleanBuilder) UnsafeAppend(v bool) {
bitutil.SetBit(b.nullBitmap.Bytes(), b.length)
if v {
bitutil.SetBit(b.rawData, b.length)
} else {
bitutil.ClearBit(b.rawData, b.length)
}
b.length++
}
func (b *BooleanBuilder) AppendValues(v []bool, valid []bool) {
if len(v) != len(valid) && len(valid) != 0 {
panic("len(v) != len(valid) && len(valid) != 0")
}
if len(v) == 0 {
return
}
b.Reserve(len(v))
for i, vv := range v {
bitutil.SetBitTo(b.rawData, b.length+i, vv)
}
b.builder.unsafeAppendBoolsToBitmap(valid, len(v))
}
func (b *BooleanBuilder) init(capacity int) {
b.builder.init(capacity)
b.data = memory.NewResizableBuffer(b.mem)
bytesN := arrow.BooleanTraits.BytesRequired(capacity)
b.data.Resize(bytesN)
b.rawData = b.data.Bytes()
}
// Reserve ensures there is enough space for appending n elements
// by checking the capacity and calling Resize if necessary.
func (b *BooleanBuilder) Reserve(n int) {
b.builder.reserve(n, b.Resize)
}
// Resize adjusts the space allocated by b to n elements. If n is greater than b.Cap(),
// additional memory will be allocated. If n is smaller, the allocated memory may reduced.
func (b *BooleanBuilder) Resize(n int) {
if n < minBuilderCapacity {
n = minBuilderCapacity
}
if b.capacity == 0 {
b.init(n)
} else {
b.builder.resize(n, b.init)
b.data.Resize(arrow.BooleanTraits.BytesRequired(n))
b.rawData = b.data.Bytes()
}
}
// NewArray creates a Boolean array from the memory buffers used by the builder and resets the BooleanBuilder
// so it can be used to build a new array.
func (b *BooleanBuilder) NewArray() Interface {
return b.NewBooleanArray()
}
// NewBooleanArray creates a Boolean array from the memory buffers used by the builder and resets the BooleanBuilder
// so it can be used to build a new array.
func (b *BooleanBuilder) NewBooleanArray() (a *Boolean) {
data := b.newData()
a = NewBooleanData(data)
data.Release()
return
}
func (b *BooleanBuilder) newData() *Data {
bytesRequired := arrow.BooleanTraits.BytesRequired(b.length)
if bytesRequired > 0 && bytesRequired < b.data.Len() {
// trim buffers
b.data.Resize(bytesRequired)
}
res := NewData(arrow.FixedWidthTypes.Boolean, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0)
b.reset()
if b.data != nil {
b.data.Release()
b.data = nil
b.rawData = nil
}
return res
}
var (
_ Builder = (*BooleanBuilder)(nil)
)
vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder.go 0 → 100644
+ 127
- 0
View file @ 636d0371
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"sync/atomic"
"github.com/apache/arrow/go/arrow/bitutil"
"github.com/apache/arrow/go/arrow/internal/debug"
"github.com/apache/arrow/go/arrow/memory"
)
// A bufferBuilder provides common functionality for populating memory with a sequence of type-specific values.
// Specialized implementations provide type-safe APIs for appending and accessing the memory.
type bufferBuilder struct {
refCount int64
mem memory.Allocator
buffer *memory.Buffer
length int
capacity int
bytes []byte
}
// Retain increases the reference count by 1.
// Retain may be called simultaneously from multiple goroutines.
func (b *bufferBuilder) Retain() {
atomic.AddInt64(&b.refCount, 1)
}
// Release decreases the reference count by 1.
// When the reference count goes to zero, the memory is freed.
// Release may be called simultaneously from multiple goroutines.
func (b *bufferBuilder) Release() {
debug.Assert(atomic.LoadInt64(&b.refCount) > 0, "too many releases")
if atomic.AddInt64(&b.refCount, -1) == 0 {
if b.buffer != nil {
b.buffer.Release()
b.buffer, b.bytes = nil, nil
}
}
}
// Len returns the length of the memory buffer in bytes.
func (b *bufferBuilder) Len() int { return b.length }
// Cap returns the total number of bytes that can be stored without allocating additional memory.
func (b *bufferBuilder) Cap() int { return b.capacity }
// Bytes returns a slice of length b.Len().
// The slice is only valid for use until the next buffer modification. That is, until the next call
// to Advance, Reset, Finish or any Append function. The slice aliases the buffer content at least until the next
// buffer modification.
func (b *bufferBuilder) Bytes() []byte { return b.bytes[:b.length] }
func (b *bufferBuilder) resize(elements int) {
if b.buffer == nil {
b.buffer = memory.NewResizableBuffer(b.mem)
}
b.buffer.Resize(elements)
oldCapacity := b.capacity
b.capacity = b.buffer.Cap()
b.bytes = b.buffer.Buf()
if b.capacity > oldCapacity {
memory.Set(b.bytes[oldCapacity:], 0)
}
}
// Advance increases the buffer by length and initializes the skipped bytes to zero.
func (b *bufferBuilder) Advance(length int) {
if b.capacity < b.length+length {
newCapacity := bitutil.NextPowerOf2(b.length + length)
b.resize(newCapacity)
}
b.length += length
}
// Append appends the contents of v to the buffer, resizing it if necessary.
func (b *bufferBuilder) Append(v []byte) {
if b.capacity < b.length+len(v) {
newCapacity := bitutil.NextPowerOf2(b.length + len(v))
b.resize(newCapacity)
}
b.unsafeAppend(v)
}
// Reset returns the buffer to an empty state. Reset releases the memory and sets the length and capacity to zero.
func (b *bufferBuilder) Reset() {
if b.buffer != nil {
b.buffer.Release()
}
b.buffer, b.bytes = nil, nil
b.capacity, b.length = 0, 0
}
// Finish TODO(sgc)
func (b *bufferBuilder) Finish() (buffer *memory.Buffer) {
if b.length > 0 {
b.buffer.ResizeNoShrink(b.length)
}
buffer = b.buffer
b.buffer = nil
b.Reset()
return
}
func (b *bufferBuilder) unsafeAppend(data []byte) {
copy(b.bytes[b.length:], data)
b.length += len(data)
}
vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder_byte.go 0 → 100644
+ 30
- 0
View file @ 636d0371
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import "github.com/apache/arrow/go/arrow/memory"
type byteBufferBuilder struct {
bufferBuilder
}
func newByteBufferBuilder(mem memory.Allocator) *byteBufferBuilder {
return &byteBufferBuilder{bufferBuilder: bufferBuilder{refCount: 1, mem: mem}}
}
func (b *byteBufferBuilder) Values() []byte { return b.Bytes() }
func (b *byteBufferBuilder) Value(i int) byte { return b.bytes[i] }
vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder_numeric.gen.go 0 → 100644
+ 58
- 0
View file @ 636d0371
// Code generated by array/bufferbuilder_numeric.gen.go.tmpl. DO NOT EDIT.
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/bitutil"
"github.com/apache/arrow/go/arrow/memory"
)
type int32BufferBuilder struct {
bufferBuilder
}
func newInt32BufferBuilder(mem memory.Allocator) *int32BufferBuilder {
return &int32BufferBuilder{bufferBuilder: bufferBuilder{refCount: 1, mem: mem}}
}
// AppendValues appends the contents of v to the buffer, growing the buffer as needed.
func (b *int32BufferBuilder) AppendValues(v []int32) { b.Append(arrow.Int32Traits.CastToBytes(v)) }
// Values returns a slice of length b.Len().
// The slice is only valid for use until the next buffer modification. That is, until the next call
// to Advance, Reset, Finish or any Append function. The slice aliases the buffer content at least until the next
// buffer modification.
func (b *int32BufferBuilder) Values() []int32 { return arrow.Int32Traits.CastFromBytes(b.Bytes()) }
// Value returns the int32 element at the index i. Value will panic if i is negative or ≥ Len.
func (b *int32BufferBuilder) Value(i int) int32 { return b.Values()[i] }
// Len returns the number of int32 elements in the buffer.
func (b *int32BufferBuilder) Len() int { return b.length / arrow.Int32SizeBytes }
// AppendValue appends v to the buffer, growing the buffer as needed.
func (b *int32BufferBuilder) AppendValue(v int32) {
if b.capacity < b.length+arrow.Int32SizeBytes {
newCapacity := bitutil.NextPowerOf2(b.length + arrow.Int32SizeBytes)
b.resize(newCapacity)
}
arrow.Int32Traits.PutValue(b.bytes[b.length:], v)
b.length += arrow.Int32SizeBytes
}
vendor/github.com/apache/arrow/go/arrow/array/bufferbuilder_numeric.gen.go.tmpl 0 → 100644
+ 61
- 0
View file @ 636d0371
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package array
import (
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/bitutil"
"github.com/apache/arrow/go/arrow/memory"
)
{{range .In}}
{{$TypeNamePrefix := .name}}
{{if .Opt.BufferBuilder}}
type {{$TypeNamePrefix}}BufferBuilder struct {
bufferBuilder
}
func new{{.Name}}BufferBuilder(mem memory.Allocator) *{{$TypeNamePrefix}}BufferBuilder {
return &{{$TypeNamePrefix}}BufferBuilder{bufferBuilder:bufferBuilder{refCount: 1, mem:mem}}
}
// AppendValues appends the contents of v to the buffer, growing the buffer as needed.
func (b *{{$TypeNamePrefix}}BufferBuilder) AppendValues(v []{{.Type}}) { b.Append(arrow.{{.Name}}Traits.CastToBytes(v)) }
// Values returns a slice of length b.Len().
// The slice is only valid for use until the next buffer modification. That is, until the next call
// to Advance, Reset, Finish or any Append function. The slice aliases the buffer content at least until the next
// buffer modification.
func (b *{{$TypeNamePrefix}}BufferBuilder) Values() []{{.Type}} { return arrow.{{.Name}}Traits.CastFromBytes(b.Bytes()) }
// Value returns the {{.Type}} element at the index i. Value will panic if i is negative or ≥ Len.
func (b *{{$TypeNamePrefix}}BufferBuilder) Value(i int) {{.Type}} { return b.Values()[i] }
// Len returns the number of {{.Type}} elements in the buffer.
func (b *{{$TypeNamePrefix}}BufferBuilder) Len() int { return b.length/arrow.{{.Name}}SizeBytes }
// AppendValue appends v to the buffer, growing the buffer as needed.
func (b *{{$TypeNamePrefix}}BufferBuilder) AppendValue(v {{.Type}}) {
if b.capacity < b.length+arrow.{{.Name}}SizeBytes {
newCapacity := bitutil.NextPowerOf2(b.length + arrow.{{.Name}}SizeBytes)
b.resize(newCapacity)
}
arrow.{{.Name}}Traits.PutValue(b.bytes[b.length:], v)
b.length+=arrow.{{.Name}}SizeBytes
}
{{end}}
{{end}}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

免费DevSecOps平台，让您的项目体验完整的DevSecOps流程，让项目更安全