@ -0,0 +1 @@ | |||
vault-envs |
@ -0,0 +1,95 @@ | |||
# vault-envs | |||
Retrieve Vault KV storage path as env variables. | |||
``` | |||
>$ ./vault-envs -h | |||
Usage of ./vault-envs: | |||
-timeout string | |||
Set timeout to connect in seconds (default "10s") | |||
-token string | |||
Set token to authorize API requests | |||
-vault-path string | |||
Set KV secrets path, like /databases/postgres-main | |||
-vault-url string | |||
Set Vault URL, like https://vault.myproject.ru:8000 | |||
``` | |||
## How to use | |||
Create sample KV secret `/databases/test` with two named values and run: | |||
``` | |||
>$ ./vault-envs -token s.UYzqUtrBpL5MX3YP7GzXapZR \ | |||
-vault-url https://vault.blindage.org \ | |||
-vault-path /databases/test | |||
``` | |||
Output: | |||
``` | |||
VAULT_RETRIEVER=vault-envs | |||
env1=value1 | |||
env2=value2 | |||
``` | |||
Ok, you got your values in ENV variables format, additional mark of vault-envs added. Now export variables to current session: | |||
``` | |||
>$ export eval `./vault-envs -token s.UYzqUtrBpL5MX3YP7GzXapZR \ | |||
-vault-url https://vault.blindage.org \ | |||
-vault-path /databases/test` | |||
>$ echo "Value of env2 is: $env2" | |||
Value of env2 is: value2 | |||
``` | |||
## How to use in Docker containers | |||
Now think about creating bash script that you can run BEFORE your application start to provide required ENV variables. | |||
Contents of `set_vars.sh` script: | |||
```bash | |||
#!/bin/bash | |||
export eval `/opt/vault-envs/vault-envs -token "$VAULT_TOKEN" \ | |||
-vault-url https://vault.blindage.org \ | |||
-vault-path /databases/test` | |||
exec "$@" | |||
``` | |||
Contents of `Dockerfile`: | |||
```Dockerfile | |||
FROM ubuntu:20.04 | |||
LABEL maintainer="Vladimir Smagin <21h@blindage.org>, https://blindage.org" | |||
RUN apt update && apt install -y ca-certificates | |||
COPY vault-envs set_vars.sh /opt/vault-envs/ | |||
RUN chmod +x /opt/vault-envs/* | |||
ENTRYPOINT ["/opt/vault-envs/set_vars.sh"] | |||
``` | |||
Now build and run! | |||
``` | |||
>$ go build && cp vault-envs sample/ && cd sample | |||
>$ docker build -t vault-envs . | |||
>$ docker run --rm -e VAULT_TOKEN=s.UYzqUtrBpL5MX3YP7GzXapZR vault-envs printenv | |||
``` | |||
Output: | |||
``` | |||
HOSTNAME=579764792a0b | |||
VAULT_TOKEN=s.UYzqUtrBpL5MX3YP7GzXapZR | |||
PWD=/ | |||
HOME=/root | |||
VAULT_RETRIEVER=vault-envs | |||
TERM=xterm | |||
SHLVL=0 | |||
env2=value2 | |||
env1=value1 | |||
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin | |||
``` | |||
Cool! You made it! | |||
--- | |||
Copyright by Vladimir Smagin (21h) 2020 | |||
http://blindage.org email: 21h@blindage.org | |||
Project page: https://git.blindage.org/21h/vault-envs |
@ -0,0 +1,5 @@ | |||
module git.blindage.org/21h/vault-envs | |||
go 1.15 | |||
require github.com/hashicorp/vault/api v1.0.4 |
@ -0,0 +1,120 @@ | |||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= | |||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= | |||
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= | |||
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= | |||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= | |||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= | |||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= | |||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= | |||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= | |||
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= | |||
github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= | |||
github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= | |||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= | |||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= | |||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= | |||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= | |||
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= | |||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= | |||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= | |||
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= | |||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= | |||
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= | |||
github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= | |||
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= | |||
github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= | |||
github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= | |||
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= | |||
github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= | |||
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= | |||
github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= | |||
github.com/hashicorp/go-retryablehttp v0.5.4 h1:1BZvpawXoJCWX6pNtow9+rpEj+3itIlutiqnntI6jOE= | |||
github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= | |||
github.com/hashicorp/go-rootcerts v1.0.1 h1:DMo4fmknnz0E0evoNYnV48RjWndOsmd6OW+09R3cEP8= | |||
github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= | |||
github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= | |||
github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= | |||
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= | |||
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= | |||
github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= | |||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= | |||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= | |||
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= | |||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= | |||
github.com/hashicorp/vault v1.6.1 h1:E0ppfQu2eUU45Bteu1aizaD7gvnFmxdkjdIwiTADovY= | |||
github.com/hashicorp/vault/api v1.0.4 h1:j08Or/wryXT4AcHj1oCbMd7IijXcKzYUGw59LGu9onU= | |||
github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q= | |||
github.com/hashicorp/vault/sdk v0.1.13 h1:mOEPeOhT7jl0J4AMl1E705+BcmeRs1VmKNb9F0sMLy8= | |||
github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M= | |||
github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= | |||
github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= | |||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= | |||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= | |||
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= | |||
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= | |||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= | |||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= | |||
github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= | |||
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= | |||
github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= | |||
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= | |||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= | |||
github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= | |||
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= | |||
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= | |||
github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= | |||
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= | |||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= | |||
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= | |||
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= | |||
github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= | |||
github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= | |||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= | |||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= | |||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= | |||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= | |||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= | |||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= | |||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= | |||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= | |||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= | |||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= | |||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= | |||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= | |||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= | |||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= | |||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= | |||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= | |||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= | |||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= | |||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= | |||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= | |||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= | |||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= | |||
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | |||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | |||
golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | |||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | |||
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | |||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= | |||
golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db h1:6/JqlYfC1CCaLnGceQTI+sDGhC9UBSPAsBqI0Gun6kU= | |||
golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= | |||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= | |||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= | |||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= | |||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= | |||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= | |||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= | |||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= | |||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= | |||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= | |||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= | |||
google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= | |||
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= | |||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= | |||
google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= | |||
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= | |||
gopkg.in/square/go-jose.v2 v2.3.1 h1:SK5KegNXmKmqE342YYN2qPHEnUYeoMiXXl1poUlI+o4= | |||
gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= | |||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= | |||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= |
@ -0,0 +1,68 @@ | |||
package main | |||
import ( | |||
"flag" | |||
"fmt" | |||
"log" | |||
"time" | |||
"github.com/hashicorp/vault/api" | |||
) | |||
var ( | |||
token = flag.String("token", "", "Set token to authorize API requests") | |||
vaultURL = flag.String("vault-url", "", "Set Vault URL, like https://vault.myproject.ru:8000") | |||
vaultPath = flag.String("vault-path", "", "Set KV secrets path, like /databases/postgres-main") | |||
vaultTimeoutFlag = flag.String("timeout", "10s", "Set timeout to connect in seconds") | |||
vaultTimeout = time.Duration(0 * time.Second) | |||
) | |||
func init() { | |||
flag.Parse() | |||
if *token == "" { | |||
log.Fatalln("all requests to API must be authorized, see help") | |||
} | |||
if *vaultURL == "" { | |||
log.Fatalln("you forgot to set vault URL, see help") | |||
} | |||
if *vaultPath == "" { | |||
log.Fatalln("set path of secrets, see help") | |||
} | |||
err := error(nil) | |||
vaultTimeout, err = time.ParseDuration(*vaultTimeoutFlag) | |||
if err != nil { | |||
log.Fatalln("timeout wrong, use time format like: 5s, 21s") | |||
} | |||
} | |||
func main() { | |||
fmt.Println("VAULT_RETRIEVER=vault-envs") | |||
client := &api.Client{} | |||
client, err := api.NewClient(&api.Config{Address: *vaultURL}) | |||
if err != nil { | |||
log.Fatalln(err) | |||
} | |||
client.SetToken(*token) | |||
client.SetClientTimeout(vaultTimeout) | |||
vaultClient := client.Logical() | |||
vaultData, err := vaultClient.Read(*vaultPath) | |||
if err != nil { | |||
log.Fatalln(err) | |||
} | |||
if vaultData == nil { | |||
log.Fatalf("No data in path, %v\n", *vaultPath) | |||
} | |||
for varname, value := range vaultData.Data { | |||
fmt.Printf("%v=%v\n", varname, value) | |||
} | |||
} |
@ -0,0 +1,8 @@ | |||
FROM ubuntu:20.04 | |||
LABEL maintainer="Vladimir Smagin <21h@blindage.org>, https://blindage.org" | |||
RUN apt update && apt install -y ca-certificates | |||
COPY vault-envs set_vars.sh /opt/vault-envs/ | |||
RUN chmod +x /opt/vault-envs/* | |||
ENTRYPOINT ["/opt/vault-envs/set_vars.sh"] |
@ -0,0 +1,7 @@ | |||
#!/bin/bash | |||
export eval `/opt/vault-envs/vault-envs -token "$VAULT_TOKEN" \ | |||
-vault-url https://vault.blindage.org \ | |||
-vault-path /databases/test` | |||
exec "$@" |
@ -0,0 +1,16 @@ | |||
cmd/snappytool/snappytool | |||
testdata/bench | |||
# These explicitly listed benchmark data files are for an obsolete version of | |||
# snappy_test.go. | |||
testdata/alice29.txt | |||
testdata/asyoulik.txt | |||
testdata/fireworks.jpeg | |||
testdata/geo.protodata | |||
testdata/html | |||
testdata/html_x_4 | |||
testdata/kppkn.gtb | |||
testdata/lcet10.txt | |||
testdata/paper-100k.pdf | |||
testdata/plrabn12.txt | |||
testdata/urls.10K |
@ -0,0 +1,15 @@ | |||
# This is the official list of Snappy-Go authors for copyright purposes. | |||
# This file is distinct from the CONTRIBUTORS files. | |||
# See the latter for an explanation. | |||
# Names should be added to this file as | |||
# Name or Organization <email address> | |||
# The email address is not required for organizations. | |||
# Please keep the list sorted. | |||
Damian Gryski <dgryski@gmail.com> | |||
Google Inc. | |||
Jan Mercl <0xjnml@gmail.com> | |||
Rodolfo Carvalho <rhcarvalho@gmail.com> | |||
Sebastien Binet <seb.binet@gmail.com> |
@ -0,0 +1,37 @@ | |||
# This is the official list of people who can contribute | |||
# (and typically have contributed) code to the Snappy-Go repository. | |||
# The AUTHORS file lists the copyright holders; this file | |||
# lists people. For example, Google employees are listed here | |||
# but not in AUTHORS, because Google holds the copyright. | |||
# | |||
# The submission process automatically checks to make sure | |||
# that people submitting code are listed in this file (by email address). | |||
# | |||
# Names should be added to this file only after verifying that | |||
# the individual or the individual's organization has agreed to | |||
# the appropriate Contributor License Agreement, found here: | |||
# | |||
# http://code.google.com/legal/individual-cla-v1.0.html | |||
# http://code.google.com/legal/corporate-cla-v1.0.html | |||
# | |||
# The agreement for individuals can be filled out on the web. | |||
# | |||
# When adding J Random Contributor's name to this file, | |||
# either J's name or J's organization's name should be | |||
# added to the AUTHORS file, depending on whether the | |||
# individual or corporate CLA was used. | |||
# Names should be added to this file like so: | |||
# Name <email address> | |||
# Please keep the list sorted. | |||
Damian Gryski <dgryski@gmail.com> | |||
Jan Mercl <0xjnml@gmail.com> | |||
Kai Backman <kaib@golang.org> | |||
Marc-Antoine Ruel <maruel@chromium.org> | |||
Nigel Tao <nigeltao@golang.org> | |||
Rob Pike <r@golang.org> | |||
Rodolfo Carvalho <rhcarvalho@gmail.com> | |||
Russ Cox <rsc@golang.org> | |||
Sebastien Binet <seb.binet@gmail.com> |
@ -0,0 +1,27 @@ | |||
Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. | |||
Redistribution and use in source and binary forms, with or without | |||
modification, are permitted provided that the following conditions are | |||
met: | |||
* Redistributions of source code must retain the above copyright | |||
notice, this list of conditions and the following disclaimer. | |||
* Redistributions in binary form must reproduce the above | |||
copyright notice, this list of conditions and the following disclaimer | |||
in the documentation and/or other materials provided with the | |||
distribution. | |||
* Neither the name of Google Inc. nor the names of its | |||
contributors may be used to endorse or promote products derived from | |||
this software without specific prior written permission. | |||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
@ -0,0 +1,107 @@ | |||
The Snappy compression format in the Go programming language. | |||
To download and install from source: | |||
$ go get github.com/golang/snappy | |||
Unless otherwise noted, the Snappy-Go source files are distributed | |||
under the BSD-style license found in the LICENSE file. | |||
Benchmarks. | |||
The golang/snappy benchmarks include compressing (Z) and decompressing (U) ten | |||
or so files, the same set used by the C++ Snappy code (github.com/google/snappy | |||
and note the "google", not "golang"). On an "Intel(R) Core(TM) i7-3770 CPU @ | |||
3.40GHz", Go's GOARCH=amd64 numbers as of 2016-05-29: | |||
"go test -test.bench=." | |||
_UFlat0-8 2.19GB/s ยฑ 0% html | |||
_UFlat1-8 1.41GB/s ยฑ 0% urls | |||
_UFlat2-8 23.5GB/s ยฑ 2% jpg | |||
_UFlat3-8 1.91GB/s ยฑ 0% jpg_200 | |||
_UFlat4-8 14.0GB/s ยฑ 1% pdf | |||
_UFlat5-8 1.97GB/s ยฑ 0% html4 | |||
_UFlat6-8 814MB/s ยฑ 0% txt1 | |||
_UFlat7-8 785MB/s ยฑ 0% txt2 | |||
_UFlat8-8 857MB/s ยฑ 0% txt3 | |||
_UFlat9-8 719MB/s ยฑ 1% txt4 | |||
_UFlat10-8 2.84GB/s ยฑ 0% pb | |||
_UFlat11-8 1.05GB/s ยฑ 0% gaviota | |||
_ZFlat0-8 1.04GB/s ยฑ 0% html | |||
_ZFlat1-8 534MB/s ยฑ 0% urls | |||
_ZFlat2-8 15.7GB/s ยฑ 1% jpg | |||
_ZFlat3-8 740MB/s ยฑ 3% jpg_200 | |||
_ZFlat4-8 9.20GB/s ยฑ 1% pdf | |||
_ZFlat5-8 991MB/s ยฑ 0% html4 | |||
_ZFlat6-8 379MB/s ยฑ 0% txt1 | |||
_ZFlat7-8 352MB/s ยฑ 0% txt2 | |||
_ZFlat8-8 396MB/s ยฑ 1% txt3 | |||
_ZFlat9-8 327MB/s ยฑ 1% txt4 | |||
_ZFlat10-8 1.33GB/s ยฑ 1% pb | |||
_ZFlat11-8 605MB/s ยฑ 1% gaviota | |||
"go test -test.bench=. -tags=noasm" | |||
_UFlat0-8 621MB/s ยฑ 2% html | |||
_UFlat1-8 494MB/s ยฑ 1% urls | |||
_UFlat2-8 23.2GB/s ยฑ 1% jpg | |||
_UFlat3-8 1.12GB/s ยฑ 1% jpg_200 | |||
_UFlat4-8 4.35GB/s ยฑ 1% pdf | |||
_UFlat5-8 609MB/s ยฑ 0% html4 | |||
_UFlat6-8 296MB/s ยฑ 0% txt1 | |||
_UFlat7-8 288MB/s ยฑ 0% txt2 | |||
_UFlat8-8 309MB/s ยฑ 1% txt3 | |||
_UFlat9-8 280MB/s ยฑ 1% txt4 | |||
_UFlat10-8 753MB/s ยฑ 0% pb | |||
_UFlat11-8 400MB/s ยฑ 0% gaviota | |||
_ZFlat0-8 409MB/s ยฑ 1% html | |||
_ZFlat1-8 250MB/s ยฑ 1% urls | |||
_ZFlat2-8 12.3GB/s ยฑ 1% jpg | |||
_ZFlat3-8 132MB/s ยฑ 0% jpg_200 | |||
_ZFlat4-8 2.92GB/s ยฑ 0% pdf | |||
_ZFlat5-8 405MB/s ยฑ 1% html4 | |||
_ZFlat6-8 179MB/s ยฑ 1% txt1 | |||
_ZFlat7-8 170MB/s ยฑ 1% txt2 | |||
_ZFlat8-8 189MB/s ยฑ 1% txt3 | |||
_ZFlat9-8 164MB/s ยฑ 1% txt4 | |||
_ZFlat10-8 479MB/s ยฑ 1% pb | |||
_ZFlat11-8 270MB/s ยฑ 1% gaviota | |||
For comparison (Go's encoded output is byte-for-byte identical to C++'s), here | |||
are the numbers from C++ Snappy's | |||
make CXXFLAGS="-O2 -DNDEBUG -g" clean snappy_unittest.log && cat snappy_unittest.log | |||
BM_UFlat/0 2.4GB/s html | |||
BM_UFlat/1 1.4GB/s urls | |||
BM_UFlat/2 21.8GB/s jpg | |||
BM_UFlat/3 1.5GB/s jpg_200 | |||
BM_UFlat/4 13.3GB/s pdf | |||
BM_UFlat/5 2.1GB/s html4 | |||
BM_UFlat/6 1.0GB/s txt1 | |||
BM_UFlat/7 959.4MB/s txt2 | |||
BM_UFlat/8 1.0GB/s txt3 | |||
BM_UFlat/9 864.5MB/s txt4 | |||
BM_UFlat/10 2.9GB/s pb | |||
BM_UFlat/11 1.2GB/s gaviota | |||
BM_ZFlat/0 944.3MB/s html (22.31 %) | |||
BM_ZFlat/1 501.6MB/s urls (47.78 %) | |||
BM_ZFlat/2 14.3GB/s jpg (99.95 %) | |||
BM_ZFlat/3 538.3MB/s jpg_200 (73.00 %) | |||
BM_ZFlat/4 8.3GB/s pdf (83.30 %) | |||
BM_ZFlat/5 903.5MB/s html4 (22.52 %) | |||
BM_ZFlat/6 336.0MB/s txt1 (57.88 %) | |||
BM_ZFlat/7 312.3MB/s txt2 (61.91 %) | |||
BM_ZFlat/8 353.1MB/s txt3 (54.99 %) | |||
BM_ZFlat/9 289.9MB/s txt4 (66.26 %) | |||
BM_ZFlat/10 1.2GB/s pb (19.68 %) | |||
BM_ZFlat/11 527.4MB/s gaviota (37.72 %) |
@ -0,0 +1,237 @@ | |||
// Copyright 2011 The Snappy-Go Authors. All rights reserved. | |||
// Use of this source code is governed by a BSD-style | |||
// license that can be found in the LICENSE file. | |||
package snappy | |||
import ( | |||
"encoding/binary" | |||
"errors" | |||
"io" | |||
) | |||
var ( | |||
// ErrCorrupt reports that the input is invalid. | |||
ErrCorrupt = errors.New("snappy: corrupt input") | |||
// ErrTooLarge reports that the uncompressed length is too large. | |||
ErrTooLarge = errors.New("snappy: decoded block is too large") | |||
// ErrUnsupported reports that the input isn't supported. | |||
ErrUnsupported = errors.New("snappy: unsupported input") | |||
errUnsupportedLiteralLength = errors.New("snappy: unsupported literal length") | |||
) | |||
// DecodedLen returns the length of the decoded block. | |||
func DecodedLen(src []byte) (int, error) { | |||
v, _, err := decodedLen(src) | |||
return v, err | |||
} | |||
// decodedLen returns the length of the decoded block and the number of bytes | |||
// that the length header occupied. | |||
func decodedLen(src []byte) (blockLen, headerLen int, err error) { | |||
v, n := binary.Uvarint(src) | |||
if n <= 0 || v > 0xffffffff { | |||
return 0, 0, ErrCorrupt | |||
} | |||
const wordSize = 32 << (^uint(0) >> 32 & 1) | |||
if wordSize == 32 && v > 0x7fffffff { | |||
return 0, 0, ErrTooLarge | |||
} | |||
return int(v), n, nil | |||
} | |||
const ( | |||
decodeErrCodeCorrupt = 1 | |||
decodeErrCodeUnsupportedLiteralLength = 2 | |||
) | |||
// Decode returns the decoded form of src. The returned slice may be a sub- | |||
// slice of dst if dst was large enough to hold the entire decoded block. | |||
// Otherwise, a newly allocated slice will be returned. | |||
// | |||
// The dst and src must not overlap. It is valid to pass a nil dst. | |||
func Decode(dst, src []byte) ([]byte, error) { | |||
dLen, s, err := decodedLen(src) | |||
if err != nil { | |||
return nil, err | |||
} | |||
if dLen <= len(dst) { | |||
dst = dst[:dLen] | |||
} else { | |||
dst = make([]byte, dLen) | |||
} | |||
switch decode(dst, src[s:]) { | |||
case 0: | |||
return dst, nil | |||
case decodeErrCodeUnsupportedLiteralLength: | |||
return nil, errUnsupportedLiteralLength | |||
} | |||
return nil, ErrCorrupt | |||
} | |||
// NewReader returns a new Reader that decompresses from r, using the framing | |||
// format described at | |||
// https://github.com/google/snappy/blob/master/framing_format.txt | |||
func NewReader(r io.Reader) *Reader { | |||
return &Reader{ | |||
r: r, | |||
decoded: make([]byte, maxBlockSize), | |||
buf: make([]byte, maxEncodedLenOfMaxBlockSize+checksumSize), | |||
} | |||
} | |||
// Reader is an io.Reader that can read Snappy-compressed bytes. | |||
type Reader struct { | |||
r io.Reader | |||
err error | |||
decoded []byte | |||
buf []byte | |||
// decoded[i:j] contains decoded bytes that have not yet been passed on. | |||
i, j int | |||
readHeader bool | |||
} | |||
// Reset discards any buffered data, resets all state, and switches the Snappy | |||
// reader to read from r. This permits reusing a Reader rather than allocating | |||
// a new one. | |||
func (r *Reader) Reset(reader io.Reader) { | |||
r.r = reader | |||
r.err = nil | |||
r.i = 0 | |||
r.j = 0 | |||
r.readHeader = false | |||
} | |||
func (r *Reader) readFull(p []byte, allowEOF bool) (ok bool) { | |||
if _, r.err = io.ReadFull(r.r, p); r.err != nil { | |||
if r.err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) { | |||
r.err = ErrCorrupt | |||
} | |||
return false | |||
} | |||
return true | |||
} | |||
// Read satisfies the io.Reader interface. | |||
func (r *Reader) Read(p []byte) (int, error) { | |||
if r.err != nil { | |||
return 0, r.err | |||
} | |||
for { | |||
if r.i < r.j { | |||
n := copy(p, r.decoded[r.i:r.j]) | |||
r.i += n | |||
return n, nil | |||
} | |||
if !r.readFull(r.buf[:4], true) { | |||
return 0, r.err | |||
} | |||
chunkType := r.buf[0] | |||
if !r.readHeader { | |||
if chunkType != chunkTypeStreamIdentifier { | |||
r.err = ErrCorrupt | |||
return 0, r.err | |||
} | |||
r.readHeader = true | |||
} | |||
chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16 | |||
if chunkLen > len(r.buf) { | |||
r.err = ErrUnsupported | |||
return 0, r.err | |||
} | |||
// The chunk types are specified at | |||
// https://github.com/google/snappy/blob/master/framing_format.txt | |||
switch chunkType { | |||
case chunkTypeCompressedData: | |||
// Section 4.2. Compressed data (chunk type 0x00). | |||
if chunkLen < checksumSize { | |||
r.err = ErrCorrupt | |||
return 0, r.err | |||
} | |||
buf := r.buf[:chunkLen] | |||
if !r.readFull(buf, false) { | |||
return 0, r.err | |||
} | |||
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 | |||
buf = buf[checksumSize:] | |||
n, err := DecodedLen(buf) | |||
if err != nil { | |||
r.err = err | |||
return 0, r.err | |||
} | |||
if n > len(r.decoded) { | |||
r.err = ErrCorrupt | |||
return 0, r.err | |||
} | |||
if _, err := Decode(r.decoded, buf); err != nil { | |||
r.err = err | |||
return 0, r.err | |||
} | |||
if crc(r.decoded[:n]) != checksum { | |||
r.err = ErrCorrupt | |||
return 0, r.err | |||
} | |||
r.i, r.j = 0, n | |||
continue | |||
case chunkTypeUncompressedData: | |||
// Section 4.3. Uncompressed data (chunk type 0x01). | |||
if chunkLen < checksumSize { | |||
r.err = ErrCorrupt | |||
return 0, r.err | |||
} | |||
buf := r.buf[:checksumSize] | |||
if !r.readFull(buf, false) { | |||
return 0, r.err | |||
} | |||
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 | |||
// Read directly into r.decoded instead of via r.buf. | |||
n := chunkLen - checksumSize | |||
if n > len(r.decoded) { | |||
r.err = ErrCorrupt | |||
return 0, r.err | |||
} | |||
if !r.readFull(r.decoded[:n], false) { | |||
return 0, r.err | |||
} | |||
if crc(r.decoded[:n]) != checksum { | |||
r.err = ErrCorrupt | |||
return 0, r.err | |||
} | |||
r.i, r.j = 0, n | |||
continue | |||
case chunkTypeStreamIdentifier: | |||
// Section 4.1. Stream identifier (chunk type 0xff). | |||
if chunkLen != len(magicBody) { | |||
r.err = ErrCorrupt | |||
return 0, r.err | |||
} | |||
if !r.readFull(r.buf[:len(magicBody)], false) { | |||
return 0, r.err | |||
} | |||
for i := 0; i < len(magicBody); i++ { | |||
if r.buf[i] != magicBody[i] { | |||
r.err = ErrCorrupt | |||
return 0, r.err | |||
} | |||
} | |||
continue | |||
} | |||
if chunkType <= 0x7f { | |||
// Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f). | |||
r.err = ErrUnsupported | |||
return 0, r.err | |||
} | |||
// Section 4.4 Padding (chunk type 0xfe). | |||
// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd). | |||
if !r.readFull(r.buf[:chunkLen], false) { | |||
return 0, r.err | |||
} | |||
} | |||
} |
@ -0,0 +1,14 @@ | |||
// Copyright 2016 The Snappy-Go Authors. All rights reserved. | |||
// Use of this source code is governed by a BSD-style | |||
// license that can be found in the LICENSE file. | |||
// +build !appengine | |||
// +build gc | |||
// +build !noasm | |||
package snappy | |||
// decode has the same semantics as in decode_other.go. | |||
// | |||
//go:noescape | |||
func decode(dst, src []byte) int |
@ -0,0 +1,490 @@ | |||
// Copyright 2016 The Go Authors. All rights reserved. | |||
// Use of this source code is governed by a BSD-style | |||
// license that can be found in the LICENSE file. | |||
// +build !appengine | |||
// +build gc | |||
// +build !noasm | |||
#include "textflag.h" | |||
// The asm code generally follows the pure Go code in decode_other.go, except | |||
// where marked with a "!!!". | |||
// func decode(dst, src []byte) int | |||
// | |||
// All local variables fit into registers. The non-zero stack size is only to | |||
// spill registers and push args when issuing a CALL. The register allocation: | |||
// - AX scratch | |||
// - BX scratch | |||
// - CX length or x | |||
// - DX offset | |||
// - SI &src[s] | |||
// - DI &dst[d] | |||
// + R8 dst_base | |||
// + R9 dst_len | |||
// + R10 dst_base + dst_len | |||
// + R11 src_base | |||
// + R12 src_len | |||
// + R13 src_base + src_len | |||
// - R14 used by doCopy | |||
// - R15 used by doCopy | |||
// | |||
// The registers R8-R13 (marked with a "+") are set at the start of the | |||
// function, and after a CALL returns, and are not otherwise modified. | |||
// | |||
// The d variable is implicitly DI - R8, and len(dst)-d is R10 - DI. | |||
// The s variable is implicitly SI - R11, and len(src)-s is R13 - SI. | |||
TEXT ยทdecode(SB), NOSPLIT, $48-56 | |||
// Initialize SI, DI and R8-R13. | |||
MOVQ dst_base+0(FP), R8 | |||
MOVQ dst_len+8(FP), R9 | |||
MOVQ R8, DI | |||
MOVQ R8, R10 | |||
ADDQ R9, R10 | |||
MOVQ src_base+24(FP), R11 | |||
MOVQ src_len+32(FP), R12 | |||
MOVQ R11, SI | |||
MOVQ R11, R13 | |||
ADDQ R12, R13 | |||
loop: | |||
// for s < len(src) | |||
CMPQ SI, R13 | |||
JEQ end | |||
// CX = uint32(src[s]) | |||
// | |||
// switch src[s] & 0x03 | |||
MOVBLZX (SI), CX | |||
MOVL CX, BX | |||
ANDL $3, BX | |||
CMPL BX, $1 | |||
JAE tagCopy | |||
// ---------------------------------------- | |||
// The code below handles literal tags. | |||
// case tagLiteral: | |||
// x := uint32(src[s] >> 2) | |||
// switch | |||
SHRL $2, CX | |||
CMPL CX, $60 | |||
JAE tagLit60Plus | |||
// case x < 60: | |||
// s++ | |||
INCQ SI | |||
doLit: | |||
// This is the end of the inner "switch", when we have a literal tag. | |||
// | |||
// We assume that CX == x and x fits in a uint32, where x is the variable | |||
// used in the pure Go decode_other.go code. | |||
// length = int(x) + 1 | |||
// | |||
// Unlike the pure Go code, we don't need to check if length <= 0 because | |||
// CX can hold 64 bits, so the increment cannot overflow. | |||
INCQ CX | |||
// Prepare to check if copying length bytes will run past the end of dst or | |||
// src. | |||
// | |||
// AX = len(dst) - d | |||
// BX = len(src) - s | |||
MOVQ R10, AX | |||
SUBQ DI, AX | |||
MOVQ R13, BX | |||
SUBQ SI, BX | |||
// !!! Try a faster technique for short (16 or fewer bytes) copies. | |||
// | |||
// if length > 16 || len(dst)-d < 16 || len(src)-s < 16 { | |||
// goto callMemmove // Fall back on calling runtimeยทmemmove. | |||
// } | |||
// | |||
// The C++ snappy code calls this TryFastAppend. It also checks len(src)-s | |||
// against 21 instead of 16, because it cannot assume that all of its input | |||
// is contiguous in memory and so it needs to leave enough source bytes to | |||
// read the next tag without refilling buffers, but Go's Decode assumes | |||
// contiguousness (the src argument is a []byte). | |||
CMPQ CX, $16 | |||
JGT callMemmove | |||
CMPQ AX, $16 | |||
JLT callMemmove | |||
CMPQ BX, $16 | |||
JLT callMemmove | |||
// !!! Implement the copy from src to dst as a 16-byte load and store. | |||
// (Decode's documentation says that dst and src must not overlap.) | |||
// | |||
// This always copies 16 bytes, instead of only length bytes, but that's | |||
// OK. If the input is a valid Snappy encoding then subsequent iterations | |||
// will fix up the overrun. Otherwise, Decode returns a nil []byte (and a | |||
// non-nil error), so the overrun will be ignored. | |||
// | |||
// Note that on amd64, it is legal and cheap to issue unaligned 8-byte or | |||
// 16-byte loads and stores. This technique probably wouldn't be as | |||
// effective on architectures that are fussier about alignment. | |||
MOVOU 0(SI), X0 | |||
MOVOU X0, 0(DI) | |||
// d += length | |||
// s += length | |||
ADDQ CX, DI | |||
ADDQ CX, SI | |||
JMP loop | |||
callMemmove: | |||
// if length > len(dst)-d || length > len(src)-s { etc } | |||
CMPQ CX, AX | |||
JGT errCorrupt | |||
CMPQ CX, BX | |||
JGT errCorrupt | |||
// copy(dst[d:], src[s:s+length]) | |||
// | |||
// This means calling runtimeยทmemmove(&dst[d], &src[s], length), so we push | |||
// DI, SI and CX as arguments. Coincidentally, we also need to spill those | |||
// three registers to the stack, to save local variables across the CALL. | |||
MOVQ DI, 0(SP) | |||
MOVQ SI, 8(SP) | |||
MOVQ CX, 16(SP) | |||
MOVQ DI, 24(SP) | |||
MOVQ SI, 32(SP) | |||
MOVQ CX, 40(SP) | |||
CALL runtimeยทmemmove(SB) | |||
// Restore local variables: unspill registers from the stack and | |||
// re-calculate R8-R13. | |||
MOVQ 24(SP), DI | |||
MOVQ 32(SP), SI | |||
MOVQ 40(SP), CX | |||
MOVQ dst_base+0(FP), R8 | |||
MOVQ dst_len+8(FP), R9 | |||
MOVQ R8, R10 | |||
ADDQ R9, R10 | |||
MOVQ src_base+24(FP), R11 | |||
MOVQ src_len+32(FP), R12 | |||
MOVQ R11, R13 | |||
ADDQ R12, R13 | |||
// d += length | |||
// s += length | |||
ADDQ CX, DI | |||
ADDQ CX, SI | |||
JMP loop | |||
tagLit60Plus: | |||
// !!! This fragment does the | |||
// | |||
// s += x - 58; if uint(s) > uint(len(src)) { etc } | |||
// | |||
// checks. In the asm version, we code it once instead of once per switch case. | |||
ADDQ CX, SI | |||
SUBQ $58, SI | |||
MOVQ SI, BX | |||
SUBQ R11, BX | |||
CMPQ BX, R12 | |||
JA errCorrupt | |||
// case x == 60: | |||
CMPL CX, $61 | |||
JEQ tagLit61 | |||
JA tagLit62Plus | |||
// x = uint32(src[s-1]) | |||
MOVBLZX -1(SI), CX | |||
JMP doLit | |||
tagLit61: | |||
// case x == 61: | |||
// x = uint32(src[s-2]) | uint32(src[s-1])<<8 | |||
MOVWLZX -2(SI), CX | |||
JMP doLit | |||
tagLit62Plus: | |||
CMPL CX, $62 | |||
JA tagLit63 | |||
// case x == 62: | |||
// x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16 | |||
MOVWLZX -3(SI), CX | |||
MOVBLZX -1(SI), BX | |||
SHLL $16, BX | |||
ORL BX, CX | |||
JMP doLit | |||
tagLit63: | |||
// case x == 63: | |||
// x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 | |||
MOVL -4(SI), CX | |||
JMP doLit | |||
// The code above handles literal tags. | |||
// ---------------------------------------- | |||
// The code below handles copy tags. | |||
tagCopy4: | |||
// case tagCopy4: | |||
// s += 5 | |||
ADDQ $5, SI | |||
// if uint(s) > uint(len(src)) { etc } | |||
MOVQ SI, BX | |||
SUBQ R11, BX | |||
CMPQ BX, R12 | |||
JA errCorrupt | |||
// length = 1 + int(src[s-5])>>2 | |||
SHRQ $2, CX | |||
INCQ CX | |||
// offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24) | |||
MOVLQZX -4(SI), DX | |||
JMP doCopy | |||
tagCopy2: | |||
// case tagCopy2: | |||
// s += 3 | |||
ADDQ $3, SI | |||
// if uint(s) > uint(len(src)) { etc } | |||
MOVQ SI, BX | |||
SUBQ R11, BX | |||
CMPQ BX, R12 | |||
JA errCorrupt | |||
// length = 1 + int(src[s-3])>>2 | |||
SHRQ $2, CX | |||
INCQ CX | |||
// offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8) | |||
MOVWQZX -2(SI), DX | |||
JMP doCopy | |||
tagCopy: | |||
// We have a copy tag. We assume that: | |||
// - BX == src[s] & 0x03 | |||
// - CX == src[s] | |||
CMPQ BX, $2 | |||
JEQ tagCopy2 | |||
JA tagCopy4 | |||
// case tagCopy1: | |||
// s += 2 | |||
ADDQ $2, SI | |||
// if uint(s) > uint(len(src)) { etc } | |||
MOVQ SI, BX | |||
SUBQ R11, BX | |||
CMPQ BX, R12 | |||
JA errCorrupt | |||
// offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1])) | |||
MOVQ CX, DX | |||
ANDQ $0xe0, DX | |||
SHLQ $3, DX | |||
MOVBQZX -1(SI), BX | |||
ORQ BX, DX | |||
// length = 4 + int(src[s-2])>>2&0x7 | |||
SHRQ $2, CX | |||
ANDQ $7, CX | |||
ADDQ $4, CX | |||
doCopy: | |||
// This is the end of the outer "switch", when we have a copy tag. | |||
// | |||
// We assume that: | |||
// - CX == length && CX > 0 | |||
// - DX == offset | |||
// if offset <= 0 { etc } | |||
CMPQ DX, $0 | |||
JLE errCorrupt | |||
// if d < offset { etc } | |||
MOVQ DI, BX | |||
SUBQ R8, BX | |||
CMPQ BX, DX | |||
JLT errCorrupt | |||
// if length > len(dst)-d { etc } | |||
MOVQ R10, BX | |||
SUBQ DI, BX | |||
CMPQ CX, BX | |||
JGT errCorrupt | |||
// forwardCopy(dst[d:d+length], dst[d-offset:]); d += length | |||
// | |||
// Set: | |||
// - R14 = len(dst)-d | |||
// - R15 = &dst[d-offset] | |||
MOVQ R10, R14 | |||
SUBQ DI, R14 | |||
MOVQ DI, R15 | |||
SUBQ DX, R15 | |||
// !!! Try a faster technique for short (16 or fewer bytes) forward copies. | |||
// | |||
// First, try using two 8-byte load/stores, similar to the doLit technique | |||
// above. Even if dst[d:d+length] and dst[d-offset:] can overlap, this is | |||
// still OK if offset >= 8. Note that this has to be two 8-byte load/stores | |||
// and not one 16-byte load/store, and the first store has to be before the | |||
// second load, due to the overlap if offset is in the range [8, 16). | |||
// | |||
// if length > 16 || offset < 8 || len(dst)-d < 16 { | |||
// goto slowForwardCopy | |||
// } | |||
// copy 16 bytes | |||
// d += length | |||
CMPQ CX, $16 | |||
JGT slowForwardCopy | |||
CMPQ DX, $8 | |||
JLT slowForwardCopy | |||
CMPQ R14, $16 | |||
JLT slowForwardCopy | |||
MOVQ 0(R15), AX | |||
MOVQ AX, 0(DI) | |||
MOVQ 8(R15), BX | |||
MOVQ BX, 8(DI) | |||
ADDQ CX, DI | |||
JMP loop | |||
slowForwardCopy: | |||
// !!! If the forward copy is longer than 16 bytes, or if offset < 8, we | |||
// can still try 8-byte load stores, provided we can overrun up to 10 extra | |||
// bytes. As above, the overrun will be fixed up by subsequent iterations | |||
// of the outermost loop. | |||
// | |||
// The C++ snappy code calls this technique IncrementalCopyFastPath. Its | |||
// commentary says: | |||
// | |||
// ---- | |||
// | |||
// The main part of this loop is a simple copy of eight bytes at a time | |||
// until we've copied (at least) the requested amount of bytes. However, | |||
// if d and d-offset are less than eight bytes apart (indicating a | |||
// repeating pattern of length < 8), we first need to expand the pattern in | |||
// order to get the correct results. For instance, if the buffer looks like | |||
// this, with the eight-byte <d-offset> and <d> patterns marked as | |||
// intervals: | |||
// | |||
// abxxxxxxxxxxxx | |||
// [------] d-offset | |||
// [------] d | |||
// | |||
// a single eight-byte copy from <d-offset> to <d> will repeat the pattern | |||
// once, after which we can move <d> two bytes without moving <d-offset>: | |||
// | |||
// ababxxxxxxxxxx | |||
// [------] d-offset | |||
// [------] d | |||
// | |||
// and repeat the exercise until the two no longer overlap. | |||
// | |||
// This allows us to do very well in the special case of one single byte | |||
// repeated many times, without taking a big hit for more general cases. | |||
// | |||
// The worst case of extra writing past the end of the match occurs when | |||
// offset == 1 and length == 1; the last copy will read from byte positions | |||
// [0..7] and write to [4..11], whereas it was only supposed to write to | |||
// position 1. Thus, ten excess bytes. | |||
// | |||
// ---- | |||
// | |||
// That "10 byte overrun" worst case is confirmed by Go's | |||
// TestSlowForwardCopyOverrun, which also tests the fixUpSlowForwardCopy | |||
// and finishSlowForwardCopy algorithm. | |||
// | |||
// if length > len(dst)-d-10 { | |||
// goto verySlowForwardCopy | |||
// } | |||
SUBQ $10, R14 | |||
CMPQ CX, R14 | |||
JGT verySlowForwardCopy | |||
makeOffsetAtLeast8: | |||
// !!! As above, expand the pattern so that offset >= 8 and we can use | |||
// 8-byte load/stores. | |||
// | |||
// for offset < 8 { | |||
// copy 8 bytes from dst[d-offset:] to dst[d:] | |||
// length -= offset | |||
// d += offset | |||
// offset += offset | |||
// // The two previous lines together means that d-offset, and therefore | |||
// // R15, is unchanged. | |||
// } | |||
CMPQ DX, $8 | |||
JGE fixUpSlowForwardCopy | |||
MOVQ (R15), BX | |||
MOVQ BX, (DI) | |||
SUBQ DX, CX | |||
ADDQ DX, DI | |||
ADDQ DX, DX | |||
JMP makeOffsetAtLeast8 | |||
fixUpSlowForwardCopy: | |||
// !!! Add length (which might be negative now) to d (implied by DI being | |||
// &dst[d]) so that d ends up at the right place when we jump back to the | |||
// top of the loop. Before we do that, though, we save DI to AX so that, if | |||
// length is positive, copying the remaining length bytes will write to the | |||
// right place. | |||
MOVQ DI, AX | |||
ADDQ CX, DI | |||
finishSlowForwardCopy: | |||
// !!! Repeat 8-byte load/stores until length <= 0. Ending with a negative | |||
// length means that we overrun, but as above, that will be fixed up by | |||
// subsequent iterations of the outermost loop. | |||
CMPQ CX, $0 | |||
JLE loop | |||
MOVQ (R15), BX | |||
MOVQ BX, (AX) | |||
ADDQ $8, R15 | |||
ADDQ $8, AX | |||
SUBQ $8, CX | |||
JMP finishSlowForwardCopy | |||
verySlowForwardCopy: | |||
// verySlowForwardCopy is a simple implementation of forward copy. In C | |||
// parlance, this is a do/while loop instead of a while loop, since we know | |||
// that length > 0. In Go syntax: | |||
// | |||
// for { | |||
// dst[d] = dst[d - offset] | |||
// d++ | |||
// length-- | |||
// if length == 0 { | |||
// break | |||
// } | |||
// } | |||
MOVB (R15), BX | |||
MOVB BX, (DI) | |||
INCQ R15 | |||
INCQ DI | |||
DECQ CX | |||
JNZ verySlowForwardCopy | |||
JMP loop | |||
// The code above handles copy tags. | |||
// ---------------------------------------- | |||
end: | |||
// This is the end of the "for s < len(src)". | |||
// | |||
// if d != len(dst) { etc } | |||
CMPQ DI, R10 | |||
JNE errCorrupt | |||
// return 0 | |||
MOVQ $0, ret+48(FP) | |||
RET | |||
errCorrupt: | |||
// return decodeErrCodeCorrupt | |||
MOVQ $1, ret+48(FP) | |||
RET |
@ -0,0 +1,101 @@ | |||
// Copyright 2016 The Snappy-Go Authors. All rights reserved. | |||
// Use of this source code is governed by a BSD-style | |||
// license that can be found in the LICENSE file. | |||
// +build !amd64 appengine !gc noasm | |||
package snappy | |||
// decode writes the decoding of src to dst. It assumes that the varint-encoded | |||
// length of the decompressed bytes has already been read, and that len(dst) | |||
// equals that length. | |||
// | |||
// It returns 0 on success or a decodeErrCodeXxx error code on failure. | |||
func decode(dst, src []byte) int { | |||
var d, s, offset, length int | |||
for s < len(src) { | |||
switch src[s] & 0x03 { | |||
case tagLiteral: | |||
x := uint32(src[s] >> 2) | |||
switch { | |||
case x < 60: | |||
s++ | |||
case x == 60: | |||
s += 2 | |||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. | |||
return decodeErrCodeCorrupt | |||
} | |||
x = uint32(src[s-1]) | |||
case x == 61: | |||
s += 3 | |||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. | |||
return decodeErrCodeCorrupt | |||
} | |||
x = uint32(src[s-2]) | uint32(src[s-1])<<8 | |||
case x == 62: | |||
s += 4 | |||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. | |||
return decodeErrCodeCorrupt | |||
} | |||
x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16 | |||
case x == 63: | |||
s += 5 | |||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. | |||
return decodeErrCodeCorrupt | |||
} | |||
x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 | |||
} | |||
length = int(x) + 1 | |||
if length <= 0 { | |||
return decodeErrCodeUnsupportedLiteralLength | |||
} | |||
if length > len(dst)-d || length > len(src)-s { | |||
return decodeErrCodeCorrupt | |||
} | |||
copy(dst[d:], src[s:s+length]) | |||
d += length | |||
s += length | |||
continue | |||
case tagCopy1: | |||
s += 2 | |||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. | |||
return decodeErrCodeCorrupt | |||
} | |||
length = 4 + int(src[s-2])>>2&0x7 | |||
offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1])) | |||
case tagCopy2: | |||
s += 3 | |||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. | |||
return decodeErrCodeCorrupt | |||
} | |||
length = 1 + int(src[s-3])>>2 | |||
offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8) | |||
case tagCopy4: | |||
s += 5 | |||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. | |||
return decodeErrCodeCorrupt | |||
} | |||
length = 1 + int(src[s-5])>>2 | |||
offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24) | |||
} | |||
if offset <= 0 || d < offset || length > len(dst)-d { | |||
return decodeErrCodeCorrupt | |||
} | |||
// Copy from an earlier sub-slice of dst to a later sub-slice. Unlike | |||
// the built-in copy function, this byte-by-byte copy always runs | |||
// forwards, even if the slices overlap. Conceptually, this is: | |||
// | |||
// d += forwardCopy(dst[d:d+length], dst[d-offset:]) | |||
for end := d + length; d != end; d++ { | |||
dst[d] = dst[d-offset] | |||
} | |||
} | |||
if d != len(dst) { | |||
return decodeErrCodeCorrupt | |||
} | |||
return 0 | |||
} |
@ -0,0 +1,285 @@ | |||
// Copyright 2011 The Snappy-Go Authors. All rights reserved. | |||
// Use of this source code is governed by a BSD-style | |||
// license that can be found in the LICENSE file. | |||
package snappy | |||
import ( | |||
"encoding/binary" | |||
"errors" | |||
"io" | |||
) | |||
// Encode returns the encoded form of src. The returned slice may be a sub- | |||
// slice of dst if dst was large enough to hold the entire encoded block. | |||
// Otherwise, a newly allocated slice will be returned. | |||
// | |||
// The dst and src must not overlap. It is valid to pass a nil dst. | |||
func Encode(dst, src []byte) []byte { | |||
if n := MaxEncodedLen(len(src)); n < 0 { | |||
panic(ErrTooLarge) | |||
} else if len(dst) < n { | |||
dst = make([]byte, n) | |||
} | |||
// The block starts with the varint-encoded length of the decompressed bytes. | |||
d := binary.PutUvarint(dst, uint64(len(src))) | |||
for len(src) > 0 { | |||
p := src | |||
src = nil | |||
if len(p) > maxBlockSize { | |||
p, src = p[:maxBlockSize], p[maxBlockSize:] | |||
} | |||
if len(p) < minNonLiteralBlockSize { | |||
d += emitLiteral(dst[d:], p) | |||
} else { | |||
d += encodeBlock(dst[d:], p) | |||
} | |||
} | |||
return dst[:d] | |||
} | |||
// inputMargin is the minimum number of extra input bytes to keep, inside | |||
// encodeBlock's inner loop. On some architectures, this margin lets us | |||
// implement a fast path for emitLiteral, where the copy of short (<= 16 byte) | |||
// literals can be implemented as a single load to and store from a 16-byte | |||
// register. That literal's actual length can be as short as 1 byte, so this | |||
// can copy up to 15 bytes too much, but that's OK as subsequent iterations of | |||
// the encoding loop will fix up the copy overrun, and this inputMargin ensures | |||
// that we don't overrun the dst and src buffers. | |||
const inputMargin = 16 - 1 | |||
// minNonLiteralBlockSize is the minimum size of the input to encodeBlock that | |||
// could be encoded with a copy tag. This is the minimum with respect to the | |||
// algorithm used by encodeBlock, not a minimum enforced by the file format. | |||
// | |||
// The encoded output must start with at least a 1 byte literal, as there are | |||
// no previous bytes to copy. A minimal (1 byte) copy after that, generated | |||
// from an emitCopy call in encodeBlock's main loop, would require at least | |||
// another inputMargin bytes, for the reason above: we want any emitLiteral | |||
// calls inside encodeBlock's main loop to use the fast path if possible, which | |||
// requires being able to overrun by inputMargin bytes. Thus, | |||
// minNonLiteralBlockSize equals 1 + 1 + inputMargin. | |||
// | |||
// The C++ code doesn't use this exact threshold, but it could, as discussed at | |||
// https://groups.google.com/d/topic/snappy-compression/oGbhsdIJSJ8/discussion | |||
// The difference between Go (2+inputMargin) and C++ (inputMargin) is purely an | |||
// optimization. It should not affect the encoded form. This is tested by | |||
// TestSameEncodingAsCppShortCopies. | |||
const minNonLiteralBlockSize = 1 + 1 + inputMargin | |||
// MaxEncodedLen returns the maximum length of a snappy block, given its | |||
// uncompressed length. | |||
// | |||
// It will return a negative value if srcLen is too large to encode. | |||
func MaxEncodedLen(srcLen int) int { | |||
n := uint64(srcLen) | |||
if n > 0xffffffff { | |||
return -1 | |||
} | |||
// Compressed data can be defined as: | |||
// compressed := item* literal* | |||
// item := literal* copy | |||
// | |||
// The trailing literal sequence has a space blowup of at most 62/60 | |||
// since a literal of length 60 needs one tag byte + one extra byte | |||
// for length information. | |||
// | |||
// Item blowup is trickier to measure. Suppose the "copy" op copies | |||
// 4 bytes of data. Because of a special check in the encoding code, | |||
// we produce a 4-byte copy only if the offset is < 65536. Therefore | |||
// the copy op takes 3 bytes to encode, and this type of item leads | |||
// to at most the 62/60 blowup for representing literals. | |||
// | |||
// Suppose the "copy" op copies 5 bytes of data. If the offset is big | |||
// enough, it will take 5 bytes to encode the copy op. Therefore the | |||
// worst case here is a one-byte literal followed by a five-byte copy. | |||
// That is, 6 bytes of input turn into 7 bytes of "compressed" data. | |||
// | |||
// This last factor dominates the blowup, so the final estimate is: | |||
n = 32 + n + n/6 | |||
if n > 0xffffffff { | |||
return -1 | |||
} | |||
return int(n) | |||
} | |||
var errClosed = errors.New("snappy: Writer is closed") | |||
// NewWriter returns a new Writer that compresses to w. | |||
// | |||
// The Writer returned does not buffer writes. There is no need to Flush or | |||
// Close such a Writer. | |||
// | |||
// Deprecated: the Writer returned is not suitable for many small writes, only | |||
// for few large writes. Use NewBufferedWriter instead, which is efficient | |||
// regardless of the frequency and shape of the writes, and remember to Close | |||
// that Writer when done. | |||
func NewWriter(w io.Writer) *Writer { | |||
return &Writer{ | |||
w: w, | |||
obuf: make([]byte, obufLen), | |||
} | |||
} | |||
// NewBufferedWriter returns a new Writer that compresses to w, using the | |||
// framing format described at | |||
// https://github.com/google/snappy/blob/master/framing_format.txt | |||
// | |||
// The Writer returned buffers writes. Users must call Close to guarantee all | |||
// data has been forwarded to the underlying io.Writer. They may also call | |||
// Flush zero or more times before calling Close. | |||
func NewBufferedWriter(w io.Writer) *Writer { | |||
return &Writer{ | |||
w: w, | |||
ibuf: make([]byte, 0, maxBlockSize), | |||
obuf: make([]byte, obufLen), | |||
} | |||
} | |||
// Writer is an io.Writer that can write Snappy-compressed bytes. | |||
type Writer struct { | |||
w io.Writer | |||
err error | |||
// ibuf is a buffer for the incoming (uncompressed) bytes. | |||
// | |||
// Its use is optional. For backwards compatibility, Writers created by the | |||
// NewWriter function have ibuf == nil, do not buffer incoming bytes, and | |||
// therefore do not need to be Flush'ed or Close'd. | |||
ibuf []byte | |||
// obuf is a buffer for the outgoing (compressed) bytes. | |||
obuf []byte | |||
// wroteStreamHeader is whether we have written the stream header. | |||
wroteStreamHeader bool | |||
} | |||
// Reset discards the writer's state and switches the Snappy writer to write to | |||
// w. This permits reusing a Writer rather than allocating a new one. | |||
func (w *Writer) Reset(writer io.Writer) { | |||
w.w = writer | |||
w.err = nil | |||
if w.ibuf != nil { | |||
w.ibuf = w.ibuf[:0] | |||
} | |||
w.wroteStreamHeader = false | |||
} | |||
// Write satisfies the io.Writer interface. | |||
func (w *Writer) Write(p []byte) (nRet int, errRet error) { | |||
if w.ibuf == nil { | |||
// Do not buffer incoming bytes. This does not perform or compress well | |||
// if the caller of Writer.Write writes many small slices. This | |||
// behavior is therefore deprecated, but still supported for backwards | |||
// compatibility with code that doesn't explicitly Flush or Close. | |||
return w.write(p) | |||
} | |||
// The remainder of this method is based on bufio.Writer.Write from the | |||
// standard library. | |||
for len(p) > (cap(w.ibuf)-len(w.ibuf)) && w.err == nil { | |||
var n int | |||
if len(w.ibuf) == 0 { | |||
// Large write, empty buffer. | |||
// Write directly from p to avoid copy. | |||
n, _ = w.write(p) | |||
} else { | |||
n = copy(w.ibuf[len(w.ibuf):cap(w.ibuf)], p) | |||
w.ibuf = w.ibuf[:len(w.ibuf)+n] | |||
w.Flush() | |||
} | |||
nRet += n | |||
p = p[n:] | |||
} | |||
if w.err != nil { | |||
return nRet, w.err | |||
} | |||
n := copy(w.ibuf[len(w.ibuf):cap(w.ibuf)], p) | |||
w.ibuf = w.ibuf[:len(w.ibuf)+n] | |||
nRet += n | |||
return nRet, nil | |||
} | |||
func (w *Writer) write(p []byte) (nRet int, errRet error) { | |||
if w.err != nil { | |||
return 0, w.err | |||
} | |||
for len(p) > 0 { | |||
obufStart := len(magicChunk) | |||
if !w.wroteStreamHeader { | |||
w.wroteStreamHeader = true | |||
copy(w.obuf, magicChunk) | |||
obufStart = 0 | |||
} | |||
var uncompressed []byte | |||
if len(p) > maxBlockSize { | |||
uncompressed, p = p[:maxBlockSize], p[maxBlockSize:] | |||
} else { | |||
uncompressed, p = p, nil | |||
} | |||
checksum := crc(uncompressed) | |||
// Compress the buffer, discarding the result if the improvement | |||
// isn't at least 12.5%. | |||
compressed := Encode(w.obuf[obufHeaderLen:], uncompressed) | |||
chunkType := uint8(chunkTypeCompressedData) | |||
chunkLen := 4 + len(compressed) | |||
obufEnd := obufHeaderLen + len(compressed) | |||
if len(compressed) >= len(uncompressed)-len(uncompressed)/8 { | |||
chunkType = chunkTypeUncompressedData | |||
chunkLen = 4 + len(uncompressed) | |||
obufEnd = obufHeaderLen | |||
} | |||
// Fill in the per-chunk header that comes before the body. | |||
w.obuf[len(magicChunk)+0] = chunkType | |||
w.obuf[len(magicChunk)+1] = uint8(chunkLen >> 0) | |||
w.obuf[len(magicChunk)+2] = uint8(chunkLen >> 8) | |||
w.obuf[len(magicChunk)+3] = uint8(chunkLen >> 16) | |||
w.obuf[len(magicChunk)+4] = uint8(checksum >> 0) | |||
w.obuf[len(magicChunk)+5] = uint8(checksum >> 8) | |||
w.obuf[len(magicChunk)+6] = uint8(checksum >> 16) | |||
w.obuf[len(magicChunk)+7] = uint8(checksum >> 24) | |||
if _, err := w.w.Write(w.obuf[obufStart:obufEnd]); err != nil { | |||
w.err = err | |||