Browse Source

Merge pull request #1704 from DataDog/aws-cache-zones-list

aws: cache zones list
master
Kubernetes Prow Robot 5 months ago
committed by GitHub
parent
commit
8b81c100fd
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 60 additions and 22 deletions
  1. +1
    -0
      CHANGELOG.md
  2. +7
    -0
      docs/tutorials/aws.md
  3. +1
    -0
      main.go
  4. +3
    -0
      pkg/apis/externaldns/types.go
  5. +26
    -22
      pkg/apis/externaldns/types_test.go
  6. +20
    -0
      provider/aws/aws.go
  7. +2
    -0
      provider/aws/aws_test.go

+ 1
- 0
CHANGELOG.md View File

@ -15,6 +15,7 @@
- Add tutorial for GKE with workload identity (#1765) @ddgenome
- Fix NodePort with externaltrafficpolicy targets duplication @codearky
- Update contributing section in README (#1760) @seanmalloy
- Option to cache AWS zones list @bpineau
## v0.7.3 - 2020-08-05


+ 7
- 0
docs/tutorials/aws.md View File

@ -420,3 +420,10 @@ Give ExternalDNS some time to clean up the DNS records for you. Then delete the
```console
$ aws route53 delete-hosted-zone --id /hostedzone/ZEWFWZ4R16P7IB
```
## Throttling
Route53 has a [5 API requests per second per account hard quota](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/DNSLimitations.html#limits-api-requests-route-53).
Running several fast polling ExternalDNS instances in a given account can easily hit that limit. Some ways to circumvent that issue includes:
* Augment the synchronization interval (`--interval`), at the cost of slower changes propagation.
* If the ExternalDNS managed zones list doesn't change frequently, set `--aws-zones-cache-duration` (zones list cache time-to-live) to a larger value. Note that zones list cache can be disabled with `--aws-zones-cache-duration=0s`.

+ 1
- 0
main.go View File

@ -175,6 +175,7 @@ func main() {
APIRetries: cfg.AWSAPIRetries,
PreferCNAME: cfg.AWSPreferCNAME,
DryRun: cfg.DryRun,
ZoneCacheDuration: cfg.AWSZoneCacheDuration,
},
)
case "aws-sd":


+ 3
- 0
pkg/apis/externaldns/types.go View File

@ -72,6 +72,7 @@ type Config struct {
AWSEvaluateTargetHealth bool
AWSAPIRetries int
AWSPreferCNAME bool
AWSZoneCacheDuration time.Duration
AzureConfigFile string
AzureResourceGroup string
AzureSubscriptionID string
@ -176,6 +177,7 @@ var defaultConfig = &Config{
AWSEvaluateTargetHealth: true,
AWSAPIRetries: 3,
AWSPreferCNAME: false,
AWSZoneCacheDuration: 0 * time.Second,
AzureConfigFile: "/etc/kubernetes/azure.json",
AzureResourceGroup: "",
AzureSubscriptionID: "",
@ -335,6 +337,7 @@ func (cfg *Config) ParseFlags(args []string) error {
app.Flag("aws-evaluate-target-health", "When using the AWS provider, set whether to evaluate the health of a DNS target (default: enabled, disable with --no-aws-evaluate-target-health)").Default(strconv.FormatBool(defaultConfig.AWSEvaluateTargetHealth)).BoolVar(&cfg.AWSEvaluateTargetHealth)
app.Flag("aws-api-retries", "When using the AWS provider, set the maximum number of retries for API calls before giving up.").Default(strconv.Itoa(defaultConfig.AWSAPIRetries)).IntVar(&cfg.AWSAPIRetries)
app.Flag("aws-prefer-cname", "When using the AWS provider, prefer using CNAME instead of ALIAS (default: disabled)").BoolVar(&cfg.AWSPreferCNAME)
app.Flag("aws-zones-cache-duration", "When using the AWS provider, set the zones list cache TTL (0s to disable).").Default(defaultConfig.AWSZoneCacheDuration.String()).DurationVar(&cfg.AWSZoneCacheDuration)
app.Flag("azure-config-file", "When using the Azure provider, specify the Azure configuration file (required when --provider=azure").Default(defaultConfig.AzureConfigFile).StringVar(&cfg.AzureConfigFile)
app.Flag("azure-resource-group", "When using the Azure provider, override the Azure resource group to use (required when --provider=azure-private-dns)").Default(defaultConfig.AzureResourceGroup).StringVar(&cfg.AzureResourceGroup)
app.Flag("azure-subscription-id", "When using the Azure provider, specify the Azure configuration file (required when --provider=azure-private-dns)").Default(defaultConfig.AzureSubscriptionID).StringVar(&cfg.AzureSubscriptionID)


+ 26
- 22
pkg/apis/externaldns/types_test.go View File

@ -29,17 +29,17 @@ import (
var (
minimalConfig = &Config{
APIServerURL: "",
KubeConfig: "",
RequestTimeout: time.Second * 30,
ContourLoadBalancerService: "heptio-contour/contour",
SkipperRouteGroupVersion: "zalando.org/v1",
Sources: []string{"service"},
Namespace: "",
FQDNTemplate: "",
Compatibility: "",
Provider: "google",
GoogleProject: "",
APIServerURL: "",
KubeConfig: "",
RequestTimeout: time.Second * 30,
ContourLoadBalancerService: "heptio-contour/contour",
SkipperRouteGroupVersion: "zalando.org/v1",
Sources: []string{"service"},
Namespace: "",
FQDNTemplate: "",
Compatibility: "",
Provider: "google",
GoogleProject: "",
GoogleBatchChangeSize: 1000,
GoogleBatchChangeInterval: time.Second,
DomainFilter: []string{""},
@ -54,6 +54,7 @@ var (
AWSEvaluateTargetHealth: true,
AWSAPIRetries: 3,
AWSPreferCNAME: false,
AWSZoneCacheDuration: 0 * time.Second,
AzureConfigFile: "/etc/kubernetes/azure.json",
AzureResourceGroup: "",
AzureSubscriptionID: "",
@ -103,17 +104,17 @@ var (
}
overriddenConfig = &Config{
APIServerURL: "http://127.0.0.1:8080",
KubeConfig: "/some/path",
RequestTimeout: time.Second * 77,
ContourLoadBalancerService: "heptio-contour-other/contour-other",
SkipperRouteGroupVersion: "zalando.org/v2",
Sources: []string{"service", "ingress", "connector"},
Namespace: "namespace",
IgnoreHostnameAnnotation: true,
FQDNTemplate: "{{.Name}}.service.example.com",
Compatibility: "mate",
Provider: "google",
APIServerURL: "http://127.0.0.1:8080",
KubeConfig: "/some/path",
RequestTimeout: time.Second * 77,
ContourLoadBalancerService: "heptio-contour-other/contour-other",
SkipperRouteGroupVersion: "zalando.org/v2",
Sources: []string{"service", "ingress", "connector"},
Namespace: "namespace",
IgnoreHostnameAnnotation: true,
FQDNTemplate: "{{.Name}}.service.example.com",
Compatibility: "mate",
Provider: "google",
GoogleProject: "project",
GoogleBatchChangeSize: 100,
GoogleBatchChangeInterval: time.Second * 2,
@ -129,6 +130,7 @@ var (
AWSEvaluateTargetHealth: false,
AWSAPIRetries: 13,
AWSPreferCNAME: true,
AWSZoneCacheDuration: 10 * time.Second,
AzureConfigFile: "azure.json",
AzureResourceGroup: "arg",
AzureSubscriptionID: "arg",
@ -261,6 +263,7 @@ func TestParseFlags(t *testing.T) {
"--aws-batch-change-interval=2s",
"--aws-api-retries=13",
"--aws-prefer-cname",
"--aws-zones-cache-duration=10s",
"--no-aws-evaluate-target-health",
"--policy=upsert-only",
"--registry=noop",
@ -348,6 +351,7 @@ func TestParseFlags(t *testing.T) {
"EXTERNAL_DNS_AWS_EVALUATE_TARGET_HEALTH": "0",
"EXTERNAL_DNS_AWS_API_RETRIES": "13",
"EXTERNAL_DNS_AWS_PREFER_CNAME": "true",
"EXTERNAL_DNS_AWS_ZONES_CACHE_DURATION": "10s",
"EXTERNAL_DNS_POLICY": "upsert-only",
"EXTERNAL_DNS_REGISTRY": "noop",
"EXTERNAL_DNS_TXT_OWNER_ID": "owner-1",


+ 20
- 0
provider/aws/aws.go View File

@ -117,6 +117,12 @@ type Route53API interface {
ListTagsForResourceWithContext(ctx context.Context, input *route53.ListTagsForResourceInput, opts ...request.Option) (*route53.ListTagsForResourceOutput, error)
}
type zonesListCache struct {
age time.Time
duration time.Duration
zones map[string]*route53.HostedZone
}
// AWSProvider is an implementation of Provider for AWS Route53.
type AWSProvider struct {
provider.BaseProvider
@ -134,6 +140,7 @@ type AWSProvider struct {
// filter hosted zones by tags
zoneTagFilter provider.ZoneTagFilter
preferCNAME bool
zonesCache *zonesListCache
}
// AWSConfig contains configuration to create a new AWS provider.
@ -149,6 +156,7 @@ type AWSConfig struct {
APIRetries int
PreferCNAME bool
DryRun bool
ZoneCacheDuration time.Duration
}
// NewAWSProvider initializes a new AWS Route53 based Provider.
@ -188,6 +196,7 @@ func NewAWSProvider(awsConfig AWSConfig) (*AWSProvider, error) {
evaluateTargetHealth: awsConfig.EvaluateTargetHealth,
preferCNAME: awsConfig.PreferCNAME,
dryRun: awsConfig.DryRun,
zonesCache: &zonesListCache{duration: awsConfig.ZoneCacheDuration},
}
return provider, nil
@ -195,6 +204,12 @@ func NewAWSProvider(awsConfig AWSConfig) (*AWSProvider, error) {
// Zones returns the list of hosted zones.
func (p *AWSProvider) Zones(ctx context.Context) (map[string]*route53.HostedZone, error) {
if p.zonesCache.zones != nil && time.Since(p.zonesCache.age) < p.zonesCache.duration {
log.Debug("Using cached zones list")
return p.zonesCache.zones, nil
}
log.Debug("Refreshing zones list cache")
zones := make(map[string]*route53.HostedZone)
var tagErr error
@ -242,6 +257,11 @@ func (p *AWSProvider) Zones(ctx context.Context) (map[string]*route53.HostedZone
log.Debugf("Considering zone: %s (domain: %s)", aws.StringValue(zone.Id), aws.StringValue(zone.Name))
}
if p.zonesCache.duration > time.Duration(0) {
p.zonesCache.zones = zones
p.zonesCache.age = time.Now()
}
return zones, nil
}


+ 2
- 0
provider/aws/aws_test.go View File

@ -500,6 +500,7 @@ func TestAWSApplyChanges(t *testing.T) {
ctx := tt.setup(provider)
provider.zonesCache = &zonesListCache{duration: 0 * time.Minute}
counter := NewRoute53APICounter(provider.client)
provider.client = counter
require.NoError(t, provider.ApplyChanges(ctx, changes))
@ -1200,6 +1201,7 @@ func newAWSProviderWithTagFilter(t *testing.T, domainFilter endpoint.DomainFilte
zoneTypeFilter: zoneTypeFilter,
zoneTagFilter: zoneTagFilter,
dryRun: false,
zonesCache: &zonesListCache{duration: 1 * time.Minute},
}
createAWSZone(t, provider, &route53.HostedZone{


Loading…
Cancel
Save