Browse Source

delete dependent PVCs (#5)

* delete dependent PVCs

* finalizers refactoring

* refactoring, added to CR purgePVC, use_service_monitor
pull/6/head
Vladimir Smagin 1 year ago
committed by GitHub
parent
commit
c9b623048d
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 314 additions and 143 deletions
  1. +2
    -2
      deploy/build-operator.sh
  2. +0
    -107
      deploy/crds/rabbitmq_v1_rabbitmq_crd.yaml
  3. +46
    -0
      deploy/deploy-instance/instance1.yaml
  4. +46
    -0
      deploy/deploy-instance/instance2.yaml
  5. +0
    -0
      deploy/deploy-operator-debug/clusterrole.yaml
  6. +12
    -0
      deploy/deploy-operator-debug/clusterrolebinding.yaml
  7. +21
    -11
      deploy/deploy-operator-debug/operator-debug.yaml
  8. +0
    -0
      deploy/deploy-operator-debug/role.yaml
  9. +0
    -0
      deploy/deploy-operator-debug/role_binding.yaml
  10. +0
    -0
      deploy/deploy-operator-debug/service_account.yaml
  11. +54
    -0
      deploy/deploy-operator-default/clusterrole.yaml
  12. +1
    -1
      deploy/deploy-operator-default/clusterrolebinding.yaml
  13. +0
    -0
      deploy/deploy-operator-default/operator.yaml
  14. +3
    -1
      deploy/deploy-operator-default/role.yaml
  15. +4
    -4
      deploy/deploy-operator-default/role_binding.yaml
  16. +1
    -1
      deploy/deploy-operator-default/service_account.yaml
  17. +6
    -2
      pkg/apis/rabbitmq/v1/rabbitmq_types.go
  18. +20
    -0
      pkg/controller/rabbitmq/helpers.go
  19. +25
    -14
      pkg/controller/rabbitmq/rabbitmq_controller.go
  20. +73
    -0
      pkg/controller/rabbitmq/rabbitmq_finalizers.go

+ 2
- 2
deploy/build-operator.sh View File

@@ -1,5 +1,5 @@
operator-sdk generate k8s
operator-sdk build 716309063777.dkr.ecr.us-east-1.amazonaws.com/rabbitmq-operator:debug
eval $(aws ecr get-login --no-include-email --region us-east-1 --profile staging | sed 's|https://||')
operator-sdk build 716309063777.dkr.ecr.us-east-1.amazonaws.com/rabbitmq-operator
docker push 716309063777.dkr.ecr.us-east-1.amazonaws.com/rabbitmq-operator
docker push 716309063777.dkr.ecr.us-east-1.amazonaws.com/rabbitmq-operator:debug
kubectl delete po -l app=rabbitmq-operator

+ 0
- 107
deploy/crds/rabbitmq_v1_rabbitmq_crd.yaml View File

@@ -12,113 +12,6 @@ spec:
scope: Namespaced
subresources:
status: {}
validation:
openAPIV3Schema:
properties:
apiVersion:
description: 'APIVersion defines the versioned schema of this representation
of an object. Servers should convert recognized schemas to the latest
internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
type: string
kind:
description: 'Kind is a string value representing the REST resource this
object represents. Servers may infer this from the endpoint the client
submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
type: string
metadata:
type: object
spec:
properties:
auth:
description: 'TODO: auth mechanisms'
properties:
enabled:
type: boolean
mechanisms:
items:
type: string
type: array
uniqueItems: true
required:
- enabled
type: object
cert:
description: 'set SSL settings TODO: add to template, issue certs with
Vault'
properties:
cacertfile:
type: string
certfile:
type: string
enabled:
type: boolean
exitingSecret:
type: string
keyfile:
type: string
required:
- enabled
type: object
env:
description: set your own ENV variables in k8s style
items:
type: object
type: array
image:
description: you can set your own image instead of official
properties:
name:
type: string
tag:
type: string
required:
- name
- tag
type: object
plugins:
description: load additional plugins
items:
type: string
type: array
policies:
description: set rabbitmq policies
items:
properties:
definition:
properties:
expires:
format: int64
type: integer
type: object
name:
type: string
pattern:
type: string
priority:
format: int64
type: integer
vhost:
type: string
required:
- name
- pattern
- definition
- priority
type: object
type: array
replicas:
format: int32
maximum: 10
minimum: 1
type: integer
required:
- replicas
- policies
- plugins
- image
type: object
status:
type: object
version: v1
versions:
- name: v1


+ 46
- 0
deploy/deploy-instance/instance1.yaml View File

@@ -0,0 +1,46 @@
---
apiVersion: rabbitmq.improvado.io/v1
kind: Rabbitmq
metadata:
name: save-rabbit
spec:
replicas: 2
image:
name: rabbitmq
tag: 3-alpine
#secret_credentials: rabbit-users
#secret_service_account: rabbit-service

# comment or set 0 if no exporter needed
prometheus_exporter_port: 9091
#prometheus_image: "kbudde/rabbitmq-exporter:v0.28.0"

memory_high_watermark: 256M

# create unprivileged serviceaccount by hands
# and write it here
k8s_serviceaccount: "rabbitmq-operator"

k8s_host: "kubernetes.default.svc.cluster.imp"
k8s_service_discovery: "svc.cluster.imp"
k8s_addrtype: hostname
cluster_node_cleanup_interval: 10
auth:
enabled: true
mechanisms:
- PLAIN
- AMQPLAIN

volume_size: 1Gi
purgePVC: false

policies:
- name: ha-three
vhost: "rabbit"
pattern: ".*"
definition:
ha-mode: "exactly"
ha-params: 3
ha-sync-mode: "automatic"
priority: 0
apply-to: all

+ 46
- 0
deploy/deploy-instance/instance2.yaml View File

@@ -0,0 +1,46 @@
---
apiVersion: rabbitmq.improvado.io/v1
kind: Rabbitmq
metadata:
name: purge-rabbit
spec:
replicas: 2
image:
name: rabbitmq
tag: 3-alpine
#secret_credentials: rabbit-users
#secret_service_account: rabbit-service

# comment or set 0 if no exporter needed
prometheus_exporter_port: 9091
#prometheus_image: "kbudde/rabbitmq-exporter:v0.28.0"

memory_high_watermark: 256M

# create unprivileged serviceaccount by hands
# and write it here
k8s_serviceaccount: "rabbitmq-operator"

k8s_host: "kubernetes.default.svc.cluster.imp"
k8s_service_discovery: "svc.cluster.imp"
k8s_addrtype: hostname
cluster_node_cleanup_interval: 10
auth:
enabled: true
mechanisms:
- PLAIN
- AMQPLAIN

volume_size: 1Gi
purgePVC: true

policies:
- name: ha-three
vhost: "rabbit"
pattern: ".*"
definition:
ha-mode: "exactly"
ha-params: 3
ha-sync-mode: "automatic"
priority: 0
apply-to: all

deploy/clusterrole.yaml → deploy/deploy-operator-debug/clusterrole.yaml View File


+ 12
- 0
deploy/deploy-operator-debug/clusterrolebinding.yaml View File

@@ -0,0 +1,12 @@
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rabbitmq-operator
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: system:serviceaccount:rabbit-operator-debug:rabbitmq-operator
roleRef:
kind: ClusterRole
name: rabbitmq-operator
apiGroup: rbac.authorization.k8s.io

deploy/operator.yaml.bak → deploy/deploy-operator-debug/operator-debug.yaml View File

@@ -1,4 +1,5 @@
apiVersion: apps/v1
---
apiVersion: apps/v1beta1
kind: Deployment
metadata:
name: rabbitmq-operator
@@ -6,27 +7,36 @@ spec:
replicas: 1
selector:
matchLabels:
name: rabbitmq-operator
app: rabbitmq-operator
template:
metadata:
labels:
name: rabbitmq-operator
app: rabbitmq-operator
spec:
serviceAccountName: rabbitmq-operator
containers:
- name: rabbitmq-operator
image: 716309063777.dkr.ecr.us-east-1.amazonaws.com/rabbitmq-operator:latest
command:
- rabbitmq-operator
image: 716309063777.dkr.ecr.us-east-1.amazonaws.com/rabbitmq-operator:debug
imagePullPolicy: Always
env:
- name: bus
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: controller
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: OPERATOR_NAME
value: "rabbitmq-operator"
- name: WATCH_NAMESPACE
value: "rabbit-operator-debug"

# create service for deployment

---
apiVersion: v1
kind: Service
metadata:
name: rabbitmq-operator
spec:
selector:
app: rabbitmq-operator
ports:
- port: 80

deploy/role.yaml → deploy/deploy-operator-debug/role.yaml View File


deploy/role_binding.yaml → deploy/deploy-operator-debug/role_binding.yaml View File


deploy/service_account.yaml → deploy/deploy-operator-debug/service_account.yaml View File


+ 54
- 0
deploy/deploy-operator-default/clusterrole.yaml View File

@@ -0,0 +1,54 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
creationTimestamp: null
name: rabbitmq-operator
rules:
- apiGroups:
- ""
resources:
- pods
- services
- endpoints
- persistentvolumeclaims
- events
- configmaps
- secrets
verbs:
- '*'
- apiGroups:
- ""
resources:
- namespaces
verbs:
- get
- apiGroups:
- apps
resources:
- deployments
- daemonsets
- replicasets
- statefulsets
verbs:
- '*'
- apiGroups:
- batch
resources:
- jobs
verbs:
- '*'
- apiGroups:
- monitoring.coreos.com
resources:
- servicemonitors
verbs:
- get
- create
- list
- watch
- apiGroups:
- rabbitmq.improvado.io
resources:
- '*'
verbs:
- '*'

deploy/clusterrolebinding.yaml → deploy/deploy-operator-default/clusterrolebinding.yaml View File

@@ -5,7 +5,7 @@ metadata:
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: system:serviceaccount:messaging:rabbitmq-operator
name: system:serviceaccount:default:rabbitmq-operator
roleRef:
kind: ClusterRole
name: rabbitmq-operator

deploy/operator.yaml → deploy/deploy-operator-default/operator.yaml View File


deploy/rabbit/role.yaml → deploy/deploy-operator-default/role.yaml View File

@@ -2,7 +2,7 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
creationTimestamp: null
name: rabbitmq-instance
name: rabbitmq-operator
rules:
- apiGroups:
- ""
@@ -25,6 +25,8 @@ rules:
- apiGroups:
- apps
resources:
- deployments
- daemonsets
- replicasets
- statefulsets
verbs:

deploy/rabbit/role_binding.yaml → deploy/deploy-operator-default/role_binding.yaml View File

@@ -1,11 +1,11 @@
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rabbitmq-instance
name: rabbitmq-operator
subjects:
- kind: ServiceAccount
name: rabbitmq-instance
name: rabbitmq-operator
roleRef:
kind: Role
name: rabbitmq-instance
kind: ClusterRole
name: rabbitmq-operator
apiGroup: rbac.authorization.k8s.io

deploy/rabbit/service_account.yaml → deploy/deploy-operator-default/service_account.yaml View File

@@ -1,4 +1,4 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: rabbitmq-instance
name: rabbitmq-operator

+ 6
- 2
pkg/apis/rabbitmq/v1/rabbitmq_types.go View File

@@ -103,6 +103,8 @@ type RabbitmqSpec struct {
// TODO: additional labels
K8SLabels []metav1.LabelSelector `json:"k8s_labels"`

// purge all PVC after CR deletion, default false
RabbitmqPurgePVC bool `json:"purgePVC,omitempty"`
// PersistentVolumeClaim in k8s style
RabbitmqVolumeSize resource.Quantity `json:"volume_size"`

@@ -114,8 +116,10 @@ type RabbitmqSpec struct {
RabbitmqK8SPeerDiscoveryBackend string `json:"k8s_peer_discovery_backend"`
RabbitmqClusterFormationNodeCleanup int64 `json:"cluster_node_cleanup_interval"`
RabbitmqClusterPartitionHandling string `json:"cluster_partition_handling"`
RabbitmqPrometheusExporterPort int32 `json:"prometheus_exporter_port,omitempty"`
RabbitmqPrometheusImage string `json:"prometheus_image,omitempty"`
RabbitmqPrometheusExporterPort int32 `json:"prometheus_exporter_port,omitempty"`
RabbitmqPrometheusImage string `json:"prometheus_image,omitempty"`

RabbitmqUseServiceMonitor bool `json:"use_service_monitor,omitempty"`
}

// RabbitmqStatus defines the observed state of Rabbitmq


+ 20
- 0
pkg/controller/rabbitmq/helpers.go View File

@@ -0,0 +1,20 @@
package rabbitmq

func containsString(slice []string, s string) bool {
for _, item := range slice {
if item == s {
return true
}
}
return false
}

func removeString(slice []string, s string) (result []string) {
for _, item := range slice {
if item == s {
continue
}
result = append(result, item)
}
return
}

+ 25
- 14
pkg/controller/rabbitmq/rabbitmq_controller.go View File

@@ -181,7 +181,7 @@ func (r *ReconcileRabbitmq) Reconcile(request reconcile.Request) (reconcile.Resu
err := r.client.Get(context.TODO(), request.NamespacedName, instance)
if err != nil {
if errors.IsNotFound(err) {
// Request object not found, could have been deleted after reconcile request.
// Request object not statefulsetFound, could have been deleted after reconcile request.
// Owned objects are automatically garbage collected. For additional cleanup logic use finalizers.
// Return and don't requeue
return reconcile.Result{}, nil
@@ -205,8 +205,8 @@ func (r *ReconcileRabbitmq) Reconcile(request reconcile.Request) (reconcile.Resu
return reconcile.Result{}, err
}

found := &v1.StatefulSet{}
err = r.client.Get(context.TODO(), types.NamespacedName{Name: statefulset.Name, Namespace: statefulset.Namespace}, found)
statefulsetFound := &v1.StatefulSet{}
err = r.client.Get(context.TODO(), types.NamespacedName{Name: statefulset.Name, Namespace: statefulset.Namespace}, statefulsetFound)
if err != nil && errors.IsNotFound(err) {
reqLogger.Info("Creating a new statefulset", "statefulset.Namespace", statefulset.Namespace, "statefulset.Name", statefulset.Name)
err = r.client.Create(context.TODO(), statefulset)
@@ -222,22 +222,22 @@ func (r *ReconcileRabbitmq) Reconcile(request reconcile.Request) (reconcile.Resu
return reconcile.Result{}, err
}

if !reflect.DeepEqual(found.Spec, statefulset.Spec) {
found.Spec.Replicas = statefulset.Spec.Replicas
found.Spec.Template = statefulset.Spec.Template
if !reflect.DeepEqual(statefulsetFound.Spec, statefulset.Spec) {
statefulsetFound.Spec.Replicas = statefulset.Spec.Replicas
statefulsetFound.Spec.Template = statefulset.Spec.Template
}

if !reflect.DeepEqual(found.Annotations, statefulset.Annotations) {
found.Annotations = statefulset.Annotations
if !reflect.DeepEqual(statefulsetFound.Annotations, statefulset.Annotations) {
statefulsetFound.Annotations = statefulset.Annotations
}

if !reflect.DeepEqual(found.Labels, statefulset.Labels) {
found.Labels = statefulset.Labels
if !reflect.DeepEqual(statefulsetFound.Labels, statefulset.Labels) {
statefulsetFound.Labels = statefulset.Labels
}

reqLogger.Info("Reconcile statefulset", "statefulset.Namespace", found.Namespace, "statefulset.Name", found.Name)
if err = r.client.Update(context.TODO(), found); err != nil {
reqLogger.Info("Reconcile statefulset error", "statefulset.Namespace", found.Namespace, "statefulset.Name", found.Name)
reqLogger.Info("Reconcile statefulset", "statefulset.Namespace", statefulsetFound.Namespace, "statefulset.Name", statefulsetFound.Name)
if err = r.client.Update(context.TODO(), statefulsetFound); err != nil {
reqLogger.Info("Reconcile statefulset error", "statefulset.Namespace", statefulsetFound.Namespace, "statefulset.Name", statefulsetFound.Name)
raven.CaptureErrorAndWait(err, nil)
return reconcile.Result{}, err
}
@@ -280,6 +280,10 @@ func (r *ReconcileRabbitmq) Reconcile(request reconcile.Request) (reconcile.Resu
raven.CaptureErrorAndWait(err, nil)
return reconcile.Result{}, err
}
}

// use ServiceMonitor?
if instance.Spec.RabbitmqUseServiceMonitor {
_, err = r.reconcilePrometheusExporterServiceMonitor(reqLogger, instance)
if err != nil {
raven.CaptureErrorAndWait(err, nil)
@@ -321,6 +325,12 @@ func (r *ReconcileRabbitmq) Reconcile(request reconcile.Request) (reconcile.Resu
}
}

_, err = r.reconcileFinalizers(reqLogger, instance)
if err != nil {
raven.CaptureErrorAndWait(err, nil)
return reconcile.Result{}, err
}

return reconcile.Result{}, nil

}
@@ -476,7 +486,8 @@ func newStatefulSet(cr *rabbitmqv1.Rabbitmq, secretNames secretResouces) *v1.Sta

PVCTemplate := corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "rabbit-data",
Name: "rabbit-data",
Finalizers: cr.ObjectMeta.Finalizers,
},
Spec: corev1.PersistentVolumeClaimSpec{
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},


+ 73
- 0
pkg/controller/rabbitmq/rabbitmq_finalizers.go View File

@@ -0,0 +1,73 @@
package rabbitmq

import (
"context"

"github.com/go-logr/logr"
rabbitmqv1 "github.com/tekliner/rabbitmq-operator/pkg/apis/rabbitmq/v1"
corev1 "k8s.io/api/core/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
)

func (r *ReconcileRabbitmq) deleteDependentResoucePVC(reqLogger logr.Logger, cr *rabbitmqv1.Rabbitmq) (reconcile.Result, error) {
reqLogger.Info("Deleting PVC")
// Finalizer PVC: remove finalizer from CR
if containsString(cr.ObjectMeta.Finalizers, "PVC") {
// get dependent PVCs
foundDependentPVCs := &corev1.PersistentVolumeClaimList{}

err := r.client.List(context.TODO(), client.InNamespace(cr.Namespace).MatchingLabels(mergeMaps(returnLabels(cr))), foundDependentPVCs)
if err != nil {
reqLogger.Info("Listing dependent PVCs error", "Namespace", cr.Namespace, ".Name", cr.Name)
return reconcile.Result{}, err
}

for _, pvc := range foundDependentPVCs.Items {
// delete dependent PVCs
if err := r.client.Delete(context.Background(), &pvc); err != nil {
reqLogger.Info("Deleting PVC failed")
return reconcile.Result{}, err
}
}
}
return reconcile.Result{}, nil
}

func (r *ReconcileRabbitmq) reconcileFinalizers(reqLogger logr.Logger, instance *rabbitmqv1.Rabbitmq) (reconcile.Result, error) {
reqLogger.Info("Processing finalizers", "Namespace", instance.Namespace, ".Name", instance.Name)
// Define finalizer strings to prevent deletion of CR before dependent resources deletion
finalizersList := []string{"PVC"}

// Check CR is being deleted or not
if instance.ObjectMeta.DeletionTimestamp.IsZero() {
// Its a new CR, add finalizers from list
instance.ObjectMeta.Finalizers = finalizersList
if err := r.client.Update(context.Background(), instance); err != nil {
return reconcile.Result{}, err
}

} else {
// The object is being deleted, removing dependencies and finalizers after it

// Finalizer "PVC": check to remove dependent PVCs
if instance.Spec.RabbitmqPurgePVC {
_, err := r.deleteDependentResoucePVC(reqLogger, instance)
if err != nil {
reqLogger.Info("PVC deletion error", "Namespace", instance.Namespace, ".Name", instance.Name)
return reconcile.Result{}, err
}
}

// Finalizer "PVC": remove "PVC" finalizer from CR
instance.ObjectMeta.Finalizers = removeString(instance.ObjectMeta.Finalizers, "PVC")
if err := r.client.Update(context.Background(), instance); err != nil {
reqLogger.Info("Removing PVC finalizer failed")
return reconcile.Result{}, err
}

// Our finalizer has finished, so the reconciler can do nothing.
}

return reconcile.Result{}, nil
}

Loading…
Cancel
Save