Browse Source

added prometheus monitoring

pull/4/head
Vladimir Smagin 1 year ago
parent
commit
15ed19fcf7
5 changed files with 211 additions and 39 deletions
  1. +2
    -0
      deploy/clusterrole.yaml
  2. +5
    -0
      deploy/crds/rabbitmq_v1_rabbitmq_cr.yaml
  3. +2
    -0
      pkg/apis/rabbitmq/v1/rabbitmq_types.go
  4. +115
    -37
      pkg/controller/rabbitmq/rabbitmq_controller.go
  5. +87
    -2
      pkg/controller/rabbitmq/rabbitmq_services.go

+ 2
- 0
deploy/clusterrole.yaml View File

@@ -44,6 +44,8 @@ rules:
verbs:
- get
- create
- list
- watch
- apiGroups:
- rabbitmq.improvado.io
resources:


+ 5
- 0
deploy/crds/rabbitmq_v1_rabbitmq_cr.yaml View File

@@ -10,6 +10,11 @@ spec:
tag: 3-alpine
#secret_credentials: rabbit-users
#secret_service_account: rabbit-service

# comment or set 0 if no exporter needed
prometheus_exporter_port: 9091
#prometheus_image: "kbudde/rabbitmq-exporter:v0.28.0"

memory_high_watermark: 256M

# create unprivileged serviceaccount by hands


+ 2
- 0
pkg/apis/rabbitmq/v1/rabbitmq_types.go View File

@@ -114,6 +114,8 @@ type RabbitmqSpec struct {
RabbitmqK8SPeerDiscoveryBackend string `json:"k8s_peer_discovery_backend"`
RabbitmqClusterFormationNodeCleanup int64 `json:"cluster_node_cleanup_interval"`
RabbitmqClusterPartitionHandling string `json:"cluster_partition_handling"`
RabbitmqPrometheusExporterPort int32 `json:"prometheus_exporter_port,omitempty"`
RabbitmqPrometheusImage string `json:"prometheus_image,omitempty"`
}

// RabbitmqStatus defines the observed state of Rabbitmq


+ 115
- 37
pkg/controller/rabbitmq/rabbitmq_controller.go View File

@@ -2,6 +2,7 @@ package rabbitmq

import (
"context"
"strconv"
"time"

rabbitmqv1 "github.com/tekliner/rabbitmq-operator/pkg/apis/rabbitmq/v1"
@@ -127,6 +128,16 @@ type ReconcileRabbitmq struct {
// The Controller will requeue the Request to be processed again if the returned error is non-nil or
// Result.Requeue is true, otherwise upon completion it will remove the work from the queue.

func mergeMaps(itermaps ...map[string]string) map[string]string {
result := make(map[string]string)
for _, rv := range itermaps {
for k, v := range rv {
result[k] = v
}
}
return result
}

func returnLabels(cr *rabbitmqv1.Rabbitmq) map[string]string {
labels := map[string]string{
"rabbitmq.improvado.io/app": "rabbitmq",
@@ -136,6 +147,21 @@ func returnLabels(cr *rabbitmqv1.Rabbitmq) map[string]string {
return labels
}

func returnAnnotationsPrometheus(cr *rabbitmqv1.Rabbitmq) map[string]string {
return map[string]string{
"prometheus.io/scrape": "true",
"prometheus.io/port": strconv.Itoa(int(cr.Spec.RabbitmqPrometheusExporterPort)),
}
}

func returnAnnotations(cr *rabbitmqv1.Rabbitmq) map[string]string {
annotations := map[string]string{}
if cr.Spec.RabbitmqPrometheusExporterPort > 0 {
annotations = mergeMaps(annotations, returnAnnotationsPrometheus(cr))
}
return annotations
}

// Reconcile method
func (r *ReconcileRabbitmq) Reconcile(request reconcile.Request) (reconcile.Result, error) {
reqLogger := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name)
@@ -155,7 +181,14 @@ func (r *ReconcileRabbitmq) Reconcile(request reconcile.Request) (reconcile.Resu
return reconcile.Result{}, err
}

statefulset := newStatefulSet(instance)
// secrets used for API requests and user control
reqLogger.Info("Reconciling secrets")
secretNames, err := r.reconcileSecrets(reqLogger, instance)
if err != nil {
return reconcile.Result{}, err
}

statefulset := newStatefulSet(instance, secretNames)
if err := controllerutil.SetControllerReference(instance, statefulset, r.scheme); err != nil {
return reconcile.Result{}, err
}
@@ -200,13 +233,6 @@ func (r *ReconcileRabbitmq) Reconcile(request reconcile.Request) (reconcile.Resu
return reconcile.Result{}, err
}

// secrets used for API requests and user control
reqLogger.Info("Reconciling secrets")
secretNames, err := r.reconcileSecrets(reqLogger, instance)
if err != nil {
return reconcile.Result{}, err
}

// configmap
reqLogger.Info("Reconciling configmap")

@@ -215,6 +241,18 @@ func (r *ReconcileRabbitmq) Reconcile(request reconcile.Request) (reconcile.Resu
return reconcile.Result{}, err
}

// check prometheus exporter flag
if instance.Spec.RabbitmqPrometheusExporterPort > 0 {
_, err = r.reconcilePrometheusExporterService(reqLogger, instance)
if err != nil {
return reconcile.Result{}, err
}
_, err = r.reconcilePrometheusExporterServiceMonitor(reqLogger, instance)
if err != nil {
return reconcile.Result{}, err
}
}

// set policies
reqLogger.Info("Setting up policies")
timeoutPolicies, _ := time.ParseDuration("30")
@@ -282,11 +320,78 @@ func appendNodeVariables(env []corev1.EnvVar, cr *rabbitmqv1.Rabbitmq) []corev1.
)
}

func newStatefulSet(cr *rabbitmqv1.Rabbitmq) *v1.StatefulSet {
func newStatefulSet(cr *rabbitmqv1.Rabbitmq, secretNames secretResouces) *v1.StatefulSet {

// prepare containers for pod
podContainers := []corev1.Container{}

// container with rabbitmq
rabbitmqContainer := corev1.Container{
Name: "rabbitmq",
Image: cr.Spec.K8SImage.Name + ":" + cr.Spec.K8SImage.Tag,
Env: append(appendNodeVariables(cr.Spec.K8SENV, cr), corev1.EnvVar{
Name: "RABBITMQ_ERLANG_COOKIE",
ValueFrom: &corev1.EnvVarSource{ConfigMapKeyRef: &corev1.ConfigMapKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: cr.Name}, Key: ".erlang.cookie"}},
}),
Resources: corev1.ResourceRequirements{
Requests: cr.Spec.RabbitmqPodRequests,
Limits: cr.Spec.RabbitmqPodLimits,
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "rabbit-etc",
MountPath: "/etc/rabbitmq",
},
{
Name: "rabbit-data",
MountPath: "/var/lib/rabbitmq",
},
{
Name: "rabbit-config",
MountPath: "/rabbit-config",
},
},
}

podContainers = append(podContainers, rabbitmqContainer)

// if prometheus exporter enabled add additional container to pod
if cr.Spec.RabbitmqPrometheusExporterPort > 0 {

exporterImageAndTag := "kbudde/rabbitmq-exporter:latest"
if cr.Spec.RabbitmqPrometheusImage != "" {
exporterImageAndTag = cr.Spec.RabbitmqPrometheusImage
}

exporterContainer := corev1.Container{
Name: "prometheus-exporter",
Image: exporterImageAndTag,
ImagePullPolicy: corev1.PullIfNotPresent,
Env: []corev1.EnvVar{
{
Name: "RABBIT_USER",
ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: secretNames.ServiceAccount}, Key: "username"}},
},
{
Name: "RABBIT_PASSWORD",
ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: secretNames.ServiceAccount}, Key: "password"}},
},
},
Ports: []corev1.ContainerPort{
{
Name: "exporter",
Protocol: corev1.ProtocolTCP,
ContainerPort: cr.Spec.RabbitmqPrometheusExporterPort,
},
},
}
podContainers = append(podContainers, exporterContainer)
}

podTemplate := corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: returnLabels(cr),
Annotations: returnAnnotations(cr),
},
Spec: corev1.PodSpec{
ServiceAccountName: cr.Spec.RabbitmqK8SServiceAccount,
@@ -311,34 +416,7 @@ func newStatefulSet(cr *rabbitmqv1.Rabbitmq) *v1.StatefulSet {
},
},
},
Containers: []corev1.Container{
{
Name: "rabbitmq",
Image: cr.Spec.K8SImage.Name + ":" + cr.Spec.K8SImage.Tag,
Env: append(appendNodeVariables(cr.Spec.K8SENV, cr), corev1.EnvVar{
Name: "RABBITMQ_ERLANG_COOKIE",
ValueFrom: &corev1.EnvVarSource{ConfigMapKeyRef: &corev1.ConfigMapKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: cr.Name}, Key: ".erlang.cookie"}},
}),
Resources: corev1.ResourceRequirements{
Requests: cr.Spec.RabbitmqPodRequests,
Limits: cr.Spec.RabbitmqPodLimits,
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "rabbit-etc",
MountPath: "/etc/rabbitmq",
},
{
Name: "rabbit-data",
MountPath: "/var/lib/rabbitmq",
},
{
Name: "rabbit-config",
MountPath: "/rabbit-config",
},
},
},
},
Containers: podContainers,
Volumes: []corev1.Volume{
{
Name: "rabbit-config",


+ 87
- 2
pkg/controller/rabbitmq/rabbitmq_services.go View File

@@ -2,6 +2,7 @@ package rabbitmq

import (
"context"
v12 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
"reflect"

"github.com/go-logr/logr"
@@ -59,6 +60,31 @@ func (r *ReconcileRabbitmq) reconcileService(reqLogger logr.Logger, cr *rabbitmq
return reconcile.Result{}, nil
}

func (r *ReconcileRabbitmq) reconcileServiceMonitor(reqLogger logr.Logger, cr *rabbitmqv1.Rabbitmq, serviceMonitorResource v12.ServiceMonitor) (reconcile.Result, error) {
reqLogger.Info("Started reconciling ServiceMonitor", "ServiceMonitor.Namespace", serviceMonitorResource.Namespace, "ServiceMonitor.Name", serviceMonitorResource.Name)

// Check if this ServiceMonitor already exists
found := v12.ServiceMonitor{}
reqLogger.Info("Getting ServiceMonitor", "Service.Namespace", serviceMonitorResource.Namespace, "Service.Name", serviceMonitorResource.Name)
err := r.client.Get(context.TODO(), types.NamespacedName{Name: serviceMonitorResource.Name, Namespace: serviceMonitorResource.Namespace}, &found)

if err != nil && apierrors.IsNotFound(err) {
reqLogger.Info("No ServiceMonitor found, creating new", "Service.Namespace", serviceMonitorResource.Namespace, "Service.Name", serviceMonitorResource.Name)
err = r.client.Create(context.TODO(), &serviceMonitorResource)
if err != nil {
reqLogger.Info("Error creating new service", "Service.Namespace", serviceMonitorResource.Namespace, "Service.Name", serviceMonitorResource.Name)
return reconcile.Result{}, err
}

} else if err != nil {
reqLogger.Info("Error getting ServiceMonitor", "Service.Namespace", serviceMonitorResource.Namespace, "Service.Name", serviceMonitorResource.Name)
return reconcile.Result{}, err
}

return reconcile.Result{}, nil
}


func (r *ReconcileRabbitmq) reconcileDiscoveryService(reqLogger logr.Logger, cr *rabbitmqv1.Rabbitmq) (reconcile.Result, error) {

service := &corev1.Service{
@@ -97,7 +123,7 @@ func (r *ReconcileRabbitmq) reconcileHAService(reqLogger logr.Logger, cr *rabbit
ObjectMeta: metav1.ObjectMeta{
Name: cr.Name,
Namespace: cr.Namespace,
Labels: returnLabels(cr),
Labels: mergeMaps(returnLabels(cr), map[string]string{"service-type": "general"}),
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeClusterIP,
@@ -136,7 +162,7 @@ func (r *ReconcileRabbitmq) reconcileHTTPService(reqLogger logr.Logger, cr *rabb
ObjectMeta: metav1.ObjectMeta{
Name: cr.Name + "-api",
Namespace: cr.Namespace,
Labels: returnLabels(cr),
Labels: mergeMaps(returnLabels(cr), map[string]string{"service-type": "api"}),
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeClusterIP,
@@ -156,3 +182,62 @@ func (r *ReconcileRabbitmq) reconcileHTTPService(reqLogger logr.Logger, cr *rabb

return reconcileResult, err
}

func (r *ReconcileRabbitmq) reconcilePrometheusExporterService(reqLogger logr.Logger, cr *rabbitmqv1.Rabbitmq) (reconcile.Result, error) {

service := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: cr.Name + "-exporter",
Namespace: cr.Namespace,
Labels: mergeMaps(returnLabels(cr), map[string]string{"service-type": "prometheus-exporter"}),
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeClusterIP,
Selector: returnLabels(cr),
Ports: []corev1.ServicePort{
{
TargetPort: intstr.IntOrString{IntVal: cr.Spec.RabbitmqPrometheusExporterPort},
Port: cr.Spec.RabbitmqPrometheusExporterPort,
Protocol: corev1.ProtocolTCP,
Name: "exporter",
},
},
},
}

reconcileResult, err := r.reconcileService(reqLogger, cr, service)

return reconcileResult, err
}

func (r *ReconcileRabbitmq) reconcilePrometheusExporterServiceMonitor(reqLogger logr.Logger, cr *rabbitmqv1.Rabbitmq) (reconcile.Result, error) {

serviceMonitor := v12.ServiceMonitor{
ObjectMeta: metav1.ObjectMeta{
Name: cr.Name,
Namespace: cr.Namespace,
Labels: returnLabels(cr),
},
Spec: v12.ServiceMonitorSpec{
Selector: metav1.LabelSelector{
MatchLabels: map[string]string{
"app": cr.Name,
"service-type": "prometheus-exporter",
},
},
Endpoints: []v12.Endpoint{
{
Port: "exporter",
Interval: "10s",
},
},
NamespaceSelector: v12.NamespaceSelector{
Any:true,
},
},
}

reconcileResult, err := r.reconcileServiceMonitor(reqLogger, cr, serviceMonitor)

return reconcileResult, err
}

Loading…
Cancel
Save