Rabbitmq HA Cluster

YAML

version: "3.7"

services:
  rabbitmq1:
    image: "rabbitmq:3.8-management"
    logging:
      driver: "json-file"
      options:
        max-size: "500m"
        max-file: "10"
    labels: [app=rabbitmq]
    volumes:
      - ./rabbitmq-plugins/enabled_plugins:/etc/rabbitmq/enabled_plugins
      - ./rabbitmq-config/rabbitmq1/:/config/
      - ./data/rabbitmq-data1/:/var/lib/rabbitmq/mnesia/
    ports:
#      - "5672:5672"
      - "15671:15672"
    networks:
      - core_network
    hostname: rabbitmq
    environment:
      #- AUTOCLUSTER_TYPE=consul
      #- AUTOCLUSTER_CLEANUP=true
      - CLEANUP_WARN_ONLY=false
      #- CONSUL_HOST=consul
      #- CONSUL_PORT=8500
      - CONSUL_SVC=rabbitmq1
      - CONSUL_SVC_ADDR_AUTO=true
      - RABBITMQ_ERLANG_COOKIE="SecretErlangCookie"
      - RABBITMQ_DEFAULT_VHOST=/
      - RABBITMQ_DEFAULT_USER=admin
      - RABBITMQ_DEFAULT_PASS=Password
      - RABBITMQ_CONFIG_FILE=/config/rabbitmq
#      - RABBITMQ_CONFIG_FILE=/tmp/rabbitmq
    deploy:
      replicas: 1
      restart_policy:
        condition: on-failure
        delay: 15s
        max_attempts: 3
        window: 120s
      update_config:
        parallelism: 2
        delay: 20s
        failure_action: continue

  rabbitmq2:
    image: "rabbitmq:3.8-management"
    logging:
      driver: "json-file"
      options:
        max-size: "500m"
        max-file: "10"
    labels: [app=rabbitmq]
    volumes:
      - ./rabbitmq-plugins/enabled_plugins:/etc/rabbitmq/enabled_plugins
      - ./rabbitmq-config/rabbitmq2/:/config/
      - ./data/rabbitmq-data2/:/var/lib/rabbitmq/mnesia/
    hostname: rabbitmq2
    ports:
#      - "5672:5672"
      - "15672:15672"
    networks:
      - core_network
    environment:
      #- AUTOCLUSTER_TYPE=consul
      #- AUTOCLUSTER_CLEANUP=true
      - CLEANUP_WARN_ONLY=false
      #- CONSUL_HOST=consul
      #- CONSUL_PORT=8500
      - CONSUL_SVC=rabbitmq
      - CONSUL_SVC_ADDR_AUTO=true
      - RABBITMQ_ERLANG_COOKIE="SecretErlangCookie"
      - RABBITMQ_DEFAULT_VHOST=/
      - RABBITMQ_DEFAULT_USER=admin
      - RABBITMQ_DEFAULT_PASS=Password
#      - RABBITMQ_CONFIG_FILE=/tmp/rabbitmq
      - RABBITMQ_CONFIG_FILE=/config/rabbitmq
    deploy:
      replicas: 1
      restart_policy:
        condition: on-failure
        delay: 15s
        max_attempts: 3
        window: 120s
      update_config:
        parallelism: 2
        delay: 20s
        failure_action: continue

  rabbitmq3:
    hostname: rabbitmq3
    image: "rabbitmq:3.8-management"
    logging:
      driver: "json-file"
      options:
        max-size: "500m"
        max-file: "10"
    labels: [app=rabbitmq]
    volumes:
      - ./rabbitmq-plugins/enabled_plugins:/etc/rabbitmq/enabled_plugins
      - ./rabbitmq-config/rabbitmq3/:/config/
      - ./data/rabbitmq-data3/:/var/lib/rabbitmq/mnesia/
    ports:
#      - "5672:5672"
      - "15673:15672"
    networks:
      - core_network
    environment:
      #- AUTOCLUSTER_TYPE=consul
      #- AUTOCLUSTER_CLEANUP=true
      - CLEANUP_WARN_ONLY=false
      #- CONSUL_HOST=consul
      #- CONSUL_PORT=8500
      - CONSUL_SVC=rabbitmq
      - CONSUL_SVC_ADDR_AUTO=true
      - RABBITMQ_ERLANG_COOKIE="SecretErlangCookie"
      - RABBITMQ_DEFAULT_VHOST=/
      - RABBITMQ_DEFAULT_USER=admin
      - RABBITMQ_DEFAULT_PASS=Password
#      - RABBITMQ_CONFIG_FILE=/tmp/rabbitmq
      - RABBITMQ_CONFIG_FILE=/config/rabbitmq
    deploy:
      replicas: 1
      restart_policy:
        condition: on-failure
        delay: 15s
        max_attempts: 3
        window: 120s
      update_config:
        parallelism: 2
        delay: 20s
        failure_action: continue

  nginxrmq:
    image: nginx
    volumes:
      - ./nginx-conf/nginx.conf:/etc/nginx/nginx.conf
    deploy:
      replicas: 1
#      resources:
#        limits:
#          memory: 512M
      restart_policy:
        condition: on-failure
    ports:
      - "5000:5000"
      - "15000:15000"
    networks:
      - core_network


networks:
  core_network:
    external: true

• Nginx config

Nginx Configuration File

events {

}
stream {
  upstream myrabbit {
    server rabbitmq1:5672;
    server rabbitmq2:5672 backup;
    server rabbitmq3:5672 backup;
  }

  server {
    listen 5000;
    proxy_pass myrabbit;
    proxy_connect_timeout 1s;
    proxy_timeout 3s;
    proxy_next_upstream on;
  }
}

• definitations.json

JSON

{"rabbit_version":"3.8.9","rabbitmq_version":"3.8.9","product_name":"RabbitMQ","product_version":"3.8.9","users":[{"name":"admin","password_hash":"nKCUQsu9SanVUh9h/lXboTBh3jBXyFPQ/4JkpqC4PFpD4ORw","hashing_algorithm":"rabbit_password_hashing_sha256","tags":"administrator"}],"vhosts":[{"name":"/"}],"permissions":[{"user":"admin","vhost":"/","configure":".*","write":".*","read":".*"}],"topic_permissions":[],"parameters":[],"global_parameters":[{"name":"cluster_name","value":"rabbit@rabbitmq"},{"name":"internal_cluster_id","value":"rabbitmq-cluster-id-muuMs9a6yWpXWTH-FwSrLQ"}],"policies":[{"vhost":"/","name":"ha-fed","pattern":"^","apply-to":"all","definition":{"ha-mode":"nodes","ha-params":["rabbit@rabbitmq","rabbit@rabbitmq2","rabbit@rabbitmq3"],"ha-sync-mode":"automatic"},"priority":0}],"queues":[],"exchanges":[],"bindings":[]}

• Rabittmq 1 config

Text Only

loopback_users.guest = false
listeners.tcp.default = 5672

cluster_formation.peer_discovery_backend = rabbit_peer_discovery_classic_config
cluster_formation.classic_config.nodes.1 = rabbit@rabbitmq1
cluster_formation.classic_config.nodes.2 = rabbit@rabbitmq2
cluster_formation.classic_config.nodes.3 = rabbit@rabbitmq3

management.load_definitions = /config/definitions.json

• Rabbitmq 2 and 3 configs

Text Only

loopback_users.guest = false
listeners.tcp.default = 5672

cluster_formation.peer_discovery_backend = rabbit_peer_discovery_classic_config
cluster_formation.classic_config.nodes.1 = rabbit@rabbitmq1
cluster_formation.classic_config.nodes.2 = rabbit@rabbitmq2
cluster_formation.classic_config.nodes.3 = rabbit@rabbitmq3

Manual:

Adding HA with command inside rabbitmq1 container:

Bash

rabbitmqctl set_policy ha-fed "^" '{"ha-mode":"nodes", "ha-params":["rabbit@rabbitmq1","rabbit@rabbitmq2","rabbit@rabbitmq3"], "ha-sync-mode":"automatic"}'

Create Definitions:

Bash

curl -s -H "Accept:application/json" -u guest:guest http://localhost:15672/api/definitions > definitions.json