# a controller node through the OCI's ENC:
#
#---
#classes:
#   oci_controller:
#      is_first_master: true
#      first_master: zigo-controller-node-3.example.com
#      first_master_ip: 192.168.100.40
#      vip_hostname: zigo-api.example.com
#      vip_ipaddr: 192.168.101.2
#      network_ipaddr: 192.168.101.0
#      network_cidr: 24
#      other_masters:
#         - zigo-controller-node-4.example.com
#         - zigo-controller-node-5.example.com
#      other_masters_ip:
#         - 192.168.100.41
#         - 192.168.100.32
#      all_masters:
#         - zigo-controller-node-3.example.com
#         - zigo-controller-node-4.example.com
#         - zigo-controller-node-5.example.com
#      all_masters_ip:
#         - 192.168.100.40
#         - 192.168.100.41
#         - 192.168.100.32
#
# This is re-used in the oci_controller class below
#
class oci::controller(
  $openstack_release        = undef,		# rocky, stein, train
  $cluster_name             = undef,		# z
  $machine_hostname         = undef,		# z-controller-1.example.com
  $machine_ip               = undef,		# 192.168.101.2
  $bridge_mapping_list      = undef,		# [ 'external:br-ex', 'external1:br-lb']
  $external_network_list    = undef,		# [ 'external', 'external1' ]
  $machine_iface            = undef,		# br-ex
  $compute_nodes            = undef,		# [ 'z-compute-1.example.com', 'z-compute-2.example.com', 'z-compute-3.example.com' ]
  $compute_nodes_ip         = undef,            # [ '192.168.101.5', '192.168.101.6', '192.168.101.7' ]
  $volume_nodes             = undef,            # [ 'z-volume-1.example.com', 'z-volume-2.example.com' ]
  $volume_nodes_ip          = undef,            # [ '192.168.101.8', '192.168.101.9' ]
  $vmnet_ip                 = undef,		# 192.168.102.2

  $messaging_nodes_for_notifs = false,
  $all_rabbits                = [],
  $all_rabbits_ips            = [],

  $neutron_global_physnet_mtu = 1500,
  $neutron_path_mtu         = 1500,

  $is_first_master          = false,		# true
  $first_master             = undef,		# z-controller-1.example.com
  $first_master_ip          = undef,		# 192.168.101.2
  $non_first_master_controllers = undef,	# All non-primary masters: [ 'z-controller-2.example.com', 'z-controller-3.example.com' ]
  $non_first_master_controllers_ip = undef,	# [ '192.168.101.3', '192.168.101.4' ]
  $other_masters            = undef,		# Other masters in this cluster, excluding this host: [ 'z-controller-2.example.com', 'z-controller-3.example.com' ]
  $other_masters_ip         = undef,		# [ '192.168.101.3', '192.168.101.4' ]
  $vip_hostname             = undef,
  $vip_ipaddr               = undef,
  $vip_netmask              = undef,
  $self_signed_api_cert     = true,
  $swiftproxy_hostname      = 'none',
  $network_ipaddr           = undef,
  $network_cidr             = undef,
  $all_masters              = undef,
  $all_masters_ip           = undef,
  $all_masters_ids          = undef,

  $has_subrole_db           = true,

  $first_sql                = undef,
  $first_sql_ip             = undef,
  $is_first_sql             = undef,
  $sql_vip_ip               = undef,
  $sql_vip_netmask          = undef,
  $sql_vip_iface            = undef,

  $initial_cluster_setup    = false,

  $amp_secgroup_list        = undef,
  $amp_boot_network_list    = undef,

  $all_sql                  = undef,
  $all_sql_ip               = undef,
  $all_sql_ids              = undef,
  $non_master_sql           = undef,
  $non_master_sql_ip        = undef,

  $has_subrole_messaging    = true,
  $has_subrole_api_keystone = true,
  $has_subrole_heat         = true,
  $has_subrole_glance       = false,
  $has_subrole_nova         = false,
  $has_subrole_neutron      = false,
  $has_subrole_swift        = true,
  $has_subrole_horizon      = true,
  $has_subrole_barbican     = true,
  $has_subrole_cinder       = false,
  $has_subrole_gnocchi      = false,
  $has_subrole_gnocchi_services = true,
  $has_subrole_ceilometer   = false,
  $has_subrole_panko        = false,
  $has_subrole_cloudkitty   = false,
  $has_subrole_aodh         = false,
  $has_subrole_octavia      = false,
  $has_subrole_magnum       = false,
  $has_subrole_designate    = false,

  $has_subrole_network_node = true,

  $glance_backend           = undef,

  $cinder_backup_backend    = 'none',

  $cinder_default_storage_availability_zone = 'nova',
  $cinder_default_volume_type       = 'LVM_1',

  $haproxy_custom_url       = undef,
  $use_ssl                  = true,
  $rabbit_env               = {},
  $all_swiftproxy           = undef,
  $all_swiftproxy_ip        = undef,
  $pass_haproxy_stats       = undef,
  $pass_mysql_rootuser      = undef,
  $pass_mysql_backup        = undef,
  $pass_rabbitmq_cookie     = undef,
  $pass_rabbitmq_monitoring = undef,
  $pass_keystone_db         = undef,
  $pass_keystone_messaging  = undef,
  $pass_keystone_adminuser  = undef,
  $pass_keystone_credkey1   = undef,
  $pass_keystone_credkey2   = undef,
  $pass_keystone_fernkey1   = undef,
  $pass_keystone_fernkey2   = undef,
  $pass_keystone_ssh_pub    = undef,
  $pass_keystone_ssh_priv   = undef,
  $pass_nova_db             = undef,
  $pass_nova_apidb          = undef,
  $pass_nova_messaging      = undef,
  $pass_nova_authtoken      = undef,
  $pass_metadata_proxy_shared_secret = undef,
  $pass_placement_db        = undef,
  $pass_placement_authtoken = undef,
  $pass_glance_db           = undef,
  $pass_glance_messaging    = undef,
  $pass_glance_authtoken    = undef,
  $pass_glance_ssh_pub      = undef,
  $pass_glance_ssh_priv     = undef,
  $pass_cinder_db           = undef,
  $pass_cinder_messaging    = undef,
  $pass_cinder_authtoken    = undef,
  $pass_neutron_db          = undef,
  $pass_neutron_messaging   = undef,
  $pass_neutron_authtoken   = undef,
  $pass_heat_encryptkey     = undef,
  $pass_heat_db             = undef,
  $pass_heat_messaging      = undef,
  $pass_heat_authtoken      = undef,
  $pass_heat_keystone_domain = undef,
  $pass_swift_authtoken     = undef,
  $pass_swift_encryption    = undef,
  $pass_horizon_secretkey   = undef,
  $pass_barbican_db         = undef,
  $pass_barbican_messaging  = undef,
  $pass_barbican_authtoken  = undef,
  $pass_gnocchi_db          = undef,
  $pass_gnocchi_messaging   = undef,
  $pass_gnocchi_authtoken   = undef,
  $pass_gnocchi_rscuuid     = undef,
  $pass_panko_db            = undef,
  $pass_panko_messaging     = undef,
  $pass_panko_authtoken     = undef,
  $pass_ceilometer_db        = undef,
  $pass_ceilometer_messaging = undef,
  $pass_ceilometer_authtoken = undef,
  $pass_ceilometer_telemetry = undef,
  $pass_cloudkitty_db       = undef,
  $pass_cloudkitty_messaging = undef,
  $pass_cloudkitty_authtoken = undef,
  $pass_redis               = undef,
  $pass_aodh_db             = undef,
  $pass_aodh_messaging      = undef,
  $pass_aodh_authtoken      = undef,
  $pass_octavia_db          = undef,
  $pass_octavia_messaging   = undef,
  $pass_octavia_authtoken   = undef,
  $pass_octavia_heatbeatkey = undef,
  $pass_magnum_db           = undef,
  $pass_magnum_messaging    = undef,
  $pass_magnum_authtoken    = undef,
  $pass_magnum_domain       = undef,
  $pass_designate_db        = undef,
  $pass_designate_messaging = undef,
  $pass_designate_authtoken = undef,
  $pass_zabbix_authtoken    = undef,

  $cluster_has_cinder_volumes = false,

  $cluster_has_osds         = false,
  $cluster_has_mons         = false,

  $ceph_fsid                = undef,
  $ceph_bootstrap_osd_key   = undef,
  $ceph_admin_key           = undef,
  $ceph_openstack_key       = undef,
  $ceph_mon_key             = undef,
  $ceph_mon_initial_members = undef,
  $ceph_mon_host            = undef,
  $ceph_mgr_key             = undef,
  $ceph_libvirtuuid         = undef,

  $disable_notifications    = false,
  $enable_monitoring_graphs = false,
  $monitoring_graphite_host = undef,
  $monitoring_graphite_port = '2003',

  $extswift_use_external        = false,
  $extswift_auth_url            = undef,
  $extswift_proxy_url           = undef,
  $extswift_project_name        = undef,
  $extswift_project_domain_name = undef,
  $extswift_user_name           = undef,
  $extswift_user_domain_name    = undef,
  $extswift_password            = undef,

  $use_gpu                  = false,
  $gpu_def                  = undef,

  $bgp_to_the_host          = false,
  $use_dynamic_routing      = false,
  $dhcp_domain              = '',

  # These are from variables.json
  $nova_limit_tenants_to_placement_aggregate = false,
  $nova_scheduler_prefers_hosts_with_more_ram = true,
  $neutron_dns_domain       = 'example.com.',
  $admin_email_address      = 'admin@example.com',
  $glance_prefer_ceph_over_swift = true,
  $install_magnum           = false,
  $neutron_use_dvr          = true,
){

  # Generic performances tweak (will default to throughput-performance)
  # class { '::oci::tuned': }

  if $use_ssl {
    $proto = 'https'
    $messaging_default_port = '5671'
    $messaging_notify_port = '5671'
    $api_port = 443
  } else {
    $proto = 'http'
    $messaging_default_port = '5672'
    $messaging_notify_port = '5672'
    $api_port = 80
  }
  $messaging_default_proto = 'rabbit'
  $messaging_notify_proto = 'rabbit'

  if $has_subrole_db {
    $sql_host = $vip_ipaddr
  }else{
    $sql_host = $sql_vip_ip
  }

  if $messaging_nodes_for_notifs {
    $notif_rabbit_servers = $all_rabbits
  }else{
    $notif_rabbit_servers = $all_masters
  }

  $base_url = "${proto}://${vip_hostname}"
  if $self_signed_api_cert {
    $api_endpoint_ca_file = "/etc/ssl/certs/oci-pki-oci-ca-chain.pem"
  }else{
    $api_endpoint_ca_file = ''
  }
  $oci_pki_root_ca_file = '/etc/ssl/certs/oci-pki-oci-ca-chain.pem'

  class { '::oci::puppet_oci_ca_cert':
    self_signed_api_cert => $self_signed_api_cert,
  }

  $keystone_auth_uri  = "${base_url}:${api_port}/identity"
  $keystone_admin_uri = "${base_url}:${api_port}/identity"
  #$memcached_servers  = ["127.0.0.1:11211"]
  $memcached_string = join([join($all_masters_ip,':11211,'), ':11211'],'')
  $memcached_servers  = ["${memcached_string}"]

    ensure_resource('file', '/root/oci-openrc', {
      'ensure'  => 'present',
      'content' => "
export OS_AUTH_TYPE=password
export OS_PROJECT_DOMAIN_NAME='default'
export OS_USER_DOMAIN_NAME='default'
export OS_PROJECT_NAME='admin'
export OS_USERNAME='admin'
export OS_PASSWORD='${pass_keystone_adminuser}'
export OS_AUTH_URL='https://${vip_hostname}/identity/v3'
export OS_IDENTITY_API_VERSION=3
export OS_IMAGE_API_VERSION=2
export OS_CACERT=${$api_endpoint_ca_file}
",
      'mode'    => '0640',
    })

    ensure_resource('file', '/root/octavia-openrc', {
      'ensure'  => 'present',
      'content' => "
export OS_AUTH_TYPE=password
export OS_PROJECT_DOMAIN_NAME='default'
export OS_USER_DOMAIN_NAME='default'
export OS_PROJECT_NAME='services'
export OS_USERNAME='octavia'
export OS_PASSWORD='${pass_octavia_authtoken}'
export OS_AUTH_URL='https://${vip_hostname}/identity/v3'
export OS_IDENTITY_API_VERSION=3
export OS_IMAGE_API_VERSION=2
export OS_CACERT=${$api_endpoint_ca_file}
",
      'mode'    => '0640',
    })

    ensure_resource('file', '/root/swift-openrc', {
      'ensure'  => 'present',
      'content' => "
export OS_AUTH_TYPE=password
export OS_PROJECT_DOMAIN_NAME='default'
export OS_USER_DOMAIN_NAME='default'
export OS_PROJECT_NAME='services'
export OS_USERNAME='swift'
export OS_PASSWORD='${pass_swift_authtoken}'
export OS_AUTH_URL='https://${vip_hostname}/identity/v3'
export OS_IDENTITY_API_VERSION=3
export OS_IMAGE_API_VERSION=2
export OS_CACERT=${$api_endpoint_ca_file}
",
      'mode'    => '0640',
    })

    ensure_resource('file', '/root/zabbix-openrc', {
      'ensure'  => 'present',
      'content' => "
export OS_AUTH_TYPE=password
export OS_PROJECT_DOMAIN_NAME='Default'
export OS_USER_DOMAIN_NAME='Default'
export OS_PROJECT_NAME='zabbix'
export OS_USERNAME='zabbix'
export OS_PASSWORD='${pass_zabbix_authtoken}'
export OS_AUTH_URL='https://${vip_hostname}/identity/v3'
export OS_IDENTITY_API_VERSION=3
export OS_IMAGE_API_VERSION=2
export OS_CACERT=${$api_endpoint_ca_file}
",
      'mode'    => '0640',
    })

    ensure_resource('group', 'zabbix', {
      'ensure' => 'present',
      'gid'    => '966',
    })

    ensure_resource('file', '/etc/oci/monitoring_openstack_api_password', {
      'ensure'  => 'present',
      'content' => "${pass_zabbix_authtoken}",
      'group'   => 'zabbix',
      'mode'    => '0640',
    })

    ensure_resource('file', '/etc/oci/monitoring_openstack_api_url', {
      'ensure'  => 'present',
      'content' => "https://${vip_hostname}/identity/v3",
      'group'   => 'zabbix',
      'mode'    => '0640',
    })

  if $vip_netmask == 32 {
    $vip_iface = 'lo'
  } else {
    $vip_iface = $machine_iface
  }

  $haproxy_api_backend_options = "check check-ssl ssl verify required ca-file ${oci_pki_root_ca_file}"
  $haproxy_api_backend_options_noverify = 'check check-ssl ssl verify none'

  if $is_first_master and $initial_cluster_setup{
    $do_db_sync = true
  }else{
    $do_db_sync = false
  }

  if $disable_notifications {
    $notification_driver = 'noop'
  }else{
    $notification_driver = 'messagingv2'
  }

  if $facts['os']['lsb'] != undef{
    $mycodename = $facts['os']['lsb']['distcodename']
  }else{
    $mycodename = $facts['os']['distro']['codename']
  }

  ##########################################################################
  ### Add a few package so that installing the Octavia service is easier ###
  ##########################################################################
  package { 'openstack-pkg-tools':
    ensure => present,
  }

  package { 'openstack-debian-images':
    ensure => present,
  }

  ################################
  ### Add a virtual IP address ###
  ################################
  # Setup corosync
  if ($openstack_release == 'rocky' or $openstack_release == 'stein') {
    class { 'corosync':
      authkey                  => '/var/lib/puppet/ssl/certs/ca.pem',
      bind_address             => $machine_ip,
      unicast_addresses        => $all_masters_ip,
      cluster_name             => 'mycluster',
      enable_secauth           => true,
      set_votequorum           => true,
      quorum_members           => $all_masters_ip,
      quorum_members_ids       => $all_masters_ids,
      log_stderr               => false,
      log_function_name        => true,
      syslog_priority          => 'debug',
      debug                    => true,
    }
    corosync::service { 'pacemaker':
      version => '0',
    }->
    cs_property { 'stonith-enabled':
      value   => 'false',
    }->
    cs_property { 'no-quorum-policy':
      value   => 'stop',
    }
  } else {
    class { 'corosync':
      authkey                  => '/var/lib/puppet/ssl/certs/ca.pem',
      bind_address             => $machine_ip,
      cluster_name             => 'mycluster',
      enable_secauth           => true,
      set_votequorum           => true,
      quorum_members           => $all_masters_ip,
      quorum_members_ids       => $all_masters_ids,
      log_stderr               => false,
      log_function_name        => true,
      syslog_priority          => 'debug',
      debug                    => true,
      enable_totem_interface   => false,
    }
    corosync::service { 'pacemaker':
      version => '0',
    }->
    cs_property { 'stonith-enabled':
      value   => 'false',
    }->
    cs_property { 'no-quorum-policy':
      value   => 'stop',
    }
  }
  if $initial_cluster_setup {
    cs_primitive { 'openstack-api-vip':
      primitive_class => 'ocf',
      primitive_type  => 'IPaddr2',
      provided_by     => 'heartbeat',
      parameters      => { 'ip' => $vip_ipaddr, 'cidr_netmask' => $vip_netmask, 'nic' => $vip_iface },
      operations      => { 'monitor' => { 'interval' => '10s' } },
      before          => Anchor['keystone::service::begin'],
      require         => Cs_property['no-quorum-policy'],
    }
  }
  # Eventually, a VIP for MariaDB
#  if $has_subrole_db {
#    oci::vip {'openstack-sql-vip':
#      vip_ip      => $sql_vip_ip,
#      vip_netmask => $sql_vip_netmask,
#      vip_iface   => $vip_sql_iface,
#    }
#  }

  #########################
  ### Setup firewalling ###
  #########################
  # Note: This only closes ports on the VIP, as everything else
  # is not accessible from the outside anyway.

  # Drop all services on public IP address, just let https APIs

  # First, general definirion (could be in global site.pp)
  resources { "firewall":
    purge   => false
  }
  class { 'firewall': }

  if $volume_nodes_ip {
    $all_sql_allowed_ips = concat($compute_nodes_ip, $volume_nodes_ip)
  }else{
    $all_sql_allowed_ips = $compute_nodes_ip
  }
  $all_sql_allowed_ips2 = concat($all_masters_ip,$all_sql_allowed_ips)
  $all_sql_allowed_ips3 = concat(['127.0.0.0/8'],$all_sql_allowed_ips2)
  $all_sql_allowed_ips4 = concat(["${vip_ipaddr}/${vip_netmask}"],$all_sql_allowed_ips3)
  $all_sql_allowed_ips5 = concat($all_rabbits_ips,$all_sql_allowed_ips4)

  $all_sql_allowed_ips5.each |Integer $index, String $value| {
    $val1 = $index+100
    firewall { "${val1} Allow ${value} to access MySQL VIP":
      proto       => tcp,
      action      => accept,
      source      => "${value}",
      dport       => [3306, 4567],
    }
  }

  # Define firewall rules
  firewall { '801 deny public access to http':
    proto       => tcp,
    action      => drop,
    destination => "${vip_ipaddr}/${vip_netmask}",
    dport       => 80,
  }->
  firewall { '802 deny public access to mysql':
    proto       => tcp,
    action      => drop,
    destination => "${vip_ipaddr}/${vip_netmask}",
    dport       => [3306, 4567],
  }->
  firewall { '803 deny public access to bgpd':
    proto       => tcp,
    action      => drop,
    destination => "${vip_ipaddr}/${vip_netmask}",
    dport       => 179,
  }->
  firewall { '804 deny public access to epmd and rabbitmq':
    proto       => tcp,
    action      => drop,
    destination => "${vip_ipaddr}/${vip_netmask}",
    dport       => [4369, 5671, 5672, 15671, 25672],
  }->
  firewall { '805 deny public access to rpcbind':
    proto       => tcp,
    action      => drop,
    destination => "${vip_ipaddr}/${vip_netmask}",
    dport       => [111],
  }->
  firewall { '806 deny public access to xinetd':
    proto       => tcp,
    action      => drop,
    destination => "${vip_ipaddr}/${vip_netmask}",
    dport       => [9200],
  }->
  firewall { '807 deny public access to horizon without haproxy':
    proto       => tcp,
    action      => drop,
    destination => "${vip_ipaddr}/${vip_netmask}",
    dport       => [7080, 7443],
  }->
  firewall { '808 deny public access to octavia API without haproxy':
    proto       => tcp,
    action      => drop,
    destination => "${vip_ipaddr}/${vip_netmask}",
    dport       => [9876],
  }->
  firewall { '809 deny public access to magnum API without haproxy':
    proto       => tcp,
    action      => drop,
    destination => "${vip_ipaddr}/${vip_netmask}",
    dport       => [9511],
  }->
  firewall { '810 deny public access to designate API without haproxy':
    proto       => tcp,
    action      => drop,
    destination => "${vip_ipaddr}/${vip_netmask}",
    dport       => [9001],
  }->
  firewall { '811 deny public access to keystone API without haproxy':
    proto       => tcp,
    action      => drop,
    destination => "${vip_ipaddr}/${vip_netmask}",
    dport       => [5000],
  }->
  firewall { '812 deny public access to SMTP':
    proto       => tcp,
    action      => drop,
    destination => "${vip_ipaddr}/${vip_netmask}",
    dport       => [25],
  }->
  firewall { '813 deny public access to Heat CFN API without haproxy':
    proto       => tcp,
    action      => drop,
    destination => "${vip_ipaddr}/${vip_netmask}",
    dport       => [8000],
  }->
  firewall { '814 deny public access to Aodh API without haproxy':
    proto       => tcp,
    action      => drop,
    destination => "${vip_ipaddr}/${vip_netmask}",
    dport       => [8042],
  }

  ####################################
  ### Setup a Zookeeper in cluster ###
  ####################################
  # needed for Telemetry / metric / rating
  if $has_subrole_gnocchi or $has_subrole_designate or $has_subrole_cloudkitty{
    $all_masters_ip.each |Integer $index, String $value| {
      if($machine_ip == $value){
        class { 'zookeeper':
          id        => String($index + 1),
          client_ip => $machine_ip,
          servers   => $all_masters_ip,
          purge_interval => 6,
        }
      }
    }
  }

  # Add haproxy that will listen on the virtual IPs, and load balance
  # to the different API daemons using tcp mode (as the APIs will do
  # full SSL already).
  if $use_ssl {
    file { "/etc/haproxy/ssl":
      ensure                  => directory,
      owner                   => 'root',
      mode                    => '0755',
      selinux_ignore_defaults => true,
      require       => Package['haproxy'],
    }->
    file { "/etc/haproxy/ssl/private":
      ensure                  => directory,
      owner                   => 'root',
      mode                    => '0755',
      selinux_ignore_defaults => true,
    }->
    file { "/etc/haproxy/ssl/private/oci-pki-api.pem":
      ensure                  => present,
      owner                   => "haproxy",
      source                  => "/etc/ssl/private/oci-pki-api.pem",
      selinux_ignore_defaults => true,
      mode                    => '0600',
      notify                  => Service['haproxy'],
    }
  }
  $haproxy_schedule1 = ['heat::db::begin', 'barbican::db::begin']
  if $has_subrole_nova {
    if $openstack_release == 'rocky' {
      $haproxy_schedule2 = concat($haproxy_schedule1, ['nova::db::begin', 'neutron::db::begin'])
    }else{
      $haproxy_schedule2 = concat($haproxy_schedule1, ['nova::db::begin', 'placement::db::begin', 'neutron::db::begin'])
    }
  } else {
    $haproxy_schedule2 = $haproxy_schedule1
  }
  if $has_subrole_cinder {
    $haproxy_schedule3 = concat($haproxy_schedule2, ['cinder::db::begin'])
  } else {
    $haproxy_schedule3 = $haproxy_schedule2
  }
  if $has_subrole_glance {
    $haproxy_schedule4 = concat($haproxy_schedule3, ['glance::db::begin'])
  } else {
    $haproxy_schedule4 = $haproxy_schedule3
  }
  if $has_subrole_aodh {
    $haproxy_schedule5 = concat($haproxy_schedule4, ['aodh::db::begin'])
  } else {
    $haproxy_schedule5 = $haproxy_schedule4
  }
  if $has_subrole_octavia {
    $haproxy_schedule6 = concat($haproxy_schedule5, ['octavia::db::begin'])
  } else {
    $haproxy_schedule6 = $haproxy_schedule5
  }
  if $has_subrole_panko {
    $haproxy_schedule7 = concat($haproxy_schedule6, ['panko::db::begin'])
  } else {
    $haproxy_schedule7 = $haproxy_schedule6
  }
  if $has_subrole_gnocchi {
    $haproxy_schedule8 = concat($haproxy_schedule7, ['gnocchi::db::begin'])
  } else {
    $haproxy_schedule8 = $haproxy_schedule7
  }
  if $has_subrole_cloudkitty {
    $haproxy_schedule9 = concat($haproxy_schedule8, ['cloudkitty::db::begin'])
  } else {
    $haproxy_schedule9 = $haproxy_schedule8
  }
  if $has_subrole_magnum and $install_magnum {
    $haproxy_schedule10 = concat($haproxy_schedule9, ['magnum::db::begin'])
  } else {
    $haproxy_schedule10 = $haproxy_schedule9
  }
  if $has_subrole_designate {
    $haproxy_schedule11 = concat($haproxy_schedule10, ['designate::db::begin'])
  } else {
    $haproxy_schedule11 = $haproxy_schedule10
  }
  $haproxy_schedule = $haproxy_schedule11

  # Here, we assume that stretch & buster don't have
  # a version of haproxy >= 2.1.
  if $mycodename == 'stretch' or $mycodename == 'buster' {
    # This used to work with haproxy 1.8,
    # let's see if we can get rid of it
    $haproxy_replace_path = 'reqrep'
    $haproxy_replace_head = 'rspirep'
    $use_haproxy_21_format = false
  }else{
    # This works with haproxy >= 2.1
    $haproxy_replace_path  = 'http-request replace-path'
    $haproxy_replace_head  = 'http-response replace-header'
    $use_haproxy_21_format = true
  }

  sysctl::value { 'net.ipv4.ip_nonlocal_bind':
    value => "1",
    target => '/etc/sysctl.d/ip-nonlocal-bind.conf',
  }->
  class { 'haproxy':
    global_options   => {
      'log'     => '/dev/log local0',
      'maxconn' => '40960',
      'user'    => 'haproxy',
      'group'   => 'haproxy',
      'stats'   => [
        'socket /var/lib/haproxy/stats',
        'timeout 30s'
      ],
      'daemon'  => '',
      'nbproc'  => '4',
      'ssl-default-bind-ciphers'   => 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256',
      'ssl-default-bind-options'   => 'no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets',
      'ssl-default-server-ciphers' => 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256',
      'ssl-default-server-options' => 'no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets',
    },
    defaults_options => {
      'mode'      => 'http',
        'timeout' => [
          'http-request 10s',
          'queue 1m',
          'connect 10s',
          'client 1m',
          'server 1m',
          'check 10s',
        ],
      'stats'     => 'enable',
    },
    merge_options => true,
    before => Anchor[$haproxy_schedule],
  }->
  haproxy::listen { 'hapstats':
    section_name => 'statshttp',
    bind         => { "${machine_ip}:8088" => 'user root'},
    mode         => 'http',
    options      => {
        'stats' => [ 'uri /', "auth admin:${pass_haproxy_stats}", 'refresh 15', 'admin if TRUE', 'realm Haproxy Statistics', ],
      },
  }->
  haproxy::listen { 'haphealthcheck':
    section_name => 'healthcheck',
    bind         => { "${machine_ip}:8081" => 'user root'},
    mode         => 'health',
    options => [
       { 'option' => 'httpchk' },
    ]
  }

  # If SQL is on the controllers, define front-end and back-end for SQL
  if $has_subrole_db {
    haproxy::frontend { 'galerafe':
      mode      => 'tcp',
      bind      => { "${vip_ipaddr}:3306" => [] },
      options   => [
        { 'timeout'         => 'client 3600s'},
        { 'default_backend' => 'galerabe'},
      ],
      before => Anchor[$haproxy_schedule],
    }
    haproxy::backend { 'galerabe':
      options => [
        { 'mode'    => 'tcp' },
        { 'balance' => 'roundrobin' },
        { 'timeout' => 'check 5000' },
        { 'timeout' => 'server 3600s' },
        { 'option'  => 'httpchk'},
      ],
      before => Anchor[$haproxy_schedule],
    }
    haproxy::balancermember { 'galerabm':
      listening_service => 'galerabe',
      ipaddresses       => $first_sql_ip,
      server_names      => $first_sql,
      options           => 'check inter 4000 port 9200 fall 3 rise 5',
      before => Anchor[$haproxy_schedule],
    }
    haproxy::balancermember { 'galerabmback':
      listening_service => 'galerabe',
      ipaddresses       => $non_master_sql_ip,
      server_names      => $non_master_sql,
      options           => 'check inter 4000 port 9200 fall 3 rise 5 backup',
      before => Anchor[$haproxy_schedule],
    }
  }

  $haproxy_options_keystone = [
    # Set HSTS (63072000 seconds = 2 years)
    { 'http-response' => 'set-header Strict-Transport-Security max-age=63072000'},
    # So that we log real customer's IPs in our logs
    { 'option'        => "forwardfor except $vip_ipaddr"},
    { 'acl'           => 'url_keystoneadmin path_beg -i /identity-admin'},
    { 'use_backend'   => 'keystonepubbe if url_keystoneadmin'},
    { 'acl'           => 'url_keystone path_beg -i /identity'},
    { 'use_backend'   => 'keystonepubbe if url_keystone'},
  ]

  $glance_haproxy_options = [
    { 'acl'         => 'url_glance path_beg -i /image'},
    { 'use_backend' => 'glancebe if url_glance'},
  ]

  $swift_haproxy_options = [
    { 'acl'         => 'url_swift path_beg -i /object'},
    { 'use_backend' => 'swiftbe if url_swift'},
  ]

  $heat_haproxy_options = [
    { 'acl'         => 'url_heatcfn path_beg -i /orchestration-cfn'},
    { 'use_backend' => 'heatcfnbe if url_heatcfn'},
    { 'acl'         => 'url_heat path_beg -i /orchestration-api'},
    { 'use_backend' => 'heatbe if url_heat'},
  ]

  $horizon_haproxy_options = [
    { 'acl'         => 'url_horizon path_beg -i /horizon'},
    { 'use_backend' => 'horizonbe if url_horizon'},
  ]

  $barbican_haproxy_options = [
    { 'acl'         => 'url_barbican path_beg -i /keymanager'},
    { 'use_backend' => 'barbicanbe if url_barbican'},
  ]

  $nova_haproxy_options = [
    { 'acl'         => 'url_nova path_beg -i /compute'},
    { 'use_backend' => 'novabe if url_nova'},
    { 'acl'         => 'url_placement path_beg -i /placement'},
    { 'use_backend' => 'placementbe if url_placement'},
    { 'acl'         => 'url_novnc path_beg -i /novnc'},
    { 'use_backend' => 'novncbe if url_novnc'},
    { 'acl'         => 'url_websockify path_beg -i /websockify'},
    { 'acl'         => 'url_websockify_token url_beg -i /?token'},
    { 'use_backend' => 'websockifybe if url_websockify or url_websockify_token'},
  ]

  $neutron_haproxy_options = [
    { 'acl'         => 'url_neutron path_beg -i /network'},
    { 'use_backend' => 'neutronbe if url_neutron'},
  ]

  $cinder_haproxy_options = [
    { 'acl'         => 'url_cinder path_beg -i /volume'},
    { 'use_backend' => 'cinderbe if url_cinder'},
  ]

  $gnocchi_haproxy_options = [
    { 'acl'         => 'url_gnocchi path_beg -i /metric'},
    { 'acl'         => 'url_gnocchi path_beg -i /v1/metric'},
    { 'use_backend' => 'gnocchibe if url_gnocchi'},
  ]

  $cloudkitty_haproxy_options = [
    { 'acl'         => 'url_cloudkitty path_beg -i /rating'},
    { 'use_backend' => 'cloudkittybe if url_cloudkitty'},
  ]

  $aodh_haproxy_options = [
    { 'acl'         => 'url_aodh path_beg -i /alarm'},
    { 'use_backend' => 'aodhbe if url_aodh'},
  ]

  $octavia_haproxy_options = [
    { 'acl'         => 'url_octavia path_beg -i /loadbalance'},
    { 'use_backend' => 'octaviabe if url_octavia'},
  ]

  $magnum_haproxy_options = [
    { 'acl'         => 'url_magnum path_beg -i /containers'},
    { 'use_backend' => 'magnumbe if url_magnum'},
    { 'acl'         => 'url_etcd_cluster path_beg -i /etcd-cluster'},
    { 'use_backend' => 'etcdclusterbe if url_etcd_cluster'},
    { 'acl'         => 'url_etcd_discovery path_beg -i /etcd-discovery'},
    { 'use_backend' => 'etcddiscoverybe if url_etcd_discovery'},
  ]

  $panko_haproxy_options = [
    { 'acl'         => 'url_panko path_beg -i /event'},
    { 'use_backend' => 'pankobe if url_panko'},
  ]

  $designate_haproxy_options = [
    { 'acl'         => 'url_designate path_beg -i /dns'},
    { 'use_backend' => 'designatebe if url_designate'},
  ]


  $custom_haproxy_options = [
    { 'acl'         => 'url_slash path /'},
    { 'redirect'    => "location $haproxy_custom_url if url_slash !url_websockify_token"},
  ]

  if $has_subrole_glance {
    $haproxy_options_with_glance = concat($haproxy_options_keystone, $glance_haproxy_options)
  } else {
    $haproxy_options_with_glance = $haproxy_options_keystone
  }

  if $has_subrole_swift and $swiftproxy_hostname == 'none' {
    $haproxy_options_with_swift = concat($haproxy_options_with_glance, $swift_haproxy_options)
  } else {
    $haproxy_options_with_swift = $haproxy_options_with_glance
  }

  if $has_subrole_heat {
    $haproxy_options_with_heat = concat($haproxy_options_with_swift, $heat_haproxy_options)
  } else {
    $haproxy_options_with_heat = $haproxy_options_with_swift
  }

  if $has_subrole_horizon {
    $haproxy_options_with_horizon = concat($haproxy_options_with_heat, $horizon_haproxy_options)
  } else {
    $haproxy_options_with_horizon = $haproxy_options_with_heat
  }

  if $has_subrole_barbican {
    $haproxy_options_with_barbican = concat($haproxy_options_with_horizon, $barbican_haproxy_options)
  } else {
    $haproxy_options_with_barbican = $haproxy_options_with_horizon
  }

  if $has_subrole_nova {
    $haproxy_options_with_nova = concat($haproxy_options_with_barbican, $nova_haproxy_options)
  } else {
    $haproxy_options_with_nova = $haproxy_options_with_barbican
  }

  if $has_subrole_neutron {
    $haproxy_options_with_neutron = concat($haproxy_options_with_nova, $neutron_haproxy_options)
  } else {
    $haproxy_options_with_neutron = $haproxy_options_with_nova
  }

  if $has_subrole_cinder {
    $haproxy_options_with_cinder = concat($haproxy_options_with_neutron, $cinder_haproxy_options)
  } else {
    $haproxy_options_with_cinder = $haproxy_options_with_neutron
  }

  if $has_subrole_gnocchi {
    $haproxy_options_with_gnocchi = concat($haproxy_options_with_cinder, $gnocchi_haproxy_options)
  } else {
    $haproxy_options_with_gnocchi = $haproxy_options_with_cinder
  }

  if $has_subrole_cloudkitty {
    $haproxy_options_with_cloudkitty = concat($haproxy_options_with_gnocchi, $cloudkitty_haproxy_options)
  } else {
    $haproxy_options_with_cloudkitty = $haproxy_options_with_gnocchi
  }

  if $has_subrole_aodh {
    $haproxy_options_with_aodh = concat($haproxy_options_with_cloudkitty, $aodh_haproxy_options)
  } else {
    $haproxy_options_with_aodh = $haproxy_options_with_cloudkitty
  }

  if $has_subrole_octavia {
    $haproxy_options_with_octavia = concat($haproxy_options_with_aodh, $octavia_haproxy_options)
  } else {
    $haproxy_options_with_octavia = $haproxy_options_with_aodh
  }

  if $has_subrole_magnum {
    $haproxy_options_with_magnum = concat($haproxy_options_with_octavia, $magnum_haproxy_options)
  } else {
    $haproxy_options_with_magnum = $haproxy_options_with_octavia
  }

  if $has_subrole_panko {
    $haproxy_options_with_panko = concat($haproxy_options_with_magnum, $panko_haproxy_options)
  } else {
    $haproxy_options_with_panko = $haproxy_options_with_magnum
  }

  if $has_subrole_designate {
    $haproxy_options_with_designate = concat($haproxy_options_with_panko, $designate_haproxy_options)
  } else {
    $haproxy_options_with_designate = $haproxy_options_with_panko
  }

  if $haproxy_custom_url {
    $haproxy_options_with_custom = concat($haproxy_options_with_designate, $custom_haproxy_options)
  } else {
    $haproxy_options_with_custom = $haproxy_options_with_designate
  }

  # Global frontend for all services with dispatch depending on the URL
  haproxy::frontend { 'openstackfe':
    mode      => 'http',
    bind      => { "${vip_ipaddr}:${api_port}" => ['ssl', 'crt', '/etc/haproxy/ssl/private/oci-pki-api.pem', 'crt', '/etc/haproxy/ssl/private/'],
                   "${machine_ip}:${api_port}" => ['ssl', 'crt', '/etc/haproxy/ssl/private/oci-pki-api.pem', 'crt', '/etc/haproxy/ssl/private/'] },
    options   => $haproxy_options_with_custom,
  }

  # Keystone public
  if $use_haproxy_21_format {
    $tweak_path_identity_admin = '^/identity-admin/?(.*) /\1'
    $tweak_path_identity       = '^/identity/?(.*) /\1'
    $tweak_head_identity       = 'Location: (https://[^/]*)/(.*)$ \1/identity/\2'
  }else{
    $tweak_path_identity_admin = '^([^\ ]*\ /)identity-admin[/]?(.*) \1\2'
    $tweak_path_identity       = '^([^\ ]*\ /)identity[/]?(.*) \1\2'
    $tweak_head_identity       = '^Location:\ (https://[^/]*)/(.*)$ Location:\ \1/identity/\2'
  }
  haproxy::backend { 'keystonepubbe':
    options => [
       { 'option'              => 'forwardfor' },
       { 'option'              => 'httpchk GET /healthcheck' },
       { 'http-request'        => "set-header X-Client-IP %[src]"},
       { 'mode'                => 'http' },
       { 'balance'             => 'roundrobin' },
       { $haproxy_replace_path => $tweak_path_identity_admin },
       { $haproxy_replace_path => $tweak_path_identity       },
       { $haproxy_replace_head => $tweak_head_identity       },
    ],
  }
  haproxy::balancermember { 'keystonepubbm':
    listening_service => 'keystonepubbe',
    ipaddresses       => $all_masters_ip,
    server_names      => $all_masters,
    ports             => 5000,
    options           => $haproxy_api_backend_options,
  }

  # glance API
  if $has_subrole_glance {
    if $use_haproxy_21_format {
      $teak_path_image = '^/image[/]?(.*) /\1'
    }else{
      $teak_path_image = '^([^\ ]*\ /)image[/]?(.*) \1\2'
    }
    if $openstack_release == 'rocky'{
      # It feels like /healthcheck is kind of broken with
      # glance and uwsgi under Rocky.
      haproxy::backend { 'glancebe':
        options => [
           { 'option'               => 'forwardfor' },
           { 'mode'                 => 'http' },
           { 'balance'              => 'source' },
           { $haproxy_replace_path  => $teak_path_image },
        ],
      }
    }else{
      haproxy::backend { 'glancebe':
        options => [
           { 'option'               => 'forwardfor' },
           { 'option'               => 'httpchk GET /healthcheck' },
           { 'mode'                 => 'http' },
           { 'balance'              => 'source' },
           { $haproxy_replace_path  => $teak_path_image},
        ],
      }
    }
    if $glance_backend == 'file'{
      haproxy::balancermember { 'glancebm':
        listening_service => 'glancebe',
        ipaddresses       => [ "${first_master_ip}", ],
        server_names      => [ "${first_master}", ],
        ports             => 9292,
        options           => $haproxy_api_backend_options,
      }
      haproxy::balancermember { 'glancebmbackup':
        listening_service => 'glancebe',
        ipaddresses       => $non_first_master_controllers_ip,
        server_names      => $non_first_master_controllers,
        ports             => 9292,
        options           => "${haproxy_api_backend_options} backup",
      }
    } else {
      haproxy::balancermember { 'glancebm':
        listening_service => 'glancebe',
        ipaddresses       => $all_masters_ip,
        server_names      => $all_masters,
        ports             => 9292,
        options           => $haproxy_api_backend_options,
      }
    }
  }

  # swift proxy
  if $has_subrole_swift and $swiftproxy_hostname == 'none' {
    if $use_haproxy_21_format {
      $tweak_path_object = '^/object/?(.*) /\1'
    }else{
      $tweak_path_object = '^([^\ ]*\ /)object[/]?(.*)     \1\2'
    }
    haproxy::backend { 'swiftbe':
      options => [
         { 'option'               => 'httpchk GET /healthcheck' },
         { 'option'               => 'forwardfor' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_object },
      ],
    }
    haproxy::balancermember { 'swiftbm':
      listening_service => 'swiftbe',
      ipaddresses       => $all_swiftproxy_ip,
      server_names      => $all_swiftproxy,
      ports             => 8080,
      options           => 'check',
    }
  }

  # heat
  if $has_subrole_heat {
    if $use_haproxy_21_format {
      $tweak_path_orchestration_api = '^/orchestration-api/?(.*) /\1'
      $tweak_head_orchestration_api = 'Location ^([^/]*)/(.*) \1/orchestration-api/\2'
      $tweak_path_orchestration_cfn = '^/orchestration-cfn/?(.*) /\1'
      $tweak_head_orchestration_cfn = 'Location ^([^/]*)/(.*) \1/orchestration-cfn/\2'
    }else{
      $tweak_path_orchestration_api = '^([^\ ]*\ /)orchestration-api[/]?(.*) \1\2' 
      $tweak_head_orchestration_api = '^Location:\ (https://[^/]*)/(.*)$ Location:\ \1/orchestration-api/\2'
      $tweak_path_orchestration_cfn = '^([^\ ]*\ /)orchestration-cfn[/]?(.*) \1\2' 
      $tweak_head_orchestration_cfn = '^Location:\ (https://[^/]*)/(.*)$ Location:\ \1/orchestration-cfn/\2'
    }
    haproxy::backend { 'heatbe':
      options => [
         { 'option'              => 'forwardfor' },
         { 'option'              => 'httpchk GET /healthcheck' },
         { 'mode'                => 'http' },
         { 'balance'             => 'roundrobin' },
         { $haproxy_replace_path => $tweak_path_orchestration_api },
         { $haproxy_replace_head => $tweak_head_orchestration_api },
      ],
    }
    haproxy::balancermember { 'heatbm':
      listening_service => 'heatbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 8004,
      options           => $haproxy_api_backend_options,
    }
    haproxy::backend { 'heatcfnbe':
      options => [
         { 'option'              => 'forwardfor' },
         { 'option'              => 'httpchk GET /healthcheck' },
         { 'mode'                => 'http' },
         { 'balance'             => 'roundrobin' },
         { $haproxy_replace_path => $tweak_path_orchestration_cfn },
         { $haproxy_replace_head => $tweak_head_orchestration_cfn },
      ],
    }
    haproxy::balancermember { 'heatcfnbm':
      listening_service => 'heatcfnbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 8000,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_horizon {
    haproxy::backend { 'horizonbe':
      options => [
         { 'option' => 'forwardfor' },
         { 'mode' => 'http' },
         { 'balance' => 'source' },
      ],
    }
    haproxy::balancermember { 'horizonbm':
      listening_service => 'horizonbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 7443,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_barbican {
    if $use_haproxy_21_format {
      $tweak_path_keymanager = '^/keymanager/?(.*) /\1'
    }else{
      $tweak_path_keymanager = '^([^\ ]*\ /)keymanager[/]?(.*) \1\2'
    }
    haproxy::backend { 'barbicanbe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'option'               => 'httpchk GET /healthcheck' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_keymanager },
      ],
    }
    haproxy::balancermember { 'barbicanbm':
      listening_service => 'barbicanbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 9311,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_nova {
    if $use_haproxy_21_format {
      $tweak_path_compute = '^/compute/?(.*) /\1'
      $tweak_head_compute = 'Location ^([^/]*)/(.*) \1/compute/\2'
    }else{
      $tweak_path_compute = '^([^\ ]*\ /)compute/?(.*) \1\2'
      $tweak_head_compute = '^Location:\ (https://[^/]*)/(.*)$ Location:\ \1/compute/\2'
    }
    haproxy::backend { 'novabe':
      options => [
         { 'option'              => 'forwardfor' },
         { 'option'              => 'httpchk GET /healthcheck' },
         { 'mode'                => 'http' },
         { 'balance'             => 'roundrobin' },
         { $haproxy_replace_path => $tweak_path_compute },
         { $haproxy_replace_head => $tweak_head_compute },
      ],
    }
    haproxy::balancermember { 'novabm':
      listening_service => 'novabe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 8774,
      options           => $haproxy_api_backend_options,
    }

    if $use_haproxy_21_format {
      $tweak_path_placement = '^/placement/?(.*) /\1'
    }else{
      $tweak_path_placement = '^([^\ ]*\ /)placement/?(.*) \1\2'
    }
    haproxy::backend { 'placementbe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'option'               => 'httpchk GET /' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_placement },
      ],
    }
    haproxy::balancermember { 'placementbm':
      listening_service => 'placementbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 8778,
      options           => $haproxy_api_backend_options,
    }

    if $use_haproxy_21_format {
      $tweak_path_novnc = '^/novnc/?(.*) /\1'
    }else{
      $tweak_path_novnc = '^([^\ ]*\ /)novnc/?(.*) \1\2'
    }
    haproxy::backend { 'novncbe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'source' },
         { $haproxy_replace_path  => $tweak_path_novnc },
      ],
    }
    haproxy::balancermember { 'novncbm':
      listening_service => 'novncbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 6080,
      options           => 'check',
    }

    haproxy::backend { 'websockifybe':
      options => [
         { 'option'  => 'forwardfor' },
         { 'mode'    => 'http' },
         { 'balance' => 'source' },
         { 'acl'     => 'hdr_connection_upgrade hdr(Connection)                 -i upgrade'},
         { 'acl'     => 'hdr_upgrade_websocket  hdr(Upgrade)                    -i websocket'},
         { 'acl'     => 'hdr_websocket_key      hdr_cnt(Sec-WebSocket-Key)      eq 1'},
         { 'acl'     => 'hdr_websocket_version  hdr_cnt(Sec-WebSocket-Version)  eq 1'},
      ],
    }
    haproxy::balancermember { 'websockifybm':
      listening_service => 'websockifybe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 6080,
      options           => 'check',
    }
  }

  if $has_subrole_neutron {
    if $use_haproxy_21_format {
      $tweak_path_network = '^/network/?(.*) /\1'
    }else{
      $tweak_path_network = '^([^\ ]*\ /)network/?(.*) \1\2'
    }
    haproxy::backend { 'neutronbe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'option'               => 'httpchk GET /healthcheck' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_network },
      ],
    }
    haproxy::balancermember { 'neutronbm':
      listening_service => 'neutronbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 9696,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_cinder {
    if $use_haproxy_21_format {
      $tweak_path_volume = '^/volume/?(.*) /\1'
    }else{
      $tweak_path_volume = '^([^\ ]*\ /)volume/?(.*) \1\2'
    }
    haproxy::backend { 'cinderbe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'option'               => 'httpchk GET /healthcheck' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_volume },
      ],
    }
    haproxy::balancermember { 'cinderbm':
      listening_service => 'cinderbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 8776,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_gnocchi {
    if $use_haproxy_21_format {
      $tweak_path_metric = '^/metric/?(.*) /\1'
    }else{
      $tweak_path_metric = '^([^\ ]*\ /)metric/?(.*) \1\2'
    }
    haproxy::backend { 'gnocchibe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'option'               => 'httpchk GET /healthcheck' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_metric },
      ],
    }
    if $has_subrole_gnocchi_services {
      $gnocchi_backend_ipaddresses = $all_masters_ip
      $gnocchi_backend_server_names = $all_masters
    }else{
      $gnocchi_backend_ipaddresses = $all_rabbits_ips
      $gnocchi_backend_server_names = $all_rabbits
    }
    haproxy::balancermember { 'gnocchibm':
      listening_service => 'gnocchibe',
      ipaddresses       => $gnocchi_backend_ipaddresses,
      server_names      => $gnocchi_backend_server_names,
      ports             => 8041,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_cloudkitty {
    if $use_haproxy_21_format {
      $tweak_path_rating = '^/rating/?(.*) /\1'
      $tweak_head_rating = 'Location ^([^/]*)/(.*) \1/rating/\2'
    }else{
      $tweak_path_rating = '^([^\ ]*\ /)rating/?(.*) \1\2'
      $tweak_head_rating = '^Location:\ (https://[^/]*)/(.*)$ Location:\ \1/rating/\2'
    }
    haproxy::backend { 'cloudkittybe':
      options => [
         { 'option'              => 'forwardfor' },
         { 'option'              => 'httpchk GET /healthcheck' },
         { 'mode'                => 'http' },
         { 'balance'             => 'roundrobin' },
         { $haproxy_replace_path => $tweak_path_rating },
         { $haproxy_replace_head => $tweak_head_rating },
      ],
    }
    haproxy::balancermember { 'cloudkittybm':
      listening_service => 'cloudkittybe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 8889,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_aodh {
    if $use_haproxy_21_format {
      $tweak_path_alarm = '^/alarm/?(.*) /\1'
    }else{
      $tweak_path_alarm = '^([^\ ]*\ /)alarm/?(.*) \1\2'
    }
    haproxy::backend { 'aodhbe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'option'               => 'httpchk GET /healthcheck' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_alarm },
      ],
    }
    haproxy::balancermember { 'aodhbm':
      listening_service => 'aodhbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 8042,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_octavia {
    if $use_haproxy_21_format {
      $tweak_path_loadbalance = '^/loadbalance/?(.*) /\1'
    }else{
      $tweak_path_loadbalance = '^([^\ ]*\ /)loadbalance/?(.*) \1\2'
    }
    haproxy::backend { 'octaviabe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'option'               => 'httpchk GET /' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_loadbalance },
      ],
    }
    haproxy::balancermember { 'octaviabm':
      listening_service => 'octaviabe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 9876,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_magnum {
    if $use_haproxy_21_format {
      $tweak_path_container = '^/container/?(.*) /\1'
      $tweak_head_container = 'Location ^([^/]*)/(.*) \1/container/\2'
    }else{
      $tweak_path_container = '^([^\ ]*\ /)container/?(.*) \1\2'
      $tweak_head_container = '^Location:\ (https://[^/]*)/(.*)$ Location:\ \1/containers/\2'
    }
    haproxy::backend { 'magnumbe':
      options => [
         { 'option'              => 'forwardfor' },
         { 'option'              => 'httpchk GET /healthcheck' },
         { 'mode'                => 'http' },
         { 'balance'             => 'roundrobin' },
         { $haproxy_replace_path => $tweak_path_container },
         { $haproxy_replace_head => $tweak_head_container },
      ],
    }
    haproxy::balancermember { 'magnumbm':
      listening_service => 'magnumbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 9511,
      options           => $haproxy_api_backend_options,
    }

    # If we're setting-up Magnum, then we also want the ETCD ...
    if $use_haproxy_21_format {
      $tweak_path_etcdc = '^/etcd-cluster/?(.*) /\1'
      $tweak_head_etcdc = 'Location ^([^/]*)/(.*) \1/etcd-cluster/\2'
    }else{
      $tweak_path_etcdc = '^([^\ ]*\ /)etcd-cluster/?(.*) \1\2'
      $tweak_head_etcdc = '^Location:\ (https://[^/]*)/(.*)$ Location:\ \1/etcd-clusters/\2'
    }
    haproxy::backend { 'etcdclusterbe':
      options => [
         { 'option'              => 'forwardfor' },
         { 'mode'                => 'http' },
         { 'balance'             => 'roundrobin' },
         { $haproxy_replace_path => $tweak_path_etcdc },
         { $haproxy_replace_head => $tweak_head_etcdc },
      ],
    }
    haproxy::balancermember { 'etcdclusterbm':
      listening_service => 'etcdclusterbe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 2379,
      options           => 'check',
    }

    # ...and etcd-discovery services.
    if $use_haproxy_21_format {
      $tweak_path_etcdd = '^/etcd-discovery/?(.*) /\1'
      $tweak_head_etcdd = 'Location ^([^/]*)/(.*) \1/etcd-discovery/\2'
    }else{
      $tweak_path_etcdd = '^([^\ ]*\ /)etcd-discovery/?(.*) \1\2'
      $tweak_head_etcdd = '^Location:\ (https://[^/]*)/(.*)$ Location:\ \1/etcd-discoverys/\2'
    }
    haproxy::backend { 'etcddiscoverybe':
      options => [
         { 'option'              => 'forwardfor' },
         { 'mode'                => 'http' },
         { 'balance'             => 'roundrobin' },
         { $haproxy_replace_path => $tweak_path_etcdd },
         { $haproxy_replace_head => $tweak_head_etcdd },
      ],
    }
    haproxy::balancermember { 'etcddiscoverybm':
      listening_service => 'etcddiscoverybe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 8087,
      options           => 'check',
    }
  }

  if $has_subrole_panko {
    if $use_haproxy_21_format {
      $tweak_path_event = '^/event/?(.*) /\1'
    }else{
      $tweak_path_event = '^([^\ ]*\ /)event/?(.*) \1\2'
    }
    haproxy::backend { 'pankobe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'option'               => 'httpchk GET /healthcheck' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_event },
      ],
    }
    haproxy::balancermember { 'pankobm':
      listening_service => 'pankobe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 8777,
      options           => $haproxy_api_backend_options,
    }
  }

  if $has_subrole_designate {
    if $use_haproxy_21_format {
      $tweak_path_dns = '^/dns/?(.*) /\1'
    }else{
      $tweak_path_dns = '^([^\ ]*\ /)dns/?(.*) \1\2'
    }
    haproxy::backend { 'designatebe':
      options => [
         { 'option'               => 'forwardfor' },
         { 'option'               => 'httpchk GET /' },
         { 'mode'                 => 'http' },
         { 'balance'              => 'roundrobin' },
         { $haproxy_replace_path  => $tweak_path_dns },
      ],
    }
    haproxy::balancermember { 'designatebm':
      listening_service => 'designatebe',
      ipaddresses       => $all_masters_ip,
      server_names      => $all_masters,
      ports             => 9001,
      options           => $haproxy_api_backend_options,
    }
  }

  #######################
  ### Setup databases ###
  #######################
  if $has_subrole_db and $initial_cluster_setup {
    if $mycodename != 'stretch'{
      package { 'mariadb-backup':
        ensure => present,
        before => Class['galera'],
      }
      $wsrep_sst_method = 'mariabackup'
    }else{
      $wsrep_sst_method = 'mariabackup'
    }

    if $is_first_sql {
      # If this is the first node, it must publish itself as working
      # before waiting for the other nodes to join.
      Class['galera::status'] -> Exec['galera-size-is-correct']
      Service['xinetd'] -> Exec['galera-size-is-correct']
    }else{
      # If this machine isn't the first Galera machine, then it must
      # wait until the first master replies it's setup.
      exec {'galera-first-master-is-up':
        command => "/usr/bin/oci-wait-for-first-galera-node ${first_master_ip} 4800",
        unless  => '/bin/false # comment to satisfy puppet syntax requirements',
        timeout => 5000,
      }
      Exec['galera-first-master-is-up'] -> Class['galera']
    }

    if $mycodename == 'stretch' or $mycodename == 'buster' {
      $galera_package_name = 'galera-3'
    }else{
      $galera_package_name = 'galera-4'
    }
    class { 'galera':
      galera_servers      => $all_masters_ip,
      galera_master       => $first_master,
      local_ip            => $machine_ip,
      mysql_package_name  => 'mariadb-server',
      client_package_name => 'default-mysql-client',
      vendor_type         => 'mariadb',
      root_password       => $pass_mysql_rootuser,
      status_password     => $pass_mysql_rootuser,
      deb_sysmaint_password => $pass_mysql_rootuser,
      configure_repo      => false,
      configure_firewall  => false,
      galera_package_name => $galera_package_name,
      override_options => {
        'mysqld' => {
          'bind_address'                    => $machine_ip,
          'wait_timeout'                    => '28800',
          'interactive_timeout'             => '30',
          'connect_timeout'                 => '30',
          'character_set_server'            => 'utf8',
          'collation_server'                => 'utf8_general_ci',
          'innodb_buffer_pool_size'         => '2G',
          'max_connections'                 => '1000',
          'max_user_connections'            => '500',
          'binlog_cache_size'               => '1M',
          'log-bin'                         => 'mysql-bin',
          'binlog_format'                   => 'ROW',
          'performance_schema'              => '1',
          'log_warnings'                    => '2',
          'wsrep_sst_auth'                  => "backup:${pass_mysql_backup}",
          'wsrep_sst_method'                => $wsrep_sst_method,
          'wsrep_cluster_name'              => $cluster_name,
          'wsrep_node_name'                 => $machine_hostname,
        }
      },
      require             => Cs_primitive['openstack-api-vip'],
    }->
    mysql_user { 'backup@%':
      ensure        => present,
      password_hash => mysql_password($pass_mysql_backup),
    }->
    mysql_grant{'backup@%/*.*':
      ensure     => 'present',
      options    => ['GRANT'],
      privileges => ['ALL'],
      table      => '*.*',
      user       => 'backup@%',
    }->

    # Wait until SHOW STATUS LIKE 'wsrep_cluster_status' shows Primary
    exec {'galera-is-up':
      command => "/usr/bin/oci-wait-for-sql \"SHOW STATUS LIKE 'wsrep_cluster_status'\" Primary mysql 4800",
      unless  => '/bin/false # comment to satisfy puppet syntax requirements',
      timeout => 5000,
    }

    # Wait until SHOW STATUS LIKE 'wsrep_connected' shows ON
    exec {'galera-wsrep-connected-on':
      command => "/usr/bin/oci-wait-for-sql \"SHOW STATUS LIKE 'wsrep_connected'\" ON mysql 4800",
      unless  => '/bin/false # comment to satisfy puppet syntax requirements',
      require => Exec['galera-is-up'],
      timeout => 5000,
    }

    # Wait until SHOW STATUS LIKE 'wsrep_local_state_comment' shows Synced
    exec {'galera-is-synced':
      command => "/usr/bin/oci-wait-for-sql \"SHOW STATUS LIKE 'wsrep_local_state_comment'\" Synced mysql 4800",
      unless  => '/bin/false # comment to satisfy puppet syntax requirements',
      require => Exec['galera-wsrep-connected-on'],
      timeout => 5000,
    }

    # Wait until all nodes are connected to the cluster
    $galera_cluster_num_of_nodes = sprintf('%i', $all_masters.size)
    exec {'galera-size-is-correct':
      command => "/usr/bin/oci-wait-for-sql \"SHOW STATUS LIKE 'wsrep_cluster_size'\" ${galera_cluster_num_of_nodes} mysql 4800",
      unless  => '/bin/false # comment to satisfy puppet syntax requirements',
      require => Exec['galera-is-synced'],
      timeout => 5000,
    }


    if $is_first_master {
      class { '::keystone::db::mysql':
        dbname   => 'keystonedb',
        password => $pass_keystone_db,
        allowed_hosts => '%',
        require  => Exec['galera-size-is-correct'],
#        require  => Class['oci::sql::galera'],
        before   => Anchor['keystone::service::begin'],
      }
    } else {
      exec {'wait-for-keystone-dbuser':
        command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='keystone'\" keystone mysql 4800",
        unless  => '/bin/false # comment to satisfy puppet syntax requirements',
        require => Exec['galera-size-is-correct'],
        before  => Anchor['keystone::service::begin'],
        timeout => 5000,
      }
    }
    if $has_subrole_glance {
      if $is_first_master {
        class { '::glance::db::mysql':
          dbname   => 'glancedb',
          password => $pass_glance_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before   => Anchor['glance::service::begin'],
        }
      } else {
        exec { 'glance-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='glance'\" glance mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
          before  => Anchor['glance::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_heat {
      if $is_first_master {
        class { '::heat::db::mysql':
          dbname   => 'heatdb',
          password => $pass_heat_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before   => Anchor['heat::service::begin'],
        }
      } else {
        exec { 'heat-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='heat'\" heat mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before  => Anchor['heat::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_barbican {
      if $is_first_master {
        class { '::barbican::db::mysql':
          dbname   => 'barbicandb',
          password => $pass_barbican_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before   => Anchor['barbican::service::begin'],
        }
      } else {
        exec { 'barbican-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='barbican'\" barbican mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before  => Anchor['barbican::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_nova {
      if $is_first_master {
        class { '::nova::db::mysql':
          user     => 'nova',
          dbname   => 'novadb',
          password => $pass_nova_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
          before   => Class['::nova::cell_v2::simple_setup'],
          notify   => Service['nova-conductor', 'nova-scheduler'],
        }
        class { '::nova::db::mysql_api':
          user     => 'novaapi',
          dbname   => 'novaapidb',
          password => $pass_nova_apidb,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
          before   => Class['::nova::cell_v2::simple_setup'],
        }
        if $openstack_release == 'rocky'{
          class { '::nova::db::mysql_placement':
            user     => 'placement',
            dbname   => 'placementdb',
            password => $pass_placement_db,
            allowed_hosts => '%',
            require  => Exec['galera-size-is-correct'],
            before   => Anchor['nova::service::begin'],
          }
        }else{
          class { '::placement::db::mysql':
            user => 'placement',
            dbname => 'placementdb',
            password => $pass_placement_db,
            allowed_hosts => '%',
            require  => Exec['galera-size-is-correct'],
            before   => Anchor['nova::service::begin'],
          }
        }
      } else {
        exec { 'nova-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='nova'\" nova mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
          before  => Class['nova'],
          timeout => 5000,
        }
        exec { 'novaapi-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='novaapi'\" novaapi mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
          before  => Class['nova'],
          timeout => 5000,
        }
        exec { 'placement-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='placement'\" placement mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
          before  => Anchor['nova::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_neutron {
      if $is_first_master {
        class { '::neutron::db::mysql':
          dbname   => 'neutrondb',
          password => $pass_neutron_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
          before   => Anchor['neutron::service::begin'],
        }
      } else {
        exec { 'neutron-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='neutron'\" neutron mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
          before  => Anchor['neutron::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_cinder {
      if $is_first_master {
        class { '::cinder::db::mysql':
          dbname        => 'cinderdb',
          password      => $pass_cinder_db,
          allowed_hosts => '%',
          require       => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before        => Anchor['cinder::service::begin'],
        }
      } else {
        exec { 'cinder-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='cinder'\" cinder mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before  => Anchor['cinder::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_gnocchi {
      if $is_first_master {
        class { '::gnocchi::db::mysql':
          dbname   => 'gnocchidb',
          password => $pass_gnocchi_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before   => Anchor['gnocchi::service::begin'],
        }
      } else {
        exec { 'gnocchi-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='gnocchi'\" gnocchi mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before  => Anchor['gnocchi::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_panko {
      if $is_first_master {
        class { '::panko::db::mysql':
          dbname   => 'pankodb',
          password => $pass_panko_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before   => Anchor['panko::service::begin'],
        }
      } else {
        exec { 'panko-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='panko'\" panko mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before  => Anchor['panko::service::begin'],
          timeout => 5000,
        }
      }
    }

    if $has_subrole_cloudkitty {
      if $is_first_master {
        class { '::cloudkitty::db::mysql':
          dbname   => 'cloudkittydb',
          password => $pass_cloudkitty_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before   => Anchor['cloudkitty::service::begin'],
        }
      } else {
        exec { 'cloudkitty-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='cloudkitty'\" cloudkitty mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before  => Anchor['cloudkitty::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_aodh {
      if $is_first_master {
        class { '::aodh::db::mysql':
          dbname   => 'aodhdb',
          password => $pass_aodh_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before   => Anchor['aodh::service::begin'],
        }
      } else {
        exec { 'aodh-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='aodh'\" aodh mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before  => Anchor['aodh::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_octavia {
      if $is_first_master {
        class { '::octavia::db::mysql':
          dbname   => 'octaviadb',
          password => $pass_octavia_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before   => Anchor['octavia::service::begin'],
        }
      } else {
        exec { 'octavia-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='octavia'\" octavia mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before  => Anchor['octavia::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_magnum and $install_magnum{
      if $is_first_master {
        class { '::magnum::db::mysql':
          dbname   => 'magnumdb',
          password => $pass_magnum_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before   => Anchor['magnum::service::begin'],
        }
      } else {
        exec { 'magnum-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='magnum'\" magnum mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before  => Anchor['magnum::service::begin'],
          timeout => 5000,
        }
      }
    }
    if $has_subrole_designate {
      if $is_first_master {
        class { '::designate::db::mysql':
          dbname   => 'designatedb',
          password => $pass_designate_db,
          allowed_hosts => '%',
          require  => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before   => Anchor['designate::service::begin'],
        }
      } else {
        exec { 'designate-db-user':
          command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='designate'\" designate mysql 4800",
          unless  => '/bin/false # comment to satisfy puppet syntax requirements',
          require => Exec['galera-size-is-correct'],
#          require  => Class['oci::sql::galera'],
          before  => Anchor['designate::service::begin'],
          timeout => 5000,
        }
      }
    }
  }  

  ######################
  ### Setup RabbitMQ ###
  ######################
  if $has_subrole_messaging {
    if $has_subrole_db and $initial_cluster_setup {
      Exec['galera-size-is-correct'] -> Class['::rabbitmq']
    }
    if $use_ssl {
      file { "/etc/rabbitmq/ssl/private":
        ensure                  => directory,
        owner                   => 'root',
        mode                    => '0755',
        require                 => File['/etc/rabbitmq/ssl'],
        selinux_ignore_defaults => true,
      }->
      file { "/etc/rabbitmq/ssl/public":
        ensure                  => directory,
        owner                   => 'root',
        mode                    => '0755',
        selinux_ignore_defaults => true,
      }->
      file { "/etc/rabbitmq/ssl/private/${::fqdn}.key":
        ensure                  => present,
        owner                   => "rabbitmq",
        source                  => "/etc/ssl/private/ssl-cert-snakeoil.key",
        selinux_ignore_defaults => true,
        mode                    => '0600',
      }->
      file { "/etc/rabbitmq/ssl/public/${::fqdn}.crt":
        ensure                  => present,
        owner                   => "rabbitmq",
        source                  => '/etc/ssl/certs/ssl-cert-snakeoil.pem',
        selinux_ignore_defaults => true,
        mode                    => '0644',
        notify        => Service['rabbitmq-server'],
      }
      $rabbit_ssl_cert = "/etc/rabbitmq/ssl/public/${::fqdn}.crt"
      $rabbit_ssl_key  = "/etc/rabbitmq/ssl/private/${::fqdn}.key"
      $rabbit_ssl_ca   = $oci_pki_root_ca_file
    } else {
      $rabbit_ssl_cert = UNSET
      $rabbit_ssl_key  = UNSET
      $rabbit_ssl_ca   = UNSET
    }

    class { '::rabbitmq':
      delete_guest_user        => true,
      node_ip_address          => $machine_ip,
      ssl_interface            => $machine_ip,
      ssl                      => $use_ssl,
      ssl_only                 => $use_ssl,
      ssl_cacert               => $rabbit_ssl_ca,
      ssl_cert                 => $rabbit_ssl_cert,
      ssl_key                  => $rabbit_ssl_key,
      environment_variables    => $rabbit_env,
      repos_ensure             => false,
      # Clustering options...
      config_cluster           => true,
      cluster_nodes            => $all_masters,
      cluster_node_type        => 'ram',
      erlang_cookie            => $pass_rabbitmq_cookie,
      wipe_db_on_cookie_change => true,
      config_variables         => {
          'vm_memory_high_watermark' => '0.4',
          'vm_memory_limit'          => '26932430438',
          'cluster_partition_handling' => 'autoheal',
        },
    }->
    rabbitmq_vhost { '/':
      provider => 'rabbitmqctl',
      require  => Class['::rabbitmq'],
    }->
    rabbitmq_user { 'keystone':
      admin    => true,
      password => $pass_keystone_messaging,
    }->
    rabbitmq_user_permissions { "keystone@/":
      configure_permission => '.*',
      write_permission     => '.*',
      read_permission      => '.*',
      provider             => 'rabbitmqctl',
      require              => Class['::rabbitmq'],
    }

    if $initial_cluster_setup {
      exec { 'auto-join-rabbit-cluster':
        command => "/usr/bin/oci-auto-join-rabbitmq-cluster ${first_master}",
        unless  => "/bin/false",
        require => Class['::rabbitmq'],
      }

      rabbitmq_user { 'monitoring':
        admin    => true,
        password => $pass_rabbitmq_monitoring,
        require  => Class['::rabbitmq'],
      }->
      rabbitmq_user_permissions { "monitoring@/":
        configure_permission => '.*',
        write_permission     => '.*',
        read_permission      => '.*',
        provider             => 'rabbitmqctl',
      }

      if $has_subrole_glance {
        rabbitmq_user { 'glance':
          admin    => true,
          password => $pass_glance_messaging,
        }->
        rabbitmq_user_permissions { "glance@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
          before               => Anchor['glance::service::begin'],
        }
      }

      if $is_first_master {
        if $has_subrole_heat {
          rabbitmq_user { 'heat':
            admin    => true,
            password => $pass_heat_messaging,
          }->
          rabbitmq_user_permissions { "heat@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['heat::service::begin'],
          }
        }

        if $has_subrole_barbican {
          rabbitmq_user { 'barbican':
            admin    => true,
            password => $pass_barbican_messaging,
          }->
          rabbitmq_user_permissions { "barbican@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['barbican::service::begin'],
          }
        }

        if $has_subrole_nova {
          rabbitmq_user { 'nova':
            admin    => true,
            password => $pass_nova_messaging,
          }->
          rabbitmq_user_permissions { "nova@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['nova::service::begin'],
          }
        }
        if $has_subrole_neutron {
          rabbitmq_user { 'neutron':
            admin    => true,
            password => $pass_neutron_messaging,
          }->
          rabbitmq_user_permissions { "neutron@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['neutron::service::begin'],
          }
        }
        if $has_subrole_cinder {
          rabbitmq_user { 'cinder':
            admin    => true,
            password => $pass_cinder_messaging,
          }->
          rabbitmq_user_permissions { "cinder@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['cinder::service::begin'],
          }
        }
        if $has_subrole_gnocchi {
          rabbitmq_user { 'gnocchi':
            admin    => true,
            password => $pass_gnocchi_messaging,
          }->
          rabbitmq_user_permissions { "gnocchi@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['gnocchi::service::begin'],
          }
        }
        if $has_subrole_panko {
          rabbitmq_user { 'panko':
            admin    => true,
            password => $pass_panko_messaging,
          }->
          rabbitmq_user_permissions { "panko@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['panko::service::begin'],
          }
        }
        if $has_subrole_ceilometer {
          rabbitmq_user { 'ceilometer':
            admin    => true,
            password => $pass_ceilometer_messaging,
          }->
          rabbitmq_user_permissions { "ceilometer@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['ceilometer::service::begin'],
          }
        }
        if $has_subrole_cloudkitty {
          rabbitmq_user { 'cloudkitty':
            admin    => true,
            password => $pass_cloudkitty_messaging,
          }->
          rabbitmq_user_permissions { "cloudkitty@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['cloudkitty::service::begin'],
          }
        }
        if $has_subrole_aodh {
          rabbitmq_user { 'aodh':
            admin    => true,
            password => $pass_aodh_messaging,
          }->
          rabbitmq_user_permissions { "aodh@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['aodh::service::begin'],
          }
        }
        if $has_subrole_octavia {
          rabbitmq_user { 'octavia':
            admin    => true,
            password => $pass_octavia_messaging,
          }->
          rabbitmq_user_permissions { "octavia@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['octavia::service::begin'],
          }
        }
        if $has_subrole_magnum and $install_magnum {
          rabbitmq_user { 'magnum':
            admin    => true,
            password => $pass_magnum_messaging,
          }->
          rabbitmq_user_permissions { "magnum@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['magnum::service::begin'],
          }
        }
        if $has_subrole_designate {
          rabbitmq_user { 'designate':
            admin    => true,
            password => $pass_designate_messaging,
          }->
          rabbitmq_user_permissions { "designate@/":
            configure_permission => '.*',
            write_permission     => '.*',
            read_permission      => '.*',
            provider             => 'rabbitmqctl',
            require              => Class['::rabbitmq'],
            before               => Anchor['designate::service::begin'],
          }
        }
      }
    }
  }

  #######################
  ### Setup memcached ###
  #######################
  class { '::memcached':
    listen_ip => $machine_ip,
    udp_port  => 0,
    max_memory => '20%',
  }

  # Configure Apache to use mod-wsgi-py3, not mod-wsgi
  if $has_subrole_api_keystone {
    # Set python3 for mod-wsgi
    include ::apache::params
    class { '::apache':
      mod_packages => merge($::apache::params::mod_packages, {
        'wsgi' => 'libapache2-mod-wsgi-py3',
      })
    }
  }

  if $has_subrole_cinder or $has_subrole_nova or $has_subrole_horizon {
    include ::apache
  }

  if $has_subrole_api_keystone {
    ##############################
    ### Setup keystone cluster ###
    ##############################
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['keystone::install::begin']
    }

    class { '::keystone::client': }
    # This cleans-up previous setup with:
    # class { '::keystone::cron::token_flush': }
    file { "/var/spool/cron/crontabs/keystone":
      ensure                  => absent,
    }

    if $use_ssl {
      if $openstack_release == 'rocky'{
        file { "/etc/keystone/ssl":
          ensure                  => directory,
          owner                   => 'root',
          mode                    => '0755',
          selinux_ignore_defaults => true,
          require       => Package['keystone'],
        }->
        file { "/etc/keystone/ssl/private":
          ensure                  => directory,
          owner                   => 'root',
          mode                    => '0755',
          selinux_ignore_defaults => true,
        }->
        file { "/etc/keystone/ssl/public":
          ensure                  => directory,
          owner                   => 'root',
          mode                    => '0755',
          selinux_ignore_defaults => true,
        }->
        file { "/etc/keystone/ssl/private/${::fqdn}.key":
          ensure                  => present,
          owner                   => "keystone",
          source                  => "/etc/ssl/private/ssl-cert-snakeoil.key",
          selinux_ignore_defaults => true,
          mode                    => '0600',
        }->
        file { "/etc/keystone/ssl/public/${::fqdn}.crt":
          ensure                  => present,
          owner                   => "keystone",
          source                  => '/etc/ssl/certs/ssl-cert-snakeoil.pem',
          selinux_ignore_defaults => true,
          mode                    => '0644',
          notify        => Service['httpd'],
        }
      }else{
        ::oci::sslkeypair {'keystone':
          notify_service_name => 'keystone',
        }
      }
      $keystone_key_file = "/etc/keystone/ssl/private/${::fqdn}.pem"
      $keystone_crt_file = "/etc/keystone/ssl/public/${::fqdn}.crt"
    } else {
      $keystone_key_file = undef
      $keystone_crt_file = undef
    }

    if $openstack_release == 'rocky'{
      ensure_resource('file', '/etc/apache2/sites-available/wsgi-keystone.conf', {
        'ensure'  => 'present',
        'content' => '',
        require   => Package['keystone'],
      })
    }

    if $disable_notifications {
      $keystone_notif_transport_url = ''
    } else {
      $keystone_notif_transport_url = os_transport_url({
                                        'transport' => 'rabbit',
                                        'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                        'port'      => '5671',
                                        'username'  => 'keystone',
                                        'password'  => $pass_keystone_messaging,
                                      })
    }
    if $openstack_release == 'rocky'{
      class { '::keystone':
        debug                      => true,
        database_connection        => "mysql+pymysql://keystone:${pass_keystone_db}@${sql_host}/keystonedb",
        database_idle_timeout      => 1800,
        catalog_type               => 'sql',
        admin_token                => $pass_keystone_adminuser,
        admin_password             => $pass_keystone_adminuser,
        enabled                    => true,
        service_name               => 'httpd',
        enable_ssl                 => $use_ssl,
        public_bind_host           => "${::fqdn}",
        admin_bind_host            => "${::fqdn}",
        manage_policyrcd           => true,
        enable_credential_setup    => true,
        credential_key_repository  => '/etc/keystone/credential-keys',
        credential_keys            => { '/etc/keystone/credential-keys/0' => { 'content' => $pass_keystone_credkey1 },
                                        '/etc/keystone/credential-keys/1' => { 'content' => $pass_keystone_credkey2 },
                                      },
        enable_fernet_setup        => true,
        fernet_replace_keys	 => false,
        fernet_key_repository      => '/etc/keystone/fernet-keys',
        fernet_max_active_keys     => 4,
        fernet_keys                => { '/etc/keystone/fernet-keys/0' => { 'content' => $pass_keystone_fernkey1 },
# With fernet_replace_keys => false, if  we don't have the "1" key
# then re-applying puppet is fine, and initial setup is ok too.
#                                      '/etc/keystone/fernet-keys/1' => { 'content' => $pass_keystone_fernkey2 },
                                      },
        token_expiration           => 604800,
        admin_endpoint             => "${keystone_admin_uri}",
        public_endpoint            => "https://${vip_hostname}:${api_port}/identity",
        default_transport_url      => os_transport_url({
          'transport' => 'rabbit',
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => '5671',
          'username'  => 'keystone',
          'password'  => $pass_keystone_messaging,
        }),
        notification_transport_url => $keystone_notif_transport_url,
        notification_driver        => $notification_driver,
        rabbit_use_ssl             => $use_ssl,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        rabbit_ha_queues           => true,
        cache_backend              => 'dogpile.cache.memcached',
        memcache_servers           => $memcached_servers,
        sync_db                    => $do_db_sync,
      }
      class { '::keystone::wsgi::apache':
        bind_host         => $machine_ip,
        admin_bind_host   => $machine_ip,
        ssl               => $use_ssl,
        ssl_key           => "/etc/keystone/ssl/private/${::fqdn}.key",
        ssl_cert          => "/etc/keystone/ssl/public/${::fqdn}.crt",
        workers           => 2,
        access_log_format => '%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"',
      }
    } elsif ($openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
      class { '::keystone':
        database_connection        => "mysql+pymysql://keystone:${pass_keystone_db}@${sql_host}/keystonedb",
        database_idle_timeout      => 1800,
        catalog_type               => 'sql',
        admin_token                => $pass_keystone_adminuser,
        admin_password             => $pass_keystone_adminuser,
        enabled                    => true,
        service_name               => 'keystone',
        enable_ssl                 => $use_ssl,
        public_bind_host           => "${::fqdn}",
        admin_bind_host            => "${::fqdn}",
        manage_policyrcd           => false,
        enable_credential_setup    => true,
        credential_key_repository  => '/etc/keystone/credential-keys',
        credential_keys            => { '/etc/keystone/credential-keys/0' => { 'content' => $pass_keystone_credkey1 },
                                        '/etc/keystone/credential-keys/1' => { 'content' => $pass_keystone_credkey2 },
                                      },
        enable_fernet_setup        => true,
        fernet_replace_keys	 => false,
        fernet_key_repository      => '/etc/keystone/fernet-keys',
        fernet_max_active_keys     => 4,
        fernet_keys                => { '/etc/keystone/fernet-keys/0' => { 'content' => $pass_keystone_fernkey1 },
# With fernet_replace_keys => false, if  we don't have the "1" key
# then re-applying puppet is fine, and initial setup is ok too.
#                                      '/etc/keystone/fernet-keys/1' => { 'content' => $pass_keystone_fernkey2 },
                                      },
        token_expiration           => 604800,
        admin_endpoint             => "${keystone_admin_uri}",
        public_endpoint            => "https://${vip_hostname}:${api_port}/identity",
        default_transport_url      => os_transport_url({
          'transport' => 'rabbit',
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => '5671',
          'username'  => 'keystone',
          'password'  => $pass_keystone_messaging,
        }),
        notification_transport_url => $keystone_notif_transport_url,
        notification_driver        => $notification_driver,
        rabbit_use_ssl             => $use_ssl,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        rabbit_ha_queues           => true,
        cache_backend              => 'dogpile.cache.memcached',
        cache_memcache_servers     => $memcached_servers,
        require                    => Service['haproxy'],
        sync_db                    => $do_db_sync,
      }
    }else{
      class { '::keystone':
        database_connection        => "mysql+pymysql://keystone:${pass_keystone_db}@${sql_host}/keystonedb",
        database_idle_timeout      => 1800,
        catalog_type               => 'sql',
        enabled                    => true,
        service_name               => 'keystone',
        enable_ssl                 => $use_ssl,
        manage_policyrcd           => false,
        enable_credential_setup    => true,
        credential_key_repository  => '/etc/keystone/credential-keys',
        credential_keys            => { '/etc/keystone/credential-keys/0' => { 'content' => $pass_keystone_credkey1 },
                                        '/etc/keystone/credential-keys/1' => { 'content' => $pass_keystone_credkey2 },
                                      },
        enable_fernet_setup        => true,
        fernet_replace_keys	 => false,
        fernet_key_repository      => '/etc/keystone/fernet-keys',
        fernet_max_active_keys     => 4,
        fernet_keys                => { '/etc/keystone/fernet-keys/0' => { 'content' => $pass_keystone_fernkey1 },
# With fernet_replace_keys => false, if  we don't have the "1" key
# then re-applying puppet is fine, and initial setup is ok too.
#                                      '/etc/keystone/fernet-keys/1' => { 'content' => $pass_keystone_fernkey2 },
                                      },
        token_expiration           => 604800,
        admin_endpoint             => "${keystone_admin_uri}",
        public_endpoint            => "${base_url}/identity",
        default_transport_url      => os_transport_url({
          'transport' => 'rabbit',
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => '5671',
          'username'  => 'keystone',
          'password'  => $pass_keystone_messaging,
        }),
        notification_transport_url => $keystone_notif_transport_url,
        notification_driver        => $notification_driver,
        rabbit_use_ssl             => $use_ssl,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        rabbit_ha_queues           => true,
        require                    => Service['haproxy'],
        sync_db                    => $do_db_sync,
      }
      class { '::keystone::cache':
        backend          => 'oslo_cache.memcache_pool',
        enabled          => true,
        memcache_servers => $memcached_servers,
      }
      if $initial_cluster_setup {
        if $is_first_master {
          class { '::keystone::bootstrap':
            password   => $pass_keystone_adminuser,
            email      => $admin_email_address,
            admin_url  => "${base_url}/identity",
            public_url => "${base_url}/identity",
            region     => 'RegionOne',
            require    => Anchor['keystone::install::end'],
            before     => Keystone_role['creator'],
          }
        }else{
          ensure_resource('file', '/etc/keystone/puppet.conf', {
            'ensure'  => 'present',
            'content' => "
[keystone_authtoken]
region_name=RegionOne
auth_url=${base_url}/identity
username=admin
password=${pass_keystone_adminuser}
project_name=admin
interface=public
",
            'mode'    => '0600',
            require   => Anchor['keystone::install::end'],
            before    => Anchor['keystone::config::begin'],
          })
        }
      }
    }
    if($is_first_master and ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri')){
      class { '::keystone::roles::admin':
        email    => $admin_email_address,
        password => $pass_keystone_adminuser,
      }->
      class { '::keystone::endpoint':
        public_url => "${base_url}/identity",
        admin_url  => "${keystone_admin_uri}",
      }
    }
    if $openstack_release == 'rocky'{
      class { '::keystone::disable_admin_token_auth': }
    }
#    class { '::openstack_extras::auth_file':
#      password       => $pass_keystone_adminuser,
#      project_domain => 'default',
#      user_domain    => 'default',
#      auth_url       => "${base_url}/identity/v3/",
#    }
    if($is_first_master){
      keystone_role { 'creator':
        ensure => present,
      }->
      keystone_role { 'member':
        ensure => present,
      }->
      keystone_role { 'SwiftOperator':
        ensure => present,
      }->
      keystone_role { 'ResellerAdmin':
        ensure => present,
      }->
      keystone_role { 'load-balancer_member':
        ensure => present,
      }->
      keystone_role { 'load-balancer_admin':
        ensure => present,
      }->
      keystone_role { 'load-balancer_global_observer':
        ensure => present,
      }->
      keystone_tenant { 'zabbix':
        ensure  => 'present',
        enabled => true,
      }->
      keystone_user { 'zabbix':
        ensure   => 'present',
        enabled  => true,
        password => $pass_zabbix_authtoken,
        email    => $admin_email_address,
        domain   => 'Default',
      }->
      keystone_user_role { 'zabbix@zabbix':
        ensure => 'present',
        roles  => [ 'member', 'creator', 'SwiftOperator', 'ResellerAdmin', 'load-balancer_member', 'load-balancer_global_observer', 'load-balancer_admin', ],
      }
    }
  }

  #######################
  ### Setup Ceph keys ###
  #######################
  if $cluster_has_osds {
    $mon_keyring_path = "/tmp/ceph-${machine_hostname}.keyring"
    class { 'ceph':
      fsid                => $ceph_fsid,
      ensure              => 'present',
      authentication_type => 'cephx',
      mon_initial_members => $ceph_mon_initial_members,
      mon_host            => $ceph_mon_host,
    }->
    ceph::key { 'client.admin':
      secret  => $ceph_admin_key,
      cap_mon => 'allow *',
      cap_osd => 'allow *',
      cap_mgr => 'allow *',
      cap_mds => 'allow',
    }->
    ceph::key { 'client.openstack':
      secret  => $ceph_openstack_key,
      mode    => '0644',
      cap_mon => 'profile rbd',
      cap_osd => 'profile rbd pool=cinder, profile rbd pool=nova, profile rbd pool=glance, profile rbd pool=gnocchi, profile rbd pool=cinderback',
    }->
    ceph::key { 'client.bootstrap-osd':
      secret       => $ceph_bootstrap_osd_key,
      keyring_path => '/var/lib/ceph/bootstrap-osd/ceph.keyring',
      cap_mon      => 'allow profile bootstrap-osd',
    }

    if $cluster_has_mons {
      warning('Cluster has OSD nodes: will not setup MON and MGR on the controller nodes.')
    }else{
      exec { 'create-tmp-keyring':
        command => "/bin/true # comment to satisfy puppet syntax requirements
set -ex

cat > /etc/ceph/ceph.conf<< EOF
[global]
fsid = ${ceph_fsid}
mon_initial_members = ${ceph_mon_initial_members}
mon_host = ${ceph_mon_host}
auth cluster required = cephx
auth service required = cephx
auth client required = cephx
osd journal size = 1024
osd pool default size = 3
osd pool default min size = 2
osd pool default pg num = 333
osd pool default pgp num = 333
osd crush chooseleaf type = 1
EOF

cat > ${mon_keyring_path}<< EOF
[mon.]
    key = ${ceph_mon_key}
    caps mon = \"allow *\"
EOF

mon_data=\$(ceph-mon --cluster ceph --id ${machine_hostname} --show-config-value mon_data)

ceph-authtool ${mon_keyring_path} --import-keyring /etc/ceph/ceph.client.admin.keyring
ceph-authtool ${mon_keyring_path} --import-keyring /etc/ceph/ceph.client.openstack.keyring
ceph-authtool ${mon_keyring_path} --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
chmod 0444 ${mon_keyring_path}
chown ceph:ceph ${mon_keyring_path}

rm -f /tmp/monmap
monmaptool --create --add ${machine_hostname} ${machine_ip} --fsid ${ceph_fsid} /tmp/monmap
MACHINE_IPS=\$(echo ${ceph_mon_host} | tr ',' ' ')
CNT=1
for i in \$(echo ${ceph_mon_initial_members} | tr ',' ' ') ; do
        if [ \$i = ${machine_hostname} ] ; then
                echo -n ''
        else
                IP=\$(echo \$MACHINE_IPS | awk '{print \$'\$CNT'}')
                monmaptool --add \$i \$IP /tmp/monmap
        fi
        CNT=\$((\${CNT} + 1))
done

chown ceph:ceph /tmp/monmap

sudo -u ceph mkdir -p \$mon_data
sudo -u ceph ceph-mon --cluster ceph --mkfs -i ${machine_hostname} --monmap /tmp/monmap --keyring ${mon_keyring_path}
touch \$mon_data/done
",
        unless  => "/bin/true # comment to satisfy puppet syntax requirements
set -ex
mon_data=\$(ceph-mon --cluster ceph --id ${machine_hostname} --show-config-value mon_data) || exit 1
# if ceph-mon fails then the mon is probably not configured yet
test -e \$mon_data/done
",
        require => Ceph::Key['client.bootstrap-osd'],
      }->
      ceph::mon { $machine_hostname:
        keyring     => $mon_keyring_path,
        public_addr => $machine_ip,
      }->
      ceph::mgr { $machine_hostname:
        key        => $ceph_mgr_key,
        inject_key => true,
      }
    }
    if $has_subrole_nova {
      $ceph_pools = ['glance', 'nova', 'cinder', 'gnocchi', 'cinderback']
      ceph::pool { $ceph_pools: }
    }
  }

  ####################
  ### Setup Glance ###
  ####################
  if $has_subrole_glance {
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['glance::install::begin']
    }
    if $use_ssl {
      ::oci::sslkeypair {'glance':
        notify_service_name => 'glance-api',
      }
      $glance_key_file = "/etc/glance/ssl/private/${::fqdn}.pem"
      $glance_crt_file = "/etc/glance/ssl/public/${::fqdn}.crt"
    } else {
      $glance_key_file = undef
      $glance_crt_file = undef
    }

    include ::glance
    include ::glance::client
    if $is_first_master {
      class { '::glance::keystone::auth':
        public_url   => "${base_url}/image",
        internal_url => "${base_url}/image",
        admin_url    => "${base_url}/image",
        password     => $pass_glance_authtoken,
      }
    }

    class { '::glance::api::authtoken':
      password             => $pass_glance_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
    }

    class { '::glance::backend::file':
      multi_store => true,
    }
    if $extswift_use_external {
      notice("Will use an external swift as a backend for Glance")
      if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'){
        $backend_stores = ['swift', 'file']
      }else{
        $backend_stores = ['swift:swift', 'file:file']
      }
      $glance_default_store = 'swift'
    }else{
      case $glance_backend {
        # This is there just in case, but doesn't really work, since
        # there's no mech to sync files between controller nodes.
        'file': {
          if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'){
            $backend_stores = ['file']
          }else{
            $backend_stores = ['file:file']
          }
          $glance_default_store = 'file'
        }
        # This will be in use in priority, if OCI sees some OSD nodes.
        'ceph': {
          class { '::glance::backend::rbd':
            rbd_store_user => 'openstack',
            rbd_store_pool => 'glance',
            multi_store    => true,
          }
          if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'){
            $backend_stores = ['rbd', 'file']
          }else{
            $backend_stores = ['rbd:rbd', 'file:file']
            ::glance::backend::multistore::rbd { 'rbd':
              rbd_store_user        => 'openstack',
              rbd_store_pool        => 'glance',
              rbd_store_chunk_size  => 8,
              rbd_thin_provisioning => true,
              rados_connect_timeout => 0,
              store_description     => 'RBD backend',
            }
          }
          $glance_default_store = 'rbd'
          # make sure ceph pool exists before running Glance API
          Exec['create-glance'] -> Service['glance-api']
        }
        # Used if no OSD nodes are setup, but some swift store are present.
        'swift': {
          if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'){
            $backend_stores = ['swift', 'file']
          }else{
            $backend_stores = ['swift:swift', 'file:file']
          }
          $glance_default_store = 'swift'
          if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'){
            class { '::glance::backend::swift':
              swift_store_user                    => 'services:glance',
              swift_store_key                     => $pass_glance_authtoken,
              swift_store_create_container_on_put => 'True',
              swift_store_auth_address            => "${keystone_auth_uri}/v3",
              swift_store_auth_version            => '3',
              multi_store                         => true,
            }
          }else{
            ::glance::backend::multistore::swift { 'swift':
              swift_store_user                    => 'services:glance',
              swift_store_auth_address            => "${keystone_auth_uri}/v3",
              swift_store_auth_project_domain_id  => 'default',
              swift_store_auth_user_domain_id     => 'default',
              swift_store_key                     => $pass_glance_authtoken,
              swift_store_create_container_on_put => 'True',
              swift_store_auth_version            => '3',
              store_description                   => 'SWIFT store',
              swift_store_config_file             => '/etc/glance/glance-swift.conf',
            }
            glance_api_config {
              "swift/swift_store_cacert": value => $api_endpoint_ca_file;
            }
          }
	  if $self_signed_api_cert {
	    glance_api_config {
              'glance_store/swift_store_auth_insecure': value => true;
            }
	  }else{
	    glance_api_config {
              'glance_store/swift_store_auth_insecure': value => false;
            }
	  }
        }
        'cinder': {
          if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'){
            $backend_stores = ['cinder', 'file']
          }else{
            $backend_stores = ['cinder:cinder', 'file:file']
          }
          class { '::glance::backend::cinder':
            cinder_ca_certificates_file => $api_endpoint_ca_file,
            multi_store                 => true,
          }
        }
      }
    }

    if $openstack_release == 'rocky'{
      class { '::glance::api':
        debug                        => true,
        database_connection          => "mysql+pymysql://glance:${pass_glance_db}@${sql_host}/glancedb?charset=utf8",
        database_idle_timeout        => 1800,
        workers                      => 2,
        use_stderr                   => true,
        stores                       => $backend_stores,
        default_store                => $glance_default_store,
        bind_host                    => $machine_ip,
        cert_file                    => $glance_crt_file,
        key_file                     => $glance_key_file,
        enable_v1_api                => false,
        enable_v2_api                => true,
        sync_db                      => $do_db_sync,
        enable_proxy_headers_parsing => true,
      }
    } else {
      if($openstack_release == 'stein' or $openstack_release == 'train'){
        class { '::glance::api':
          database_connection          => "mysql+pymysql://glance:${pass_glance_db}@${sql_host}/glancedb?charset=utf8",
          database_idle_timeout        => 1800,
          workers                      => 2,
          stores                       => $backend_stores,
          default_store                => $glance_default_store,
          bind_host                    => $machine_ip,
          cert_file                    => $glance_crt_file,
          key_file                     => $glance_key_file,
          enable_v1_api                => false,
          enable_v2_api                => true,
          multi_store                  => true,
          sync_db                      => $do_db_sync,
          enable_proxy_headers_parsing => true,
        }
      }else{
        class { '::glance::api':
          database_connection          => "mysql+pymysql://glance:${pass_glance_db}@${sql_host}/glancedb?charset=utf8",
          database_idle_timeout        => 1800,
          workers                      => 2,
          bind_host                    => $machine_ip,
          cert_file                    => $glance_crt_file,
          key_file                     => $glance_key_file,
          enable_v1_api                => false,
          enable_v2_api                => true,
          sync_db                      => $do_db_sync,
          enable_proxy_headers_parsing => true,
          enabled_backends             => $backend_stores,
          default_backend              => $glance_default_store,
          multi_store                  => true,
        }
      }
      class { '::glance::api::logging':
        debug => true,
      }
    }

    if $extswift_use_external {
      class { '::glance::backend::swift':
        swift_store_user                    => "${extswift_project_name}:${extswift_user_name}",
        swift_store_key                     => $extswift_password,
        swift_store_create_container_on_put => 'True',
        swift_store_auth_address            => "${extswift_auth_url}/v3",
        swift_store_auth_version            => '3',
        multi_store                         => true,
      }
    }

    if $disable_notifications {
      $glance_notif_transport_url = ''
    } else {
      $glance_notif_transport_url = os_transport_url({
                                      'transport' => $messaging_notify_proto,
                                      'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                      'port'      => $messaging_notify_port,
                                      'username'  => 'glance',
                                      'password'  => $pass_glance_messaging,
                                    })
    }
    class { '::glance::notify::rabbitmq':
      default_transport_url      => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'glance',
        'password'  => $pass_glance_messaging,
      }),
      notification_transport_url => $glance_notif_transport_url,
      notification_driver        => $notification_driver,
      rabbit_use_ssl             => $use_ssl,
      rabbit_ha_queues           => true,
      kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
    }

    class { '::glance::config':
      api_config => {
        'DEFAULT/public_endpoint' => { value => "${base_url}/image"},
      }
    }
  }

  ###################
  ### Setup Swift ###
  ###################
  if $has_subrole_swift  {
    if $swiftproxy_hostname == 'none' {
      $swiftproxy_baseurl = $base_url
    } else {
      $swiftproxy_baseurl = "${proto}://${swiftproxy_hostname}"
    }
    if $is_first_master {
      class { '::swift::keystone::auth':
        public_url            => "${swiftproxy_baseurl}/object/v1/AUTH_%(tenant_id)s",
        admin_url             => "${swiftproxy_baseurl}/object",
        internal_url          => "${swiftproxy_baseurl}/object/v1/AUTH_%(tenant_id)s",
        password              => $pass_swift_authtoken,
        operator_roles        => ['admin', 'SwiftOperator', 'ResellerAdmin'],
        configure_s3_endpoint => false,
      }
    }
  }

  ##################
  ### Setup Heat ###
  ##################
  if $has_subrole_heat {
    if $has_subrole_db  and $initial_cluster_setup {
      Class['galera'] -> Anchor['heat::install::begin']
      Exec['galera-size-is-correct'] -> Package['heat dashboard']
#      Class['oci::sql::galera'] -> Package['heat dashboard']
    }
    if $use_ssl {
      oci::sslkeypair { 'heat':
        notify_service_name => 'heat-api',
      }
      $heat_key_file = "/etc/heat/ssl/private/${::fqdn}.pem"
      $heat_crt_file = "/etc/heat/ssl/public/${::fqdn}.crt"
    } else {
      $heat_key_file = undef
      $heat_crt_file = undef
    }
    class { '::heat::keystone::authtoken':
      password             => $pass_heat_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
    }
    if $is_first_master {
      class { '::heat::keystone::auth':
        public_url   => "${base_url}/orchestration-api/v1/%(tenant_id)s",
        internal_url => "${base_url}/orchestration-api/v1/%(tenant_id)s",
        admin_url    => "${base_url}/orchestration-api/v1/%(tenant_id)s",
        password     => $pass_heat_authtoken,
      }
      class { '::heat::keystone::auth_cfn':
        public_url   => "${base_url}/orchestration-cfn/v1",
        internal_url => "${base_url}/orchestration-cfn/v1",
        admin_url    => "${base_url}/orchestration-cfn/v1",
        password     => $pass_heat_authtoken,
      }
    }

    if $disable_notifications {
      $heat_notif_transport_url = ''
    } else {
      $heat_notif_transport_url = os_transport_url({
                                    'transport' => $messaging_notify_proto,
                                    'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                    'port'      => $messaging_notify_port,
                                    'username'  => 'heat',
                                    'password'  => $pass_heat_messaging,
                                  })
    }
    if $openstack_release == 'rocky'{
      class { '::heat':
        debug                        => true,
        default_transport_url        => os_transport_url({
                                          'transport' => $messaging_default_proto,
                                          'hosts'     => fqdn_rotate($all_masters),
                                          'port'      => $messaging_default_port,
                                          'username'  => 'heat',
                                          'password'  => $pass_heat_messaging,
                                        }),
        notification_transport_url   => $heat_notif_transport_url,
        host                         => $machine_hostname,
        rabbit_use_ssl               => $use_ssl,
        rabbit_ha_queues             => true,
        kombu_ssl_ca_certs           => $oci_pki_root_ca_file,
        database_connection          => "mysql+pymysql://heat:${pass_heat_db}@${sql_host}/heatdb?charset=utf8",
        database_idle_timeout        => 1800,
        notification_driver          => $notification_driver,
        sync_db                      => $do_db_sync,
        max_stacks_per_tenant        => 5000,
        enable_proxy_headers_parsing => true,
      }
    } else {
      class { '::heat':
        default_transport_url      => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'heat',
          'password'  => $pass_heat_messaging,
        }),
        notification_transport_url => $heat_notif_transport_url,
        host                       => $machine_hostname,
        rabbit_use_ssl             => $use_ssl,
        rabbit_ha_queues           => true,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        database_connection        => "mysql+pymysql://heat:${pass_heat_db}@${sql_host}/heatdb?charset=utf8",
        database_idle_timeout      => 1800,
        notification_driver        => $notification_driver,
        sync_db                    => $do_db_sync,
        max_stacks_per_tenant      => 5000,
        enable_stack_abandon       => true,
      }
      class { '::heat::logging':
        debug => true,
      }
    }
    if $initial_cluster_setup {
      class { '::heat::keystone::domain':
        domain_password => $pass_heat_keystone_domain,
      }
    }
    class { '::heat::client': }
    class { '::heat::api':
      service_name => 'heat-api',
    }
    class { '::heat::engine':
      auth_encryption_key           => $pass_heat_encryptkey[0,32],
#      heat_metadata_server_url      => "${base_url}:8000/orchestration-cfn",
#      heat_waitcondition_server_url => "${base_url}:8000/orchestration-cfn/v1/waitcondition",
    }
    class { '::heat::api_cfn':
      service_name => 'heat-api-cfn',
    }
    class { '::heat::cron::purge_deleted': }
    package { 'heat dashboard':
      name => 'python3-heat-dashboard',
      ensure => installed,
    }
    heat_config {
      'clients/ca_file':           value => $api_endpoint_ca_file;
      'clients_aodh/ca_file':      value => $api_endpoint_ca_file;
      'clients_barbican/ca_file':  value => $api_endpoint_ca_file;
      'clients_cinder/ca_file':    value => $api_endpoint_ca_file;
      'clients_designate/ca_file': value => $api_endpoint_ca_file;
      'clients_glance/ca_file':    value => $api_endpoint_ca_file;
      'clients_heat/ca_file':      value => $api_endpoint_ca_file;
      'clients_keystone/ca_file':  value => $api_endpoint_ca_file;
      'clients_magnum/ca_file':    value => $api_endpoint_ca_file;
      'clients_manila/ca_file':    value => $api_endpoint_ca_file;
      'clients_mistral/ca_file':   value => $api_endpoint_ca_file;
      'clients_monasca/ca_file':   value => $api_endpoint_ca_file;
      'clients_neutron/ca_file':   value => $api_endpoint_ca_file;
      'clients_nova/ca_file':      value => $api_endpoint_ca_file;
      'clients_octavia/ca_file':   value => $api_endpoint_ca_file;
      'clients_sahara/ca_file':    value => $api_endpoint_ca_file;
      'clients_senlin/ca_file':    value => $api_endpoint_ca_file;
      'clients_swift/ca_file':     value => $api_endpoint_ca_file;
      'clients_trove/ca_file':     value => $api_endpoint_ca_file;
      'clients_zaqar/ca_file':     value => $api_endpoint_ca_file;
    }


  }

  #####################
  ### Setup Horizon ###
  #####################
  if $has_subrole_horizon {
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['horizon::install::begin']
    }
    if $use_ssl {
      file { "/etc/openstack-dashboard/ssl":
        ensure                  => directory,
        owner                   => 'root',
        mode                    => '0755',
        selinux_ignore_defaults => true,
        require                 => Class['::horizon'],
      }->
      file { "/etc/openstack-dashboard/ssl/private":
        ensure                  => directory,
        owner                   => 'root',
        mode                    => '0755',
        selinux_ignore_defaults => true,
      }->
      file { "/etc/openstack-dashboard/ssl/public":
        ensure                  => directory,
        owner                   => 'root',
        mode                    => '0755',
        selinux_ignore_defaults => true,
      }->
      file { "/etc/openstack-dashboard/ssl/private/${::fqdn}.pem":
        ensure                  => present,
        owner                   => "www-data",
        source                  => "/etc/ssl/private/ssl-cert-snakeoil.key",
        selinux_ignore_defaults => true,
        mode                    => '0600',
      }->
      file { "/etc/openstack-dashboard/ssl/public/${::fqdn}.crt":
        ensure                  => present,
        owner                   => "www-data",
        source                  => '/etc/ssl/certs/ssl-cert-snakeoil.pem',
        selinux_ignore_defaults => true,
        mode                    => '0644',
        notify                  => Service['httpd'],
      }

      $horizon_key_file = "/etc/openstack-dashboard/ssl/private/${::fqdn}.pem"
      $horizon_crt_file = "/etc/openstack-dashboard/ssl/public/${::fqdn}.crt"
    } else {
      $horizon_key_file = undef
      $horizon_crt_file = undef
    }

    case $cinder_backup_backend {
      'ceph': {
        $enable_backup             = true
        $create_volume             = true
        $disable_instance_snapshot = false
        $disable_volume_snapshot   = false
      }
      'swift': {
        $enable_backup             = true
        $create_volume             = true
        $disable_instance_snapshot = false
        $disable_volume_snapshot   = false
      }
      'none': {
        $enable_backup             = false
        $create_volume             = false
        $disable_instance_snapshot = true
        $disable_volume_snapshot   = true
      }
    }

    if $openstack_release == 'rocky' or $openstack_release == 'stein'{
      $horizon_log_hanlder = 'file'
    }else{
      $horizon_log_hanlder = 'console'
    }
    if $openstack_release == 'rocky' or $openstack_release == 'stein'{
      class { '::horizon':
        secret_key       => $pass_horizon_secretkey,
        # TODO: Fix this with a more secure stuff, probably the FQDN of the API.
        allowed_hosts    => '*',
        listen_ssl       => true,
        ssl_redirect     => false,
        http_port        => '7080',
        https_port       => '7443',
        horizon_cert     => $horizon_crt_file,
        horizon_key      => $horizon_key_file,
  # Is this $api_endpoint_ca_file or $oci_pki_root_ca_file ?
        horizon_ca       => '/etc/ssl/certs/oci-pki-oci-ca-chain.pem',
        keystone_url     => "${keystone_auth_uri}/v3",
        compress_offline => true,
        cinder_options   => { 'enable_backup' => $enable_backup },
        neutron_options  => { 'enable_ha_router' => true },
        instance_options => {
          'create_volume'             => $create_volume,
          'disable_instance_snapshot' => $disable_instance_snapshot,
          'disable_volume_snapshot'   => $disable_volume_snapshot },
        # Horizon is broken in Train otherwise:
        log_handler      => $horizon_log_hanlder,
      }
    }else{
      class { '::horizon':
        secret_key       => $pass_horizon_secretkey,
        # TODO: Fix this with a more secure stuff, probably the FQDN of the API.
        allowed_hosts    => '*',
        listen_ssl       => true,
        ssl_redirect     => false,
        http_port        => '7080',
        https_port       => '7443',
        horizon_cert     => $horizon_crt_file,
        horizon_key      => $horizon_key_file,
  # Is this $api_endpoint_ca_file or $oci_pki_root_ca_file ?
        horizon_ca       => '/etc/ssl/certs/oci-pki-oci-ca-chain.pem',
        keystone_url     => "${keystone_auth_uri}/v3",
        compress_offline => true,
        cinder_options   => { 'enable_backup' => $enable_backup },
        neutron_options  => { 'enable_ha_router' => true },
        instance_options => {
          'create_volume'             => $create_volume,
          'disable_instance_snapshot' => $disable_instance_snapshot,
          'disable_volume_snapshot'   => $disable_volume_snapshot },
        # Horizon is broken in Train otherwise:
        log_handler      => $horizon_log_hanlder,
        manage_memcache_package => false,
        django_session_engine => 'django.contrib.sessions.backends.cache',
        cache_backend    => 'django.core.cache.backends.memcached.MemcachedCache',
        cache_server_url => "${machine_hostname}:11211",
      }
    }
  }
  ######################
  ### Setup Barbican ###
  ######################
  if $has_subrole_barbican {
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['barbican::install::begin']
    }
    if $use_ssl {
      oci::sslkeypair {'barbican':
        notify_service_name => 'barbican-api',
      }
      $barbican_key_file = "/etc/barbican/ssl/private/${::fqdn}.pem"
      $barbican_crt_file = "/etc/barbican/ssl/public/${::fqdn}.crt"
    } else {
      $barbican_key_file = undef
      $barbican_crt_file = undef
    }
    include ::barbican
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'{
      class { '::barbican::db':
        database_connection   => "mysql+pymysql://barbican:${pass_barbican_db}@${sql_host}/barbicandb?charset=utf8",
        database_idle_timeout => 1800,
      }
    }else{
      class { '::barbican::db':
        database_connection              => "mysql+pymysql://barbican:${pass_barbican_db}@${sql_host}/barbicandb?charset=utf8",
        database_connection_recycle_time => 1800,
      }
    }
    if $is_first_master {
      class { '::barbican::keystone::auth':
        public_url   => "${base_url}/keymanager",
        internal_url => "${base_url}/keymanager",
        admin_url    => "${base_url}/keymanager",
        password     => $pass_barbican_authtoken,
      }
    }
    include ::barbican::quota
    include ::barbican::keystone::notification
    class { '::barbican::api::logging':
      debug => true,
    }
    class { '::barbican::keystone::authtoken':
      password             => $pass_barbican_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
    }

    if $disable_notifications {
      $barbican_notif_transport_url = ''
    } else {
      $barbican_notif_transport_url = os_transport_url({
                                        'transport' => $messaging_default_proto,
                                        'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                        'port'      => $messaging_default_port,
                                        'username'  => 'barbican',
                                        'password'  => $pass_barbican_messaging,
                                      })
    }
    class { '::barbican::api':
      default_transport_url       => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'barbican',
        'password'  => $pass_barbican_messaging,
      }),
      notification_transport_url  => $barbican_notif_transport_url,
      host_href                   => "${base_url}/keymanager",
      auth_strategy               => 'keystone',
      enabled_certificate_plugins => ['simple_certificate'],
      db_auto_create              => false,
      rabbit_use_ssl              => $use_ssl,
      rabbit_ha_queues            => true,
      kombu_ssl_ca_certs          => $oci_pki_root_ca_file,
      sync_db                     => $do_db_sync,
    }
  }

  #######################
  ### Setup Placement ###
  #######################
  if $has_subrole_nova and $openstack_release != 'rocky'{
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['placement::install::begin']
    }
    if $use_ssl {
      oci::sslkeypair {'placement':
        notify_service_name => 'placement-api',
      }
      $placement_key_file = "/etc/placement/ssl/private/${::fqdn}.pem"
      $placement_crt_file = "/etc/placement/ssl/public/${::fqdn}.crt"
    } else {
      $placement_key_file = undef
      $placement_crt_file = undef
    }

    class { '::placement':
      sync_db => $do_db_sync,
    }
  
    if $is_first_master {
      class { '::placement::keystone::auth':
        public_url   => "${base_url}/placement",
        internal_url => "${base_url}/placement",
        admin_url    => "${base_url}/placement",
        password     => $pass_placement_authtoken,
      }
    }
    class { '::placement::keystone::authtoken':
      password             => $pass_placement_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
    }
    class { '::placement::logging':
      debug => true,
    }
    class { '::placement::db':
      database_connection   => "mysql+pymysql://placement:${pass_placement_db}@${sql_host}/placementdb?charset=utf8",
    }
    package { 'python3-osc-placement':
      ensure => 'present',
    }
    package { 'nova-placement-api':
      ensure => 'absent',
    }->
    class { '::placement::api':
      sync_db => true,
    }
  }

  ##################
  ### Setup Nova ###
  ##################
  if $has_subrole_nova {
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['nova::install::begin']
    }
    if $use_ssl {
      oci::sslkeypair {'nova':
        notify_service_name => 'httpd',
      }
      $nova_key_file = "/etc/nova/ssl/private/${::fqdn}.pem"
      $nova_crt_file = "/etc/nova/ssl/public/${::fqdn}.crt"
    } else {
      $nova_key_file = undef
      $nova_crt_file = undef
    }
    if $is_first_master and $initial_cluster_setup {
      class { '::nova::cell_v2::simple_setup':
        require => Exec['nova-db-sync-api'],
      }
      Class['nova'] -> Class['::nova::cell_v2::simple_setup']
    }

    if $is_first_master {
      class { '::nova::keystone::auth':
        public_url   => "${base_url}/compute/v2.1",
        internal_url => "${base_url}/compute/v2.1",
        admin_url    => "${base_url}/compute/v2.1",
        password     => $pass_nova_authtoken,
      }
    }

    if $is_first_master {
      if $openstack_release == 'rocky'{
        class { '::nova::keystone::auth_placement':
          public_url   => "${base_url}/placement",
          internal_url => "${base_url}/placement",
          admin_url    => "${base_url}/placement",
          password     => $pass_placement_authtoken,
        }
      }
    }

    class { '::nova::keystone::authtoken':
      password             => $pass_nova_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
    }

    class { '::nova::logging':
      debug => true,
    }

    if $disable_notifications {
      $nova_notif_transport_url = ''
    } else {
      $nova_notif_transport_url = os_transport_url({
                                    'transport' => $messaging_notify_proto,
                                    'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                    'port'      => $messaging_notify_port,
                                    'username'  => 'nova',
                                    'password'  => $pass_nova_messaging,
                                  })
    }
    if $openstack_release == 'rocky'{
      class { '::nova':
        default_transport_url      => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'nova',
          'password'  => $pass_nova_messaging,
        }),
        notification_transport_url    => $nova_notif_transport_url,
        database_connection           => "mysql+pymysql://nova:${pass_nova_db}@${sql_host}/novadb?charset=utf8",
        api_database_connection       => "mysql+pymysql://novaapi:${pass_nova_apidb}@${sql_host}/novaapidb?charset=utf8",
        placement_database_connection => "mysql+pymysql://placement:${pass_placement_db}@${sql_host}/placementdb?charset=utf8",
        database_idle_timeout         => 1800,
        rabbit_use_ssl                => $use_ssl,
        rabbit_ha_queues              => true,
        kombu_ssl_ca_certs            => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms          => 'PLAIN',
        use_ipv6                      => false,
        glance_api_servers            => "${base_url}/image",
        notification_driver           => $notification_driver,
        notify_on_state_change        => 'vm_and_task_state',
      }
    }else{
      class { '::nova':
        default_transport_url      => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'nova',
          'password'  => $pass_nova_messaging,
        }),
        notification_transport_url    => $nova_notif_transport_url,
        database_connection           => "mysql+pymysql://nova:${pass_nova_db}@${sql_host}/novadb?charset=utf8",
        api_database_connection       => "mysql+pymysql://novaapi:${pass_nova_apidb}@${sql_host}/novaapidb?charset=utf8",
        database_idle_timeout         => 1800,
        rabbit_use_ssl                => $use_ssl,
        rabbit_ha_queues              => true,
        kombu_ssl_ca_certs            => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms          => 'PLAIN',
        glance_api_servers            => "${base_url}/image",
        notification_driver           => $notification_driver,
        notify_on_state_change        => 'vm_and_task_state',
      }
    }

    # The nova-api and nova-api-metadata services.
    if $openstack_release == 'rocky'{
      class { '::nova::api':
        api_bind_address                     => $machine_ip,
        neutron_metadata_proxy_shared_secret => $pass_metadata_proxy_shared_secret,
        metadata_workers                     => 2,
        sync_db_api                          => $do_db_sync,
        service_name                         => 'httpd',
        allow_resize_to_same_host            => true,
        enable_proxy_headers_parsing         => true,
      }

      class { '::nova::wsgi::apache_api':
        bind_host => $machine_ip,
        ssl_key   => $nova_key_file,
        ssl_cert  => $nova_crt_file,
        ssl       => $use_ssl,
        workers   => '2',
      }

      class { '::nova::placement':
        auth_url => $keystone_admin_uri,
        password => $pass_placement_authtoken,
      }
    }else{
      # In >= Stein, use uwsgi, not httpd.
      if $openstack_release == 'stein' {
        class { '::nova::api':
          api_bind_address                     => $machine_ip,
          sync_db                              => $do_db_sync,
          sync_db_api                          => $do_db_sync,
          allow_resize_to_same_host            => true,
          neutron_metadata_proxy_shared_secret => $pass_metadata_proxy_shared_secret,
          enable_proxy_headers_parsing         => true,
        }
      }else{
        class { '::nova::api':
          api_bind_address                     => $machine_ip,
          sync_db                              => $do_db_sync,
          sync_db_api                          => $do_db_sync,
          allow_resize_to_same_host            => true,
          enable_proxy_headers_parsing         => true,
        }
        class { '::nova::metadata':
          neutron_metadata_proxy_shared_secret => $pass_metadata_proxy_shared_secret,
          dhcp_domain                          => $dhcp_domain,
          enable_proxy_headers_parsing         => true,
        }
      }
      nova_config {
        'placement/auth_type':           value => 'password';
        'placement/auth_url':            value => $keystone_admin_uri;
        'placement/password':            value => $pass_placement_authtoken, secret => true;
        'placement/project_domain_name': value => 'Default';
        'placement/project_name':        value => 'services';
        'placement/user_domain_name':    value => 'Default';
        'placement/username':            value => 'placement';
        'placement/region_name':         value => 'RegionOne';
      }
    }

    class { '::nova::client': }
    class { '::nova::conductor':
      workers => 4,
    }
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'{
      class { '::nova::consoleauth': }
    }
    class { '::nova::cron::archive_deleted_rows': }

    if $openstack_release == 'rocky'{
      class { '::nova::scheduler': }
    }else{
      if $openstack_release == 'stein'{
        class { '::nova::scheduler':
          workers => '4',
        }
      } else {
        class { '::nova::scheduler':
          workers                              => '4',
          limit_tenants_to_placement_aggregate => $nova_limit_tenants_to_placement_aggregate,
        }
      }
    }
    if $openstack_release == 'rocky' or $openstack_release == 'stein'{
      nova_config {
        'scheduler/limit_tenants_to_placement_aggregate':   value => $nova_limit_tenants_to_placement_aggregate;
      }
    }

    if $nova_scheduler_prefers_hosts_with_more_ram {
      $ram_weight_multiplier = 1
    }else{
      $ram_weight_multiplier = -1
    }

    if $use_gpu {
      class { '::nova::scheduler::filter':
        scheduler_default_filters => [ 'AvailabilityZoneFilter', 'ComputeFilter', 'ComputeCapabilitiesFilter', 'ImagePropertiesFilter', 'ServerGroupAntiAffinityFilter', 'ServerGroupAffinityFilter', 'PciPassthroughFilter', ],
        ram_weight_multiplier     => $ram_weight_multiplier,
      }
    }else{
      class { '::nova::scheduler::filter':
        ram_weight_multiplier     => $ram_weight_multiplier,
      }
    }
    class { '::nova::vncproxy':
      host          => $machine_ip,
      vncproxy_path => "/novnc/vnc_auto.html",
    }
    class { '::nova::vncproxy::common':
	vncproxy_protocol => 'https',
	vncproxy_host     => "${vip_hostname}",
	vncproxy_path     => "/novnc/vnc_auto.html",
	vncproxy_port     => "443",
    }

    nova_config {
      'neutron/cafile':                   value => $api_endpoint_ca_file;
      'glance/cafile':                    value => $api_endpoint_ca_file;
      'keystone/cafile':                  value => $api_endpoint_ca_file;
      'placement/cafile':                 value => $api_endpoint_ca_file;
      'cinder/cafile':                    value => $api_endpoint_ca_file;
      'service_user/cafile':              value => $api_endpoint_ca_file;
      'DEFAULT/default_ephemeral_format': value => 'ext4';
    }
    if $openstack_release == 'rocky'{
      nova_config {
        'scheduler/workers':         value => '4';
      }
    }

    if $openstack_release == 'rocky'{
      class { '::nova::network::neutron':
        neutron_auth_url      => "${keystone_auth_uri}/v3",
        neutron_url           => "${base_url}/network",
        neutron_password      => $pass_neutron_authtoken,
        default_floating_pool => 'public',
        dhcp_domain           => $dhcp_domain,
      }
      nova_config {
        'neutron/endpoint_override': value => "${base_url}/network";
      }
    }else{
      if $openstack_release == 'stein' or $openstack_release == 'train'{
        class { '::nova::network::neutron':
          neutron_auth_url          => "${keystone_auth_uri}/v3",
          neutron_url               => "${base_url}/network",
          neutron_password          => $pass_neutron_authtoken,
          default_floating_pool     => 'public',
          neutron_endpoint_override => "${base_url}/network",
          dhcp_domain               => $dhcp_domain,
        }
      }else{
        class { '::nova::network::neutron':
          neutron_auth_url          => "${keystone_auth_uri}/v3",
          neutron_password          => $pass_neutron_authtoken,
          default_floating_pool     => 'public',
          neutron_endpoint_override => "${base_url}/network",
        }
      }
    }
    if $use_gpu {
      $gpu_alias_table = $gpu_def.map |Integer $index, String $value| {
        {"vendor_id" => values_at(split($value, '[.]'),1)[0], "product_id" => values_at(split($value, '[.]'),2)[0], "name" => values_at(split($value, '[.]'),0)[0], "device_type" => values_at(split($value, '[.]'),3)[0]}
      }
#      notify { "GPU alias table finished: ${gpu_alias_table}": withpath => true }
      class { '::nova::pci':
        aliases => $gpu_alias_table,
      }
    }
  }
  #####################
  ### Setup Neutron ###
  #####################
  if $has_subrole_neutron {
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['neutron::install::begin']
    }
    if $use_ssl {
      oci::sslkeypair {'neutron':
        notify_service_name => 'neutron-api',
      }
      $neutron_key_file = "/etc/neutron/ssl/private/${::fqdn}.pem"
      $neutron_crt_file = "/etc/neutron/ssl/public/${::fqdn}.crt"
    } else {
      $neutron_key_file = undef
      $neutron_crt_file = undef
    }

    if $is_first_master {
      class { '::neutron::keystone::auth':
        public_url   => "${base_url}/network",
        internal_url => "${base_url}/network",
        admin_url    => "${base_url}/network",
        password     => $pass_neutron_authtoken,
      }
    }

    if $disable_notifications {
      $neutron_notif_transport_url = ''
    } else {
      $neutron_notif_transport_url = os_transport_url({
                                       'transport' => $messaging_notify_proto,
                                       'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                       'port'      => $messaging_notify_port,
                                       'username'  => 'neutron',
                                       'password'  => $pass_neutron_messaging,
                                     })
    }

    $bgp_dr_plugin = $use_dynamic_routing ? {
      true    => 'bgp',
      default => undef,
    }
    $plugins_list = delete_undef_values(['router', 'metering', 'qos', 'trunk', 'segments', $bgp_dr_plugin])

    if $openstack_release == 'rocky'{
      class { '::neutron':
        debug                      => true,
        default_transport_url      => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'neutron',
          'password'  => $pass_neutron_messaging,
        }),
        notification_transport_url => $neutron_notif_transport_url,
        rabbit_use_ssl             => $use_ssl,
        rabbit_ha_queues           => true,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms       => 'PLAIN',
        allow_overlapping_ips      => true,
        core_plugin                => 'ml2',
        service_plugins            => $plugins_list,
        bind_host                  => $machine_ip,
        use_ssl                    => $use_ssl,
        cert_file                  => $neutron_crt_file,
        key_file                   => $neutron_key_file,
        notification_driver        => $notification_driver,
        global_physnet_mtu         => $neutron_global_physnet_mtu,
        dns_domain                 => $neutron_dns_domain,
      }
    }else{
      class { '::neutron':
        default_transport_url      => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'neutron',
          'password'  => $pass_neutron_messaging,
        }),
        notification_transport_url => $neutron_notif_transport_url,
        rabbit_use_ssl             => $use_ssl,
        rabbit_ha_queues           => true,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms       => 'PLAIN',
        allow_overlapping_ips      => true,
        core_plugin                => 'ml2',
        service_plugins            => $plugins_list,
        bind_host                  => $machine_ip,
        use_ssl                    => $use_ssl,
        cert_file                  => $neutron_crt_file,
        key_file                   => $neutron_key_file,
        notification_driver        => $notification_driver,
        global_physnet_mtu         => $neutron_global_physnet_mtu,
        dns_domain                 => $neutron_dns_domain,
      }
      class { '::neutron::logging':
        debug => true,
      }
    }
    class { '::neutron::client': }
    class { '::neutron::keystone::authtoken':
      password             => $pass_neutron_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
    }
    Service<| title == 'neutron-server'|> -> Openstacklib::Service_validation<| title == 'neutron-server' |> -> Neutron_network<||>
    if $use_dynamic_routing {
      $ensure_dr_package = true
    }else{
      $ensure_dr_package = false
    }

    if $neutron_use_dvr {
      $router_distributed         = true
      $enable_distributed_routing = true
      $l3_agent_mode              = 'dvr_snat'
    }else{
      $router_distributed         = false
      $enable_distributed_routing = false
      $l3_agent_mode              = 'legacy'
    }

    if $openstack_release == 'rocky'{
      class { '::neutron::server':
        database_connection              => "mysql+pymysql://neutron:${pass_neutron_db}@${sql_host}/neutrondb?charset=utf8",
        database_idle_timeout            => 1800,
        sync_db                          => $do_db_sync,
        api_workers                      => 2,
        rpc_workers                      => 2,
        validate                         => true,
        router_distributed               => $router_distributed,
        allow_automatic_l3agent_failover => true,
        enable_dvr                       => true,
        ensure_fwaas_package             => true,
        service_providers                => [ 'LOADBALANCERV2:Octavia:neutron_lbaas.drivers.octavia.driver.OctaviaDriver:default', ],
        enable_proxy_headers_parsing     => true,
      }
    }elsif($openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri'){      
      class { '::neutron::server':
        database_connection              => "mysql+pymysql://neutron:${pass_neutron_db}@${sql_host}/neutrondb?charset=utf8",
        database_idle_timeout            => 1800,
        sync_db                          => $do_db_sync,
        api_workers                      => 2,
        rpc_workers                      => 2,
        validate                         => true,
        router_distributed               => $router_distributed,
        allow_automatic_l3agent_failover => true,
        enable_dvr                       => true,
        ensure_fwaas_package             => true,
        ensure_dr_package                => $ensure_dr_package,
        service_providers                => [ 'LOADBALANCERV2:Octavia:neutron_lbaas.drivers.octavia.driver.OctaviaDriver:default', ],
        enable_proxy_headers_parsing     => true,
      }
    }else{
      class { 'neutron::db':
        database_connection              => "mysql+pymysql://neutron:${pass_neutron_db}@${sql_host}/neutrondb?charset=utf8",
        database_connection_recycle_time => 1800,
      }
      class { '::neutron::server':
        sync_db                          => $do_db_sync,
        api_workers                      => 2,
        rpc_workers                      => 2,
        validate                         => true,
        router_distributed               => $router_distributed,
        allow_automatic_l3agent_failover => true,
        enable_dvr                       => true,
        ensure_fwaas_package             => false,
        ensure_dr_package                => $ensure_dr_package,
        service_providers                => [ 'LOADBALANCERV2:Octavia:neutron_lbaas.drivers.octavia.driver.OctaviaDriver:default', ],
        enable_proxy_headers_parsing     => true,
      }
    }
    # If we have dynamic routing agent installed anywhere
    # in the cluster, then we must also install the plugin
    # python files in the controller.
    if $use_dynamic_routing {
      ensure_packages('neutron-dynamic-routing', {
        ensure => installed,
        name   => $::neutron::params::dynamic_routing_package,
        tag    => ['openstack', 'neutron-package'],
      })
      class { '::neutron::server::placement':
        password            => $pass_nova_authtoken,
        auth_type           => 'password',
        username            => 'nova',
        project_domain_name => 'Default',
        project_name        => 'services',
        user_domain_name    => 'Default',
        auth_url            => "${base_url}:${api_port}/identity",
      }
    }else{
      package { 'Python 3 neutron dr agent package':
        name => 'python3-neutron-dynamic-routing',
        ensure => purged,
      }
    }

    neutron_config {
      'nova/cafile':                       value => $api_endpoint_ca_file;
    }

    if $has_subrole_network_node {
      class { '::neutron::agents::ml2::ovs':
        local_ip                   => $vmnet_ip,
        tunnel_types               => ['vxlan'],
        bridge_uplinks             => ['eth0'],
        bridge_mappings            => $bridge_mapping_list,
        extensions                 => '',
        l2_population              => true,
        arp_responder              => true,
        firewall_driver            => 'iptables_hybrid',
        drop_flows_on_start        => false,
        enable_distributed_routing => $enable_distributed_routing,
        manage_vswitch             => false,
      }

      class { '::neutron::agents::l3':
        interface_driver => 'openvswitch',
        debug            => true,
        agent_mode       => $l3_agent_mode,
        ha_enabled       => false,
        extensions       => '',
      }
      neutron_l3_agent_config {
        'DEFAULT/external_network_bridge': value => '';
      }
    }

    if $has_subrole_designate {
      $extension_drivers = 'port_security,qos,dns'
    }else{
      $extension_drivers = 'port_security,qos'
    }

    if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri'){
      class { '::neutron::plugins::ml2':
        type_drivers         => ['flat', 'vxlan', 'vlan', ],
        tenant_network_types => ['flat', 'vxlan', 'vlan', ],
        extension_drivers    => $extension_drivers,
        mechanism_drivers    => 'openvswitch,l2population',
        firewall_driver      => 'iptables_v2',
        flat_networks        => $external_network_list,
        network_vlan_ranges  => $external_network_list,
        vni_ranges           => '1000:9999',
        path_mtu             => $neutron_path_mtu,
      }
    }else{
      class { '::neutron::plugins::ml2':
        type_drivers         => ['flat', 'vxlan', 'vlan', ],
        tenant_network_types => ['flat', 'vxlan', 'vlan', ],
        extension_drivers    => $extension_drivers,
        mechanism_drivers    => 'openvswitch,l2population',
        flat_networks        => $external_network_list,
        network_vlan_ranges  => $external_network_list,
        vni_ranges           => '1000:9999',
        path_mtu             => $neutron_path_mtu,
      }
    }

    if $openstack_release == 'rocky'{
      class { '::neutron::services::lbaas::octavia':
        base_url          => "${base_url}/loadbalance",
        allocates_vip     => true,
        auth_url          => "${base_url}:${api_port}/identity/v3/",
        admin_user        => 'octavia',
        admin_tenant_name => 'services',
        admin_password    => $pass_octavia_authtoken,
      }
    }
    class { '::neutron::server::notifications':
      auth_url => $keystone_admin_uri,
      password => $pass_nova_authtoken,
    }
    if $has_subrole_designate {
      class { '::neutron::designate':
        url                 => "${base_url}/dns/",
        username            => 'admin',
        project_domain_name => 'admin',
        password            => $pass_keystone_adminuser,
        auth_url            => "${base_url}/identity/v3/",
        ptr_zone_email      => $admin_email_address,
      }
    }
  }
  ####################
  ### Setup Cinder ###
  ####################
  if $has_subrole_cinder {
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['cinder::install::begin']
    }
    if $use_ssl {
      oci::sslkeypair {'cinder':
        notify_service_name => 'httpd',
      }
      $cinder_key_file = "/etc/cinder/ssl/private/${::fqdn}.pem"
      $cinder_crt_file = "/etc/cinder/ssl/public/${::fqdn}.crt"
    } else {
      $cinder_key_file = undef
      $cinder_crt_file = undef
    }

    # Cinder endpoints
    if $is_first_master {
      class { '::cinder::keystone::auth':
        public_url_v2   => "${base_url}/volume/v2/%(tenant_id)s",
        internal_url_v2 => "${base_url}/volume/v2/%(tenant_id)s",
        admin_url_v2    => "${base_url}/volume/v2/%(tenant_id)s",
        public_url_v3   => "${base_url}/volume/v3/%(tenant_id)s",
        internal_url_v3 => "${base_url}/volume/v3/%(tenant_id)s",
        admin_url_v3    => "${base_url}/volume/v3/%(tenant_id)s",
        password        => $pass_cinder_authtoken,
      }
    }
    if $disable_notifications {
      $cinder_notif_transport_url = ''
    }else{
      $cinder_notif_transport_url = os_transport_url({
                                    'transport' => $messaging_notify_proto,
                                    'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                    'port'      => $messaging_notify_port,
                                    'username'  => 'cinder',
                                    'password'  => $pass_cinder_messaging,
                                  })
    }

    if $openstack_release == 'rocky'{
      class { '::cinder':
        debug                 => true,
        default_transport_url => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'cinder',
          'password'  => $pass_cinder_messaging,
        }),
        database_connection       => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_host}/cinderdb?charset=utf8",
        database_idle_timeout     => 1800,
        rabbit_use_ssl            => $use_ssl,
        rabbit_ha_queues          => true,
        kombu_ssl_ca_certs        => $oci_pki_root_ca_file,
        storage_availability_zone => $cinder_default_storage_availability_zone,
      }
    }else{
      if($openstack_release == 'stein' or $openstack_release == 'train'){
        class { '::cinder':
          default_transport_url => os_transport_url({
            'transport' => $messaging_default_proto,
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => $messaging_default_port,
            'username'  => 'cinder',
            'password'  => $pass_cinder_messaging,
          }),
          database_connection       => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_host}/cinderdb?charset=utf8",
          database_idle_timeout     => 1800,
          rabbit_use_ssl            => $use_ssl,
          rabbit_ha_queues          => true,
          kombu_ssl_ca_certs        => $oci_pki_root_ca_file,
          storage_availability_zone => $cinder_default_storage_availability_zone,
        }
      }else{
        class { '::cinder':
          default_transport_url => os_transport_url({
            'transport' => $messaging_default_proto,
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => $messaging_default_port,
            'username'  => 'cinder',
            'password'  => $pass_cinder_messaging,
          }),
          database_connection       => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_host}/cinderdb?charset=utf8",
          database_idle_timeout     => 1800,
          rabbit_use_ssl            => $use_ssl,
          rabbit_ha_queues          => true,
          kombu_ssl_ca_certs        => $oci_pki_root_ca_file,
          storage_availability_zone => $cinder_default_storage_availability_zone,
          notification_transport_url => $cinder_notif_transport_url,
          notification_driver        => $notification_driver,
        }
      }
      class { '::cinder::logging':
        debug => true,
      }
    }
    if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'){
      # Starting with puppet-cinder 16.2.0, this has to go in the main init.pp call.
      class { 'cinder::ceilometer':
        notification_transport_url => $cinder_notif_transport_url,
        notification_driver        => $notification_driver,
      }
    }

    class { '::cinder::keystone::authtoken':
      password             => $pass_cinder_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
    }
    if $cluster_has_cinder_volumes{
      $default_volume_type = $cinder_default_volume_type
    }else{
      $default_volume_type = 'CEPH_1'
    }
    if $openstack_release == 'rocky'{
      class { '::cinder::api':
        # Must match what we have in the volumes as enabled backend.
        default_volume_type          => $default_volume_type,
        public_endpoint              => "${base_url}/volume",
        service_name                 => 'httpd',
        keymgr_backend               => 'barbican',
        keymgr_encryption_api_url    => "${base_url}/keymanager",
        keymgr_encryption_auth_url   => "${keystone_auth_uri}/v3",
        sync_db                      => $do_db_sync,
        enable_proxy_headers_parsing => true,
      }
      class { '::cinder::wsgi::apache':
        bind_host => $machine_ip,
        ssl       => $use_ssl,
        ssl_key   => $cinder_key_file,
        ssl_cert  => $cinder_crt_file,
        workers   => 2,
      }
    }else{
      class { '::cinder::api':
        # Must match what we have in the volumes as enabled backend.
        default_volume_type          => $default_volume_type,
        public_endpoint              => "${base_url}/volume",
        service_name                 => 'cinder-api',
        keymgr_backend               => 'barbican',
        keymgr_encryption_api_url    => "${base_url}/keymanager",
        keymgr_encryption_auth_url   => "${keystone_auth_uri}/v3",
        sync_db                      => $do_db_sync,
        enable_proxy_headers_parsing => true,
      }
    }
    class { '::cinder::quota': }
    class { '::cinder::scheduler': }
    class { '::cinder::scheduler::filter': }
    class { '::cinder::cron::db_purge': }
    class { '::cinder::glance':
      glance_api_servers => "${base_url}/image",
    }
    if $extswift_use_external {
      class { '::cinder::backup::swift':
        backup_swift_url             => $extswift_proxy_url,
        backup_swift_auth_url        => "${extswift_auth_url}",
        backup_swift_container       => 'volumebackups',
#        backup_swift_object_size     => 52428800,
#        backup_swift_retry_attempts  => $::os_service_default,
#        backup_swift_retry_backoff   => $::os_service_default,
        backup_swift_user_domain     => $extswift_user_domain_name,
        backup_swift_project_domain  => $extswift_project_domain_name,
        backup_swift_project         => $extswift_project_name,
        backup_compression_algorithm => 'None',
      }
      cinder_config {
        'DEFAULT/backup_swift_user': value => $extswift_user_name;
        'DEFAULT/backup_swift_key':  value => $extswift_password;
        'DEFAULT/backup_swift_auth':         value => 'single_user';
        'DEFAULT/backup_swift_auth_version': value => '3';
      }
    }else{
      cinder_config {
        'DEFAULT/backup_swift_auth':         value => 'per_user';
      }
    }
    cinder_config {
      'DEFAULT/scheduler_default_weighers': value => 'GoodnessWeigher';
    }
  }
  #####################
  ### Setup Gnocchi ###
  #####################
  if $has_subrole_gnocchi {
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['gnocchi::install::begin']
    }
    if $use_ssl {
      if $has_subrole_gnocchi_services {
        oci::sslkeypair {'gnocchi':
          notify_service_name => 'gnocchi-api',
        }
      }else{
        oci::sslkeypair {'gnocchi': }
      }
      $gnocchi_key_file = "/etc/gnocchi/ssl/private/${::fqdn}.pem"
      $gnocchi_crt_file = "/etc/gnocchi/ssl/public/${::fqdn}.crt"
    } else {
      $gnocchi_key_file = undef
      $gnocchi_crt_file = undef
    }
    if $openstack_release == 'rocky'{
      class { '::gnocchi':
        debug                 => true,
        database_connection   => "mysql+pymysql://gnocchi:${pass_gnocchi_db}@${sql_host}/gnocchidb?charset=utf8",
      }
    }else{
      if ($openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
        class { '::gnocchi':
          database_connection   => "mysql+pymysql://gnocchi:${pass_gnocchi_db}@${sql_host}/gnocchidb?charset=utf8",
        }
      } else {
        class { '::gnocchi':
          coordination_url => "zookeeper://${first_master_ip}:2181/",
        }
        class { 'gnocchi::db':
          database_connection => "mysql+pymysql://gnocchi:${pass_gnocchi_db}@${sql_host}/gnocchidb?charset=utf8",
        }
      }
      class { '::gnocchi::logging':
        debug => true,
      }
    }
    if $is_first_master {
      class { '::gnocchi::keystone::auth':
        public_url   => "${base_url}/metric",
        internal_url => "${base_url}/metric",
        admin_url    => "${base_url}/metric",
        password     => $pass_gnocchi_authtoken,
      }
    }
    class { '::gnocchi::keystone::authtoken':
      password             => $pass_gnocchi_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
    }
    class { '::gnocchi::client': }
    if $has_subrole_gnocchi_services {
      class { '::gnocchi::api':
        enabled                      => true,
        service_name                 => 'gnocchi-api',
        sync_db                      => true,
        enable_proxy_headers_parsing => true,
      }
      class { '::gnocchi::metricd':
        workers       => 2,
        cleanup_delay => 20,
      }
    }else{
      package { 'gnocchi api removed':
        name   => 'gnocchi-api',
        ensure => purged,
      }
      package { 'gnocchi metricd removed':
        name   => 'gnocchi-metricd',
        ensure => purged,
      }
    }
    class { '::gnocchi::storage':
      metric_processing_delay => 10,
      coordination_url        => "zookeeper://${first_master_ip}:2181/",
    }
    class { '::gnocchi::storage::ceph':
      ceph_username => 'openstack',
      ceph_keyring  => '/etc/ceph/ceph.client.openstack.keyring',
      manage_cradox => false,
      manage_rados  => true,
    }

    gnocchi_config {
      'database/connection': value => "mysql+pymysql://gnocchi:${pass_gnocchi_db}@${sql_host}/gnocchidb?charset=utf8", secret => true;
    }

    # make sure ceph pool exists before running gnocchi (dbsync & services)
    if $has_subrole_gnocchi_services {
      Exec['create-gnocchi'] -> Exec['gnocchi-db-sync']

      class { '::gnocchi::statsd':
        archive_policy_name => 'high',
        flush_delay         => '100',
        # random datas:
        resource_id         => $pass_gnocchi_rscuuid,
      }
    }else{
      package { 'gnocchi statsd removed':
        name   => 'gnocchi-statsd',
        ensure => purged,
      }
    }
  }
  ###################
  ### Setup Panko ###
  ###################
  if $has_subrole_panko {
    if $use_ssl {
      oci::sslkeypair {'panko':
        notify_service_name => 'panko-api',
      }
      $panko_key_file = "/etc/panko/ssl/private/${::fqdn}.pem"
      $panko_crt_file = "/etc/panko/ssl/public/${::fqdn}.crt"
    } else {
      $panko_key_file = undef
      $panko_crt_file = undef
    }
    include ::panko
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'{
      class { '::panko::db':
        database_connection   => "mysql+pymysql://panko:${pass_panko_db}@${sql_host}/pankodb?charset=utf8",
        database_idle_timeout => 1800,
      }
    }else{
      class { '::panko::db':
        database_connection              => "mysql+pymysql://panko:${pass_panko_db}@${sql_host}/pankodb?charset=utf8",
        database_connection_recycle_time => 1800,
      }
    }
    if $is_first_master {
      class { '::panko::keystone::auth':
        public_url   => "${base_url}/event",
        internal_url => "${base_url}/event",
        admin_url    => "${base_url}/event",
        password     => $pass_panko_authtoken,
      }
    }
    class { '::panko::keystone::authtoken':
      password             => $pass_panko_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
    }
    class { '::panko::api':
      sync_db                      => $do_db_sync,
      enabled                      => true,
      enable_proxy_headers_parsing => true,
    }
    if $openstack_release != 'rocky'{
      class { '::panko::logging':
        debug => true,
      }
    }
  }

  ########################
  ### Setup Ceilometer ###
  ########################
  if $has_subrole_ceilometer {
    if $openstack_release == 'rocky'{
      class { '::ceilometer':
        debug                      => true,
        telemetry_secret           => $pass_ceilometer_telemetry,
        default_transport_url      => os_transport_url({
          'transport' => 'rabbit',
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => '5671',
          'username'  => 'ceilometer',
          'password'  => $pass_ceilometer_messaging,
        }),
        notification_transport_url => os_transport_url({
          'transport' => 'rabbit',
          'hosts'     => fqdn_rotate($notif_rabbit_servers),
          'port'      => '5671',
          'username'  => 'ceilometer',
          'password'  => $pass_ceilometer_messaging,
        }),
        rabbit_use_ssl             => $use_ssl,
        rabbit_ha_queues           => true,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms       => 'PLAIN',
        memcache_servers           => $memcached_servers,
      }
    }else{
      class { '::ceilometer':
        telemetry_secret           => $pass_ceilometer_telemetry,
        default_transport_url      => os_transport_url({
          'transport' => 'rabbit',
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => '5671',
          'username'  => 'ceilometer',
          'password'  => $pass_ceilometer_messaging,
        }),
        notification_transport_url => os_transport_url({
          'transport' => 'rabbit',
          'hosts'     => fqdn_rotate($notif_rabbit_servers),
          'port'      => '5671',
          'username'  => 'ceilometer',
          'password'  => $pass_ceilometer_messaging,
        }),
        rabbit_use_ssl             => $use_ssl,
        rabbit_ha_queues           => true,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms       => 'PLAIN',
        memcache_servers           => $memcached_servers,
      }
      class { '::ceilometer::logging':
        debug => true,
      }

    }

    # We don't need the endpoint, but we need it as this
    # configures the user and pass for keystone_authtoken.
    class { '::ceilometer::keystone::auth':
      password           => $pass_ceilometer_authtoken,
      configure_endpoint => false,
    }

    if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'){
      class { '::ceilometer::keystone::authtoken':
        password             => $pass_ceilometer_authtoken,
        auth_url             => $keystone_admin_uri,
        www_authenticate_uri => $keystone_auth_uri,
        memcached_servers    => $memcached_servers,
        cafile               => $api_endpoint_ca_file,
      }
    }
    include ::ceilometer::db::sync
    Class['ceilometer::agent::auth'] -> Exec['ceilometer-upgrade']
    if $is_first_master {
      Class['gnocchi::keystone::auth']    -> Exec['ceilometer-upgrade']
      Class['ceilometer::keystone::auth'] -> Exec['ceilometer-upgrade']
      if $has_subrole_gnocchi_services {
        Service['gnocchi-api']              -> Exec['ceilometer-upgrade']
      }
    }

    $sample_pipeline_publishers = ['gnocchi://']
    $event_pipeline_publishers = ['gnocchi://', 'panko://']

    class { '::ceilometer::agent::notification':
      notification_workers      => '2',
      manage_pipeline           => true,
      pipeline_publishers       => $sample_pipeline_publishers,
      manage_event_pipeline     => true,
      event_pipeline_publishers => $event_pipeline_publishers,
    }
    if $openstack_release == 'rocky'{
      class { '::ceilometer::agent::central':
        coordination_url => "zookeeper://${first_master_ip}:2181/",
      }
    }else{
      class { '::ceilometer::agent::polling':
        central_namespace => true,
        compute_namespace => false,
        ipmi_namespace    => false,
        manage_polling    => true,
        # That's each 5 minutes:
        polling_interval  => 300,
        coordination_url  => "zookeeper://${first_master_ip}:2181/",
      }
    }
    class { '::ceilometer::agent::auth':
      auth_password => $pass_ceilometer_authtoken,
      auth_url      => $keystone_auth_uri,
      auth_cacert   => $api_endpoint_ca_file,
    }
    # Looks like the above sets ca_file instead of cafile.
    if $openstack_release == 'rocky'{
      ceilometer_config {
        'service_credentials/cafile': value => $api_endpoint_ca_file;
      }
    }
    ceilometer_config {
      'service_credentials/www_authenticate_uri': value => "${keystone_auth_uri}";
    }
  }
  ########################
  ### Setup CloudKitty ###
  ########################
  if $has_subrole_cloudkitty {
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['cloudkitty::install::begin']
      Exec['galera-size-is-correct'] -> Package['cloudkitty dashboard']
#      Class['oci::sql::galera'] -> Package['cloudkitty dashboard']
    }
    if $use_ssl {
      oci::sslkeypair {'cloudkitty':
        notify_service_name => 'cloudkitty-api',
      }
      $cloudkitty_key_file = "/etc/cloudkitty/ssl/private/${::fqdn}.pem"
      $cloudkitty_crt_file = "/etc/cloudkitty/ssl/public/${::fqdn}.crt"
    } else {
      $cloudkitty_key_file = undef
      $cloudkitty_crt_file = undef
    }
    # Note: ::cloudkitty::db *MUST* be declared before ::cloudkitty,
    # because of the include in the ::cloudkitty class.
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'{
      class { '::cloudkitty::db':
        database_connection   => "mysql+pymysql://cloudkitty:${pass_cloudkitty_db}@${sql_host}/cloudkittydb?charset=utf8",
        database_idle_timeout => 1800,
      }
    }else{
      class { '::cloudkitty::db':
        database_connection              => "mysql+pymysql://cloudkitty:${pass_cloudkitty_db}@${sql_host}/cloudkittydb?charset=utf8",
        database_connection_recycle_time => 1800,
      }
    }
    if $disable_notifications {
      $cloudkitty_notif_transport_url = ''
    } else {
      $cloudkitty_notif_transport_url = os_transport_url({
                                          'transport' => 'rabbit',
                                          'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                          'port'      => '5671',
                                          'username'  => 'cloudkitty',
                                          'password'  => $pass_cloudkitty_messaging,
                                        })
    }
    if $openstack_release == 'rocky'{
      class { '::cloudkitty':
        default_transport_url      => os_transport_url({
                                                        'transport' => 'rabbit',
                                                        'hosts'     => fqdn_rotate($all_masters),
                                                        'port'      => '5671',
                                                        'username'  => 'cloudkitty',
                                                        'password'  => $pass_cloudkitty_messaging,
                                                      }),
        notification_transport_url => $cloudkitty_notif_transport_url,
        rabbit_use_ssl             => $use_ssl,
        rabbit_ha_queues           => true,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms       => 'PLAIN',
        storage_backend            => 'sqlalchemy',
        tenant_fetcher_backend     => 'keystone',
        auth_section               => 'keystone_authtoken',
        host                       => $machine_hostname,
      }
    }else{
      if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
        class { '::cloudkitty':
          default_transport_url      => os_transport_url({
                                                          'transport' => 'rabbit',
                                                          'hosts'     => fqdn_rotate($all_masters),
                                                          'port'      => '5671',
                                                          'username'  => 'cloudkitty',
                                                          'password'  => $pass_cloudkitty_messaging,
                                                        }),
          notification_transport_url => $cloudkitty_notif_transport_url,
          rabbit_use_ssl             => $use_ssl,
          rabbit_ha_queues           => true,
          kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
          amqp_sasl_mechanisms       => 'PLAIN',
          storage_backend            => 'sqlalchemy',
          storage_version            => '1',
          tenant_fetcher_backend     => 'keystone',
          auth_section               => 'keystone_authtoken',
          host                       => $machine_hostname,
        }
      }else{
        class { '::cloudkitty':
          default_transport_url      => os_transport_url({
                                                          'transport' => 'rabbit',
                                                          'hosts'     => fqdn_rotate($all_masters),
                                                          'port'      => '5671',
                                                          'username'  => 'cloudkitty',
                                                          'password'  => $pass_cloudkitty_messaging,
                                                        }),
          notification_transport_url => $cloudkitty_notif_transport_url,
          rabbit_use_ssl             => $use_ssl,
          rabbit_ha_queues           => true,
          kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
          amqp_sasl_mechanisms       => 'PLAIN',
          storage_backend            => 'sqlalchemy',
          storage_version            => '1',
          fetcher_backend            => 'gnocchi',
          auth_section               => 'ociadmin_authtoken',
          host                       => $machine_hostname,
        }
        class { '::cloudkitty::logging':
          debug => true,
        }
      }
    }
    if $is_first_master {
      class { '::cloudkitty::keystone::auth':
        public_url   => "${base_url}/rating",
        internal_url => "${base_url}/rating",
        admin_url    => "${base_url}/rating",
        password     => $pass_cloudkitty_authtoken,
      }
    }
    class { '::cloudkitty::keystone::authtoken':
      password             => $pass_cloudkitty_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
    }
    class { '::cloudkitty::client': }
    class { '::cloudkitty::api':
      sync_db      => $do_db_sync,
    }

    cloudkitty_config {
      'keystone_fetcher/cafile':       value => $api_endpoint_ca_file;
      'gnocchi_collector/cafile':      value => $api_endpoint_ca_file;
      'coordination/backend_url':      value => "zookeeper://${first_master_ip}:2181/";
      'orchestrator/coordination_url': value => "zookeeper://${first_master_ip}:2181/";
      'fetcher_gnocchi/cafile':        value => $api_endpoint_ca_file;
      'fetcher_gnocchi/auth_section':  value => 'keystone_authtoken';
    }

    # This is needed for the Keystone fetcher to list users
    cloudkitty_config {
      'ociadmin_authtoken/auth_type':            value => 'password';
      'ociadmin_authtoken/auth_url':             value => $keystone_admin_uri;
      'ociadmin_authtoken/project_name':         value => 'admin';
      'ociadmin_authtoken/project_domain_name':  value => 'Default';
      'ociadmin_authtoken/username':             value => 'admin';
      'ociadmin_authtoken/user_domain_name':     value => 'Default';
      'ociadmin_authtoken/password':             value => $pass_keystone_adminuser;
      'ociadmin_authtoken/www_authenticate_uri': value => $keystone_auth_uri;
      'ociadmin_authtoken/cafile':               value => $api_endpoint_ca_file;
      'ociadmin_authtoken/memcached_servers':    value => $memcached_string;
    }
    if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
      cloudkitty_config {
        'fetcher/backend': value => 'gnocchi';
      }
    }

    class { '::cloudkitty::processor':
      auth_section => 'ociadmin_authtoken',
    }

    package { 'cloudkitty dashboard':
      name => 'python3-cloudkitty-dashboard',
      ensure => installed,
    }
  }

  ##################
  ### Setup Aodh ###
  ##################
  if $has_subrole_aodh {
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['aodh::install::begin']
    }
    if $use_ssl {
      oci::sslkeypair {'aodh':
        notify_service_name => 'aodh-api',
      }
      $aodh_key_file = "/etc/aodh/ssl/private/${::fqdn}.pem"
      $aodh_crt_file = "/etc/aodh/ssl/public/${::fqdn}.crt"
    } else {
      $aodh_key_file = undef
      $aodh_crt_file = undef
    }
    if $disable_notifications {
      $aodh_notif_transport_url = ''
    }else{
      $aodh_notif_transport_url = os_transport_url({
                                  'transport' => $messaging_notify_proto,
                                  'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                  'port'      => $messaging_notify_port,
                                  'username'  => 'aodh',
                                  'password'  => $pass_aodh_messaging,
                                })
    }
    if $openstack_release == 'rocky'{
      class { '::aodh':
        debug                      => true,
        default_transport_url      => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'aodh',
          'password'  => $pass_aodh_messaging,
        }),
        notification_transport_url => $aodh_notif_transport_url,
        rabbit_use_ssl             => $use_ssl,
        rabbit_ha_queues           => true,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms       => 'PLAIN',
        database_connection        => "mysql+pymysql://aodh:${pass_aodh_db}@${sql_host}/aodhdb?charset=utf8",
        database_idle_timeout      => 1800,
        notification_driver        => $notification_driver,
      }
    }else{
      class { '::aodh':
        default_transport_url      => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'aodh',
          'password'  => $pass_aodh_messaging,
        }),
        notification_transport_url => $aodh_notif_transport_url,
        rabbit_use_ssl             => $use_ssl,
        rabbit_ha_queues           => true,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms       => 'PLAIN',
        database_connection        => "mysql+pymysql://aodh:${pass_aodh_db}@${sql_host}/aodhdb?charset=utf8",
        database_idle_timeout      => 1800,
        notification_driver        => $notification_driver,
      }
      class { '::aodh::logging':
        debug => true,
      }
    }
    if $is_first_master {
      class { '::aodh::keystone::auth':
        public_url   => "${base_url}/alarm",
        internal_url => "${base_url}/alarm",
        admin_url    => "${base_url}/alarm",
        password     => $pass_aodh_authtoken,
      }
    }
    class { '::aodh::keystone::authtoken':
      password             => $pass_aodh_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
    }
    class { '::aodh::api':
      enabled                      => true,
      service_name                 => 'aodh-api',
      sync_db                      => $do_db_sync,
      enable_proxy_headers_parsing => true,
    }
    class { '::aodh::auth':
      auth_url      => $keystone_auth_uri,
      auth_cacert   => $api_endpoint_ca_file,
      auth_password => $pass_aodh_authtoken,
    }
    class { '::aodh::client': }
    class { '::aodh::notifier': }
    class { '::aodh::listener': }
    class { '::aodh::evaluator':
      evaluation_interval => 10,
    }
  }
  #####################
  ### Setup Octavia ###
  #####################
  if $has_subrole_octavia {
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['octavia::install::begin']
      Exec['galera-size-is-correct'] -> Package['octavia dashboard']
#      Class['oci::sql::galera'] -> Package['octavia dashboard']
    }
    if $use_ssl {
      oci::sslkeypair {'octavia':
        notify_service_name => 'octavia-api',
      }
      $octavia_key_file = "/etc/octavia/ssl/private/${::fqdn}.pem"
      $octavia_crt_file = "/etc/octavia/ssl/public/${::fqdn}.crt"
    } else {
      $octavia_key_file = undef
      $octavia_crt_file = undef
    }
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'{
      class { '::octavia::db':
        database_connection   => "mysql+pymysql://octavia:${pass_octavia_db}@${sql_host}/octaviadb?charset=utf8",
        database_idle_timeout => 1800,
      }
    }else{
      class { '::octavia::db':
        database_connection              => "mysql+pymysql://octavia:${pass_octavia_db}@${sql_host}/octaviadb?charset=utf8",
        database_connection_recycle_time => 1800,
      }
    }
    if $disable_notifications {
      $octavia_notif_transport_url = ''
    }else{
      $octavia_notif_transport_url = os_transport_url({
                                       'transport' => $messaging_notify_proto,
                                       'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                       'port'      => $messaging_notify_port,
                                       'username'  => 'octavia',
                                       'password'  => $pass_octavia_messaging,
                                     })
    }
    class { '::octavia':
      default_transport_url      => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'octavia',
        'password'  => $pass_octavia_messaging,
      }),
      notification_transport_url => $octavia_notif_transport_url,
      rabbit_use_ssl             => $use_ssl,
      rabbit_ha_queues           => true,
      kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
      amqp_sasl_mechanisms       => 'PLAIN',
      notification_driver        => $notification_driver,
    }
    if $is_first_master {
      class { '::octavia::keystone::auth':
        public_url   => "${base_url}/loadbalance",
        internal_url => "${base_url}/loadbalance",
        admin_url    => "${base_url}/loadbalance",
        password     => $pass_octavia_authtoken,
      }
    }
    class { '::octavia::keystone::authtoken':
      password             => $pass_octavia_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
    }
    class { '::octavia::service_auth':
      auth_url            => $keystone_admin_uri,
      username            => 'octavia',
      project_name        => 'services',
      password            => $pass_octavia_authtoken,
      user_domain_name    => 'default',
      project_domain_name => 'default',
      auth_type           => 'password',
    }

    class { '::octavia::api':
      enabled      => true,
      sync_db      => $do_db_sync,
    }

    octavia_config {
      'certificates/ca_certificates_file':      value => '/etc/octavia/certs/server_ca.cert.pem';
      'certificates/ca_certificate':            value => '/etc/octavia/certs/server_ca.cert.pem';
      'certificates/ca_private_key':            value => '/etc/octavia/certs/server_ca.key.pem';
      'certificates/ca_private_key_passphrase': value => 'octavia';
      'controller_worker/client_ca':            value => '/etc/octavia/certs/client_ca.cert.pem';
      'haproxy_amphora/client_cert':            value => '/etc/octavia/certs/client.cert-and-key.pem';
      'haproxy_amphora/server_ca':              value => '/etc/octavia/certs/server_ca.cert.pem';
      'glance/ca_certificates_file':            value => $api_endpoint_ca_file;
      'neutron/ca_certificates_file':           value => $api_endpoint_ca_file;
      'nova/ca_certificates_file':              value => $api_endpoint_ca_file;
      'service_auth/cafile':                    value => $api_endpoint_ca_file;
    }

    class { '::octavia::health_manager':
      heartbeat_key => $pass_octavia_heatbeatkey,
      ip            => $machine_ip,
    }
    $octavia_controller_ip_port_list = join([join($all_masters_ip,':5555,'), ':5555'],'')
    #$all_masters_ip.map |Integer $index, String $value| {
    #  [ "${value}:555" ]
    #}
    # Flavor creation is currently broken in Victoria
    if ($initial_cluster_setup and ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri')){
      $manage_nova_flavor = true
    }else{
      $manage_nova_flavor = false
    }
    if $openstack_release == 'rocky' {
      class { '::octavia::worker':
        amp_image_tag         => 'amphora',
        loadbalancer_topology => 'ACTIVE_STANDBY',
        amp_secgroup_list     => $amp_secgroup_list,
        amp_boot_network_list => $amp_boot_network_list,
        manage_nova_flavor    => $manage_nova_flavor,
        # The default is not enough, haproxy logs may fill the HDD quickly.
        nova_flavor_config    => { 'disk' => '4', 'ram' => '4096' },
      }
      # puppet-openstack in Rocky doesn't have a way to configure controller_ip_port_list
      octavia_config {
        'health_manager/controller_ip_port_list': value => $octavia_controller_ip_port_list;
      }
    }else{
      # Take care: ::octavia::controller *MUST* be declared before
      # the ::octavia::worker, otherwise it's called twice and
      # compilation of the catalogue will failt, because the
      # ::octavia::worker will include it.
      class { '::octavia::controller':
        amp_image_tag           => 'amphora',
        loadbalancer_topology   => 'ACTIVE_STANDBY',
        amp_secgroup_list       => $amp_secgroup_list,
        amp_boot_network_list   => $amp_boot_network_list,
        controller_ip_port_list => $octavia_controller_ip_port_list,
      }
      class { '::octavia::worker':
        manage_nova_flavor    => $manage_nova_flavor,
        # The default is not enough, haproxy logs may fill the HDD quickly.
        nova_flavor_config    => { 'disk' => '4', 'ram' => '2048' },
      }
    }
    if $openstack_release == 'rocky' {
      octavia_config {
        'nova/enable_anti_affinity': value => 'true';
      }
    }else{
      class { '::octavia::nova':
        enable_anti_affinity => 'true',
      }
    }
    package { 'octavia dashboard':
      name => 'python3-octavia-dashboard',
      ensure => installed,
    }
  }
  ####################
  ### Setup Magnum ###
  ####################
  if $has_subrole_magnum and $install_magnum{
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['magnum::install::begin']
      Exec['galera-size-is-correct'] -> Package['magnum ui']
    }
    if $use_ssl {
      oci::sslkeypair {'magnum':
        notify_service_name => 'magnum-api',
      }
      $magnum_key_file = "/etc/magnum/ssl/private/${::fqdn}.pem"
      $magnum_crt_file = "/etc/magnum/ssl/public/${::fqdn}.crt"
    } else {
      $magnum_key_file = undef
      $magnum_crt_file = undef
    }
    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'{
      class { '::magnum::db':
        database_connection   => "mysql+pymysql://magnum:${pass_magnum_db}@${sql_host}/magnumdb?charset=utf8",
        database_idle_timeout => 1800,
      }
    }else{
      class { '::magnum::db':
        database_connection              => "mysql+pymysql://magnum:${pass_magnum_db}@${sql_host}/magnumdb?charset=utf8",
        database_connection_recycle_time => 1800,
      }
    }
    if $initial_cluster_setup {
      if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'{
        class { '::magnum::keystone::domain':
          cluster_user_trust => 'true',
          domain_admin       => 'magnum_admin',
          domain_admin_email => 'root@localhost',
          domain_password    => $pass_magnum_domain,
          roles              => 'admin',
        }
      }else{
        class { '::magnum::keystone::domain':
          cluster_user_trust   => 'true',
          domain_admin         => 'magnum_admin',
          domain_admin_email   => 'root@localhost',
          domain_password      => $pass_magnum_domain,
          roles                => 'admin',
#          keystone_region_name => 'RegionOne',
        }
      }
    }
    class { '::magnum::keystone::authtoken':
      password             => $pass_magnum_authtoken,
      auth_url             => "${keystone_admin_uri}/v3",
      www_authenticate_uri => "${keystone_auth_uri}/v3",
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
      region_name          => 'RegionOne',
    }
    class { '::magnum::api':
      service_name  => 'magnum-api',
      host          => $machine_ip,
      enabled_ssl   => $use_ssl,
      ssl_cert_file => $magnum_crt_file,
      ssl_key_file  => $magnum_key_file,
      sync_db       => $do_db_sync,
    }

    if $is_first_master {
      class { '::magnum::keystone::auth':
        public_url   => "${base_url}/containers/v1",
        internal_url => "${base_url}/containers/v1",
        admin_url    => "${base_url}/containers/v1",
        password     => $pass_magnum_authtoken,
      }
    }

    if $disable_notifications {
      $magnum_notif_transport_url = ''
    }else{
      $magnum_notif_transport_url = os_transport_url({
                                      'transport' => $messaging_notify_proto,
                                      'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                      'port'      => $messaging_notify_port,
                                      'username'  => 'magnum',
                                      'password'  => $pass_magnum_messaging,
                                    })
    }
    if $openstack_release == 'rocky' {
      class { '::magnum':
        default_transport_url      => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'magnum',
          'password'  => $pass_magnum_messaging,
        }),
        notification_transport_url => $magnum_notif_transport_url,
        rabbit_use_ssl             => $use_ssl,
        rabbit_ha_queues           => true,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        notification_driver        => $notification_driver,
      }
    }else{

/*      $initial_cluster = $all_masters.map |Integer $indexetcd, String $valueetcd| {
        "${valueetcd}=http://${valueetcd}:2380"
      }

      user { 'etcd':
        ensure   => present,
        system   => true,
      }->
      file { "/etc/etcd":
        ensure                  => directory,
        owner                   => 'root',
        mode                    => '0755',
        selinux_ignore_defaults => true,
      }->
      file { "/etc/etcd/ssl":
        ensure                  => directory,
        owner                   => 'root',
        mode                    => '0755',
        selinux_ignore_defaults => true,
      }->
      file { "/etc/etcd/ssl/private":
        ensure                  => directory,
        owner                   => 'root',
        mode                    => '0755',
        selinux_ignore_defaults => true,
      }->
      file { "/etc/etcd/ssl/public":
        ensure                  => directory,
        owner                   => 'root',
        mode                    => '0755',
        selinux_ignore_defaults => true,
      }->
      file { "/etc/etcd/ssl/private/${::fqdn}.pem":
        ensure                  => present,
        owner                   => 'etcd',
        source                  => "/etc/ssl/private/ssl-cert-snakeoil.key",
        selinux_ignore_defaults => true,
        mode                    => '0600',
      }->
      file { "/etc/etcd/ssl/public/${::fqdn}.crt":
        ensure                  => present,
        owner                   => 'etcd',
        source                  => '/etc/ssl/certs/ssl-cert-snakeoil.pem',
        selinux_ignore_defaults => true,
        mode                    => '0644',
        notify                  => Service['etcd'],
      }

      # Etcd is only available in Buster, so we only set it up there...
      $etcd_key_file = "/etc/etcd/ssl/private/${::fqdn}.pem"
      $etcd_crt_file = "/etc/etcd/ssl/public/${::fqdn}.crt"

      file { '/var/lib/etcd':
        ensure                  => present,
        owner                   => 'etcd',
        mode                    => '0755',
        selinux_ignore_defaults => true,
      }->
      file { '/var/lib/etcd/default':
        ensure                  => present,
        owner                   => 'etcd',
        mode                    => '0755',
        selinux_ignore_defaults => true,
      }->
      class { 'etcd':
        ensure                      => 'present',
        etcd_name                   => $machine_hostname,
        # clients
        listen_client_urls          => "http://${machine_ip}:2379",
        advertise_client_urls       => "http://${machine_hostname}:2379",
        # clients ssl
#        cert_file                   => $etcd_crt_file,
#        key_file                    => $etcd_key_file,
        # authorize clients
#        client_cert_auth            => true,
        # verify clients certificates
        trusted_ca_file             => '/etc/ssl/certs/oci-pki-oci-ca-chain.pem',
        # cluster
        listen_peer_urls            => "http://${machine_ip}:2380",
        initial_advertise_peer_urls => "http://${machine_hostname}:2380",
        initial_cluster             => $initial_cluster,
#            [ "${::hostname}=http://${::fqdn}:2380",
#            'infra1=http://infra1.domain.net:2380',
#            'infra2=http://infra2.domain.net:2380'],
        # peers ssl
#        peer_cert_file              => $etcd_crt_file,
#        peer_key_file               => $etcd_key_file,
        # authorize peers
#        peer_client_cert_auth       => true,
        # verify peers certificates
#        peer_trusted_ca_file        => '/etc/ssl/certs/oci-pki-oci-ca-chain.pem',
        debug     => true,
        require                    => Service['haproxy'],
      }->
      class { '::oci::etcddiscovery':
        prefix_url             => "${base_url}/etcd-discovery",
        web_service_address    => "${machine_ip}:8087",
        etcd_endpoint_location => "http://${machine_hostname}:2379",
        require                => Service['haproxy'],
      }
*/
      class { '::magnum':
        default_transport_url      => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'magnum',
          'password'  => $pass_magnum_messaging,
        }),
        notification_transport_url => $magnum_notif_transport_url,
        rabbit_use_ssl             => $use_ssl,
        rabbit_ha_queues           => true,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms       => 'PLAIN',
        notification_driver        => $notification_driver,
      }
    }

    class { '::magnum::conductor':
      workers => '4',
    }

    class { '::magnum::client': }

    class { '::magnum::clients::barbican':
      region_name => 'RegionOne',
    }
    class { '::magnum::clients::cinder':
      region_name => 'RegionOne',
    }
    class { '::magnum::clients::glance':
      region_name => 'RegionOne',
      ca_file     => $api_endpoint_ca_file,
    }
    class { '::magnum::clients::heat':
      region_name => 'RegionOne',
      ca_file     => $api_endpoint_ca_file,
    }
    class { '::magnum::clients::magnum':
      region_name => 'RegionOne',
    }
    class { '::magnum::clients::neutron':
      region_name => 'RegionOne',
      ca_file     => $api_endpoint_ca_file,
    }
    class { '::magnum::clients::nova':
      region_name => 'RegionOne',
      ca_file     => $api_endpoint_ca_file,
    }

    class { '::magnum::certificates':
      cert_manager_type => 'barbican'
#      cert_manager_type => 'local'
    }

    if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'{
      magnum_config {
        'trust/trustee_keystone_region_name':             value => 'RegionOne';
      }
    }
    magnum_config {
      'keystone_auth/auth_section': value => 'keystone_authtoken';
      'octavia_client/region_name': value => 'RegionOne';
      'octavia_client/ca_file':     value => $api_endpoint_ca_file;
      'cluster/etcd_discovery_service_endpoint_format': value => "${base_url}/etcd-discovery/new?size=%(size)d";
    }

    package { 'magnum ui':
      name => 'python3-magnum-ui',
      ensure => installed,
    }

    class { '::etcddiscovery':
      prefix_url          => "${base_url}/etcd-discovery",
      web_service_address => "${machine_ip}:8087",
      endpoint_location   => "${base_url}/etcd-cluster",
    }
  }

  #######################
  ### Setup Designate ###
  #######################
  if $has_subrole_designate {
    ### SETUP BIND ###
    # Need some particular options that aren't default in ::dns when using
    # designate with bind9 backend. Set them up.
    $controls = {
      '127.0.0.1' => {
        'port'              => 953,
        'allowed_addresses' => [ '127.0.0.1' ],
        'keys'              => [ 'rndc-key' ]
      },
    }
    # NOTE (dmsimard): listen_on_v6 is false and overridden due to extended port
    # configuration in additional_options
    class { 'dns':
      recursion          => 'no',
      allow_recursion    => [],
      controls           => $controls,
      listen_on_v6       => false,
      additional_options => {
        'listen-on'     => 'port 5322 { any; }',
        'listen-on-v6'  => 'port 5322 { any; }',
        'auth-nxdomain' => 'no',
      }
    }

    # ::dns creates the rndc key but not a rndc.conf.
    # Contribute this in upstream ::dns ?
    file { '/etc/rndc.conf':
      ensure  => present,
      owner   => $::dns::params::owner,
      group   => $::dns::params::group,
      content => template("${module_name}/rndc.conf.erb"),
      require => Package[$dns::params::dns_server_package]
    }

    ### SETUP DESIGNATE ###
    if $has_subrole_db and $initial_cluster_setup {
      Class['galera'] -> Anchor['designate::install::begin']
    }
    if $use_ssl {
      oci::sslkeypair {'designate':
        notify_service_name => 'designate-api',
      }
      $designate_key_file = "/etc/designate/ssl/private/${::fqdn}.pem"
      $designate_crt_file = "/etc/designate/ssl/public/${::fqdn}.crt"
    } else {
      $designate_key_file = undef
      $designate_crt_file = undef
    }
    class { '::designate::db':
      database_connection   => "mysql+pymysql://designate:${pass_designate_db}@${sql_host}/designatedb?charset=utf8",
    }
    class { 'designate::logging':
      debug => true,
    }
    if $disable_notifications {
      $designate_notif_transport_url = ''
    }else{
      $designate_notif_transport_url = os_transport_url({
                                       'transport' => $messaging_notify_proto,
                                       'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                       'port'      => $messaging_notify_port,
                                       'username'  => 'designate',
                                       'password'  => $pass_designate_messaging,
                                     })
    }

    class { '::designate':
      default_transport_url      => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'designate',
        'password'  => $pass_designate_messaging,
      }),
      notification_transport_url => $designate_notif_transport_url,
      notification_driver        => $notification_driver,
      rabbit_use_ssl             => $use_ssl,
      rabbit_ha_queues           => true,
      kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
#      amqp_sasl_mechanisms       => 'PLAIN',
    }
    designate_config {
      'coordination/backend_url':     value => "zookeeper://${first_master_ip}:2181/";
    }

    include 'designate::client'

    class { '::designate::keystone::authtoken':
      password             => $pass_designate_authtoken,
      user_domain_name     => 'Default',
      project_domain_name  => 'Default',
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
    }

    if $is_first_master {
      class { '::designate::keystone::auth':
        public_url   => "${base_url}/dns",
        internal_url => "${base_url}/dns",
        admin_url    => "${base_url}/dns",
        password     => $pass_designate_authtoken,
      }
    }

    class { 'designate::api':
      listen             => "${machine_ip}:9001",
      api_base_uri       => "${base_url}/dns",
      auth_strategy      => 'keystone',
      enable_api_v2      => true,
      enable_api_admin   => true,
      enable_host_header => false
    }

    class { 'designate::mdns':
      listen => "${machine_ip}:53"
    }

    class { 'designate::central': }

    class { 'designate::producer': }

    class { 'designate::worker': }

    class { 'designate::backend::bind9':
      rndc_host        => $machine_ip,
      rndc_port        => '953',
      rndc_config_file => '/etc/rndc.conf',
      rndc_key_file    => $::dns::params::rndckeypath,
    }

    $command = "openstack --os-auth-url ${base_url}/identity/v3 \
--os-identity-api-version 3 \
--os-project-name services --os-username designate --os-password ${pass_designate_authtoken} \
--os-project-domain-name Default --os-user-domain-name Default zone list"
    openstacklib::service_validation { 'designate-central':
      command     => $command,
      timeout     => '15',
      refreshonly => true,
      subscribe   => Anchor['designate::service::end'],
    }
    file { '/etc/designate/pools.yaml':
      ensure  => present,
      content => template("${module_name}/pools.yaml.erb"),
      require => Service['designate-central'],
    }
    exec { 'Update designate pools':
      command     => 'designate-manage pool update --file /etc/designate/pools.yaml',
      path        => '/usr/bin',
      refreshonly => true,
      logoutput   => 'on_failure',
      subscribe   => File['/etc/designate/pools.yaml'],
      require     => Openstacklib::Service_validation['designate-central'],
    }
    designate_config {
      'network_api:neutron/ca_certificates_file': value => $api_endpoint_ca_file;
      'keystone/cafile':                          value => $api_endpoint_ca_file;
    }
  }

  ##############################################
  ### Manage the cron jobs of the controller ###
  ##############################################
  # First, provision the ssh keys for Glance and Keystone
  if $has_subrole_glance {
    ::oci::oci_ssh_keypair { 'glance-keypair':
      path    => '/var/lib/glance',
      type    => 'ssh-rsa',
      user    => 'glance',
      group   => 'glance',
      pubkey  => $pass_glance_ssh_pub,
      privkey => base64('decode', $pass_glance_ssh_priv),
      require => Package['glance-api'],
    }
  }

  if $has_subrole_api_keystone {
    ::oci::oci_ssh_keypair { 'keystone-keypair':
      path    => '/var/lib/keystone',
      type    => 'ssh-rsa',
      user    => 'keystone',
      group   => 'keystone',
      pubkey  => $pass_keystone_ssh_pub,
      privkey => base64('decode', $pass_keystone_ssh_priv),
      require => Package['keystone'],
    }
  }

  if $is_first_master {
    if $has_subrole_glance {
      cron { 'oci-glance-images-rsync':
        command => '/usr/bin/oci-glance-image-rsync',
        user    => 'glance',
        minute  => 0,
      }
    }
    if $has_subrole_api_keystone {
      cron { 'oci-fernet-keys-rotate':
        command => '/usr/bin/oci-fernet-keys-rotate',
        user    => 'keystone',
        minute  => 40,
        hour    => 6,
        weekday => 2,
      }
    }
  }else{
    file { "/var/spool/cron/crontabs/root":
      ensure                  => absent,
    }
  }
}
