class oci::compute(
  $openstack_release        = undef,
  $cluster_name             = undef,
  $machine_hostname         = undef,
  $machine_ip               = undef,
  $bridge_mapping_list      = undef,
  $external_network_list    = undef,
  $first_master             = undef,
  $first_master_ip          = undef,
  $cluster_domain           = undef,
  $vmnet_ip                 = undef,

  $messaging_nodes_for_notifs = false,
  $all_rabbits                = [],
  $all_rabbits_ips            = [],


  $neutron_global_physnet_mtu = 1500,
  $neutron_path_mtu         = 1500,

  $all_masters              = undef,
  $all_masters_ip           = undef,
  $vip_hostname             = undef,
  $vip_ipaddr               = undef,
  $vip_netmask              = undef,
  $self_signed_api_cert     = true,
  $sql_vip_ip               = undef,
  $network_ipaddr           = undef,
  $network_cidr             = undef,
  $use_ssl                  = true,

  $pass_haproxy_stats       = undef,

  $pass_nova_messaging      = undef,
  $pass_nova_authtoken      = undef,
  $pass_nova_ssh_pub        = undef,
  $pass_nova_ssh_priv       = undef,
  $pass_neutron_authtoken   = undef,
  $pass_metadata_proxy_shared_secret = undef,
  $pass_neutron_messaging   = undef,
  $pass_placement_authtoken = undef,

  $has_subrole_ceilometer   = false,
  $pass_ceilometer_messaging = undef,
  $pass_ceilometer_authtoken = undef,
  $pass_ceilometer_telemetry = undef,

  $pass_cinder_db           = undef,
  $pass_cinder_authtoken    = undef,
  $pass_cinder_messaging    = undef,

  $cluster_has_osds         = undef,
  $use_ceph_if_available    = false,
  $ceph_fsid                = undef,
  $ceph_libvirtuuid         = undef,
  $ceph_bootstrap_osd_key   = undef,
  $ceph_admin_key           = undef,
  $ceph_openstack_key       = undef,
  $ceph_mon_key             = undef,
  $ceph_mon_initial_members = undef,
  $ceph_mon_host            = undef,

  $disable_notifications    = false,
  $enable_monitoring_graphs = false,
  $monitoring_graphite_host = undef,
  $monitoring_graphite_port = '2003',

  $use_gpu                  = false,
  $gpu_name                 = undef,
  $gpu_vendor_id            = undef,
  $gpu_product_id           = undef,
  $gpu_device_type          = undef,
  $vfio_ids                 = undef,

  $has_subrole_dhcp_agent   = true,

  $nested_virt              = false,

  $cpu_mode                 = 'host-passthrough',
  $cpu_model                = undef,
  $cpu_model_extra_flags    = undef,

  $cpu_allocation_ratio     = undef,
  $ram_allocation_ratio     = undef,
  $disk_allocation_ratio    = undef,

  $bgp_to_the_host          = false,
  $dhcp_domain              = '',
  $backup_backend           = 'ceph',
  $has_subrole_designate    = false,
  $neutron_dns_domain       = 'example.com.',
  $admin_email_address      = 'admin@example.com',

  $use_dynamic_routing      = false,

  $neutron_install_dragent  = false,

  $neutron_use_dvr          = true,
  # This one is unused, because we use the agregate of
  # network + bridge instead. However, it's transmitted
  # by the ENC because it is a machine variable, so this
  # variable must be present.
  $neutron_external_network_name = undef,

  $compute_is_cephosd       = false,

  $ceph_osd_initial_setup   = false,
  $block_devices            = undef,
){

  # Tweak performances for KVM host
  # class { '::oci::tuned': profile => 'virtual-host' }

  $packages = ['numactl','numad']
  package { $packages:
    ensure => installed,
  }->
  service { 'numad':
    ensure => 'running',
  }

  if $cpu_mode == 'host-passthrough'{
    $cpu_model_real = undef
  }else{
    $cpu_model_real = $cpu_model
  }

  if $messaging_nodes_for_notifs {
    $notif_rabbit_servers = $all_rabbits
  }else{
    $notif_rabbit_servers = $all_masters
  }

  if $use_ssl {
    $proto = 'https'
    $messaging_default_port = '5671'
    $messaging_notify_port = '5671'
    $api_port = 443
  } else {
    $proto = 'http'
    $messaging_default_port = '5672'
    $messaging_notify_port = '5672'
    $api_port = 80
  }
  $messaging_default_proto = 'rabbit'
  $messaging_notify_proto = 'rabbit'

  $base_url = "${proto}://${vip_hostname}"
  $keystone_auth_uri  = "${base_url}:${api_port}/identity"
  $keystone_admin_uri = "${base_url}:${api_port}/identity"

  if $self_signed_api_cert {
    $api_endpoint_ca_file = '/etc/ssl/certs/oci-pki-oci-ca-chain.pem'
  }else{
    $api_endpoint_ca_file = ''
  }
  $oci_pki_root_ca_file = '/etc/ssl/certs/oci-pki-oci-ca-chain.pem'

  class { '::oci::puppet_oci_ca_cert':
    self_signed_api_cert => $self_signed_api_cert,
  }

  if $facts['os']['lsb'] != undef{
    $mycodename = $facts['os']['lsb']['distcodename']
  }else{
    $mycodename = $facts['os']['distro']['codename']
  }
  if $mycodename != 'stretch'{
    $libvirt_security_driver_content = 'security_driver = "apparmor"'
  }else{
    $libvirt_security_driver_content = '#security_driver = "apparmor"'
  }

  if $disable_notifications {
    $notification_driver = 'noop'
  }else{
    $notification_driver = 'messagingv2'
  }
  ############################################################
  ### HAProxy (used for metadata server proxying and such) ###
  ############################################################
  class { 'haproxy':
    global_options   => {
      'log'     => '/dev/log local0',
      'maxconn' => '256',
      'user'    => 'haproxy',
      'group'   => 'haproxy',
      'daemon'  => '',
      'nbproc'  => '4',
    },
    defaults_options => {
      'mode'    => 'http',
    },
    merge_options => true,
  }->
  haproxy::listen { 'hapstats':
    section_name => 'statshttp',
    bind         => { "${machine_ip}:8088" => 'user root'},
    mode         => 'http',
    options      => {
        'stats' => [ 'uri /', "auth admin:${pass_haproxy_stats}", 'refresh 15', 'admin if TRUE', 'realm Haproxy Statistics', ],
      },
  }

  $haproxy_options_for_metadata_proxy = [
    { 'use_backend' => 'metadatabe' },
  ]
  haproxy::frontend { 'openstackfe':
    mode      => 'http',
    bind      => { "127.0.0.1:8775" => [] },
    options   => $haproxy_options_for_metadata_proxy,
  }
  haproxy::backend { 'metadatabe':
    options => [
       { 'option' => 'forwardfor' },
       { 'option' => 'httpchk GET /healthcheck' },
       { 'mode' => 'http' },
       { 'balance' => 'roundrobin' },
    ],
  }
  if $openstack_release == 'rocky'{
    $metadatabm_options = 'check'
  }else{
    $metadatabm_options = 'check check-ssl ssl verify required ca-file /etc/ssl/certs/oci-pki-oci-ca-chain.pem'
  }
  haproxy::balancermember { 'metadatabm':
    listening_service => 'metadatabe',
    ipaddresses       => $all_masters_ip,
    server_names      => $all_masters,
    ports             => 8775,
    options           => $metadatabm_options
  }

  ############
  ### Nova ###
  ############
  if $disable_notifications {
    $nova_notif_transport_url = ''
  } else {
    $nova_notif_transport_url = os_transport_url({
                                  'transport' => $messaging_notify_proto,
                                  'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                  'port'      => $messaging_notify_port,
                                  'username'  => 'nova',
                                  'password'  => $pass_nova_messaging,
                                })
  }

  # This is needed for Buster, so live migration can work
  package { 'apparmor':
    ensure => present,
    before => Class['::nova'],
  }

  class { '::nova':
    default_transport_url      => os_transport_url({
      'transport' => $messaging_default_proto,
      'hosts'     => fqdn_rotate($all_masters),
      'port'      => $messaging_default_port,
      'username'  => 'nova',
      'password'  => $pass_nova_messaging,
    }),
    notification_transport_url => $nova_notif_transport_url,
    database_connection    => '',
    rabbit_use_ssl         => $use_ssl,
    rabbit_ha_queues       => true,
    kombu_ssl_ca_certs     => $oci_pki_root_ca_file,
    amqp_sasl_mechanisms   => 'PLAIN',
    glance_api_servers     => "${base_url}/image",
    notification_driver    => $notification_driver,
    notify_on_state_change => 'vm_and_task_state',
    nova_public_key        => { type => 'ssh-rsa', key => $pass_nova_ssh_pub },
    nova_private_key       => { type => 'ssh-rsa', key => base64('decode', $pass_nova_ssh_priv) },
    cpu_allocation_ratio   => $cpu_allocation_ratio,
    ram_allocation_ratio   => $ram_allocation_ratio,
    disk_allocation_ratio  => $disk_allocation_ratio,
  }

  class { '::nova::keystone::authtoken':
    password             => $pass_nova_authtoken,
    auth_url             => $keystone_admin_uri,
    www_authenticate_uri => $keystone_auth_uri,
    cafile               => $api_endpoint_ca_file,
  }

  # The +0 is there for converting the string to an int,
  # 1536 is for the PoC, 8192 is for real deployments where we do expect
  # hosts to hold more than 16GB of RAM.
  if (($::memorysize_mb + 0) < 16000) {
    $reserved_host_memory_mb = 1536
  }else{
    $reserved_host_memory_mb = 8192
  }

  nova_config {
    'placement/cafile':                 value => $api_endpoint_ca_file;
    'placement/auth_type':              value => 'password';
    'placement/auth_url':               value => $keystone_admin_uri;
    'placement/password':               value => $pass_placement_authtoken, secret => true;
    'placement/project_domain_name':    value => 'Default';
    'placement/project_name':           value => 'services';
    'placement/user_domain_name':       value => 'Default';
    'placement/username':               value => 'placement';
    'placement/region_name':            value => 'RegionOne';
    'glance/cafile':                    value => $api_endpoint_ca_file;
    'keystone/cafile':                  value => $api_endpoint_ca_file;
    'neutron/cafile':                   value => $api_endpoint_ca_file;
    'cinder/cafile':                    value => $api_endpoint_ca_file;
    'service_user/cafile':              value => $api_endpoint_ca_file;
    'DEFAULT/use_cow_images':           value => 'False';
    'DEFAULT/default_ephemeral_format': value => 'ext4';
  }
  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
    nova_config {
      'DEFAULT/reserved_host_disk_mb':    value => '10240';
    }
  }
  # Needed, so puppet-openstack can restart virtlogd and libvirtd after configuration
  # so it listen to TCP before nova-compute starts.
  include ::nova::compute::libvirt::services
  Service['virtlogd'] -> Service['libvirtd']

  # Note: this *must* be set *before* calling ::nova::compute, as ::nova::compute is
  # doing an include ::nova::pci.
  if $use_gpu {
    class { '::nova::pci':
      aliases => [
        {"vendor_id" => $gpu_vendor_id, "product_id" => $gpu_product_id, "name" => $gpu_name, "device_type" => $gpu_device_type},
      ],
    }
    class {'::nova::compute::pci':
      passthrough => [
        {"vendor_id" => $gpu_vendor_id, "product_id" => $gpu_product_id},
      ]
    }
    file { '/etc/modprobe.d/blacklist-nvidia.conf':
      ensure                  => present,
      owner                   => 'root',
      content                 => 'blacklist nouveau
blacklist nvidiafb
blacklist snd_hda_intel
',
      selinux_ignore_defaults => true,
      mode                    => '0644',
    }
    file { '/etc/modprobe.d/vfio.conf':
      ensure                  => present,
      owner                   => 'root',
      content                 => "options vfio-pci ids=${vfio_ids}",
      selinux_ignore_defaults => true,
      mode                    => '0644',
    }
  }else{
    file { '/etc/modprobe.d/blacklist-nvidia.conf':
      ensure                  => absent,
    }
    file { '/etc/modprobe.d/vfio.conf':
      ensure                  => absent,
    }
  }
  # Needed to expose physical devices to VMs
  file { '/etc/modules-load.d/vfio.conf':
    ensure                  => present,
    owner                   => 'root',
    content                 => "# Managed by puppet: do not touch

vfio-pci
",
    selinux_ignore_defaults => true,
    mode                    => '0644',
  }

  if $openstack_release == 'rocky'{
    class { '::nova::compute':
      vnc_enabled                      => true,
      vncproxy_host                    => "${vip_hostname}",
      vncproxy_protocol                => 'https',
      vncproxy_port                    => '443',
      vncproxy_path                    => '/novnc/vnc_auto.html',
      vncserver_proxyclient_address    => $machine_ip,
      instance_usage_audit             => true,
      instance_usage_audit_period      => 'hour',
      barbican_auth_endpoint           => "${base_url}/identity/v3",
      barbican_endpoint                => "${base_url}/keymanager",
      keymgr_backend                   => 'barbican',
      resume_guests_state_on_host_boot => true,
      allow_resize_to_same_host        => true,
      reserved_host_memory             => $reserved_host_memory_mb,
#     resize_confirm_window            => '60',
    }
  } else {
    if ($openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
      class { '::nova::compute':
        vnc_enabled                      => true,
        vncproxy_host                    => "${vip_hostname}",
        vncproxy_protocol                => 'https',
        vncproxy_port                    => '443',
        vncproxy_path                    => '/novnc/vnc_auto.html',
        vncserver_proxyclient_address    => $machine_ip,
        instance_usage_audit             => true,
        instance_usage_audit_period      => 'hour',
        barbican_auth_endpoint           => "${base_url}/identity/v3",
        barbican_endpoint                => "${base_url}/keymanager",
        keymgr_backend                   => 'barbican',
        resume_guests_state_on_host_boot => true,
        allow_resize_to_same_host        => true,
        reserved_host_memory             => $reserved_host_memory_mb,
  #     resize_confirm_window            => '60',
      }
    }else{
      class { '::nova::compute':
        vnc_enabled                      => true,
        vncproxy_host                    => "${vip_hostname}",
        vncproxy_protocol                => 'https',
        vncproxy_port                    => '443',
        vncproxy_path                    => '/novnc/vnc_auto.html',
        vncserver_proxyclient_address    => $machine_ip,
        instance_usage_audit             => true,
        instance_usage_audit_period      => 'hour',
        barbican_auth_endpoint           => "${base_url}/identity/v3",
        barbican_endpoint                => "${base_url}/keymanager",
        keymgr_backend                   => 'barbican',
        resume_guests_state_on_host_boot => true,
        allow_resize_to_same_host        => true,
        reserved_host_memory             => $reserved_host_memory_mb,
#       resize_confirm_window            => '60',
        reserved_host_disk               => '10240',
      }
    }
    class { '::nova::logging':
      debug => true,
    }
  }

  class { '::nova::compute::libvirt':
    libvirt_virt_type       => 'kvm',
#    libvirt_cpu_mode        => 'host-passthrough',
    # Setting-up cpu_model doesn't seem to work in the KVM PoC.
    # This is to be investigated.
    libvirt_cpu_mode        => $cpu_mode,
    libvirt_cpu_model       => $cpu_model_real,
    libvirt_cpu_model_extra_flags => $cpu_model_extra_flags,
    migration_support       => true,
    # False is needed because of the above include ::nova::compute::libvirt::services
    manage_libvirt_services => false,
    preallocate_images      => 'space',
    # This is one week retention.
    remove_unused_original_minimum_age_seconds => '604800',
    vncserver_listen        => '0.0.0.0',
    libvirt_hw_disk_discard => 'unmap',
  }->
  file_line { 'parallel-shutdown-of-vms':
    path   => '/etc/default/libvirt-guests',
    match  => '.*PARALLEL_SHUTDOWN.*=.*',
    line   => 'PARALLEL_SHUTDOWN=8',
  }->
  file_line { 'shutdown-timeout-of-vms':
    path   => '/etc/default/libvirt-guests',
    match  => '.*SHUTDOWN_TIMEOUT.*=.*',
    line   => 'SHUTDOWN_TIMEOUT=120',
  }->
  file_line { 'start-delay-of-vms':
    path   => '/etc/default/libvirt-guests',
    match  => '.*START_DELAY.*=.*',
    line   => 'START_DELAY=4',
  }->
  file_line { 'use-apparmor-not-selinux':
    path   => '/etc/libvirt/qemu.conf',
    match  => '^#?security_driver.*=.*',
    line   => $libvirt_security_driver_content,
    notify => Service['libvirtd'],
  }

  if $nested_virt{
    file { '/etc/modprobe.d/kvm.conf':
      ensure  => present,
      content => 'options kvm_intel nested=1',
    }
    file { '/etc/modprobe.d/kvm_amd.conf':
      ensure  => present,
      content => 'options kvm_amd nested=1',
    }
  }else{
    file { '/etc/modprobe.d/kvm.conf':
      ensure  => present,
      content => 'options kvm_intel nested=0',
    }
    file { '/etc/modprobe.d/kvm_amd.conf':
      ensure  => present,
      content => 'options kvm_intel nested=0',
    }
  }

  if $openstack_release == 'rocky'{
    class { '::nova::network::neutron':
      neutron_auth_url      => "${base_url}/identity/v3",
      neutron_url           => "${base_url}/network",
      neutron_password      => $pass_neutron_authtoken,
      default_floating_pool => 'public',
      dhcp_domain           => $dhcp_domain,
    }
    nova_config {
      'neutron/endpoint_override': value => "${base_url}/network";
    }
  }else{
    if $openstack_release == 'stein' or $openstack_release == 'train'{
      class { '::nova::network::neutron':
        neutron_auth_url          => "${base_url}/identity/v3",
        neutron_url               => "${base_url}/network",
        neutron_password          => $pass_neutron_authtoken,
        default_floating_pool     => 'public',
        dhcp_domain               => $dhcp_domain,
        neutron_endpoint_override => "${base_url}/network",
      }
    }else{
      class { '::nova::network::neutron':
        neutron_auth_url          => "${base_url}/identity/v3",
        neutron_password          => $pass_neutron_authtoken,
        default_floating_pool     => 'public',
        neutron_endpoint_override => "${base_url}/network",
      }
    }
  }

  if $cluster_has_osds {
    class { 'ceph':
      fsid                => $ceph_fsid,
      ensure              => 'present',
      authentication_type => 'cephx',
      mon_initial_members => $ceph_mon_initial_members,
      mon_host            => $ceph_mon_host,
    }->
    ceph::key { 'client.admin':
      secret  => $ceph_admin_key,
      cap_mon => 'allow *',
      cap_osd => 'allow *',
      cap_mds => 'allow',
    }->
    ceph::key { 'client.openstack':
      secret  => $ceph_openstack_key,
      mode    => '0644',
      cap_mon => 'profile rbd',
      cap_osd => 'profile rbd pool=cinder, profile rbd pool=nova, profile rbd pool=glance, profile rbd pool=gnocchi, profile rbd pool=cinderback',
    }->
    ceph::key { 'client.bootstrap-osd':
      secret       => $ceph_bootstrap_osd_key,
      keyring_path => '/var/lib/ceph/bootstrap-osd/ceph.keyring',
      cap_mon      => 'allow profile bootstrap-osd',
    }->
    class { '::ceph::profile::params':
      fsid                => $ceph_fsid,
      manage_repo         => false,
      release             => 'nautilus',
      authentication_type => 'cephx',
      mon_host            => $ceph_mon_host,
      mon_initial_members => $ceph_mon_initial_members,
      client_keys         => {
        'client.admin' => {
          secret  => $ceph_admin_key,
          cap_mon => 'allow *',
          cap_osd => 'allow *',
          cap_mds => 'allow',
        },
        'client.openstack' => {
          secret  => $ceph_openstack_key,
          mode    => '0644',
          cap_mon => 'profile rbd',
          cap_osd => 'profile rbd pool=cinder, profile rbd pool=nova, profile rbd pool=glance, profile rbd pool=gnocchi, profile rbd pool=cinderback',
        },
        'client.bootstrap-osd' => {
          secret       => $ceph_bootstrap_osd_key,
          keyring_path => '/var/lib/ceph/bootstrap-osd/ceph.keyring',
          cap_mon      => 'allow profile bootstrap-osd',
        }
      }
    }

    if $compute_is_cephosd and $ceph_osd_initial_setup{
      $block_devices.each |Integer $index, String $value| {
        ceph::osd { "/dev/${value}":
          dmcrypt => false,
          require => Class['::ceph::profile::params'],
        }
      }
    }

    if $use_ceph_if_available or $compute_is_cephosd{
      class { '::nova::compute::rbd':
        libvirt_rbd_user        => 'openstack',
        libvirt_rbd_secret_uuid => $ceph_libvirtuuid,
        libvirt_rbd_secret_key  => $ceph_openstack_key,
        libvirt_images_rbd_pool => 'nova',
        rbd_keyring             => 'client.openstack',
        # ceph packaging is already managed by puppet-ceph
        manage_ceph_client      => false,
        require => Ceph::Key['client.openstack'],
      }
    }else{
      # If we have Ceph OSDs but we don't use /var/lib/nova/instances,
      # we still need the secret to be defined on libvirt: let's do it
      # without ::nova::compute::rbd, and by the way, let's disable
      # RBD on nova.conf, just in case someone changed his mind, and
      # disabled such config on the node.

      $rbd_keyring = 'client.openstack'
      $libvirt_rbd_secret_uuid = $ceph_libvirtuuid
      file { '/etc/nova/secret.xml':
        content => template('nova/secret.xml-compute.erb'),
        require => Ceph::Key['client.openstack'],
      }

      #Variable name shrunk in favor of removing
      #the more than 140 chars puppet-lint warning.
      #variable used in the get-or-set virsh secret
      #resource.
      $cm = '/usr/bin/virsh secret-define --file /etc/nova/secret.xml | /usr/bin/awk \'{print $2}\' | sed \'/^$/d\' > /etc/nova/virsh.secret'
      exec { 'get-or-set virsh secret':
        command => $cm,
        unless  => "/usr/bin/virsh secret-list | grep -i ${libvirt_rbd_secret_uuid}",
        require => File['/etc/nova/secret.xml'],
      }
      Service<| title == 'libvirt' |> -> Exec['get-or-set virsh secret']
      nova_config {
        'libvirt/images_type':          ensure => absent;
        'libvirt/images_rbd_pool':      ensure => absent;
        'libvirt/images_rbd_ceph_conf': ensure => absent;
        'libvirt/rbd_secret_uuid':      value => $ceph_libvirtuuid;
        'libvirt/rbd_user':             value => 'openstack';
      }
      # make sure ceph pool exists before running nova-compute
      # Note: cannot be done on the compute nodes, since we aren't doing pools here
      # but on the controller.
      #Exec['create-nova'] -> Service['nova-compute']
    }
  }

  ###############
  ### Neutron ###
  ###############
  if $disable_notifications {
    $neutron_notif_transport_url = ''
  } else {
    $neutron_notif_transport_url = os_transport_url({
                                     'transport' => $messaging_notify_proto,
                                     'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                     'port'      => $messaging_notify_port,
                                     'username'  => 'neutron',
                                     'password'  => $pass_neutron_messaging,
                                   })
  }

  if $neutron_use_dvr {
    $l3_agent_mode              = 'dvr'
    $enable_distributed_routing = true
    $install_l3_agent           = true
  }else{
    $l3_agent_mode              = 'legacy'
    $enable_distributed_routing = false
    $install_l3_agent           = false
  }

  $bgp_dr_plugin = $use_dynamic_routing ? {
    true    => 'bgp',
    default => undef,
  }
  $plugins_list = delete_undef_values(['router', 'metering', 'qos', 'trunk', 'segments', $bgp_dr_plugin])
  if $neutron_install_dragent {
    class { '::neutron::agents::bgp_dragent':
      bgp_router_id => $machine_ip,
    }
  }else{
    package { 'neutron dynamic routing':
      name   => 'neutron-bgp-dragent',
      ensure => purged,
    }
    if $use_dynamic_routing {
      package { 'neutron-dynamic-routing-common':
        name   => 'neutron-dynamic-routing-common',
        ensure => present,
      }
    }else{
      package { 'python3-neutron-dynamic-routing':
        name   => 'python3-neutron-dynamic-routing',
        ensure => purged,
      }
    }
  }

  if $openstack_release == 'rocky'{
    class { '::neutron':
      debug                      => true,
      default_transport_url      => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'neutron',
        'password'  => $pass_neutron_messaging,
      }),
      notification_transport_url => $neutron_notif_transport_url,
      rabbit_use_ssl             => $use_ssl,
      rabbit_ha_queues           => true,
      kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
      amqp_sasl_mechanisms       => 'PLAIN',
      allow_overlapping_ips      => true,
      core_plugin                => 'ml2',
      service_plugins            => $plugins_list,
      bind_host                  => $machine_ip,
      notification_driver        => $notification_driver,
      global_physnet_mtu         => $neutron_global_physnet_mtu,
      dns_domain                 => $neutron_dns_domain,
    }
  }else{
    class { '::neutron':
      default_transport_url      => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'neutron',
        'password'  => $pass_neutron_messaging,
      }),
      notification_transport_url => $neutron_notif_transport_url,
      rabbit_use_ssl             => $use_ssl,
      rabbit_ha_queues           => true,
      kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
      amqp_sasl_mechanisms       => 'PLAIN',
      allow_overlapping_ips      => true,
      core_plugin                => 'ml2',
      service_plugins            => $plugins_list,
      bind_host                  => $machine_ip,
      notification_driver        => $notification_driver,
      global_physnet_mtu         => $neutron_global_physnet_mtu,
      dns_domain                 => $neutron_dns_domain,
    }
  }
  neutron_config {
    'nova/cafile':         value => $api_endpoint_ca_file;
    'database/connection': value => '';
  }

  class { '::neutron::server::notifications':
    auth_url => $keystone_admin_uri,
    password => $pass_nova_authtoken,
  }

  class { '::neutron::client': }
  class { '::neutron::keystone::authtoken':
    password             => $pass_neutron_authtoken,
    auth_url             => $keystone_admin_uri,
    www_authenticate_uri => $keystone_auth_uri,
    cafile               => $api_endpoint_ca_file,
  }
  class { '::neutron::agents::ml2::ovs':
    local_ip                   => $vmnet_ip,
    tunnel_types               => ['vxlan'],
    bridge_uplinks             => ['eth0'],
    bridge_mappings            => $bridge_mapping_list,
    extensions                 => '',
    l2_population              => true,
    arp_responder              => true,
    firewall_driver            => 'iptables_hybrid',
    drop_flows_on_start        => false,
    enable_distributed_routing => $enable_distributed_routing,
    manage_vswitch             => false,
  }
  if $install_l3_agent {
    class { '::neutron::agents::l3':
      interface_driver => 'openvswitch',
      debug            => true,
      agent_mode       => $l3_agent_mode,
      ha_enabled       => false,
      extensions       => '',
    }
    neutron_l3_agent_config {
      'DEFAULT/external_network_bridge': value => '';
    }
  } else {
    package { 'neutron-l3-agent':
      ensure => absent,
    }
  }

  if $has_subrole_designate {
    $extension_drivers = 'port_security,qos,dns'
  }else{
    $extension_drivers = 'port_security,qos'
  }

  if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri'){
    class { '::neutron::plugins::ml2':
      type_drivers         => ['flat', 'vxlan', 'vlan', ],
      tenant_network_types => ['flat', 'vxlan', 'vlan', ],
      extension_drivers    => $extension_drivers,
      mechanism_drivers    => 'openvswitch,l2population',
      firewall_driver      => 'iptables_v2',
      flat_networks        => $external_network_list,
      network_vlan_ranges  => $external_network_list,
      vni_ranges           => '1000:9999',
      path_mtu             => $neutron_path_mtu,
    }
  }else{
    class { '::neutron::plugins::ml2':
      type_drivers         => ['flat', 'vxlan', 'vlan', ],
      tenant_network_types => ['flat', 'vxlan', 'vlan', ],
      extension_drivers    => $extension_drivers,
      mechanism_drivers    => 'openvswitch,l2population',
      flat_networks        => $external_network_list,
      network_vlan_ranges  => $external_network_list,
      vni_ranges           => '1000:9999',
      path_mtu             => $neutron_path_mtu,
    }
  }
  if $has_subrole_dhcp_agent {
    class { '::neutron::agents::dhcp':
      interface_driver         => 'openvswitch',
      debug                    => true,
      enable_isolated_metadata => true,
      enable_metadata_network  => true,
    }
  }else{
    package { 'neutron-dhcp-agent':
      ensure => absent,
    }
  }
  class { '::neutron::agents::metering':
    interface_driver => 'openvswitch',
    driver           => 'iptables',
    debug            => true,
  }
  class { '::neutron::agents::metadata':
    debug             => true,
    shared_secret     => $pass_metadata_proxy_shared_secret,
    metadata_workers  => 2,
    metadata_protocol => 'http',
    metadata_host     => 'localhost',
  }
  package { 'l2gw networking':
    name => 'python3-networking-l2gw',
    ensure => installed,
  }
  ##############
  ### Cinder ###
  ##############
  # If using Ceph, then we setup cinder-volume and cinder-backup over Ceph
  # on each and every compute nodes. The goal is to spread the load away
  # from the controller nodes.
  if $cluster_has_osds {
    include ::cinder::client

    if $disable_notifications {
      $cinder_notif_transport_url = ''
    } else {
      $cinder_notif_transport_url = os_transport_url({
                                      'transport' => $messaging_notify_proto,
                                      'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                      'port'      => $messaging_notify_port,
                                      'username'  => 'cinder',
                                      'password'  => $pass_cinder_messaging,
                                    })
    }

    # Cinder main class (ie: cinder-common config)
    if $openstack_release == 'rocky'{
      class { '::cinder':
        debug                 => true,
        default_transport_url => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'cinder',
          'password'  => $pass_cinder_messaging,
        }),
        database_connection   => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_vip_ip}/cinderdb?charset=utf8",
        rabbit_use_ssl        => $use_ssl,
        rabbit_ha_queues      => true,
        kombu_ssl_ca_certs    => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms  => 'PLAIN',
      }
    }else{
      if($openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri'){
        class { '::cinder':
          default_transport_url => os_transport_url({
            'transport' => $messaging_default_proto,
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => $messaging_default_port,
            'username'  => 'cinder',
            'password'  => $pass_cinder_messaging,
          }),
          database_connection   => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_vip_ip}/cinderdb?charset=utf8",
          rabbit_use_ssl        => $use_ssl,
          rabbit_ha_queues      => true,
          kombu_ssl_ca_certs    => $oci_pki_root_ca_file,
          amqp_sasl_mechanisms  => 'PLAIN',
        }
        class { 'cinder::ceilometer':
          notification_transport_url => $cinder_notif_transport_url,
          notification_driver        => $notification_driver,
        }
      }else{
        class { '::cinder':
          default_transport_url => os_transport_url({
            'transport' => $messaging_default_proto,
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => $messaging_default_port,
            'username'  => 'cinder',
            'password'  => $pass_cinder_messaging,
          }),
          database_connection   => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_vip_ip}/cinderdb?charset=utf8",
          rabbit_use_ssl        => $use_ssl,
          rabbit_ha_queues      => true,
          kombu_ssl_ca_certs    => $oci_pki_root_ca_file,
          amqp_sasl_mechanisms  => 'PLAIN',
          notification_transport_url => $cinder_notif_transport_url,
          notification_driver        => $notification_driver,
        }
      }
    }


    case $backup_backend {
      'ceph': {
        $backup_backend_real = 'cinder.backup.drivers.ceph.CephBackupDriver'
      }
      'swift': {
        $backup_backend_real = 'cinder.backup.drivers.swift.SwiftBackupDriver'
      }
      'none': {
        $backup_backend_real = ''
      }
    }

    cinder_config {
      'nova/cafile':                       value => $api_endpoint_ca_file;
      'service_user/cafile':               value => $api_endpoint_ca_file;
      'DEFAULT/backup_driver':             value => $backup_backend_real;
      'DEFAULT/snapshot_clone_size':       value => '200';
      'DEFAULT/rbd_pool':                  value => 'cinder';
      'DEFAULT/rbd_ceph_conf':             value => '/etc/ceph/ceph.conf';
      'DEFAULT/rbd_secret_uuid':           value => $ceph_libvirtuuid;
      'DEFAULT/backup_swift_auth_url':     value => $keystone_auth_uri;
      'DEFAULT/backup_ceph_user':          value => 'openstack';
      'DEFAULT/backup_ceph_pool':          value => 'cinderback';
      'DEFAULT/backup_swift_ca_cert_file': value => $api_endpoint_ca_file;
      'ssl/ca_file':                       value => $api_endpoint_ca_file;
    }

    # Configure the authtoken
    class { '::cinder::keystone::authtoken':
      password             => $pass_cinder_authtoken,
      auth_url             => $keystone_admin_uri,
      www_authenticate_uri => $keystone_auth_uri,
      memcached_servers    => $memcached_servers,
      cafile               => $api_endpoint_ca_file,
    }

    class { '::cinder::backends':
      enabled_backends => ['CEPH_1'],
    }

    cinder::backend::rbd { 'CEPH_1':
      rbd_user           => 'openstack',
      rbd_pool           => 'cinder',
      rbd_secret_uuid    => $ceph_libvirtuuid,
      manage_volume_type => true,
      backend_host       => $machine_hostname,
    }

    # Clear volumes on delete (for data security)
    class { '::cinder::volume':
      volume_clear => 'zero',
    }

    # A cinder-backup service on each volume nodes
    class { '::cinder::backup': }

    # Avoids Cinder to lookup for the catalogue
    class { '::cinder::glance':
      glance_api_servers => "${base_url}/image",
    }
  }

  ##################
  ### Ceilometer ###
  ##################
  if $has_subrole_ceilometer {
    if $openstack_release == 'rocky'{
      class { '::ceilometer':
        debug                      => true,
        telemetry_secret           => $pass_ceilometer_telemetry,
        default_transport_url      => os_transport_url({
          'transport' => 'rabbit',
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => '5671',
          'username'  => 'ceilometer',
          'password'  => $pass_ceilometer_messaging,
        }),
        notification_transport_url => os_transport_url({
          'transport' => 'rabbit',
          'hosts'     => fqdn_rotate($notif_rabbit_servers),
          'port'      => '5671',
          'username'  => 'ceilometer',
          'password'  => $pass_ceilometer_messaging,
        }),
        rabbit_use_ssl             => $use_ssl,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms       => 'PLAIN',
      }
    }else{
      class { '::ceilometer':
        telemetry_secret           => $pass_ceilometer_telemetry,
        default_transport_url      => os_transport_url({
          'transport' => 'rabbit',
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => '5671',
          'username'  => 'ceilometer',
          'password'  => $pass_ceilometer_messaging,
        }),
        notification_transport_url => os_transport_url({
          'transport' => 'rabbit',
          'hosts'     => fqdn_rotate($notif_rabbit_servers),
          'port'      => '5671',
          'username'  => 'ceilometer',
          'password'  => $pass_ceilometer_messaging,
        }),
        rabbit_use_ssl             => $use_ssl,
        kombu_ssl_ca_certs         => $oci_pki_root_ca_file,
        amqp_sasl_mechanisms       => 'PLAIN',
      }
    }
    if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri'){
      class { '::ceilometer::keystone::authtoken':
        password             => $pass_ceilometer_authtoken,
        auth_url             => $keystone_admin_uri,
        www_authenticate_uri => $keystone_auth_uri,
        cafile               => $api_endpoint_ca_file,
      }
    }
    class { '::ceilometer::agent::auth':
      auth_password => $pass_ceilometer_authtoken,
      auth_url      => $keystone_auth_uri,
      auth_cacert   => $api_endpoint_ca_file,
    }
    # Looks like the above sets ca_file instead of cafile.
    if $openstack_release == 'rocky' or $openstack_release == 'stein'{
      ceilometer_config {
        'service_credentials/cafile': value => $api_endpoint_ca_file;
      }
    }
    if $openstack_release == 'rocky' or $openstack_release == 'stein'{
      class { '::ceilometer::agent::compute':
        instance_discovery_method => 'libvirt_metadata',
      }
    }else{
      class { '::ceilometer::agent::polling':
        central_namespace         => false,
        compute_namespace         => true,
        ipmi_namespace            => false,
        instance_discovery_method => 'libvirt_metadata',
      }
    }
  }
  ############################
  ### OpenVSwitch defaults ###
  ############################
  file { '/etc/default/openvswitch-switch':
    content => '# Managed by Puppet
OVS_CTL_OPTS="--delete-bridges"',
  }
}
