<?php

function get_cluster_password($con, $conf, $cluster_id, $service_type, $password_type){
    $json["status"] = "success";
    $json["message"] = "Successfuly queried API.";
    $q = "SELECT pass,passtxt1,passtxt2 FROM passwords WHERE cluster='$cluster_id' AND service='$service_type' AND passtype='$password_type'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        # If password doesn't exist, generate it!
        insert_cluster_pass($con, $conf, $cluster_id, $service_type, $password_type);
        $q = "SELECT pass,passtxt1,passtxt2 FROM passwords WHERE cluster='$cluster_id' AND service='$service_type' AND passtype='$password_type'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "Cannot find password $password_type for service $service_type.";
            return $json;
        }
    }
    $pass_a = mysqli_fetch_array($r);
    if(($service_type == "nova" || $service_type == "keystone" || $service_type == "glance") && $password_type == "ssh"){
        $json["data"]["ssh_pub"] = $pass_a["passtxt1"];
        $json["data"]["ssh_priv"] = $pass_a["passtxt2"];
    }else{
        $json["data"] = $pass_a["pass"];
    }
    return $json;
}

function get_ssh_keypair($con, $conf, $cluster_id, $hostname, $username){
    $json["status"] = "success";
    $json["message"] = "Successfuly queried API.";
    $q = "SELECT * FROM sshkeypairs WHERE clusterid='$cluster_id' AND hostname='$hostname' AND username='$username'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        # If ssh key doesn't exist, generate it!
        insert_ssh_key($con, $conf, $cluster_id, $hostname, $username);
        $q = "SELECT * FROM sshkeypairs WHERE clusterid='$cluster_id' AND hostname='$hostname' AND username='$username'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "Cannot find ssh keypair for $hostname and user $username.";
            return $json;
        }
    }
    $a = mysqli_fetch_array($r);
    $json["data"]["pubkey"] = $a["pubkey"];
    $json["data"]["privatekey"] = $a["privatekey"];
    return $json;
}

# $role can be either 'cephmon' or 'billmon'
function enc_get_mon_nodes($con,$conf,$cluster_id,$role='cephmon'){
    if($role == 'cephmon'){
        $q = "SELECT * FROM machines WHERE role='cephmon' AND cluster='$cluster_id'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n < 1){
            // If there's no specific cephmon/billmon nodes, then controllers will assume that role
            $q = "SELECT * FROM machines WHERE role='controller' AND cluster='$cluster_id'";
            $r = mysqli_query($con, $q);
            if($r === FALSE){
                $json["status"] = "error";
                $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
                return $json;
            }
            $n = mysqli_num_rows($r);
            if($n < 1){
                $json["status"] = "error";
                $json["message"] = "No Ceph MON or controllers in database.";
                return $json;
            }
            $role_to_select = "controller";
        }else{
            $role_to_select = "cephmon";
        }
    }else{
        $role_to_select = "billmon";
    }

    $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='$role_to_select' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi' AND machines.cluster='$cluster_id'";

    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);

    $osd_members = "";
    $osd_ips = "";
    for($i=0;$i<$n;$i++){
        $machine = mysqli_fetch_array($r);
        $machine_hostname = $machine["hostname"];
        $machine_ipaddr   = $machine["ipaddr"];
        if($osd_members == ""){
            $osd_members = $machine_hostname;
            $osd_ips     = $machine_ipaddr;
        }else{
            $osd_members = $osd_members . "," . $machine_hostname;
            $osd_ips     = $osd_ips . "," . $machine_ipaddr;
        }
    }
    $json["status"] = "success";
    $json["message"] = "Successfuly queried API.";
    $json["data"]["osd_members"] = $osd_members;
    $json["data"]["osd_ips"] = $osd_ips;
    return $json;
}

function puppet_enc($con,$conf){
    $json["status"] = "success";
    $json["message"] = "Successfuly queried API.";

    // This returns the external node classifyer yaml file, to be used by puppet.

    // Get our machine object in db, using hostname to search, as this is
    // what the enc script is given as parameter
    $safe_hostname = safe_fqdn("hostname");
    $q = "SELECT * FROM machines WHERE hostname='$safe_hostname'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        $json["status"] = "error";
        $json["message"] = "No such hostname in database.";
        return $json;
    }
    $machine = mysqli_fetch_array($r);
    $machine_id   = $machine["id"];
    $machine_role = $machine["role"];
    $cluster_id   = $machine["cluster"];
    $machine_location = $machine["location_id"];
    $machine_force_no_bgp2host = $machine["force_no_bgp2host"];

    $machine_networks = slave_fetch_networks($con, $conf, $machine_id);
    if(sizeof($machine_networks["networks"]) == 0){
        $json["status"] = "error";
        $json["message"] = "Machine has no network.";
        return $json;
    }

    // Fetch matching cluster object
    $q = "SELECT * FROM clusters WHERE id='$cluster_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        $json["status"] = "error";
        $json["message"] = "Cluster not found in database.";
        return $json;
    }
    $cluster = mysqli_fetch_array($r);
    $cluster_domain = $cluster["domain"];
    $cluster_name = $cluster["name"];
    $cluster_bgp_to_the_host = $cluster["bgp_to_the_host"];
    if($cluster_bgp_to_the_host == "yes" && $machine_force_no_bgp2host == "no"){
        $machine_bgp2host = "yes";
        $machine_puppet_bgp2host = "true";
    }else{
        $machine_bgp2host = "no";
        $machine_puppet_bgp2host = "false";
    }
    $cluster_initial_cluster_setup = $cluster["initial_cluster_setup"];
    if($cluster_initial_cluster_setup == "yes"){
        $cluster_initial_cluster_setup = "true";
    }else{
        $cluster_initial_cluster_setup = "false";
    }
    $cluster_statsd_hostname = $cluster["statsd_hostname"];
    $cluster_time_server_host = $cluster["time_server_host"];
    $amp_secgroup_list        = $cluster["amp_secgroup_list"];
    $amp_boot_network_list    = $cluster["amp_boot_network_list"];
    $disable_notifications    = $cluster["disable_notifications"];
    if($disable_notifications == "yes"){
        $disable_notifications = "true";
    }else{
        $disable_notifications = "false";
    }
    $self_signed_api_cert = $cluster["self_signed_api_cert"];
    if($self_signed_api_cert == "yes"){
        $self_signed_api_cert = "true";
    }else{
        $self_signed_api_cert = "false";
    }
    $enable_monitoring_graphs = $cluster["enable_monitoring_graphs"];
    if($enable_monitoring_graphs == "yes"){
        $enable_monitoring_graphs = "true";
    }else{
        $enable_monitoring_graphs = "false";
    }
    $monitoring_graphite_host = $cluster["monitoring_graphite_host"];
    $monitoring_graphite_port = $cluster["monitoring_graphite_port"];

    $swift_object_replicator_concurrency = $cluster["swift_object_replicator_concurrency"];
    $swift_rsync_connection_limit = $cluster["swift_rsync_connection_limit"];

    $machine_networks = slave_fetch_network_config($con, $conf, $machine_id);
    if(sizeof($machine_networks["networks"]) == 0){
        $json["status"]  = "error";
        $json["message"] = "No network configured for this machine.";
        return $out;
    }

    // We just fetch the first network that's not public and vm-trafic for now, maybe we'll need to be
    // more selective later, let's see...
    for($i=0;$i<sizeof($machine_networks["networks"]);$i++){
        if($machine_networks["networks"][$i]["is_public"] == 'no' && $machine_networks["networks"][$i]["role"] != "vm-net" && $machine_networks["networks"][$i]["role"] != "ovs-bridge" && $machine_networks["networks"][$i]["role"] != "ceph-cluster" && $machine_networks["networks"][$i]["role"] != "ipmi"){
            $network_id = $machine_networks["networks"][$i]["id"];
            continue;
        }
    }

    // Get this machine's IP and hostname
    $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips WHERE machines.id='$machine_id' AND machines.cluster='$cluster_id' AND machines.id=ips.machine  AND ips.network='$network_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        $json["status"] = "error";
        $json["message"] = "No such hostname in database when doing $q.";
        return $json;
    }
    $machine_netconf = mysqli_fetch_array($r);
    $machine_hostname = $machine_netconf["hostname"];
    $machine_ip = $machine_netconf["ipaddr"];

    // Get the API's VIP ip address
    $q = "SELECT * FROM networks WHERE cluster='$cluster_id' AND is_public='yes'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = "MySQL error when doing $q: " .mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        $json["status"] = "error";
        $json["message"] = "Cannot find public network to find VIP when doing $q.";
        return $json;
    }

    $public_network = mysqli_fetch_array($r);
    $public_network_id = $public_network["id"];

    if($public_network["cidr"] == "32"){
        $vip_ipaddr  = $public_network["ip"];
        $vip_netmask = "32";
    }else{
        $q = "SELECT INET_NTOA(ips.ip) AS ipaddr FROM ips WHERE network='$public_network_id' AND usefor='vip' AND vip_usage='api'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "No such hostname in database when doing $q.";
            return $json;
        }
        $public_ip = mysqli_fetch_array($r);
        $vip_ipaddr  = $public_ip["ipaddr"];
        $vip_netmask = $public_network["cidr"];
    }
    if($cluster["vip_hostname"] == ""){
        $vip_hostname = $cluster["name"] . "-api." . $cluster["domain"];
    }else{
        $vip_hostname = $cluster["vip_hostname"];
    }

    $q = "SELECT * FROM networks WHERE id='$network_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = "MySQL error when doing $q: " .mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        $json["status"] = "error";
        $json["message"] = "Cannot find network in database when doing $q.";
        return $json;
    }
    $network = mysqli_fetch_array($r);
    $network_ip = $network["ip"];
    $network_cidr = $network["cidr"];

    // Fetch all controllers
    
    $enc_amhn = "      all_masters:\n";
    $enc_amip = "      all_masters_ip:\n";
    $enc_nids = "      all_masters_ids:\n";
    $q = "SELECT machines.hostname AS hostname, machines.id AS nodeid, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='controller' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    for($i=0;$i<$n;$i++){
        $one_master = mysqli_fetch_array($r);
        $one_master_hostname = $one_master["hostname"];
        $one_master_ipaddr = $one_master["ipaddr"];
        $one_node_id = $one_master["nodeid"];
        $enc_amhn .= "         - $one_master_hostname\n";
        $enc_amip .= "         - $one_master_ipaddr\n";
        $enc_nids .= "         - $one_node_id\n";
    }

    // Fetch all networks that contain an IP with a swiftstore or a swiftproxy
    $q_allowed_network = "SELECT DISTINCT networks.ip AS ip,networks.cidr AS cidr FROM networks,ips,machines WHERE ips.network = networks.id AND machines.id = ips.machine AND (machines.role='swiftstore' OR machines.role='swiftproxy') AND machines.cluster='$cluster_id'";
    $r_allowed_network = mysqli_query($con, $q_allowed_network);
    if($r_allowed_network === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n_allowed_network = mysqli_num_rows($r_allowed_network);
    $enc_swift_storage_allowed_networks = "      swift_storage_allowed_networks:\n";;
    for($i=0;$i<$n_allowed_network;$i++){
        $one_allowed_network = mysqli_fetch_array($r_allowed_network);
        $net = $one_allowed_network["ip"];
        $cidr = $one_allowed_network["cidr"];
        $enc_swift_storage_allowed_networks .= "         - $net/$cidr\n";;
    }

    // Fetch all swift stores
    $q = "SELECT machines.hostname AS hostname, machines.id AS nodeid, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE ips.network=networks.id AND networks.role!='ipmi'  AND networks.role!='vm-net' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND machines.role='swiftstore' AND machines.cluster='$cluster_id' AND machines.id=ips.machine";
    //        $q = "SELECT machines.hostname AS hostname, machines.id AS nodeid, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips WHERE machines.role='swiftstore' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network='$network_id'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    $enc_allswiftstore_hostanme = "";
    $enc_allswiftstore_ip = "";
    for($i=0;$i<$n;$i++){
        $one_swiftstore = mysqli_fetch_array($r);
        $one_swiftstore_hostname = $one_swiftstore["hostname"];
        $one_swiftstore_ipaddr = $one_swiftstore["ipaddr"];
        $enc_allswiftstore_hostanme .= "         - $one_swiftstore_hostname\n";
        $enc_allswiftstore_ip .= "         - $one_swiftstore_ipaddr\n";
    }

    // Fetch all swift proxies
    $q = "SELECT machines.hostname AS hostname, machines.id AS nodeid, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE networks.id=ips.network AND networks.role!='ipmi' AND networks.role!='vm-net' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND machines.role='swiftproxy' AND machines.cluster='$cluster_id' AND machines.id=ips.machine";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    $enc_allswiftproxies_hostname = "";
    $enc_allswiftproxies_ip = "";
    for($i=0;$i<$n;$i++){
        $one_swiftproxy = mysqli_fetch_array($r);
        $one_swiftproxy_hostname = $one_swiftproxy["hostname"];
        $one_swiftproxy_ipaddr = $one_swiftproxy["ipaddr"];
        $one_node_id = $one_swiftproxy["nodeid"];
        $enc_allswiftproxies_hostname .= "         - $one_swiftproxy_hostname\n";
        $enc_allswiftproxies_ip .= "         - $one_swiftproxy_ipaddr\n";
    }

    // Get the number of compute nodes
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='compute'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_compute_nodes = mysqli_num_rows($r);

    // Get the number of network nodes
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='network'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_network_nodes = mysqli_num_rows($r);

    // Get the number of swiftstore nodes
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='swiftstore'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_swiftstore_nodes = mysqli_num_rows($r);

    // Get the number of volume nodes
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='volume'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_volume_nodes = mysqli_num_rows($r);

    // Get the number of cephosd nodes
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='cephosd' OR (role='compute' AND compute_is_cephosd='yes')";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_cephosd_nodes = mysqli_num_rows($r);

    // Get the number of billosd nodes
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='billosd'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_billosd_nodes = mysqli_num_rows($r);

    // Get the number of billosd nodes
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='billmon'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_billmon_nodes = mysqli_num_rows($r);

    // Get the number of cephmon nodes
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='cephmon'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_cephmon_nodes = mysqli_num_rows($r);

    $enc_file = "---\n";
    $enc_file .= "classes:\n";

    ##############################################################
    ### Load hiera from /etc/openstack-cluster-installer/hiera ###
    ##############################################################
    $hiera_dir  = "/etc/openstack-cluster-installer/hiera";
    $hiera_role = $hiera_dir . "/roles/" . $machine_role . ".yaml";
    if(file_exists($hiera_role) === TRUE){
        $enc_file .= file_get_contents($hiera_role);
    }
    $hiera_node = $hiera_dir . "/nodes/" . $safe_hostname . ".yaml";
    if(file_exists($hiera_node) === TRUE){
        $enc_file .= file_get_contents($hiera_node);
    }
    $hiera_all  = $hiera_dir . "/all.yaml";
    if(file_exists($hiera_all) === TRUE){
        $enc_file .= file_get_contents($hiera_all);
    }
    $hiera_cluster_role = $hiera_dir . "/clusters/" . $cluster["name"] . "/roles/" . $machine_role . ".yaml";
    if(file_exists($hiera_cluster_role) === TRUE){
        $enc_file .= file_get_contents($hiera_cluster_role);
    }
    $hiera_cluster_node = $hiera_dir . "/clusters/" . $cluster["name"] . "/nodes/" . $safe_hostname . ".yaml";
    if(file_exists($hiera_cluster_node) === TRUE){
        $enc_file .= file_get_contents($hiera_cluster_node);
    }
    $hiera_cluster_all = $hiera_dir . "/clusters/" . $cluster["name"] . "/all.yaml";
    if(file_exists($hiera_cluster_all) === TRUE){
        $enc_file .= file_get_contents($hiera_cluster_all);
    }

    // Get the first_master controler node
    $first_master_machine_id = $cluster["first_master_machine_id"];
    $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.id='$first_master_machine_id' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    if($n != 1){
        $json["status"] = "error";
        $json["message"] = "Cannot find a single IP (found more or less than one IP) for this hostname's management network when doing: $q line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $first_master = mysqli_fetch_array($r);
    $first_master_hostname = $first_master["hostname"];
    $first_master_ipaddr   = $first_master["ipaddr"];

    // Do we have some messaging machines in the cluster
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='messaging'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $num_messaging_nodes = mysqli_num_rows($r);
    if($num_messaging_nodes > 0){
        $first_rabbit_machine_id = $cluster["first_rabbit_machine_id"];

        $q = "SELECT hostname FROM machines WHERE machines.id='$first_rabbit_machine_id' AND machines.cluster='$cluster_id'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "Cannot find first rabbit node when doing: $q line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $first_rabbit = mysqli_fetch_array($r);
        $first_rabbit_hostname = $first_rabbit["hostname"];
        $first_rabbit_ipaddr   = $first_rabbit["ipaddr"];

        $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='messaging' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)."  line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        $enc_rabbit_hostnames = "      all_rabbits:\n";
        $enc_rabbit_ips = "      all_rabbits_ips:\n";
        for($i=0;$i<$n;$i++){
            $one_rabbit = mysqli_fetch_array($r);
            $one_rabbit_hostname = $one_rabbit["hostname"];
            $enc_rabbit_hostnames .= "         - $one_rabbit_hostname\n";
            $one_rabbit_ipaddr = $one_rabbit["ipaddr"];
            $enc_rabbit_ips .= "         - $one_rabbit_ipaddr\n";
        }
        $enc_rabbits = $enc_rabbit_hostnames;
        $enc_rabbits .= $enc_rabbit_ips;
    }

    # Calculate the bridge list. If there's no network with bridge for this cluster, then we have a single br-ex.
    # Otherwise, we have a list of bridge from db.
    $q = "SELECT bridgename FROM networks WHERE cluster='$cluster_id' AND role='ovs-bridge'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $bridge_list = array();
    $n = mysqli_num_rows($r);
    if($n > 0){
        for($i=0;$i<$n;$i++){
            $one_network = mysqli_fetch_array($r);
            $bridge_list[] = $one_network["bridgename"];
        }
    }else{
        $bridge_list[] = "br-ex";
    }

    if($machine["role"] == "compute" || $machine["role"] == "network"){
        $enc_bridge_list = "      bridge_mapping_list:\n";
        $enc_external_netlist = "      external_network_list:\n";
        $neutron_external_network_name = $machine["neutron_external_network_name"];
        for($i=0;$i<sizeof($bridge_list);$i++){
            if($i == 0){
                $enc_bridge_list .= "         - $neutron_external_network_name:" . $bridge_list[$i] ."\n";
                $enc_external_netlist .= "         - $neutron_external_network_name\n";
            }else{
                $enc_bridge_list .= "         - $neutron_external_network_name$i:" . $bridge_list[$i] ."\n";
                $enc_external_netlist .= "         - $neutron_external_network_name$i\n";
            }
        }
        $enc_bridge_list .= $enc_external_netlist;
    }else{
        $enc_bridge_list = "      bridge_mapping_list:\n";
        $enc_external_netlist = "      external_network_list:\n";

        $q_ext_net_name = "SELECT DISTINCT neutron_external_network_name FROM machines WHERE (role='compute' OR role='network') AND cluster='$cluster_id'";
        $r_ext_net_name = mysqli_query($con, $q_ext_net_name);
        if($r_ext_net_name === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n_ext_net_name = mysqli_num_rows($r_ext_net_name);
        for($i_ext_net_name=0;$i_ext_net_name<$n_ext_net_name;$i_ext_net_name++){
            $a_ext_net_name = mysqli_fetch_array($r_ext_net_name);
            $neutron_external_network_name = $a_ext_net_name["neutron_external_network_name"];
            for($i=0;$i<sizeof($bridge_list);$i++){
                if($i == 0){
                    $enc_external_netlist .= "         - $neutron_external_network_name\n";
                    $enc_bridge_list .= "         - $neutron_external_network_name:" . $bridge_list[$i] ."\n";
                }else{
                    $enc_external_netlist .= "         - $neutron_external_network_name$i\n";
                    $enc_bridge_list .= "         - $neutron_external_network_name$i:" . $bridge_list[$i] ."\n";
                }
            }
        }
        $enc_bridge_list .= $enc_external_netlist;
    }

    $openstack_release = $conf["releasenames"]["openstack_release"];

    # See if we have machines with role SQL, if yes, find who's first_sql_machine_id.
    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='sql'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n_sql_machines = mysqli_num_rows($r);

    $q = "SELECT * FROM machines WHERE cluster='$cluster_id' AND role='controller'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n_controller_machines = mysqli_num_rows($r);

    if($n_sql_machines > 0 || $n_controller_machines > 0){
        if($n_sql_machines > 0){
            $first_sql_machine_id = $cluster["first_sql_machine_id"];
            $vip_usage = "sql";
        }else{
            $first_sql_machine_id = $cluster["first_master_machine_id"];
            $vip_usage = "api";
        }
        if($first_sql_machine_id == $machine_id){
            $is_first_sql = "true";
        }else{
            $is_first_sql = "false";
        }

        // Get the first_sql sql node
        $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.id='$first_sql_machine_id' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "Cannot find a single IP (found more or less than one IP) for this hostname's management network when doing: $q line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $first_sql = mysqli_fetch_array($r);
        $first_sql_hostname = $first_sql["hostname"];
        $first_sql_ipaddr   = $first_sql["ipaddr"];

        $enc_ashn = "      all_sql:\n";
        $enc_asip = "      all_sql_ip:\n";
        $enc_snids = "      all_sql_ids:\n";

        # If there's SQL machines, that's what we search for,
        # otherwise, we do the query on controllers.
        if($n_sql_machines > 0){
            $qrole = 'sql';
        }else{
            $qrole = 'controller';
        }
        $q = "SELECT machines.hostname AS hostname, machines.id AS nodeid, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='$qrole' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        for($i=0;$i<$n;$i++){
            $one_sql = mysqli_fetch_array($r);
            $one_sql_hostname = $one_sql["hostname"];
            $one_sql_ipaddr = $one_sql["ipaddr"];
            $one_sql_id = $one_sql["nodeid"];
            $enc_ashn .= "         - $one_sql_hostname\n";
            $enc_asip .= "         - $one_sql_ipaddr\n";
            $enc_snids .= "         - $one_sql_id\n";
        }

        $enc_non_master_sql = "      non_master_sql:\n";
        $enc_non_master_sql_ip = "      non_master_sql_ip:\n";
        $q = "SELECT machines.hostname AS hostname, machines.id AS nodeid, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='$qrole' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi' AND machines.hostname!='$first_sql_hostname'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        for($i=0;$i<$n;$i++){
            $one_sql = mysqli_fetch_array($r);
            $one_sql_hostname = $one_sql["hostname"];
            $one_sql_ipaddr = $one_sql["ipaddr"];
            $one_sql_id = $one_sql["nodeid"];
            $enc_non_master_sql .= "         - $one_sql_hostname\n";
            $enc_non_master_sql_ip .= "         - $one_sql_ipaddr\n";
        }

        $q = "SELECT INET_NTOA(ips.ip) AS ipaddr, networks.iface1 AS iface1, networks.iface2 AS iface2, networks.vlan AS vlan, networks.cidr AS cidr FROM ips,networks WHERE ips.usefor='vip' AND ips.vip_usage='$vip_usage' AND networks.cluster='$cluster_id' AND ips.network=networks.id";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "Cannot find a single IP (found more or less than one IP) for this SQL's VIP when doing: $q line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $vip_sql = mysqli_fetch_array($r);
        $vip_sql_ip = $vip_sql["ipaddr"];
        $vip_sql_iface1 = $vip_sql["iface1"];
        $vip_sql_iface2 = $vip_sql["iface2"];
        $vip_sql_vlan   = $vip_sql["vlan"];
        $vip_sql_netmask= $vip_sql["cidr"];
        if(is_null($vip_sql_vlan) && $vip_sql_vlan != ""){
            $vip_sql_iface = "vlan" . $vip_sql_vlan;
        }else{
            if($vip_sql_iface2 != "none"){
                $vip_sql_iface = "bond0";
            }else{
                $vip_sql_iface = $vip_sql_iface1;
            }
        }
    }elseif($n_controller_machines > 0){
        $first_sql_hostname = $first_master_hostname;
        $first_sql_ipaddr   = $first_master_ipaddr;
    }

    ###########################################
    ### Generic configuration for all nodes ###
    ###########################################
    $enc_file .= "   oci::generic:\n";
#    $enc_file .= "      machine_hostname: $machine_hostname\n";
#    $enc_file .= "      machine_ip: $machine_ip\n";

    $enc_file .= "      ssh_listen_ips:\n";
    $enc_file .= "         - 127.0.0.1\n";
    $q = "SELECT INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.id='$machine_id' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='ovs-bridge' AND networks.role!='ipmi'";
    $r = mysqli_query($con, $q);
    if($r === FALSE){
        $json["status"] = "error";
        $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
        return $json;
    }
    $n = mysqli_num_rows($r);
    for($i=0;$i<$n;$i++){
        $one_ssh_listen_ip = mysqli_fetch_array($r);
        $ssh_listen_ip = $one_ssh_listen_ip["ipaddr"];
        $enc_file .= "         - $ssh_listen_ip\n";
    }

    $enc_file .= "      etc_hosts: ".base64_etc_hosts($con, $conf, $machine_id)."\n";
    $enc_file .= "      time_server_host: $cluster_time_server_host\n";


    if($cluster["generate_root_ssh_keys"] == "yes"){
        $enc_file .= "      generate_root_ssh_keys: true\n";
        $json = get_ssh_keypair($con, $conf, $cluster_id, $machine_hostname, 'root');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_root_ssh_pub: " . unserialize($json["data"]["pubkey"]) . "\n";
        $enc_file .= "      pass_root_ssh_priv: " . base64_encode(unserialize($json["data"]["privatekey"])) . "\n";
    }else{
        $enc_file .= "      generate_root_ssh_keys: false\n";
    }

    if($machine["role"] == "volume" && $cluster["generate_root_ssh_keys"] == "yes"){
        $q = "SELECT hostname FROM machines WHERE role='volume' AND cluster='$cluster_id' AND id!='$machine_id'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        $enc_file .= "      authorized_keys_hash:\n";
        for($i=0;$i<$n;$i++){
            $one_volume = mysqli_fetch_array($r);
            $one_volume_hostname = $one_volume["hostname"];
            $json = get_ssh_keypair($con, $conf, $cluster_id, $one_volume_hostname, 'root');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "         $one_volume_hostname: " . unserialize($json["data"]["pubkey"]) . "\n";
        }
    }

    $enc_file .= "\n";

    $skip_variables_dot_json_enc = "no";

    ###############################
    ### Role specific ENC output ##
    ###############################
    switch($machine["role"]){
    case "tempest":
        $enc_file .= "   oci::tempest:\n";
        $enc_file .= "      openstack_release: $openstack_release\n";
        $enc_file .= "      cluster_name: $cluster_name\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";

        $enc_file .= "      vip_hostname: $vip_hostname\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";

        if($num_volume_nodes > 0 OR $num_cephosd_nodes > 0){
            $enc_file .= "      cluster_has_cinder: true\n";
        }

        if($num_cephosd_nodes > 0){
            $enc_file .= "      cluster_has_ceph: true\n";
        }

        if($num_compute_nodes > 0){
            $enc_file .= "      cluster_has_compute: true\n";
        }

        if($num_swiftstore_nodes > 0){
            $enc_file .= "      cluster_has_swift: true\n";
        }

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'adminuser');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_adminuser: " . $json["data"] . "\n";
        break;

    case "controller":
        if($machine_id == $first_master_machine_id){
            $is_first_master = "true";
        }else{
            $is_first_master = "false";
        }

        // Start writing oci::controller class parameters
        $enc_file .= "   oci::controller:\n";
        $enc_file .= "      openstack_release: $openstack_release\n";
        $enc_file .= "      cluster_name: $cluster_name\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        $enc_file .= $enc_bridge_list;
        if($num_compute_nodes > 0 && $num_network_nodes == 0){
            $enc_file .= "      machine_iface: br-ex\n";
        }else{
            $enc_file .= "      machine_iface: eth0\n";
        }

        $enc_file .= "      bgp_to_the_host: $machine_puppet_bgp2host\n";

        $enc_file .= "      first_sql: $first_sql_hostname\n";
        $enc_file .= "      first_sql_ip: $first_sql_ipaddr\n";
        $enc_file .= "      is_first_sql: $is_first_sql\n";
        $enc_file .= "      sql_vip_ip: $vip_sql_ip\n";
        $enc_file .= "      sql_vip_netmask: $vip_sql_netmask\n";
        $enc_file .= "      sql_vip_iface: $vip_sql_iface\n";

        $enc_file .= "      initial_cluster_setup: $cluster_initial_cluster_setup\n";

        if($num_messaging_nodes > 0){
            $enc_file .= "      messaging_nodes_for_notifs: true\n";
            $enc_file .= $enc_rabbits;
        }

        $enc_file .= $enc_ashn;
        $enc_file .= $enc_asip;
        $enc_file .= $enc_snids;
        $enc_file .= $enc_non_master_sql;
        $enc_file .= $enc_non_master_sql_ip;

        $enc_file .= "      amp_secgroup_list: $amp_secgroup_list\n";
        $enc_file .= "      amp_boot_network_list: $amp_boot_network_list\n";

        if($cluster["cinder_backup_prefer_swift"] == "yes"){
            if($num_swiftstore_nodes > 0){
                $enc_file .= "      cinder_backup_backend: swift\n";
            }elseif($num_cephosd_nodes > 0){
                $enc_file .= "      cinder_backup_backend: ceph\n";
            }else{
                $enc_file .= "      cinder_backup_backend: none\n";
            }
        }else{
            if($num_cephosd_nodes > 0){
                $enc_file .= "      cinder_backup_backend: ceph\n";
            }elseif($num_swiftstore_nodes > 0){
                $enc_file .= "      cinder_backup_backend: swift\n";
            }else{
                $enc_file .= "      cinder_backup_backend: none\n";
            }
        }

        # No need for swift haproxy backend if no swift store.
        if($num_swiftstore_nodes == 0){
            $enc_file .= "      has_subrole_swift: false\n";
        }

        // Get all IPs from compute and volume nodes, that's needed to allow
        // SQL queries from them, as cinder-volume may run there.
        $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='compute' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        $enc_comp_list = "      compute_nodes:\n";
        $enc_compip_list = "      compute_nodes_ip:\n";
        for($i=0;$i<$n;$i++){
            $one_compute = mysqli_fetch_array($r);
            $one_compute_hostname = $one_compute["hostname"];
            $one_compute_ip = $one_compute["ipaddr"];
            $enc_comp_list .= "         - $one_compute_hostname\n";
            $enc_compip_list .= "         - $one_compute_ip\n";
        }
        $enc_file .= $enc_comp_list;
        $enc_file .= $enc_compip_list;

        // Same with volume nodes
        $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='volume' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        $enc_volu_list = "      volume_nodes:\n";
        $enc_voluip_list = "      volume_nodes_ip:\n";
        for($i=0;$i<$n;$i++){
            $one_volume = mysqli_fetch_array($r);
            $enc_volu_hostname = $one_volume["hostname"];
            $enc_volu_ip = $one_volume["ipaddr"];
            $enc_volu_list .= "         - $enc_volu_hostname\n";
            $enc_voluip_list .= "         - $enc_volu_ip\n";
        }
        $enc_file .= $enc_volu_list;
        $enc_file .= $enc_voluip_list;

        // Get the IP for VM trafic.
        $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr, networks.mtu AS mtu FROM machines, ips, networks WHERE machines.id='$machine_id' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role='vm-net'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n == 1){
            $vmnet_ip_array = mysqli_fetch_array($r);
            $vmnet_ip = $vmnet_ip_array["ipaddr"];
            $vmnet_mtu = $vmnet_ip_array["mtu"];
            $enc_file .= "      vmnet_ip: $vmnet_ip\n";
        // If we don't find a VMNet IP, then let's use the management network IP instead.
        }else{
            $enc_file .= "      vmnet_ip: $machine_ip\n";
        }

        $enc_file .= "      is_first_master: $is_first_master\n";
        $enc_file .= "      first_master: $first_master_hostname\n";
        $enc_file .= "      first_master_ip: $first_master_ipaddr\n";

        $enc_nfmc = "      non_first_master_controllers:\n";
        $enc_nfmc_ip = "      non_first_master_controllers_ip:\n";
        // Fetch all other controllers (ie: all but this machine we're setting-up)
        $qnfmc = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='controller' AND machines.cluster='$cluster_id' AND machines.id != '".$cluster["first_master_machine_id"]."' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
        $rnfmc = mysqli_query($con, $qnfmc);
        if($rnfmc === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $nnfmc = mysqli_num_rows($rnfmc);
        for($i=0;$i<$nnfmc;$i++){
            $one_other_master = mysqli_fetch_array($rnfmc);
            $one_other_master_hostname = $one_other_master["hostname"];
            $one_other_master_ipaddr = $one_other_master["ipaddr"];
            $enc_nfmc .= "         - $one_other_master_hostname\n";
            $enc_nfmc_ip .= "         - $one_other_master_ipaddr\n";
        }
        $enc_file .= $enc_nfmc;
        $enc_file .= $enc_nfmc_ip;

        $enc_file .= "      vip_hostname: $vip_hostname\n";
        $enc_file .= "      vip_ipaddr: $vip_ipaddr\n";
        $enc_file .= "      vip_netmask: $vip_netmask\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";
        if($cluster["swift_proxy_hostname"] == ""){
            $enc_file .= "      swiftproxy_hostname: none\n";
        }else{
            $enc_file .= "      swiftproxy_hostname: " . $cluster["swift_proxy_hostname"] ."\n";
        }
        $enc_file .= "      haproxy_custom_url: " . $cluster["haproxy_custom_url"] . "\n";
        $enc_file .= "      network_ipaddr: $network_ip\n";
        $enc_file .= "      network_cidr: $network_cidr\n";
        $enc_file .= "      other_masters:\n";

        $enc_omip  = "      other_masters_ip:\n";
        // Fetch all other controllers (ie: all but this machine we're setting-up)
        $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr FROM machines, ips, networks WHERE machines.role='controller' AND machines.cluster='$cluster_id' AND machines.id != '$machine_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role!='vm-net' AND networks.role!='ovs-bridge' AND networks.role!='ceph-cluster' AND networks.role!='ipmi'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        for($i=0;$i<$n;$i++){
            $one_other_master = mysqli_fetch_array($r);
            $one_other_master_hostname = $one_other_master["hostname"];
            $one_other_master_ipaddr = $one_other_master["ipaddr"];
            $enc_file .= "         - $one_other_master_hostname\n";
            $enc_omip .= "         - $one_other_master_ipaddr\n";
        }
        $enc_file .= $enc_omip;

        $enc_file .= $enc_amhn;
        $enc_file .= $enc_amip;
        $enc_file .= $enc_nids;

        $enc_file .= "      all_swiftproxy:\n";
        $enc_file .= $enc_allswiftproxies_hostname;
        $enc_file .= "      all_swiftproxy_ip:\n";
        $enc_file .= $enc_allswiftproxies_ip;

        if($n_sql_machines > 0){
            $enc_file .= "      has_subrole_db: false\n";
        }

        if($num_compute_nodes > 0){
            $enc_file .= "      has_subrole_glance: true\n";
            $enc_file .= "      has_subrole_nova: true\n";
            $enc_file .= "      has_subrole_neutron: true\n";
            $enc_file .= "      has_subrole_aodh: true\n";
            $enc_file .= "      has_subrole_octavia: true\n";
            $enc_file .= "      has_subrole_magnum: true\n";
        }

        if($num_cephmon_nodes > 0){
            $enc_file .= "      cluster_has_mons: true\n";
        }else{
            $enc_file .= "      cluster_has_mons: false\n";
        }

        if($num_cephosd_nodes > 0 || $num_volume_nodes > 0){
            if($num_compute_nodes > 0){
                $enc_file .= "      has_subrole_cinder: true\n";
            }
            if($num_volume_nodes > 0){
                $enc_file .= "      cluster_has_cinder_volumes: true\n";
            }
            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'libvirtuuid');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_libvirtuuid: " . $json["data"] . "\n";
        }
        if($num_cephosd_nodes > 0){
            $enc_file .= "      cluster_has_osds: true\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'fsid');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_fsid: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'bootstraposdkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_bootstrap_osd_key: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'adminkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_admin_key: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'openstackkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_openstack_key: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'monkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_mon_key: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'mgrkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_mgr_key: " . $json["data"] . "\n";

            $ret = enc_get_mon_nodes($con,$conf,$cluster_id,'cephmon');
            if($ret["status"] != "success"){
                return $ret;
            }
            $enc_file .= "      ceph_mon_initial_members: " . $ret["data"]["osd_members"] . "\n";
            $enc_file .= "      ceph_mon_host: " . $ret["data"]["osd_ips"] . "\n";
            if($num_compute_nodes > 0){
                if($num_messaging_nodes > 0){
                    $enc_file .= "      has_subrole_gnocchi_services: false\n";
                }else{
                    $enc_file .= "      has_subrole_gnocchi_services: true\n";
                }
                $enc_file .= "      has_subrole_gnocchi: true\n";
                $enc_file .= "      has_subrole_ceilometer: true\n";
                $enc_file .= "      has_subrole_panko: true\n";
                $enc_file .= "      has_subrole_cloudkitty: true\n";
            }

            $json = get_cluster_password($con, $conf, $cluster_id, 'gnocchi', 'uuid');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_gnocchi_rscuuid: " . $json["data"] . "\n";

        }else{
            $enc_file .= "      cluster_has_osds: false\n";
        }

        if($num_network_nodes == 0){
            $enc_file .= "      has_subrole_network_node: true\n";
        }else{
            $enc_file .= "      has_subrole_network_node: false\n";
        }

        if($cluster["glance_prefer_ceph_over_swift"] == "yes"){
            if($num_cephosd_nodes > 0){
                $glance_backend = 'ceph';
            }elseif($num_swiftstore_nodes > 0){
                $glance_backend = 'swift';
            }else{
                $glance_backend = 'file';
            }
        }else{
            if($num_swiftstore_nodes > 0){
                $glance_backend = 'swift';
            }elseif($num_cephosd_nodes > 0){
                $glance_backend = 'ceph';
            }else{
                $glance_backend = 'file';
            }
        }
        $enc_file .= "      glance_backend: " . $glance_backend . "\n";

        // Send all passwords
        $json = get_cluster_password($con, $conf, $cluster_id, 'haproxy', 'stats');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_haproxy_stats: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'mysql', 'rootuser');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_mysql_rootuser: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'mysql', 'backup');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_mysql_backup: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'rabbitmq', 'cookie');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_rabbitmq_cookie: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'rabbitmq', 'monitoring');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_rabbitmq_monitoring: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'adminuser');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_adminuser: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'credential1');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_credkey1: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'credential2');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_credkey2: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'fernetkey1');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_fernkey1: " . base64_encode(substr($json["data"],0,32)) . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'fernetkey2');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_fernkey2: " . base64_encode(substr($json["data"],0,32)) . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'ssh');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_ssh_pub: " . unserialize($json["data"]["ssh_pub"]) . "\n";
        $enc_file .= "      pass_keystone_ssh_priv: " . base64_encode(unserialize($json["data"]["ssh_priv"])) . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_nova_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'apidb');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_nova_apidb: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_nova_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_nova_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'novaneutron', 'shared_secret');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_metadata_proxy_shared_secret: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'placement', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_placement_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'placement', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_placement_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'glance', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_glance_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'glance', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_glance_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'glance', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_glance_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'glance', 'ssh');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_glance_ssh_pub: " . unserialize($json["data"]["ssh_pub"]) . "\n";
        $enc_file .= "      pass_glance_ssh_priv: " . base64_encode(unserialize($json["data"]["ssh_priv"])) . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'neutron', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_neutron_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'neutron', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_neutron_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'neutron', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_neutron_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cinder_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cinder_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cinder_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'heat', 'encryptkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_heat_encryptkey: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'heat', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_heat_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'heat', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_heat_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'heat', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_heat_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'heat', 'keystone_domain');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_heat_keystone_domain: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'swift', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_swift_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'swift', 'encryption');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_swift_encryption: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'horizon', 'secretkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_horizon_secretkey: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'barbican', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_barbican_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'barbican', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_barbican_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'barbican', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_barbican_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'openstackkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_openstack_key: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'gnocchi', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_gnocchi_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'gnocchi', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_gnocchi_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'gnocchi', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_gnocchi_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'panko', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_panko_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'panko', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_panko_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'panko', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_panko_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'ceilometer', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_ceilometer_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'ceilometer', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_ceilometer_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'ceilometer', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_ceilometer_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'ceilometer', 'telemetry');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_ceilometer_telemetry: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cloudkitty', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cloudkitty_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cloudkitty', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cloudkitty_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cloudkitty', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cloudkitty_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'redis', 'redis');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_redis: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'aodh', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_aodh_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'aodh', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_aodh_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'aodh', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_aodh_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'octavia', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_octavia_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'octavia', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_octavia_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'octavia', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_octavia_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'octavia', 'heatbeatkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_octavia_heatbeatkey: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'magnum', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_magnum_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'magnum', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_magnum_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'magnum', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_magnum_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'magnum', 'domain');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_magnum_domain: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'designate', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_designate_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'designate', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_designate_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'designate', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_designate_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'zabbix', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_zabbix_authtoken: " . $json["data"] . "\n";

        $enc_file .= "      disable_notifications: " . $disable_notifications . "\n";
        $enc_file .= "      enable_monitoring_graphs: " . $enable_monitoring_graphs . "\n";
        $enc_file .= "      monitoring_graphite_host: " . $monitoring_graphite_host . "\n";
        $enc_file .= "      monitoring_graphite_port: " . $monitoring_graphite_port . "\n";

        $qgpu = "SELECT use_gpu,gpu_name,gpu_vendor_id,gpu_product_id,gpu_device_type FROM machines WHERE cluster='$cluster_id' AND use_gpu='yes'";
        $rgpu = mysqli_query($con, $qgpu);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $ngpu = mysqli_num_rows($rgpu);
        if($ngpu == 0){
            $enc_file .= "      use_gpu: false\n";
        }else{
            $enc_file .= "      use_gpu: true\n";
            $qgpu = "SELECT DISTINCT use_gpu,gpu_name,gpu_vendor_id,gpu_product_id,gpu_device_type FROM machines WHERE cluster='$cluster_id' AND use_gpu='yes'";
            $rgpu = mysqli_query($con, $qgpu);
            if($rgpu === FALSE){
                $json["status"] = "error";
                $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
                return $json;
            }
            $ngpu = mysqli_num_rows($rgpu);
            $enc_gpu_def = "      gpu_def:\n";
            for($i=0;$i<$ngpu;$i++){
                $a_gpu = mysqli_fetch_array($rgpu);
                $enc_gpu_def .= "         - " . $a_gpu["gpu_name"] . "." . $a_gpu["gpu_vendor_id"] . "." . $a_gpu["gpu_product_id"]  . "." . $a_gpu["gpu_device_type"] . "\n";
            }
            $enc_file .= $enc_gpu_def;
        }

        if($cluster["extswift_use_external"] == "yes"){
            $enc_file .= "      extswift_use_external: true\n";
            $enc_file .= "      extswift_auth_url: " . $cluster["extswift_auth_url"] . "\n";
            $enc_file .= "      extswift_proxy_url: " . $cluster["extswift_proxy_url"] . "\n";
            $enc_file .= "      extswift_project_name: " . $cluster["extswift_project_name"] . "\n";
            $enc_file .= "      extswift_project_domain_name: " . $cluster["extswift_project_domain_name"] . "\n";
            $enc_file .= "      extswift_user_name: " . $cluster["extswift_user_name"] . "\n";
            $enc_file .= "      extswift_user_domain_name: " . $cluster["extswift_user_domain_name"] . "\n";
            $enc_file .= "      extswift_password: " . $cluster["extswift_password"] . "\n";
        }

        if($num_compute_nodes > 0 && $cluster["install_designate"] == "yes"){
            $enc_file .= "      has_subrole_designate: true\n";
        }

        # See if one node has Neutron dynamic routing, and install the extension in the
        # controller if we have one.
        $qdragent = "SELECT id FROM machines WHERE cluster='$cluster_id' AND neutron_install_dragent='yes'";
        $rdragent = mysqli_query($con, $qdragent);
        if($rdragent === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $ndragent = mysqli_num_rows($rdragent);
        if($ndragent == 0){
            $enc_file .= "      use_dynamic_routing: false\n";
        }else{
            $enc_file .= "      use_dynamic_routing: true\n";
        }

        break;

    case "messaging":
        if($machine_id == $first_rabbit_machine_id){
            $is_first_rabbit = "true";
        }else{
            $is_first_rabbit = "false";
        }

        // Start writing oci::controller class parameters
        $enc_file .= "   oci::messaging:\n";
        $enc_file .= "      openstack_release: $openstack_release\n";
        $enc_file .= "      cluster_name: $cluster_name\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";

        $enc_file .= "      first_master: $first_master_hostname\n";
        $enc_file .= "      first_master_ip: $first_master_ipaddr\n";

        $enc_file .= $enc_amhn;
        $enc_file .= $enc_amip;

        $enc_file .= "      is_first_rabbit: $is_first_rabbit\n";
        $enc_file .= "      first_rabbit: $first_rabbit_hostname\n";
        $enc_file .= $enc_rabbits;

        $enc_file .= "      initial_cluster_setup: $cluster_initial_cluster_setup\n";

        if($num_compute_nodes > 0){
            $enc_file .= "      has_subrole_glance: true\n";
            $enc_file .= "      has_subrole_nova: true\n";
            $enc_file .= "      has_subrole_neutron: true\n";
            $enc_file .= "      has_subrole_aodh: true\n";
            $enc_file .= "      has_subrole_octavia: true\n";
            $enc_file .= "      has_subrole_magnum: true\n";
        }

        if($num_cephosd_nodes > 0){
            $enc_file .= "      has_subrole_gnocchi: true\n";
            $enc_file .= "      has_subrole_ceilometer: true\n";
            $enc_file .= "      has_subrole_cloudkitty: true\n";
            $enc_file .= "      has_subrole_panko: true\n";
        }
        if($num_cephosd_nodes > 0 || $num_volume_nodes > 0){
            $enc_file .= "      has_subrole_cinder: true\n";
        }
        if($cluster["install_designate"] == "yes"){
            $enc_file .= "      has_subrole_designate: true\n";
        }

        $enc_file .= "      vip_hostname: $vip_hostname\n";
        $enc_file .= "      vip_ipaddr: $vip_ipaddr\n";
        $enc_file .= "      vip_netmask: $vip_netmask\n";

        $enc_file .= "      sql_vip_ip: $vip_sql_ip\n";
        $enc_file .= "      sql_vip_netmask: $vip_sql_netmask\n";
        $enc_file .= "      sql_vip_iface: $vip_sql_iface\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'rabbitmq', 'cookie');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_rabbitmq_cookie: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'rabbitmq', 'monitoring');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_rabbitmq_monitoring: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_nova_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'glance', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_glance_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'neutron', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_neutron_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cinder_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'heat', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_heat_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'barbican', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_barbican_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'gnocchi', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_gnocchi_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'panko', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_panko_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'ceilometer', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_ceilometer_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cloudkitty', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cloudkitty_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'aodh', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_aodh_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'octavia', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_octavia_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'magnum', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_magnum_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'designate', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_designate_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'gnocchi', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_gnocchi_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'gnocchi', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_gnocchi_authtoken: " . $json["data"] . "\n";

        # Ceph configuration
        if($num_billosd_nodes == 0){
            $servicetype = "ceph";
            $mon_node_type = 'cephmon';
        }else{
            $servicetype = "bill";
            $mon_node_type = 'billmon';
        }

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'fsid');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_fsid: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'bootstraposdkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_bootstrap_osd_key: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'adminkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_admin_key: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'openstackkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_openstack_key: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'monkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_mon_key: " . $json["data"] . "\n";

        $ret = enc_get_mon_nodes($con,$conf,$cluster_id,$mon_node_type);
        if($ret["status"] != "success"){
            return $ret;
        }
        $enc_file .= "      ceph_mon_initial_members: " . $ret["data"]["osd_members"] . "\n";
        $enc_file .= "      ceph_mon_host: " . $ret["data"]["osd_ips"] . "\n";

        break;

    case "sql":
        $enc_file .= "   oci::sql:\n";
        $enc_file .= "      cluster_name: $cluster_name\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        $enc_file .= "      network_ipaddr: $network_ip\n";
        $enc_file .= "      network_cidr: $network_cidr\n";
        $enc_file .= "      first_sql: $first_sql_hostname\n";
        $enc_file .= "      first_sql_ip: $first_sql_ipaddr\n";
        $enc_file .= "      is_first_sql: $is_first_sql\n";
        $enc_file .= "      sql_vip_ip: $vip_sql_ip\n";
        $enc_file .= "      sql_vip_netmask: $vip_sql_netmask\n";
        $enc_file .= "      sql_vip_iface: $vip_sql_iface\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";

        $enc_file .= $enc_ashn;
        $enc_file .= $enc_asip;
        $enc_file .= $enc_snids;
        $enc_file .= $enc_non_master_sql;
        $enc_file .= $enc_non_master_sql_ip;

        if($num_compute_nodes > 0){
            $enc_file .= "      has_subrole_glance: true\n";
            $enc_file .= "      has_subrole_nova: true\n";
            $enc_file .= "      has_subrole_neutron: true\n";
            $enc_file .= "      has_subrole_aodh: true\n";
            $enc_file .= "      has_subrole_octavia: true\n";
        }else{
            $enc_file .= "      has_subrole_glance: false\n";
            $enc_file .= "      has_subrole_nova: false\n";
            $enc_file .= "      has_subrole_neutron: false\n";
            $enc_file .= "      has_subrole_aodh: false\n";
            $enc_file .= "      has_subrole_octavia: false\n";
        }
        if($num_cephosd_nodes > 0 || $num_volume_nodes > 0){
            $enc_file .= "      has_subrole_cinder: true\n";
        }else{
            $enc_file .= "      has_subrole_cinder: false\n";
        }
        if($num_cephosd_nodes > 0){
            $enc_file .= "      has_subrole_gnocchi: true\n";
            $enc_file .= "      has_subrole_ceilometer: true\n";
            $enc_file .= "      has_subrole_panko: true\n";
            $enc_file .= "      has_subrole_cloudkitty: true\n";
        }else{
            $enc_file .= "      has_subrole_gnocchi: false\n";
            $enc_file .= "      has_subrole_ceilometer: false\n";
            $enc_file .= "      has_subrole_panko: false\n";
            $enc_file .= "      has_subrole_cloudkitty: false\n";
        }
        $enc_file .= "      has_subrole_heat: true\n";
        $enc_file .= "      has_subrole_barbican: true\n";


        // Send all passwords
        $json = get_cluster_password($con, $conf, $cluster_id, 'mysql', 'rootuser');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_mysql_rootuser: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'mysql', 'backup');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_mysql_backup: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_nova_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'apidb');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_nova_apidb: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'placement', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_placement_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'glance', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_glance_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'neutron', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_neutron_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cinder_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'heat', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_heat_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'barbican', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_barbican_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'gnocchi', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_gnocchi_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'panko', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_panko_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'ceilometer', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_ceilometer_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cloudkitty', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cloudkitty_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'aodh', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_aodh_db: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'octavia', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_octavia_db: " . $json["data"] . "\n";
        break;

    case "swiftproxy":
        $enc_file .= "   oci::swiftproxy:\n";
        $enc_file .= "      openstack_release: $openstack_release\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        $enc_file .= "      first_master: $first_master_hostname\n";
        $enc_file .= "      first_master_ip: $first_master_ipaddr\n";
        $enc_file .= "      vip_hostname: $vip_hostname\n";
        $enc_file .= "      vip_ipaddr: $vip_ipaddr\n";
        $enc_file .= "      vip_netmask: $vip_netmask\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";
        $enc_file .= "      network_ipaddr: $network_ip\n";
        $enc_file .= "      network_cidr: $network_cidr\n";
        $enc_file .= "      statsd_hostname: $cluster_statsd_hostname\n";
        $enc_file .= "      monitoring_graphite_host: " . $monitoring_graphite_host . "\n";
        $enc_file .= "      monitoring_graphite_port: " . $monitoring_graphite_port . "\n";
        $enc_file .= $enc_amhn;
        $enc_file .= $enc_amip;

        if($num_messaging_nodes > 0){
            $enc_file .= "      messaging_nodes_for_notifs: true\n";
            $enc_file .= $enc_rabbits;
        }

        $enc_file .= "      all_swiftstore:\n";
        $enc_file .= $enc_allswiftstore_hostanme;
        $enc_file .= "      all_swiftstore_ip:\n";
        $enc_file .= $enc_allswiftstore_ip;
        $enc_file .= "      all_swiftproxy:\n";
        $enc_file .= $enc_allswiftproxies_hostname;
        $enc_file .= "      all_swiftproxy_ip:\n";
        $enc_file .= $enc_allswiftproxies_ip;
        if($cluster["swift_proxy_hostname"] == ""){
            $enc_file .= "      swiftproxy_hostname: none\n";
        }else{
            $enc_file .= "      swiftproxy_hostname: " . $cluster["swift_proxy_hostname"] ."\n";
        }

        $enc_file .= "      swift_disable_encryption: " . $cluster["swift_disable_encryption"] ."\n";
        $enc_file .= $enc_swift_storage_allowed_networks;


        $q = "SELECT swiftregions.id AS region_id FROM machines,locations,swiftregions WHERE locations.swiftregion=swiftregions.name AND locations.id=machines.location_id AND machines.id='$machine_id'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "Cannot find swiftregion ID";
            return $json;
        }
        $swiftregion = mysqli_fetch_array($r);
        $swiftregion_id = $swiftregion["region_id"];
        $enc_file .= "      swiftregion_id: $swiftregion_id\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'swift', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_swift_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'swift', 'hashpathsuffix');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_swift_hashpathsuffix: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'swift', 'hashpathprefix');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_swift_hashpathprefix: " . $json["data"] . "\n";

        $enc_file .= "      swift_encryption_key_id: \"" . $cluster["swift_encryption_key_id"] . "\"\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'haproxy', 'stats');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_haproxy_stats: " . $json["data"] . "\n";

        $qexpir = "SELECT hostname FROM machines WHERE cluster='$cluster_id' AND role='swiftproxy' LIMIT 1";
        $rexpir = mysqli_query($con, $qexpir);
        if($rexpir === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($rexpir);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "Cannot find expirer";
            return $json;
        }
        $expirer_machine = mysqli_fetch_array($rexpir);
        $expirer_hostname = $expirer_machine["hostname"];
        if($machine_hostname == $expirer_hostname){
            $enc_file .= "      object_expirer_enable: true" . "\n";
        }else{
            $enc_file .= "      object_expirer_enable: false" . "\n";
        }

        # What type of storage on the proxy?
        if($machine["swift_store_account"] == "yes"){
            $enc_file .= "      swift_store_account: true" . "\n";
        }else{
            $enc_file .= "      swift_store_account: false" . "\n";
        }
        if($machine["swift_store_container"] == "yes"){
            $enc_file .= "      swift_store_container: true" . "\n";
        }else{
            $enc_file .= "      swift_store_container: false" . "\n";
        }
        if($machine["swift_store_object"] == "yes"){
            $enc_file .= "      swift_store_object: true" . "\n";
        }else{
            $enc_file .= "      swift_store_object: false" . "\n";
        }

        if($machine["swift_store_account"] == "yes" || $machine["swift_store_container"] == "yes" || $machine["swift_store_object"] == "yes"){
            $enc_file .= "      swift_object_replicator_concurrency: " . $swift_object_replicator_concurrency . "\n";
            $enc_file .= "      swift_rsync_connection_limit: " . $swift_rsync_connection_limit . "\n";

            $enc_file .= "      block_devices:\n";
            if($machine["install_on_raid"] == "no"){
                $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '%da'";
            }else{
                switch($machine["raid_type"]){
                case "0":
                case "1":
                    $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '".$machine["raid_dev0"]."' AND name NOT LIKE '".$machine["raid_dev1"]."'";
                    break;
                case "10":
                    $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '".$machine["raid_dev0"]."' AND name NOT LIKE '".$machine["raid_dev1"]."' AND name NOT LIKE '".$machine["raid_dev2"]."' AND name NOT LIKE '".$machine["raid_dev3"]."'";
                    break;
                case "5":
                default:
                    die("Not supported yet.");
                    break;
                }
            }
            $r = mysqli_query($con, $q);
            $n = mysqli_num_rows($r);
            if($n < 1){
                $json["status"] = "error";
                $json["message"] = "Cannot find block devices line ".__LINE__." file ".__FILE__;
                return $json;
            }
            for($i=0;$i<$n;$i++){
                $a = mysqli_fetch_array($r);
                $hdd_name = $a["name"];
                $enc_file .= "         - $hdd_name\n";
            }
        }

        break;

    case "swiftstore":
        $enc_file .= "   oci::swiftstore:\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        $enc_file .= "      network_ipaddr: $network_ip\n";
        $enc_file .= "      network_cidr: $network_cidr\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";
        $enc_file .= "      zoneid: $machine_location\n";
        $enc_file .= "      block_devices:\n";

        if($machine["install_on_raid"] == "no"){
            $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '%da'";
        }else{
            switch($machine["raid_type"]){
            case "0":
            case "1":
                $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '".$machine["raid_dev0"]."' AND name NOT LIKE '".$machine["raid_dev1"]."'";
                break;
            case "10":
                $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '".$machine["raid_dev0"]."' AND name NOT LIKE '".$machine["raid_dev1"]."' AND name NOT LIKE '".$machine["raid_dev2"]."' AND name NOT LIKE '".$machine["raid_dev3"]."'";
                break;
            case "5":
            default:
                die("Not supported yet.");
                break;
            }
        }
        $r = mysqli_query($con, $q);
        $n = mysqli_num_rows($r);
        if($n < 1){
            $json["status"] = "error";
            $json["message"] = "Cannot find block devices line ".__LINE__." file ".__FILE__;
            return $json;
        }
        for($i=0;$i<$n;$i++){
            $a = mysqli_fetch_array($r);
            $hdd_name = $a["name"];
            $enc_file .= "         - $hdd_name\n";
        }

        # What type of storage on the store?
        if($machine["swift_store_account"] == "yes"){
            $enc_file .= "      swift_store_account: true" . "\n";
        }else{
            $enc_file .= "      swift_store_account: false" . "\n";
        }
        if($machine["swift_store_container"] == "yes"){
            $enc_file .= "      swift_store_container: true" . "\n";
        }else{
            $enc_file .= "      swift_store_container: false" . "\n";
        }
        if($machine["swift_store_object"] == "yes"){
            $enc_file .= "      swift_store_object: true" . "\n";
        }else{
            $enc_file .= "      swift_store_object: false" . "\n";
        }


        $enc_file .= "      statsd_hostname: $cluster_statsd_hostname\n";
        $enc_file .= "      monitoring_graphite_host: " . $monitoring_graphite_host . "\n";
        $enc_file .= "      monitoring_graphite_port: " . $monitoring_graphite_port . "\n";

        $enc_file .= "      swift_object_replicator_concurrency: " . $swift_object_replicator_concurrency . "\n";
        $enc_file .= "      swift_rsync_connection_limit: " . $swift_rsync_connection_limit . "\n";

        $enc_file .= "      all_swiftstore:\n";
        $enc_file .= $enc_allswiftstore_hostanme;
        $enc_file .= "      all_swiftstore_ip:\n";
        $enc_file .= $enc_allswiftstore_ip;
        $enc_file .= "      all_swiftproxy:\n";
        $enc_file .= $enc_allswiftproxies_hostname;
        $enc_file .= "      all_swiftproxy_ip:\n";
        $enc_file .= $enc_allswiftproxies_ip;
        $enc_file .= $enc_swift_storage_allowed_networks;

        $json = get_cluster_password($con, $conf, $cluster_id, 'swift', 'hashpathsuffix');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_swift_hashpathsuffix: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'swift', 'hashpathprefix');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_swift_hashpathprefix: " . $json["data"] . "\n";
        break;

    case "network":
        $enc_file .= "   oci::network:\n";
        $enc_file .= "      openstack_release: $openstack_release\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      cluster_name: $cluster_name\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        $enc_file .= $enc_bridge_list;
        $enc_file .= "      first_master: $first_master_hostname\n";
        $enc_file .= "      first_master_ip: $first_master_ipaddr\n";
        $enc_file .= "      cluster_domain: $cluster_domain\n";
        $enc_file .= "      sql_vip_ip: $vip_sql_ip\n";

        if($num_messaging_nodes > 0){
            $enc_file .= "      messaging_nodes_for_notifs: true\n";
            $enc_file .= $enc_rabbits;
        }

        // Get the IP for VM trafic.
        $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr, networks.mtu AS mtu FROM machines, ips, networks WHERE machines.id='$machine_id' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role='vm-net'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n == 1){
            $vmnet_ip_array = mysqli_fetch_array($r);
            $vmnet_ip = $vmnet_ip_array["ipaddr"];
            $enc_file .= "      vmnet_ip: $vmnet_ip\n";
        // If we don't find a VMNet IP, then let's use the management network IP instead.
        }else{
            $enc_file .= "      vmnet_ip: $machine_ip\n";
        }

        $enc_file .= $enc_amhn;
        $enc_file .= $enc_amip;
        $enc_file .= "      vip_hostname: $vip_hostname\n";
        $enc_file .= "      vip_ipaddr: $vip_ipaddr\n";
        $enc_file .= "      vip_netmask: $vip_netmask\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";
        $enc_file .= "      network_ipaddr: $network_ip\n";
        $enc_file .= "      network_cidr: $network_cidr\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_nova_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'neutron', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_neutron_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'neutron', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_neutron_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'novaneutron', 'shared_secret');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_metadata_proxy_shared_secret: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'keystone', 'adminuser');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_keystone_adminuser: " . $json["data"] . "\n";

        $enc_file .= "      disable_notifications: " . $disable_notifications . "\n";

        if($num_compute_nodes > 0 && $cluster["install_designate"] == "yes"){
            $enc_file .= "      has_subrole_designate: true\n";
        }

        # See if one node has Neutron dynamic routing, and install the extension in the
        # compute if we have one.
        $qdragent = "SELECT id FROM machines WHERE cluster='$cluster_id' AND neutron_install_dragent='yes'";
        $rdragent = mysqli_query($con, $qdragent);
        if($rdragent === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $ndragent = mysqli_num_rows($rdragent);
        if($ndragent == 0){
            $enc_file .= "      use_dynamic_routing: false\n";
        }else{
            $enc_file .= "      use_dynamic_routing: true\n";
        }

        break;

    case "compute":
        # See if there's some controllers in the clusters that aren't full installed yet
        $q = "SELECT COUNT(id) AS numnotinstalled FROM machines WHERE role='controller' AND cluster='$cluster_id' AND (status!='installed' OR puppet_status!='success');";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n != 1){
            $json["status"] = "error";
            $json["message"] = "Num of line is not one.";
            return $json;
        }
        $a = mysqli_fetch_array($r);
        $n = $a["numnotinstalled"];
        # If that's the case, and we're setting-up an hyper-converged node,
        # then we install it as cephosd first.
        if(($n != 0) && ($machine["compute_is_cephosd"] == "yes")){

            # As we're artificially using another role, the
            # ENC variables from variables.json will be added
            # to the wrong class. So we do this workaround:
            $skip_variables_dot_json_enc = "yes";

            $enc_file .= "   oci::cephosd:\n";
            $enc_file .= "      machine_hostname: $machine_hostname\n";
            $enc_file .= "      machine_ip: $machine_ip\n";
            $enc_file .= "      vip_hostname: $vip_hostname\n";
            $enc_file .= "      vip_ipaddr: $vip_ipaddr\n";
            $enc_file .= "      vip_netmask: $vip_netmask\n";
            $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";
            $enc_file .= "      network_ipaddr: $network_ip\n";
            $enc_file .= "      network_cidr: $network_cidr\n";
            $enc_file .= "      bgp_to_the_host: $machine_puppet_bgp2host\n";

            if($machine["ceph_osd_initial_setup"] == "yes"){
                    $ceph_osd_initial_setup = "true";
            }else{
                    $ceph_osd_initial_setup = "false";
            }
            $enc_file .= "      ceph_osd_initial_setup: $ceph_osd_initial_setup\n";

            $enc_file .= "      block_devices:\n";

            $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '%da'";
            $r = mysqli_query($con, $q);
            $n = mysqli_num_rows($r);
            if($n < 1){
                $json["status"] = "error";
                $json["message"] = "Cannot find block devices line ".__LINE__." file ".__FILE__;
                return $json;
            }
            for($i=0;$i<$n;$i++){
                $a = mysqli_fetch_array($r);
                $hdd_name = $a["name"];
                $enc_file .= "         - $hdd_name\n";
            }

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'fsid');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_fsid: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'bootstraposdkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_bootstrap_osd_key: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'adminkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_admin_key: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'openstackkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_openstack_key: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'monkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_mon_key: " . $json["data"] . "\n";

            $ret = enc_get_mon_nodes($con,$conf,$cluster_id,'cephmon');
            if($ret["status"] != "success"){
                return $ret;
            }
            $enc_file .= "      ceph_mon_initial_members: " . $ret["data"]["osd_members"] . "\n";
            $enc_file .= "      ceph_mon_host: " . $ret["data"]["osd_ips"] . "\n";

            // Get the IP for the cluster network.
            $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr, networks.mtu AS mtu, networks.ip AS networkaddr, networks.cidr AS networkcidr FROM machines, ips, networks WHERE machines.id='$machine_id' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role='ceph-cluster' AND networks.role='ipmi'";
            $r = mysqli_query($con, $q);
            if($r === FALSE){
                $json["status"] = "error";
                $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
                return $json;
            }
            $n = mysqli_num_rows($r);
            if($n == 1){
                $cephnet_ip_array = mysqli_fetch_array($r);
                $cephnet_ip          = $cephnet_ip_array["ipaddr"];
                $cephnet_mtu         = $cephnet_ip_array["mtu"];
                $cephnet_networkaddr = $cephnet_ip_array["networkaddr"];
                $cephnet_networkcidr = $cephnet_ip_array["networkcidr"];
                $enc_file .= "      use_ceph_cluster_net: true\n";
                $enc_file .= "      cephnet_ip: $cephnet_ip\n";
                $enc_file .= "      cephnet_network_addr: $cephnet_networkaddr\n";
                $enc_file .= "      cephnet_network_cidr: $cephnet_networkcidr\n";
                $enc_file .= "      cephnet_mtu: $cephnet_mtu\n";
            // If we don't find a ceph-cluster IP, then let's use the management network IP instead.
            }else{
                $enc_file .= "      use_ceph_cluster_net: false\n";
                $enc_file .= "      cephnet_ip: $machine_ip\n";
                $enc_file .= "      cephnet_mtu: 0\n";
            }
        }else{
            $enc_file .= "   oci::compute:\n";
            $enc_file .= "      openstack_release: $openstack_release\n";
            $enc_file .= "      machine_hostname: $machine_hostname\n";
            $enc_file .= "      cluster_name: $cluster_name\n";
            $enc_file .= "      machine_ip: $machine_ip\n";
            $enc_file .= $enc_bridge_list;
            $enc_file .= "      first_master: $first_master_hostname\n";
            $enc_file .= "      first_master_ip: $first_master_ipaddr\n";
            $enc_file .= "      cluster_domain: $cluster_domain\n";
            $enc_file .= "      sql_vip_ip: $vip_sql_ip\n";
            $enc_file .= "      bgp_to_the_host: $machine_puppet_bgp2host\n";

            if($num_messaging_nodes > 0){
                $enc_file .= "      messaging_nodes_for_notifs: true\n";
                $enc_file .= $enc_rabbits;
            }

            if($machine["nested_virt"] == "cluster_value"){
                if($cluster["nested_virt"] == "yes"){
                    $nested_virt = "true";
                }else{
                    $nested_virt = "false";
                }
            }elseif($machine["nested_virt"] == "yes"){
                    $nested_virt = "true";
            }else{
                    $nested_virt = "false";
            }
            $enc_file .= "      nested_virt: $nested_virt\n";

            // Get the IP for VM trafic.
            $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr, networks.mtu AS mtu FROM machines, ips, networks WHERE machines.id='$machine_id' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role='vm-net'";
            $r = mysqli_query($con, $q);
            if($r === FALSE){
                $json["status"] = "error";
                $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
                return $json;
            }
            $n = mysqli_num_rows($r);
            if($n == 1){
                $vmnet_ip_array = mysqli_fetch_array($r);
                $vmnet_ip = $vmnet_ip_array["ipaddr"];
                $enc_file .= "      vmnet_ip: $vmnet_ip\n";
            // If we don't find a VMNet IP, then let's use the management network IP instead.
            }else{
                $enc_file .= "      vmnet_ip: $machine_ip\n";
            }

            $enc_file .= $enc_amhn;
            $enc_file .= $enc_amip;
            $enc_file .= "      vip_hostname: $vip_hostname\n";
            $enc_file .= "      vip_ipaddr: $vip_ipaddr\n";
            $enc_file .= "      vip_netmask: $vip_netmask\n";
            $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";
            $enc_file .= "      network_ipaddr: $network_ip\n";
            $enc_file .= "      network_cidr: $network_cidr\n";

            if($machine["compute_is_cephosd"] == "yes"){
                $enc_file .= "      use_ceph_if_available: true\n";
            }else{
                if($machine["use_ceph_if_available"] == "yes"){
                    $enc_file .= "      use_ceph_if_available: ".$machine["use_ceph_if_available"]."\n";
                }else{
                    $enc_file .= "      use_ceph_if_available: false\n";
                }
            }

            $json = get_cluster_password($con, $conf, $cluster_id, 'haproxy', 'stats');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_haproxy_stats: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'messaging');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_cinder_messaging: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'authtoken');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_cinder_authtoken: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'db');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_cinder_db: " . $json["data"] . "\n";

            if($num_cephosd_nodes > 0){
                $enc_file .= "      cluster_has_osds: true\n";
                $enc_file .= "      has_subrole_ceilometer: true\n";

                $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'fsid');
                if($json["status"] != "success"){ return $json; }
                $enc_file .= "      ceph_fsid: " . $json["data"] . "\n";

                $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'libvirtuuid');
                if($json["status"] != "success"){ return $json; }
                $enc_file .= "      ceph_libvirtuuid: " . $json["data"] . "\n";

                $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'bootstraposdkey');
                if($json["status"] != "success"){ return $json; }
                $enc_file .= "      ceph_bootstrap_osd_key: " . $json["data"] . "\n";

                $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'adminkey');
                if($json["status"] != "success"){ return $json; }
                $enc_file .= "      ceph_admin_key: " . $json["data"] . "\n";

                $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'openstackkey');
                if($json["status"] != "success"){ return $json; }
                $enc_file .= "      ceph_openstack_key: " . $json["data"] . "\n";

                $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'monkey');
                if($json["status"] != "success"){ return $json; }
                $enc_file .= "      ceph_mon_key: " . $json["data"] . "\n";

                $ret = enc_get_mon_nodes($con,$conf,$cluster_id,'cephmon');
                if($ret["status"] != "success"){
                    return $ret;
                }
                $enc_file .= "      ceph_mon_initial_members: " . $ret["data"]["osd_members"] . "\n";
                $enc_file .= "      ceph_mon_host: " . $ret["data"]["osd_ips"] . "\n";

                $json = get_cluster_password($con, $conf, $cluster_id, 'ceilometer', 'telemetry');
                if($json["status"] != "success"){ return $json; }
                $enc_file .= "      pass_ceilometer_telemetry: " . $json["data"] . "\n";

                $json = get_cluster_password($con, $conf, $cluster_id, 'ceilometer', 'messaging');
                if($json["status"] != "success"){ return $json; }
                $enc_file .= "      pass_ceilometer_messaging: " . $json["data"] . "\n";

                $json = get_cluster_password($con, $conf, $cluster_id, 'ceilometer', 'authtoken');
                if($json["status"] != "success"){ return $json; }
                $enc_file .= "      pass_ceilometer_authtoken: " . $json["data"] . "\n";

            }else{
                $enc_file .= "      cluster_has_osds: false\n";
            }

            $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'messaging');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_nova_messaging: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'authtoken');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_nova_authtoken: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'nova', 'ssh');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_nova_ssh_pub: " . unserialize($json["data"]["ssh_pub"]) . "\n";
            $enc_file .= "      pass_nova_ssh_priv: " . base64_encode(unserialize($json["data"]["ssh_priv"])) . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'novaneutron', 'shared_secret');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_metadata_proxy_shared_secret: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'neutron', 'messaging');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_neutron_messaging: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'neutron', 'authtoken');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_neutron_authtoken: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'placement', 'authtoken');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      pass_placement_authtoken: " . $json["data"] . "\n";

            $enc_file .= "      disable_notifications: " . $disable_notifications . "\n";
            $enc_file .= "      enable_monitoring_graphs: " . $enable_monitoring_graphs . "\n";
            $enc_file .= "      monitoring_graphite_host: " . $monitoring_graphite_host . "\n";
            $enc_file .= "      monitoring_graphite_port: " . $monitoring_graphite_port . "\n";

            if($machine["use_gpu"] == "yes"){
                $enc_file .= "      use_gpu: true\n";
                $enc_file .= "      gpu_name: " . $machine["gpu_name"] . "\n";
                $enc_file .= "      gpu_vendor_id: " . $machine["gpu_vendor_id"] . "\n";
                $enc_file .= "      gpu_product_id: " . $machine["gpu_product_id"] . "\n";
                $enc_file .= "      gpu_device_type: " . $machine["gpu_device_type"] . "\n";
                $enc_file .= "      vfio_ids: " . str_replace(' ', ',', $machine["vfio_ids"]) . "\n";
            }else{
                $enc_file .= "      use_gpu: false\n";
            }

            if($num_network_nodes == 0 || $machine["force_dhcp_agent"] == "yes"){
                $enc_file .= "      has_subrole_dhcp_agent: true\n";
            }else{
                $enc_file .= "      has_subrole_dhcp_agent: false\n";
            }

            # CPU mode & model(s)
            if($machine["cpu_mode"] == "cluster_value"){
                $cpu_mode = $cluster["cpu_mode"];
            }else{
                $cpu_mode = $machine["cpu_mode"];
            }
            if($machine["cpu_model"] == "cluster_value"){
                $cpu_model = $cluster["cpu_model"];
            }else{
                $cpu_model = $machine["cpu_model"];
            }
            if($machine["cpu_model_extra_flags"] == "cluster_value"){
                $cpu_model_extra_flags = $cluster["cpu_model_extra_flags"];
            }else{
                $cpu_model_extra_flags = $machine["cpu_model_extra_flags"];
            }
            $enc_file .= "      cpu_mode: $cpu_mode\n";
            $enc_file .= "      cpu_model: $cpu_model\n";
            if($cpu_model_extra_flags != "" && $cpu_model_extra_flags != "none"){
                $enc_file .= "      cpu_model_extra_flags: $cpu_model_extra_flags\n";
            }

            if($cluster["cinder_backup_prefer_swift"] == "yes"){
                if($num_swiftstore_nodes > 0){
                    $enc_file .= "      backup_backend: swift\n";
                }elseif($num_cephosd_nodes > 0){
                    $enc_file .= "      backup_backend: ceph\n";
                }else{
                    $enc_file .= "      backup_backend: none\n";
                }
            }else{
                if($num_cephosd_nodes > 0){
                    $enc_file .= "      backup_backend: ceph\n";
                }elseif($num_swiftstore_nodes > 0){
                    $enc_file .= "      backup_backend: swift\n";
                }else{
                    $enc_file .= "      backup_backend: none\n";
                }
            }

            if($cluster["install_designate"] == "yes"){
                $enc_file .= "      has_subrole_designate: true\n";
            }

            if($machine["compute_is_cephosd"] == "yes"){
                $enc_file .= "      block_devices:\n";

                $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '%da'";
                $r = mysqli_query($con, $q);
                $n = mysqli_num_rows($r);
                if($n < 1){
                    $json["status"] = "error";
                    $json["message"] = "Cannot find block devices line ".__LINE__." file ".__FILE__;
                    return $json;
                }
                for($i=0;$i<$n;$i++){
                    $a = mysqli_fetch_array($r);
                    $hdd_name = $a["name"];
                    $enc_file .= "         - $hdd_name\n";
                }
            }

            # See if one node has Neutron dynamic routing, and install the extension in the
            # compute if we have one.
            $qdragent = "SELECT id FROM machines WHERE cluster='$cluster_id' AND neutron_install_dragent='yes'";
            $rdragent = mysqli_query($con, $qdragent);
            if($rdragent === FALSE){
                $json["status"] = "error";
                $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
                return $json;
            }
            $ndragent = mysqli_num_rows($rdragent);
            if($ndragent == 0){
                $enc_file .= "      use_dynamic_routing: false\n";
            }else{
                $enc_file .= "      use_dynamic_routing: true\n";
            }
        }
        break;

    case "billosd":
    case "cephosd":
        $enc_file .= "   oci::cephosd:\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        $enc_file .= "      vip_hostname: $vip_hostname\n";
        $enc_file .= "      vip_ipaddr: $vip_ipaddr\n";
        $enc_file .= "      vip_netmask: $vip_netmask\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";
        $enc_file .= "      network_ipaddr: $network_ip\n";
        $enc_file .= "      network_cidr: $network_cidr\n";
        $enc_file .= "      bgp_to_the_host: $machine_puppet_bgp2host\n";

        if($machine["ceph_osd_initial_setup"] == "yes"){
                $ceph_osd_initial_setup = "true";
        }else{
                $ceph_osd_initial_setup = "false";
        }
        $enc_file .= "      ceph_osd_initial_setup: $ceph_osd_initial_setup\n";

        $enc_file .= "      block_devices:\n";
        $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '%da'";
        $r = mysqli_query($con, $q);
        $n = mysqli_num_rows($r);
        if($n < 1){
            $json["status"] = "error";
            $json["message"] = "Cannot find block devices line ".__LINE__." file ".__FILE__;
            return $json;
        }
        for($i=0;$i<$n;$i++){
            $a = mysqli_fetch_array($r);
            $hdd_name = $a["name"];
            $enc_file .= "         - $hdd_name\n";
        }

        if($machine["role"] == "cephosd"){
            $servicetype = "ceph";
            $mon_node_type = 'cephmon';
        }else{
            $servicetype = "bill";
            $mon_node_type = 'billmon';
        }

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'fsid');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_fsid: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'bootstraposdkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_bootstrap_osd_key: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'adminkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_admin_key: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'openstackkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_openstack_key: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'monkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_mon_key: " . $json["data"] . "\n";

        $ret = enc_get_mon_nodes($con,$conf,$cluster_id,$mon_node_type);
        if($ret["status"] != "success"){
            return $ret;
        }
        $enc_file .= "      ceph_mon_initial_members: " . $ret["data"]["osd_members"] . "\n";
        $enc_file .= "      ceph_mon_host: " . $ret["data"]["osd_ips"] . "\n";

        // Get the IP for the cluster network.
        $q = "SELECT machines.hostname AS hostname, INET_NTOA(ips.ip) AS ipaddr, networks.mtu AS mtu, networks.ip AS networkaddr, networks.cidr AS networkcidr FROM machines, ips, networks WHERE machines.id='$machine_id' AND machines.cluster='$cluster_id' AND machines.id=ips.machine AND ips.network=networks.id AND networks.is_public='no' AND networks.role='ceph-cluster' AND networks.role='ipmi'";
        $r = mysqli_query($con, $q);
        if($r === FALSE){
            $json["status"] = "error";
            $json["message"] = mysqli_error($con)." line ".__LINE__." file ".__FILE__;
            return $json;
        }
        $n = mysqli_num_rows($r);
        if($n == 1){
            $cephnet_ip_array = mysqli_fetch_array($r);
            $cephnet_ip          = $cephnet_ip_array["ipaddr"];
            $cephnet_mtu         = $cephnet_ip_array["mtu"];
            $cephnet_networkaddr = $cephnet_ip_array["networkaddr"];
            $cephnet_networkcidr = $cephnet_ip_array["networkcidr"];
            $enc_file .= "      use_ceph_cluster_net: true\n";
            $enc_file .= "      cephnet_ip: $cephnet_ip\n";
            $enc_file .= "      cephnet_network_addr: $cephnet_networkaddr\n";
            $enc_file .= "      cephnet_network_cidr: $cephnet_networkcidr\n";
            $enc_file .= "      cephnet_mtu: $cephnet_mtu\n";
        // If we don't find a ceph-cluster IP, then let's use the management network IP instead.
        }else{
            $enc_file .= "      use_ceph_cluster_net: false\n";
            $enc_file .= "      cephnet_ip: $machine_ip\n";
            $enc_file .= "      cephnet_mtu: 0\n";
        }

        break;

    case "billmon":
    case "cephmon":
        $enc_file .= "   oci::cephmon:\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        $enc_file .= "      vip_hostname: $vip_hostname\n";
        $enc_file .= "      vip_ipaddr: $vip_ipaddr\n";
        $enc_file .= "      vip_netmask: $vip_netmask\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";
        $enc_file .= "      network_ipaddr: $network_ip\n";
        $enc_file .= "      network_cidr: $network_cidr\n";

        if($machine["role"] == "cephmon"){
            $servicetype = "ceph";
            $mon_node_type = 'cephmon';
        }else{
            $servicetype = "bill";
            $mon_node_type = 'billmon';
        }

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'fsid');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_fsid: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'bootstraposdkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_bootstrap_osd_key: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'adminkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_admin_key: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'openstackkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_openstack_key: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'monkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_mon_key: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, $servicetype, 'mgrkey');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      ceph_mgr_key: " . $json["data"] . "\n";

        $ret = enc_get_mon_nodes($con,$conf,$cluster_id,$mon_node_type);
        if($ret["status"] != "success"){
            return $ret;
        }
        $enc_file .= "      ceph_mon_initial_members: " . $ret["data"]["osd_members"] . "\n";
        $enc_file .= "      ceph_mon_host: " . $ret["data"]["osd_ips"] . "\n";
        break;

    case "volume":
        $enc_file .= "   oci::volume:\n";
        $enc_file .= "      openstack_release: $openstack_release\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        $enc_file .= $enc_amhn;
        $enc_file .= $enc_amip;
        $enc_file .= "      first_master: $first_master_hostname\n";
        $enc_file .= "      first_master_ip: $first_master_ipaddr\n";
        $enc_file .= "      sql_vip_ip: $vip_sql_ip\n";
        $enc_file .= "      vip_hostname: $vip_hostname\n";
        $enc_file .= "      vip_ipaddr: $vip_ipaddr\n";
        $enc_file .= "      vip_netmask: $vip_netmask\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";
        $enc_file .= "      network_ipaddr: $network_ip\n";
        $enc_file .= "      network_cidr: $network_cidr\n";

        $enc_file .= "      block_devices:\n";

        if($num_messaging_nodes > 0){
            $enc_file .= "      messaging_nodes_for_notifs: true\n";
            $enc_file .= $enc_rabbits;
        }

        if($machine["install_on_raid"] == "no"){
            $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '%da'";
        }else{
            switch($machine["raid_type"]){
            case "0":
            case "1":
                $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '".$machine["raid_dev0"]."' AND name NOT LIKE '".$machine["raid_dev1"]."'";
                break;
            case "10":
                $q = "SELECT * FROM blockdevices WHERE machine_id='$machine_id' AND name NOT LIKE '".$machine["raid_dev0"]."' AND name NOT LIKE '".$machine["raid_dev1"]."' AND name NOT LIKE '".$machine["raid_dev2"]."' AND name NOT LIKE '".$machine["raid_dev3"]."'";
                break;
            case "5":
            default:
                die("Not supported yet.");
                break;
            }
        }
        $r = mysqli_query($con, $q);
        $n = mysqli_num_rows($r);
        if($n < 1){
            $json["status"] = "error";
            $json["message"] = "Cannot find block devices line ".__LINE__." file ".__FILE__;
            return $json;
        }
        for($i=0;$i<$n;$i++){
            $a = mysqli_fetch_array($r);
            $hdd_name = $a["name"];
            $enc_file .= "         - $hdd_name\n";
        }

        $enc_file .= "      enabled_backend_list:\n";
        if($machine["cinder_enabled_backends"] != "no-override"){
            $backend_names_array = explode(":", $machine["cinder_enabled_backends"]);
            foreach($backend_names_array as &$backend){
                $enc_file .= "         - $backend\n";
            }
        }

        $volume_node_number = explode("-", explode(".", $machine_hostname)[0])[2];
        if($machine["cinder_enabled_backends"] != "no-override"){
            $enc_file .= "      vgname: " . $cluster_name . "vol" . $volume_node_number . "vg" ."\n";
        }else{
            $enc_file .= "      vgname: " . $cluster_name . "vol" . $volume_node_number . "vg0" ."\n";
        }

        if($cluster["cinder_backup_prefer_swift"] == "yes"){
            if($num_swiftstore_nodes > 0){
                $enc_file .= "      backup_backend: swift\n";
            }elseif($num_cephosd_nodes > 0){
                $enc_file .= "      backup_backend: ceph\n";
            }else{
                $enc_file .= "      backup_backend: none\n";
            }
        }else{
            if($num_cephosd_nodes > 0){
                $enc_file .= "      backup_backend: ceph\n";
            }elseif($num_swiftstore_nodes > 0){
                $enc_file .= "      backup_backend: swift\n";
            }else{
                $enc_file .= "      backup_backend: none\n";
            }
        }

        # Ceph stuff
        if($num_cephosd_nodes > 0){
            $enc_file .= "      cluster_has_osds: true\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'fsid');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_fsid: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'adminkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_admin_key: " . $json["data"] . "\n";

            $json = get_cluster_password($con, $conf, $cluster_id, 'ceph', 'openstackkey');
            if($json["status"] != "success"){ return $json; }
            $enc_file .= "      ceph_openstack_key: " . $json["data"] . "\n";

            $ret = enc_get_mon_nodes($con,$conf,$cluster_id);
            if($ret["status"] != "success"){
                return $ret;
            }
            $enc_file .= "      ceph_mon_initial_members: " . $ret["data"]["osd_members"] . "\n";
            $enc_file .= "      ceph_mon_host: " . $ret["data"]["osd_ips"] . "\n";
        }else{
            $enc_file .= "      cluster_has_osds: false\n";
        }

        # Normal cinder password stuff
        $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'messaging');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cinder_messaging: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'authtoken');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cinder_authtoken: " . $json["data"] . "\n";

        $json = get_cluster_password($con, $conf, $cluster_id, 'cinder', 'db');
        if($json["status"] != "success"){ return $json; }
        $enc_file .= "      pass_cinder_db: " . $json["data"] . "\n";

        if($cluster["extswift_use_external"] == "yes"){
            $enc_file .= "      extswift_use_external: true\n";
            $enc_file .= "      extswift_auth_url: " . $cluster["extswift_auth_url"] . "\n";
            $enc_file .= "      extswift_proxy_url: " . $cluster["extswift_proxy_url"] . "\n";
            $enc_file .= "      extswift_project_name: " . $cluster["extswift_project_name"] . "\n";
            $enc_file .= "      extswift_project_domain_name: " . $cluster["extswift_project_domain_name"] . "\n";
            $enc_file .= "      extswift_user_name: " . $cluster["extswift_user_name"] . "\n";
            $enc_file .= "      extswift_user_domain_name: " . $cluster["extswift_user_domain_name"] . "\n";
            $enc_file .= "      extswift_password: " . $cluster["extswift_password"] . "\n";
        }
        $enc_file .= "      disable_notifications: " . $disable_notifications . "\n";

        break;

    case "debmirror":
        $enc_file .= "   oci::debmirror:\n";
        $enc_file .= "      machine_hostname: $machine_hostname\n";
        $enc_file .= "      machine_ip: $machine_ip\n";
        $enc_file .= "      self_signed_api_cert: $self_signed_api_cert\n";
        break;

    default:
        // If we don't know the role, then it's a custom one. Just output the
        // "standard" enc yaml file.
        break;
    }

    if($skip_variables_dot_json_enc == "no"){
        # Add to the role whatever's in the db matching the json file
        $json_file = file_get_contents("/usr/share/openstack-cluster-installer/variables.json");
        $dec_json = json_decode($json_file, TRUE);

        # Parse the per-machine attributes in variables.json
        $items_json_array = $dec_json["machines"];
        $a_keys = array_keys($items_json_array);
        $n = sizeof($a_keys);
        for($i=0;$i<$n;$i++){
            $col = $a_keys[$i];
            $attr = $items_json_array[$col];

            # First, see if the current custom variable includes the role of the machine
            $col_roles = $attr["roles"];                                                                                                                                                                                                     
            $nroles = sizeof($col_roles);                                                                                                                                                                                                    
            for($j=0;$j<$nroles;$j++){
                $one_role_name = $col_roles[$j];
                if($machine["role"] == $one_role_name || $one_role_name == "all"){
                    switch($attr["type"]){
                    case "int":
                    case "float":
                    case "string":
                    case "fqdn":
                        $custom_var_value = $machine[$col];
                        break;
                    case "boolean":
                        if($machine[$col] == "yes"){
                            $custom_var_value = "true";
                        }else{
                            $custom_var_value = "false";
                        }
                        break;
                    default:
                        die("Variable type not handled by OCI's ENC.\n");
                        break;
                    }
                    $enc_file .= "      $col: $custom_var_value\n";
                }
            }
        }

        # Parse the per-cluster attributes in variables.json
        $items_json_array = $dec_json["clusters"];
        $a_keys = array_keys($items_json_array);
        $n = sizeof($a_keys);
        for($i=0;$i<$n;$i++){
            $col = $a_keys[$i];
            $attr = $items_json_array[$col];

            # First, see if the current custom variable includes the role of the machine
            if(isset($attr["roles"])){
                $col_roles = $attr["roles"];
                $nroles = sizeof($col_roles);
            }else{
                $nroles = 0;
            }
            for($j=0;$j<$nroles;$j++){
                $one_role_name = $col_roles[$j];
                if($machine["role"] == $one_role_name || $one_role_name == "all"){
                    switch($attr["type"]){
                    case "int":
                    case "float":
                    case "string":
                    case "fqdn":
                        $custom_var_value = $cluster[$col];
                        break;
                    case "boolean":
                        if($cluster[$col] == "yes"){
                            $custom_var_value = "true";
                        }else{
                            $custom_var_value = "false";
                        }
                        break;
                    default:
                        die("Variable type not handled by OCI's ENC.\n");
                        break;
                    }
                    $enc_file .= "      $col: $custom_var_value\n";
                }
            }
        }
    }

    # Finally send the ENC data
    $json["data"] = $enc_file;
    return $json;
}

?>
