bhyve-api.ovh

#!/usr/local/bin/cbsd
#v12.2.2
DIST_MODULE_PATH="${distmoduledir}/api.d"
. ${DIST_MODULE_PATH}/share/bhyve.conf
MYOPTARG="imgsize jname image ram cpus pubkey host_hostname extras ci_user_pw_user ci_user_pw_root bhyve_vnc_tcp_bind"
MYARG="mode"
MYDESC="Operate with bhyve via API"
CBSDMODULE="sys"
ADDHELP="\
"
EXTHELP="wf_bhyve"
MODULE_PATH="${dbdir}/bhyve"

. ${subrdir}/nc.subr
dryrun=0
checkip=1
gold=0
develop=0
runasap=0
ip4_gw=
readconf bhyve-default-default.conf
readconf bhyve.conf

# ssh host
ssh_host=

readconf api.conf
readconf bhyve-api.conf

[ -n "${ip4_gw}" ] && oip4_gw="${ip4_gw}"

set -e
. ${distmoduledir}/api.d/api.subr
set +e
MY_LOG="${cbsd_api_logdir}/bhyve-api.log"
log "wakeup: $*"

# Max VM
MY_RANGE=$( ${SEQ_CMD} 0 20 | ${XARGS_CMD} )
[ -z "${K8S_IP_OCTET1}" ] && K8S_IP_OCTET1="10"
[ -z "${K8S_IP_OCTET2}" ] && K8S_IP_OCTET2="10"

ip4_addr=
vm_os_type=
vm_os_profile=
ip4_addr=
ip4_gw=

. ${cbsdinit}
. ${subrdir}/system.subr
. ${subrdir}/strings.subr

[ -z "${ip4_gw}" -a -n "${oip4_gw}" ] && ip4_gw="${oip4_gw}"

create_env()
{
	ssh_user=
	ssh_port=

	eval vm_os_type="\$${image}_vm_os_type"
	eval vm_os_profile="\$${image}_vm_os_profile"
	eval ssh_user="\$${image}_ssh_user"

	[ -z "${vm_os_type}" ] && err 1 "vm_os_type empty? check img/mapping in bhyve-api.conf for: ${image}"
	[ -z "${vm_os_profile}" ] && err 1 "vm_os_profile empty? check img/mapping in bhyve-api.conf for: ${image}"
	[ -z "${ssh_user}" ] && err 1 "ssh_user empty? check img/mapping in bhyve-api.conf for: ${image}"

	# extras area
	v6_addr=
	v4_addr=
	primary_ip=
	ip4_addr=
	ossh_port=
	port=

	ext_v4=$( dhcpd ip4pool=51.195.77.160/27 )
	ssh_host="${ext_v4}"

	MYNET=
	MYID=
	for i in ${MY_RANGE}; do
		# check for local iface only
		_res=$( checkip check=3 ip="${K8S_IP_OCTET1}.${K8S_IP_OCTET2}.${i}.1" )
		_ret=$?
		if [ ${_ret} -eq 1 ]; then
			MYNET="${K8S_IP_OCTET1}.${K8S_IP_OCTET2}.${i}"
			MYID="${i}"
			break
		fi
	done

	[ -z "${MYNET}" ] && err 1 "no such free net"
	echo "my net: ${MYNET}.0/24, ID: [${MYID}], EXT_V4: [${ext_v4}]"
	# bridge section

	int_br="${MYID}"

#	int_br=$(( MYID * 2 ))
#	ext_br=$(( int_br + 1 ))

	int_bridge_name="bridge${int_br}"
#	ext_bridge_name="bridge${ext_br}"

	int_br_ip="${K8S_IP_OCTET1}.${K8S_IP_OCTET2}.${MYID}.1"

#	ext_br_ip=$( dhcpdv6 )
#	ext_ip=$( dhcpdv6 )

	# check for ext_ip/ext_br_ip IP here!!
	# to post/pre hooks + destroy

	echo "my ifaces: int: ${int_bridge_name} -> ${int_br_ip}, ext: ${ext_bridge_name} -> ${ext_br_ip}"

	IPFW_RULE=$(( MYID + 1024 ))

#	echo "/sbin/ipfw -q add ${IPFW_RULE} deny ip from ${ext_br_ip} to any via igb0"
#	echo "/sbin/ipfw -q add ${IPFW_RULE} deny ip from any to ${ext_br_ip} via ${_uplink_iface4}"
#	/sbin/ipfw -q add ${IPFW_RULE} deny ip from ${ext_br_ip} to any via ${_uplink_iface4}
#	/sbin/ipfw -q add ${IPFW_RULE} deny ip from any to ${ext_br_ip} via ${_uplink_iface4}
#	/sbin/ipfw -q add ${IPFW_RULE} deny ip from ${K8S_IP_OCTET1}.${K8S_IP_OCTET2}.${MYID}.2,${K8S_IP_OCTET1}.${K8S_IP_OCTET2}.${MYID}.3,${int_br_ip} to any out via ${_uplink_iface4}
#	/sbin/ipfw -q add ${IPFW_RULE} deny ip6 from ${ext_ip} to any out via ${_uplink_iface4}

	# rc.conf
	${LOCKF_CMD} -s -t10 /tmp/sysrc_rc.conf.lock /usr/local/cbsd/misc/cbsdsysrc -qf /etc/rc.conf cloned_interfaces+="${int_bridge_name}"
	${LOCKF_CMD} -s -t10 /tmp/sysrc_rc.conf.lock /usr/local/cbsd/misc/cbsdsysrc -qf /etc/rc.conf ifconfig_${int_bridge_name}="${int_br_ip}/24 up description k8s-${MYID}-int"

	${IFCONFIG_CMD} ${int_bridge_name} create description k8s-${MYID}-int
	${IFCONFIG_CMD} ${int_bridge_name} ${int_br_ip}/24 up

#	${IFCONFIG_CMD} ${ext_bridge_name} create description k8s-${MYID}-ext
#	${IFCONFIG_CMD} ${ext_bridge_name} inet6 ${ext_br_ip}/64 up

	# rc.conf
#	${LOCKF_CMD} -s -t10 /tmp/sysrc_rc.conf.lock /usr/local/cbsd/misc/cbsdsysrc -qf /etc/rc.conf cloned_interfaces+="${ext_bridge_name}"
#	${LOCKF_CMD} -s -t10 /tmp/sysrc_rc.conf.lock /usr/local/cbsd/misc/cbsdsysrc -qf /etc/rc.conf ifconfig_${ext_bridge_name}="inet6 ${ext_br_ip}/64 up description k8s-${MYID}-ext"

	# to post/pre hooks + destroy
	# todo: route flags in CBSD 13.1.0+
#	${ROUTE_CMD} -6 add ${ext_ip} -interface ${ext_bridge_name}

	# save MYID
	echo "${MYID}" > ${cbsd_api_dbdir}/${jname}.myd
	echo "${ext_v4}" > ${cbsd_api_dbdir}/${jname}.ext_v4

	${IFCONFIG_CMD} ix0 ${ext_v4}/32 alias

	v4_addr="${MYNET}.2"
	#v4_addr=$( dhcpd ip4pool=${ip4pool} )
	primary_ip="${v4_addr}"
	ip4_addr="${v4_addr}"

#	v6_addr="${ext_ip}"
#	ci_gw42="${ext_br_ip}"

	ip4_gw="${MYNET}.1"

	ci_jname="${jname}"

	if [ -n "${host_hostname}" ]; then
		ci_fqdn="${host_hostname}"
	else
		ci_fqdn="${jname}.my.domain"
	fi

	ci_ip4_addr="${ip4_addr}"
	ci_gw4="${ip4_gw}"

	[ -z "${vm_os_type}" ] && err 1 "no vm_os_type"
	[ -z "${vm_os_profile}" ] && err 1 "no vm_os_profile"

	# additional check for profile/type?

	cid=$( ${miscdir}/cbsd_md5 "${pubkey}" )
	if [ ! -d ${cbsd_api_dbdir}/${cid} ]; then
		${MKDIR_CMD} -m 0770 -p ${cbsd_api_dbdir}/${cid}
		${CHOWN_CMD} ${cbsduser}:${cbsduser} ${cbsd_api_dbdir}/${cid}
	fi

	if [ ! -d ${cbsd_api_dbdir}/${cid}/.ssh ]; then
		${MKDIR_CMD} -m 0770 -p ${cbsd_api_dbdir}/${cid}/.ssh
		${CHOWN_CMD} ${cbsduser}:${cbsduser} ${cbsd_api_dbdir}/${cid}/.ssh
	fi

	if [ ! -r ${cbsd_api_dbdir}/${cid}/.ssh/authorized_keys ]; then
		echo "${pubkey}" > ${cbsd_api_dbdir}/${cid}/.ssh/authorized_keys
		${CHOWN_CMD} ${cbsduser}:${cbsduser} ${cbsd_api_dbdir}/${cid}/.ssh/authorized_keys
		sync
		#fflush?
		sleep 1
	fi
	if [ ! -d ${cbsd_api_dbdir}/${cid}/vms ]; then
		${MKDIR_CMD} -m 0770 -p ${cbsd_api_dbdir}/${cid}/vms
		${CHOWN_CMD} ${cbsduser}:${cbsduser} ${cbsd_api_dbdir}/${cid}/vms
	fi

	#str="cbsd bcreate jname=${jname} runasap=1 vm_cpus=${cpus} vm_ram=${ram} imgsize=${imgsize} vm_os_type=\"${vm_os_type}\" vm_os_profile=\"${vm_os_profile}\" ci_jname=\"${jname}\" ci_fqdn=\"${ci_fqdn}\" ci_ip4_addr=\"${ip4_addr}\" ci_gw4=\"${ip4_gw}\" ip4_addr=\"${ip4_addr}\" ci_user_pubkey=\"${cbsd_api_dbdir}/${cid}/.ssh/authorized_keys\" ci_jname=\"${ci_fqdn}\" ci_ip4_addr2=\"${v6_addr}\" ci_user_pw_root=\"cbsd\" ci_interface2=\"${ext_bridge_name}\" ci_gw42=\"${ci_gw42}\""
	str="cbsd bcreate jname=${jname} runasap=1 vm_cpus=${cpus} vm_ram=${ram} imgsize=${imgsize} vm_os_type=\"${vm_os_type}\" vm_os_profile=\"${vm_os_profile}\" ci_jname=\"${jname}\" ci_fqdn=\"${ci_fqdn}\" ci_ip4_addr=\"${ip4_addr}\" ci_gw4=\"${ip4_gw}\" ip4_addr=\"${ip4_addr}\" ci_user_pubkey=\"${cbsd_api_dbdir}/${cid}/.ssh/authorized_keys\" ci_jname=\"${ci_fqdn}\""

	[ -n "${bhyve_vnc_tcp_bind}" ] && str="${str} bhyve_vnc_tcp_bind=\"${bhyve_vnc_tcp_bind}\""
	[ -n "${ci_user_pw_user}" ] && str="${str} ci_user_pw_user=\"${ci_user_pw_user}\""
	[ -n "${ci_user_pw_root}" ] && str="${str} ci_user_pw_root=\"${ci_user_pw_root}\""

	log "${str}"
	${str}

	expose mode=add jname=${jname} inaddr=${ext_v4}

	max_retry=20
	retry=0

	# inherit workdir from CBSD/API?
	. /etc/rc.conf

	while [ ${retry} -lt ${max_retry} ]; do
		if [ -r ${cbsd_workdir}/vm/${jname}/vnc_port ]; then
			vnc_file_stat=$( ${STAT_CMD} -f "%z" ${cbsd_workdir}/vm/${jname}/vnc_port 2>/dev/null )
			[ ${vnc_file_stat} -gt 0 ] && break
		fi
		sleep 1
		retry=$(( retry + 1 ))
	done

	if [ ${retry} -gt ${max_retry} ]; then
		log "unable to read vnc port file: ${cbsd_workdir}/vm/${jname}/vnc_port"
		err 1 "unable to read vnc port file: ${cbsd_workdir}/vm/${jname}/vnc_port"
	fi

	# for slow hdd/vm startup
	sleep 5

	port=22

	if is_number "${imgsize}"; then
		if conv2bytes ${imgsize}; then
			imgsize_bytes="${convval}"
		else
			imgsize_bytes="0"
		fi
	else
		# already on bytes?
		imgsize_bytes="${imgsize}"
	fi

	if is_number "${ram}"; then
		if conv2bytes ${ram}; then
			ram_bytes="${convval}"
		else
			ram_bytes="0"
		fi
	else
		# already on bytes?
		ram_bytes="${ram}"
	fi

	create_time=$( ${DATE_CMD} -u "+%Y-%m-%dT%H:%M:%S" )
	[ -z "${port4}" ] && port4="${port}"

	case "${vm_os_type}" in
		windows)
			port="3389"
			port4="3389"
			;;
	esac

	[ -z "${port6}" ] && port6="0"
	# overwrite primary/int ip by ext
	ssh4_host="${ssh_host}"
	#[ -z "${ssh6_host}" ] && ssh6_host="${v6_addr}"

	hostname=$( hostname )

	echo "${hostname}" > ${cbsd_api_dbdir}/${cid}/${jname}.node
	${CAT_CMD} > ${cbsd_api_dbdir}/${cid}/${jname}-bhyve.ssh <<EOF
{
  "instanceid": "${host_hostname}",
  "type": "vm",
  "is_power_on": true,
  "created": "${create_time}",
  "cpus": ${cpus},
  "ram_bytes": ${ram_bytes},
  "ram_human": "${ram}",
  "imgsize_bytes": ${imgsize_bytes},
  "imgsize_human": "${imgsize}",
  "vm_os_type": "${vm_os_type}",
  "vm_os_profile": "${vm_os_profile}",
  "cloud_user": "${ssh_user}",
  "primary_ip": "${ssh_host}",
  "ip4": "${ssh4_host}",
  "ip6": "${ssh6_host}",
  "remote_port": ${port},
  "remote_port4": ${port4},
  "remote_port6": ${port6},
EOF

	case "${vm_os_type}" in
		windows)
			${CAT_CMD} >> ${cbsd_api_dbdir}/${cid}/${jname}-bhyve.ssh <<EOF
  "rdp_string": "xfreerdp +clipboard +fonts /rfx /size:1280x720 /u:Admin /v:${ssh_host}",
  "rdp4_string": "xfreerdp +clipboard +fonts /rfx /size:1280x720 /u:Admin /v:${ssh4_host}",
  "rdp6_string": ""
}
EOF
			;;
		*)

			${CAT_CMD} >> ${cbsd_api_dbdir}/${cid}/${jname}-bhyve.ssh <<EOF
  "ssh_string": "ssh ${ssh_user}@${ssh_host}",
  "ssh4_string": "ssh ${ssh_user}@${ssh4_host}",
  "rdp6_string": ""
}
EOF
			;;
	esac

        # dup to param-values
	${CAT_CMD} > ${cbsd_api_dbdir}/${cid}/vms/${jname} <<EOF
host_hostname="${host_hostname}"
gid="${jname}"
hostname="${hostname}"
type="vm"
is_power_on="true"
created="${create_time}"
cpus="${cpus}"
ram_bytes="${ram_bytes}"
ram_human="${ram}"
imgsize_bytes="${imgsize_bytes}"
imgsize_human="${imgsize}"
vm_os_type="${vm_os_type}"
vm_os_profile="${vm_os_profile}"
ssh_user="${ssh_user}"
ssh_host="${ssh_host}"
ssh4_host="${ssh4_host}"
ssh6_host="${ssh6_host}"
ssh_port="${port}"
ssh_port4="${port4}"
ssh_port6="${port6}"
EOF

	case "${vm_os_type}" in
		windows)
			${CAT_CMD} >> ${cbsd_api_dbdir}/${cid}/vms/${jname} <<EOF
rdp_string="xfreerdp +clipboard +fonts /rfx /size:1280x720 /u:Admin /v:${ssh_host}"
rdp4_string="xfreerdp +clipboard +fonts /rfx /size:1280x720 /u:Admin /v:${ssh4_host}"
rdp6_string=""
EOF
			;;
		*)
			${CAT_CMD} >> ${cbsd_api_dbdir}/${cid}/vms/${jname} <<EOF
ssh_string="ssh ${ssh_user}@${ssh_host} -p${port}"
ssh4_string="ssh ${ssh_user}@${ssh4_host} -p${port4}"
ssh6_string=""
EOF
			;;
	esac

	${CHOWN_CMD} ${cbsduser}:${cbsduser} ${cbsd_api_dbdir}/${cid}/vms/${jname} ${cbsd_api_dbdir}/${cid}/${jname}.node ${cbsd_api_dbdir}/${cid}/${jname}-bhyve.ssh
	[ -n "${api_postcreate_hook}" -a -x "${api_postcreate_hook}" ] && ${api_postcreate_hook} -c ${cid} -j ${jname} -m create -r ${cbsd_api_dbdir}

	# save route
	#${RM_CMD} -f /usr/jails/vm/${jname}/master_poststop.d/route_del.sh || true
	#${RM_CMD} -f /usr/jails/vm/${jname}/master_poststart.d/route_add.sh || true

	# route add
	#echo "${ROUTE_CMD} -6 add ${ext_ip} -interface ${ext_bridge_name}" > /usr/jails/vm/${jname}/master_poststart.d/route_add.sh
	#echo "${ROUTE_CMD} -6 add ${ext_ip2} -interface ${ext_bridge_name}" >> /usr/jails/vm/${jname}/master_poststart.d/route_add.sh
	echo "${IFCONFIG_CMD} ix0 ${ext_v4}/32 alias" >> /usr/jails/vm/${jname}/master_poststart.d/route_add.sh

	${CHMOD_CMD} +x /usr/jails/vm/${jname}/master_poststart.d/route_add.sh

	# route del
	#echo "${ROUTE_CMD} -6 delete ${ext_ip} -interface ${ext_bridge_name}" > /usr/jails/vm/${jname}/master_poststop.d/route_del.sh
	#echo "${ROUTE_CMD} -6 delete ${ext_ip2} -interface ${ext_bridge_name}" >> /usr/jails/vm/${jname}/master_poststop.d/route_del.sh
	#${CHMOD_CMD} +x /usr/jails/vm/${jname}/master_poststop.d/route_del.sh
}

## MAIN
case "${mode}" in
	create)
		create_env
		;;
esac

# temporary hack/mock for standalone no-DB API version
[ -x /root/bin/tubestat.sh ] && /root/bin/tubestat.sh > /dev/null 2>&1
[ -x /root/bin/update_cluster_status.sh ] && /root/bin/update_cluster_status.sh > /dev/null 2>&1

exit 0