KVM

Install KVM/libvirt

yum install qemu-kvm libvirt libvirt-python virt-install libguestfs-tools

Create MacVtap Bridge

cat > /usr/share/libvirt/networks/internal.xml <<EOF
<network>
  <name>internal</name>
  <forward mode="bridge">
    <interface dev="enp4s0"/>
  </forward>
</network>
EOF
virsh net-define /usr/share/libvirt/networks/internal.xml
virsh net-start internal
virsh net-autostart internal
virsh net-list

Add Guest's VM interface to MacVtap Bridge

virsh edit DCOSM03
<interface type='direct'>
  <mac address='52:54:00:d9:fc:4f'/>
  <source dev='enp4s0' mode='bridge'/>
  <model type='virtio'/>
  <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</interface>

Create Host Bridge

brctl addbr inSW
brctl addif inSW em3
brctl addif inSW em4
brctl stp inSW on

Clone KVM image

virt-clone -o COS7x64 -n COS7x64.4G.40G.Docker.Template -f COS7x64.4G.40G.Docker.qcow2
virsh dumpxml COS7x64 > COS7x64.4G.40G.Docker.xml

Resize QCOW2 image

qemu-img resize COS7x64.4G.40G.Docker.qcow2 +200G
cp COS7x64.4G.40G.Docker.qcow2 COS7x64.4G.40G.Docker-orig.qcow2
virt-resize --expand /dev/sda2 COS7x64.4G.40G.Docker-orig.qcow2 COS7x64.4G.40G.Docker.qcow2
virsh start COS7x64
virsh console COS7x64
lvextend --extents +100%FREE /dev/centos/root /dev/vda2  
xfs_growfs /dev/centos/root

There are three options that modern `qemu-img` supports:

(1) 'preallocation=metadata': allocates qcow2 metadata, and it's still
    a sparse image.

    $ qemu-img create -f qcow2 -o preallocation=metadata test1-metadata.qcow2 1G
    Formatting 'test1-metadata.qcow2', fmt=qcow2 size=1073741824 encryption=off cluster_size=65536 preallocation='metadata' lazy_refcounts=off refcount_bits=16
    328K -rw-r--r--. 1 root root 1.1G Jun  3 03:20 copy-test1-metadata.qcow2

(2) 'preallocation=full': allocates zeroes and makes a non-sparse image.

    $ qemu-img create -f qcow2 -o preallocation=full test2-full.qcow2 1G
    Formatting 'test2-full.qcow2', fmt=qcow2 size=1073741824 encryption=off cluster_size=65536 preallocation='full' lazy_refcounts=off refcount_bits=16
    $ ls -lash test2-full.qcow2 
    1.1G -rw-r--r--. 1 root root 1.1G Jun  3 03:31 test2-full.qcow2

(3) 'preallocation=falloc': which uses posix_fallocate() to "allocate
    blocks and marking them as uninitialized", and is relatively faster
    than writing out zeroes to a file:

    $ qemu-img create -f qcow2 -o preallocation=falloc test3-falloc.qcow2 1G
    Formatting 'test3-falloc.qcow2', fmt=qcow2 size=1073741824 encryption=off cluster_size=65536 preallocation='falloc' lazy_refcounts=off refcount_bits=16
    $ ls -lash test3-falloc.qcow2 
    1.1G -rw-r--r--. 1 root root 1.1G Jun  3 03:32 test3-falloc.qcow2

You can test and compare between (2) and (3) what works best for you.

Create Virtual NAT Network

cat > /usr/share/libvirt/networks/internal.xml <<EOF
<network>
  <name>internal</name>
  <bridge name="inSW"/>
  <forward/>
  <ip address="192.168.3.1" netmask="255.255.255.0">
    <dhcp>
      <range start="192.168.3.100" end="192.168.3.199"/>
    </dhcp>
  </ip>
</network>
EOF
virsh net-define /usr/share/libvirt/networks/internal.xml
virsh net-start internal
virsh net-autostart internal
virsh net-list

Deactivate Network

virsh net-destroy default
virsh net-autostart --network default --disable

Create Virtual Bridge Network

cat > /usr/share/libvirt/networks/internal.xml <<EOF
<network>
  <name>internal</name>
  <forward mode="bridge"/>
  <bridge name="inSW"/>
</network>
EOF
virsh net-define /usr/share/libvirt/networks/internal.xml
virsh net-start internal
virsh net-autostart internal
virsh net-list

Register Domain(Virtual Machine)

virsh define DCOSBoot.xml
virsh start DCOSBoot
virsh autostart DCOSBoot

Add an interface to Domain（Guest VM）

virsh attach-interface --domain SEOut --type bridge --source InternetSW --model virtio --persistent

Dump Domain Configuration to a File

virsh dumpxml SEOut > SEOut.xml

Notes for pfSense VM Guest

With the current state of VirtIO network drivers in FreeBSD, it is necessary to check Disable hardware checksum offload under System > Advanced on the Networking tab and to manually reboot pfSense

https://www.netgate.com/docs/pfsense/virtualization/virtio-driver-support.html

Update VM's Device setting

cat > /usr/share/libvirt/networks/ChangeNet.xml <<EOF
<interface type='bridge'>
    <source bridge='outSW'/>
    <model type='virtio'/>
</interface>
EOF
virsh update-device COSv7.2x64.PIAF ChangeNet.xml