generate minigraph based on topology file

ansible/README.testbed.md

@@ -4,6 +4,7 @@
 - [Setup](doc/README.testbed.Setup.md)
 - [Topology](doc/README.testbed.Topology.md)
 - [Configuration](doc/README.testbed.Config.md)
+- [Minigraph](doc/README.testbed.Minigraph.md)
 - [Command Line](doc/README.testbed.Cli.md)
 - [Example](doc/README.testbed.Example.md)
 - [FAQ](doc/README.testbed.FAQ.md)

ansible/config_sonic_basedon_testbed.yml

@@ -1,5 +1,5 @@
-# This Playbook run time generate matching configuration file for SONiC switch(Minigraph) based on a specific testbed topology specified in testbed.csv  
-# When user call testbed-cli to deploy a testbed topology, use this playbook to generate matching SONiC minigraph file and deploy it into SONiC switch under test. 
+# This Playbook run time generate matching configuration file for SONiC switch(Minigraph) based on a specific testbed topology specified in testbed.csv
+# When user call testbed-cli to deploy a testbed topology, use this playbook to generate matching SONiC minigraph file and deploy it into SONiC switch under test.
 # Or when you know your topology name, you may use this playbook alone to generate a minigraph matching your topology name without deploy it.
 #
 # VM Topologies are defined inside of vars/ directory in files vars/topo_{{ topology_name}}.yml
@@ -13,12 +13,12 @@
 # To generate and deploy minigraph for SONiC switch matching the VM topology please use following command
 # ansible-playbook -i lab config_sonic_basedon_testbed.yml -l sonic_dut_name -e vm_base=VM0300 -e topo=t0 [-e deploy=true -e save=true]
 # ansible-playbook -i lab config_sonic_basedon_testbed.yml -l sonic_dut_name -e testbed_name=vms1-1 [-e deploy=true -e save=true]
-#
+
 # Parameters
 # -l str-msn2700-01          - the sonic_dut_name you are going to generate minigraph for
 # -e vm_base=VM0300          - the VM name which is used to as base to calculate VM name for this set
 # -e topo=t0                 - the name of topology to generate minigraph file
-# -e testbed_name=vms1-1     - the testbed name specified in testbed.csv file 
+# -e testbed_name=vms1-1     - the testbed name specified in testbed.csv file
 #                              (if you give 'testbed_name' option, will use info from testbed and ignore topo and vm_base options)
 # -e deploy=True             - if deploy the newly generated minigraph to the targent DUT, default is false if not defined
 # -e save=True               - if save the newly generated minigraph to the targent DUT as starup-config, default is false if not defined
@@ -31,12 +31,9 @@
 ####################################################################################################################################################################################
 
 - hosts: sonic
-  gather_facts: yes
+  gather_facts: no
   tasks:
 
-  - fail: msg="need hwsku, interface speed, netmask and interface prefix/postfix defined to generate configuration file"
-    when: (hwsku is not defined) or (iface_speed is not defined) or (mgmt_subnet_mask_length is not defined)
-
   - block:
     - name: Gathering testbed information
       test_facts: testbed_name="{{ testbed_name }}"
@@ -55,39 +52,55 @@
       when: "testbed_facts['vm_base'] != ''"
     when: testbed_name is defined
 
+  - topo_facts: topo={{ topo }}
+    connection: local
+
   - set_fact:
       VM_topo: "{% if 'ptf' in topo %}False{% else %}True{% endif %}"
       remote_dut: "{{ ansible_ssh_host }}"
-      template_name: "{{ 't1' if 'ptf' in topo else topo }}"
 
-  - testbed_vm_info: base_vm="{{ vm_base }}"  topo="{{ topo }}"
+  - name: gather testbed VM informations
+    testbed_vm_info: base_vm={{ testbed_facts['vm_base'] }} topo={{ testbed_facts['topo'] }}
     connection: local
-    when: VM_topo
+    when: "VM_topo | bool"
 
-  - name: find interface name mapping
+  - name: find interface name mapping and individual interface speed if defined
     port_alias: hwsku="{{ hwsku }}"
-
-  - debug: var=port_alias
-
-  - name: save original minigraph file (if original file does not exist, then ignore errors)
-    shell: mv minigraph/{{ inventory_hostname }}.xml minigraph/{{ inventory_hostname }}.xml.orig
     connection: local
-    ignore_errors: true
 
-  - name: create minigraph file in minigraph folder
-    become: true
-    template: src=templates/topo/{{ template_name }}.j2
-              dest=minigraph/{{ inventory_hostname}}.xml
+  - name: find all enabled host_interfaces
+    set_fact:
+      host_if_indexes: "{{ vm_topo_config['host_interfaces'] | difference(vm_topo_config['disabled_host_interfaces']) }}"
+
+  - name: find all vlan interface names for T0 topology
+    set_fact:
+      vlan_intfs: "{{ vlan_intfs|default([])}} + ['{{ port_alias[item] }}' ]"
+    with_items: "{{ host_if_indexes }}"
+    when: ("'host_interfaces' in vm_topo_config") and ("'tor' in vm_topo_config['dut_type'] | lower")
+
+  - name: find all interface indexes connecting to VM
+    set_fact:
+      interface_to_vms: "{{ interface_to_vms|default({}) | combine({ item.key: item.value['interface_indexes'] }) }}"
+    with_dict: vm_topo_config['vm']
+
+  - name: find all interface names
+    set_fact:
+      intf_names: "{{ intf_names | default({}) | combine({item.key: port_alias[item.value[0]|int:item.value[-1]|int+1] }) }}"
+    with_dict: interface_to_vms
+
+  - name: create minigraph file in ansible minigraph folder
+    template: src=templates/minigraph_template.j2
+              dest=minigraph/{{ inventory_hostname}}.{{ topo }}.xml
     connection: local
 
   - block:
-      - name: saved original minigraph file (if original file may don't exist, then ignore errors)
+      - name: saved original minigraph file in SONiC DUT(ignore errors when file doesnot exist)
         shell: mv /etc/sonic/minigraph.xml /etc/sonic/minigraph.xml.orig
         become: true
         ignore_errors: true
 
-      - name: create minigraph file for SONiC device
-        template: src=templates/topo/{{ template_name }}.j2
+      - name: create new minigraph file for SONiC device
+        template: src=templates/minigraph_template.j2
                   dest=/etc/sonic/minigraph.xml
         become: true
 

ansible/doc/README.testbed.Config.md

@@ -14,7 +14,7 @@
 
 - [```ansible/files/lab_connection_graph.xml```](../files/lab_connection_graph.xml): This is the lab graph file for library/conn_graph_facts.py to parse and get all lab fanout switch connections information. If you have only one fanout switch, you may go head manually modify the sample lab_connection_graph.xml file to set bot your fanout leaf and fanout root switch management IP point to the same fanout switch management IP and make sure all DUT and Fanout name and IP are matching your testbed.
 
-- ```ansible/files/creategraph.py```: Helper file helps you generate a lab_connection_graph.xml based on the device file and link file specified above.
+- [```ansible/files/creategraph.py```](../files/creategraph.py): Helper file helps you generate a lab_connection_graph.xml based on the device file and link file specified above.
 
      Based on ansible_facts,  you may write ansible playbooks to deploy fanout switches or run test which requires to know the DUT physical connections to fanout switch
 
@@ -63,44 +63,4 @@ Must be strictly checked in code reviews
 
 TODO: check this constraints in testbed-cli.sh
 
-## Generate and load SONiC configuration file
 
-There is an ansible playbook `config_sonic_basedon_testbed.yml` which can help you generate a minigraph for your SONiC testbed based on the topology you specified and load the configuration minigraph.xml to SONiC DUT. 
-
-When user call testbed-cli to deploy a testbed topology, use this playbook to generate matching SONiC minigraph file and deploy it into SONiC switch under test. 
-
-Or when you know your topology name, you may use this playbook alone to generate a minigraph matching your topology name without deploy it.
-
-VM Topologies are defined inside of vars/ directory in files vars/topo_{{ topology_name}}.yml
-
-Every topology should have a name to distinct one topology from another on the server
-
-Every topology contains a ptf container which will be used as placeholder for the injected interfaces from VMs, or direct connections to PTF host
-
-VMs inventory file is also required to have all VMs ready for generating the minigraph file
-
-VMs inventory is in file 'veos'
-
-Template files for generating minigraph.xml are defined in template/topo directory
-
-`TODO: Create xml graph template files for all available topologies; and create config_db style json configuration files to match all available topologies. No all checked in topologies have the correct xml graph template. T0, T1, T1-lag for 32 ports and t1-lag for 64 ports are supported for now.`
-
-To generate and deploy minigraph for SONiC switch matching the VM topology please use following command:
-
-`ansible-playbook -i lab config_sonic_basedon_testbed.yml -l sonic_dut_name -e vm_base=VM0300 -e topo=t0 [-e deploy=true]`
-
-```Parameters
--l str-msn2700-01          - the sonic_dut_name you are going to generate minigraph for
--e vm_base=VM0300          - the VM name which is used to as base to calculate VM name for this set
--e topo=t0                 - the name of topology to generate minigraph file
--e deploy=True             - if deploy the newly generated minigraph to the targent DUT, default is false if not defined
--e save=True               - if save the newly generated minigraph to the targent DUT as starup-config, default is false if not defined
-```
-
-After minigraph.xml is generated, the playbook will replace the original minigraph file under ansible/minigraph/ with the newly generated minigraph file for the SONiC device.
-
-The playbook will based on deploy=True or False to deside if load the SONiC device with new minigraph or not.
-```
-If deploy=true, the playbook will apply the newly generated minigraph to the SONiC switch
-If save=true, the playbook will save the newly generated minigraph to SONiC switch as startup-config
-```

ansible/doc/README.testbed.Minigraph.md

@@ -0,0 +1,38 @@
+# Device Minigraph Generation and Deployment
+
+Different topologies requires different configuration/minigraph on the DUT.
+```testbed-cli.sh gen-mg|deploy-mg``` allows to generate and deploy minigraph to
+the DUT based on the topology type.
+
+Before you run the command, first prepare [inventory](../lab) file and
+[testbed.csv](../testbed.csv) file. In the command line, `vms-sn2700-t0` 
+is the testbed name defined in `testbed.csv`, `lab` is the inventory file.
+`password.txt` is the vault password file.
+
+```
+./testbed-cli.sh deploy-mg vms-sn2700-t0 lab password.txt
+```
+
+**Note**
+
+- To configure your SONiC switch with different port speeds, you need to specify port speed in `port_config.ini`. Example [port_config.ini](https://github.com/Azure/sonic-buildimage/blob/master/device/arista/x86_64-arista_7260cx3_64/Arista-7260CX3-D108C8/port_config.ini).
+
+## How it works
+
+The tool works as follows:
+
+- Read the testbed info from `testbed.csv` using the testbed name.
+
+- Read the topology information defined in topology file `vars/topo_{{ topology_name}}.yml`
+
+- Read the VMs information from the `veos` inventory file.
+
+- Generate minigraph based on minigraph templates in `templates` folder.
+
+- Deploy minigraph to the DUT.
+
+- Load the minigraph on DUT.
+
+- Save the configuration in config db.
+
+For more details, please refer the comments in [`config_sonic_basedon_testbed.yml`](../config_sonic_basedon_testbed.yml).

ansible/lab

@@ -1,13 +1,35 @@
-[sonic_mlnx_40]
+[sonic_sn2700_40]
 str-msn2700-01    ansible_host=10.251.0.188
 
-[sonic_mlnx_40:vars]
+[sonic_sn2700_40:vars]
 hwsku="ACS-MSN2700"
 iface_speed='40000'
-mgmt_subnet_mask_length="24"
+
+[sonic_s6000]
+lab-s6000-01      ansible_host=10.251.0.189
+
+[sonic_s6000:vars]
+hwsku="Force10-S6000"
+iface_speed='40000'
+
+[sonic_s6100]
+lab-s6100-01      ansible_host=10.251.0.190
+
+[sonic_s6100:vars]
+hwsku="Force10-S6100"
+iface_speed='40000'
+
+[sonic_a7260]
+lab-a7260-01      ansible_host=10.251.0.191    hwsku="Arista-7260CX3-D108C8"
+
+[sonic_a7260:vars]
+iface_speed='100000'
 
 [sonic:children]
-sonic_mlnx_40
+sonic_sn2700_40
+sonic_s6000
+sonic_s6100
+sonic_a7260
 
 [ptf]
 ptf_ptf1    ansible_host=10.255.0.188 ansible_ssh_user=root ansible_ssh_pass=root
@@ -17,6 +39,9 @@ ptf_vms1-1  ansible_host=10.255.0.178 ansible_ssh_user=root ansible_ssh_pass=roo
 sonic
 fanout
 
+[lab:vars]
+mgmt_subnet_mask_length="24"
+
 [fanout]
 str-7260-10       ansible_host=10.251.0.13
 str-7260-11       ansible_host=10.251.0.234

ansible/library/port_alias.py

@@ -16,7 +16,7 @@
         Minigraph file is using SONiC deivce alias to describe the interface name, it's vendor and and hardware platform dependent
         This module is used to find the correct port_config.ini for the hwsku and return Ansible ansible_facts.port_alias
         The definition of this mapping is specified in http://github.com/azure/sonic-buildimage/device
-        You should build docker-sonic-mgmt from sonic-buildimage and run Ansible from sonic-mgmt docker container 
+        You should build docker-sonic-mgmt from sonic-buildimage and run Ansible from sonic-mgmt docker container
     Input:
         hwsku
 
@@ -30,14 +30,21 @@
       port_alias: hwsku='ACS-MSN2700'
 '''
 
-### TODO:  we could eventually use sonic config package to replace this port_alias module later ###############
-### Here are the expectation of files of device port_config.ini located, in case changed please modify it here 
+RETURN = '''
+      ansible_facts{
+        port_alias: [Ethernet0, Ethernet4, ....],
+        port_speed: {'Ethernet0':'40000', 'Ethernet4':'40000', ......]
+      }
+'''
+
+### Here are the expectation of files of device port_config.ini located, in case changed please modify it here
 FILE_PATH = '/usr/share/sonic/device'
 PORTMAP_FILE = 'port_config.ini'
+ALLOWED_HEADER = ['name', 'lanes', 'alias', 'index', 'speed']
 
 class SonicPortAliasMap():
     """
-    Retrieve SONiC device interface port alias mapping
+    Retrieve SONiC device interface port alias mapping and port speed if they are definded
 
     """
     def __init__(self, hwsku):
@@ -48,56 +55,67 @@ def __init__(self, hwsku):
     def findfile(self):
         for (rootdir, dirnames, filenames) in os.walk(FILE_PATH, followlinks=True):
             if self.hwsku == rootdir.split('/')[-1] and len(dirnames) == 0 and PORTMAP_FILE in filenames:
-                self.filename = rootdir+'/'+PORTMAP_FILE
+                self.filename = rootdir + '/' + PORTMAP_FILE
 
     def get_portmap(self):
         aliases = []
         portmap = {}
         aliasmap = {}
+        portspeed = {}
         self.findfile()
         if self.filename == '':
             raise Exception("Something wrong when trying to find the portmap file, either the hwsku is not available or file location is not correct")
         with open(self.filename) as f:
             lines = f.readlines()
-        has_alias=False
+        alias_index = -1
+        speed_index = -1
         while len(lines) != 0:
             line = lines.pop(0)
             if re.match('^#', line):
                 title=re.sub('#', '', line.strip().lower()).split()
-                if 'alias' in title:
-                    index = title.index('alias')
-                    has_alias = True
-            else: 
+                for text in title:
+                    if text in ALLOWED_HEADER:
+                        index = title.index(text)
+                        if 'alias' in text:
+                            alias_index = index
+                        if 'speed' in text:
+                            speed_index = index
+            else:
                 if re.match('^Ethernet', line):
                     mapping = line.split()
                     name = mapping[0]
-                    if has_alias and len(mapping) > index:
-                        alias = mapping[index]
+                    if alias_index != -1 and len(mapping) > alias_index:
+                        alias = mapping[alias_index]
                     else:
                         alias = name
                     aliases.append(alias)
                     portmap[name] = alias
                     aliasmap[alias] = name
+                    if speed_index != -1:
+                        portspeed[alias] = mapping[speed_index]
 
-        return (aliases, portmap, aliasmap)
+        return (aliases, portmap, aliasmap, portspeed)
 
 def main():
     module = AnsibleModule(
         argument_spec=dict(
             hwsku=dict(required=True, type='str')
         ),
-        supports_check_mode=False
+        supports_check_mode=True
     )
     m_args = module.params
     try:
         allmap = SonicPortAliasMap(m_args['hwsku'])
-        (aliases, portmap, aliasmap) = allmap.get_portmap()
-        module.exit_json(ansible_facts={'port_alias': aliases, 'port_name_map': portmap, 'port_alias_map': aliasmap})
+        (aliases, portmap, aliasmap, portspeed) = allmap.get_portmap()
+        module.exit_json(ansible_facts={'port_alias': aliases,
+                                        'port_name_map': portmap,
+                                        'port_alias_map': aliasmap,
+                                        'port_speed': portspeed})
     except (IOError, OSError), e:
         fail_msg = "IO error" + str(e)
         module.fail_json(msg=fail_msg)
     except Exception, e:
-        fail_msg = "failed to find the correct port names for "+m_args['hwsku'] + str(e)
+        fail_msg = "failed to find the correct port config for "+m_args['hwsku'] + str(e)
         module.fail_json(msg=fail_msg)
 
 from ansible.module_utils.basic import *

ansible/library/testbed_vm_info.py

@@ -76,7 +76,7 @@ def main():
             base_vm=dict(required=True, type='str'),
             topo=dict(required=True, type='str'),
         ),
-        supports_check_mode=False
+        supports_check_mode=True
     )
     m_args = module.params
     topo_type = m_args['topo']