Updates to support interface address collection. Requires telegraf >=…

… 1.17.0 now

bin/tsds-output.py

@@ -2,6 +2,7 @@
 
 import sys
 import json, yaml
+import re
 import requests
 
 print("args are %s " % sys.argv)
@@ -31,8 +32,8 @@ def main():
             print("Unable to parse line = \"%s\", skipping" % line)
             continue
 
-        # rename everything, apply counters, etc. Returns a fully
-        # formed TSDS update message
+        # rename everything, apply counters, etc. Adds a fully
+        # formed TSDS update message to the `block` list
         datatransformer.update(data, block)
 
         if len(block) >= 10:
@@ -41,14 +42,19 @@ def main():
 
 
 def _send_data(config, data):
-    json_data  = json.dumps(data)
+    try:
+        json_data  = json.dumps(data)
+    except:
+        print("***************** Bad data was %s ***************" % data)
+        sys.exit(1)
+        return
     tsds_creds = config.credentials()
     result     = requests.post(tsds_creds['url'] + "/services/push.cgi", 
                                data = {"method": "add_data", "data": json_data}, 
                                auth = (tsds_creds['username'], tsds_creds['password']))
 
     # TODO: error handling here, at least log it
-    print(result.text)
+    #print(result.text)
 
 
 # Some helper classes
@@ -130,7 +136,7 @@ def update(self, data, block):
             "interval": interval,
             "type": tsds_data_name
         }
-        #print(tsds_data)
+        #print("Data = %s" % tsds_data)
 
         block.append(tsds_data)
 
@@ -144,7 +150,22 @@ def update(self, data, block):
                 opt_to   = opt_config['to']
                 opt_from = opt_config['from']
 
-                if opt_from in tags:
+                # Handle wildcarding, we're building an array
+                if "*" in opt_from:
+                    array = []                                   
+
+                    for tag in tags:
+                        if re.match(opt_from, tag):
+                            if opt_config.get('field_name'):
+                                array.append({opt_config['field_name']: tags[tag]})
+                            else:
+                                array.append(tags[tag])
+
+                    if array:
+                        metadata[opt_to] = array
+                        has_any = True
+
+                elif opt_from in tags:
                     metadata[opt_to] = tags[opt_from]
                     has_any = True
 
@@ -155,6 +176,8 @@ def update(self, data, block):
                     "type": tsds_data_name + ".metadata"
                 }
 
+                #print("Optional meta = %s" % metadata_data)
+
                 block.append(metadata_data)
 
 

conf/config.yaml

@@ -16,6 +16,10 @@
       optional_metadata:
           - from: ifAlias
             to: description
+          # Take all the ip addresses and roll them up
+          - from: ip_address_*
+            to: interface_address
+            field_name: value
       fields:
           - from: ifInErrors
             to: inerror
@@ -29,10 +33,10 @@
           - from: ifHCOutOctets
             to: output
             rate: true
-          - from: ifInUcastPkts
+          - from: ifHCInUcastPkts
             to: inUcast
             rate: true
-          - from: ifOutUcastPkts
+          - from: ifHCOutUcastPkts
             to: outUcast
             rate: true
           - from: ifInDiscards

conf/telegraf_interface_example.conf

@@ -83,20 +83,65 @@
 
 
 [[outputs.execd]]
-  command = ["/usr/bin/python3", "/path/to/tsds-output.py/", "/path/to/config.yaml"]
+  namepass = ["interface"]
+  command = ["/usr/bin/python3", "/path/to/tsds-output.py/", "/path/to/config.yaml"]  
   data_format = "json"
   restart_delay = "10s"
 
 
+# A starlark script that fixes bytes->bits to be more engineer friendly.
+# This also provides an in telegraf "mapping" between the ifTable and ipTables
+# since they use a slightly different indexing scheme. There might be a 
+# better way to do this.
 [[processors.starlark]]
-namepass = ["interface"]
-# Reads the Starlark script embedded
+namepass = ["interface", "interface_address"]
 source = '''
+state = {}
+
 def apply(metric):
- # Convert from bytes to bits
- metric.fields['ifHCInOctets'] *= 8
- metric.fields['ifHCOutOctets'] *= 8
- return metric
+   #print("metric = %s" % metric) 
+
+   # Convert from bytes to bits 
+   if "ifHCInOctets" in metric.fields:
+       metric.fields["ifHCInOctets"] *= 8
+   if "ifHCOutOctets" in metric.fields:
+       metric.fields["ifHCOutOctets"] *= 8
+
+   # Store the ifIndex out of the ifXTable, we can use
+   # use this to associate the interface name with the IP info
+   if "ifIndex" in metric.tags:
+       ipInfo = state.get(metric.tags["ifIndex"], [])
+       ipInfo = sorted(ipInfo, key = lambda x: x.tags["ipAdEntAddr"])
+
+       # We cannot store "arrays" here so we instead are going to create
+       # multiple fields. This is very ugly but we can fix this in the
+       # TSDS driver
+       i = 0    
+       for result in ipInfo:    
+           value = result.tags["ipAdEntAddr"]
+	   metric.tags["ip_address_%s" % i] = value
+	   i += 1 
+
+   # If we are handling an ipAddr metric, see if we can
+   # grab the ifTable info to associate it with an interface
+   if "ipAdEntIfIndex" in metric.tags:
+       ifIndex = metric.tags["ipAdEntIfIndex"]
+       ip_list = state.setdefault(ifIndex, [])
+
+       # Filter out any that might have "timed out" since this is done
+       # asynchronously and one at a time, we never have the full picture.
+       # TODO: The expire time really should not be hardcoded?
+       # Apparently there is no "filter" in starlark either
+       ip_list = [x for x in ip_list if x.time > metric.time - (60*10)]
+
+       # If we have not seen this IP before, store it
+       if not [x for x in ip_list if x.tags["ipAdEntAddr"] == metric.tags["ipAdEntAddr"]]:
+           ip_list.append(deepcopy(metric))
+
+       #print("STATE = %s" % ip_list)
+       state[ifIndex] = ip_list
+
+   return metric
 '''
 
 
@@ -111,7 +156,7 @@ def apply(metric):
 
   ## Host addresses to retrieve values for (hostname or IP address)
   agents = [
-    "udp://yourhost:161"
+    "udp://yourhost:161"    
   ]
 
   ## Timeout for each request.
@@ -132,9 +177,8 @@ def apply(metric):
   ## Add fields and tables defining the variables you wish to collect.  This
   ## example collects the system uptime and interface variables.  Reference the
   ## full plugin documentation for configuration details.
-  ## Add fields and tables defining the variables you wish to collect.  This
-  ## example collects the system uptime and interface variables.  Reference the
-  ## full plugin documentation for configuration details.
+
+  # Walk the ifXTable for counters, names, etc.
   [[inputs.snmp.table]]
     oid = "IF-MIB::ifXTable"
     name = "interface"
@@ -145,6 +189,11 @@ def apply(metric):
       name = "ifName"
       is_tag = true
 
+    [[inputs.snmp.table.field]]
+      oid = "IF-MIB::ifIndex"
+      name = "ifIndex"
+      is_tag = true
+
     [[inputs.snmp.table.field]]
       oid = "IF-MIB::ifAlias"
       name = "ifAlias"
@@ -166,5 +215,44 @@ def apply(metric):
       oid = "IF-MIB::ifOutDiscards"
       name = "ifOutDiscards"
 
+    [[inputs.snmp.table.field]]
+      oid = "IF-MIB::ifOperStatus"
+      name = "ifOperStatus"
+
     [inputs.snmp.tagpass]
       ifName = ["et*", "xe*", "lo*"]
+
+
+[[inputs.snmp]]
+  agents = [
+    "udp://yourhost:161"    
+  ]
+
+  ## Timeout for each request.
+  timeout = "15s"
+
+  ## SNMP version; can be 1, 2, or 3.
+  version = 2
+
+  ## SNMP community string.
+  community = "public"
+
+
+  # Walk ipAddrTable separately. It doesn't use the same
+  # mapping schema so we'll handle that in the Starlark configuration
+  [[inputs.snmp.table]]
+      oid = "IP-MIB::ipAddrTable"
+      name = "interface_address"
+
+    [[inputs.snmp.table.field]]
+      oid = "IP-MIB::ipAdEntIfIndex"
+      name = "ipAdEntIfIndex"
+      is_tag = true
+
+    [[inputs.snmp.table.field]]
+      oid = "IP-MIB::ipAdEntAddr"
+      name = "ipAdEntAddr"
+      is_tag = true
+
+     [inputs.snmp.tagdrop]
+       ipAdEntAddr = ["192.168.*", "127.*", "10.*"]