diff --git a/bin/tsds-output.py b/bin/tsds-output.py index 315d24e..e2ed894 100644 --- a/bin/tsds-output.py +++ b/bin/tsds-output.py @@ -2,6 +2,7 @@ import sys import json, yaml +import re import requests print("args are %s " % sys.argv) @@ -31,8 +32,8 @@ def main(): print("Unable to parse line = \"%s\", skipping" % line) continue - # rename everything, apply counters, etc. Returns a fully - # formed TSDS update message + # rename everything, apply counters, etc. Adds a fully + # formed TSDS update message to the `block` list datatransformer.update(data, block) if len(block) >= 10: @@ -41,14 +42,19 @@ def main(): def _send_data(config, data): - json_data = json.dumps(data) + try: + json_data = json.dumps(data) + except: + print("***************** Bad data was %s ***************" % data) + sys.exit(1) + return tsds_creds = config.credentials() result = requests.post(tsds_creds['url'] + "/services/push.cgi", data = {"method": "add_data", "data": json_data}, auth = (tsds_creds['username'], tsds_creds['password'])) # TODO: error handling here, at least log it - print(result.text) + #print(result.text) # Some helper classes @@ -130,7 +136,7 @@ def update(self, data, block): "interval": interval, "type": tsds_data_name } - #print(tsds_data) + #print("Data = %s" % tsds_data) block.append(tsds_data) @@ -144,7 +150,22 @@ def update(self, data, block): opt_to = opt_config['to'] opt_from = opt_config['from'] - if opt_from in tags: + # Handle wildcarding, we're building an array + if "*" in opt_from: + array = [] + + for tag in tags: + if re.match(opt_from, tag): + if opt_config.get('field_name'): + array.append({opt_config['field_name']: tags[tag]}) + else: + array.append(tags[tag]) + + if array: + metadata[opt_to] = array + has_any = True + + elif opt_from in tags: metadata[opt_to] = tags[opt_from] has_any = True @@ -155,6 +176,8 @@ def update(self, data, block): "type": tsds_data_name + ".metadata" } + #print("Optional meta = %s" % metadata_data) + block.append(metadata_data) diff --git a/conf/config.yaml b/conf/config.yaml index 71a68a4..07f2d61 100644 --- a/conf/config.yaml +++ b/conf/config.yaml @@ -16,6 +16,10 @@ optional_metadata: - from: ifAlias to: description + # Take all the ip addresses and roll them up + - from: ip_address_* + to: interface_address + field_name: value fields: - from: ifInErrors to: inerror @@ -29,10 +33,10 @@ - from: ifHCOutOctets to: output rate: true - - from: ifInUcastPkts + - from: ifHCInUcastPkts to: inUcast rate: true - - from: ifOutUcastPkts + - from: ifHCOutUcastPkts to: outUcast rate: true - from: ifInDiscards diff --git a/conf/telegraf_interface_example.conf b/conf/telegraf_interface_example.conf index b102b44..087292d 100644 --- a/conf/telegraf_interface_example.conf +++ b/conf/telegraf_interface_example.conf @@ -83,20 +83,65 @@ [[outputs.execd]] - command = ["/usr/bin/python3", "/path/to/tsds-output.py/", "/path/to/config.yaml"] + namepass = ["interface"] + command = ["/usr/bin/python3", "/path/to/tsds-output.py/", "/path/to/config.yaml"] data_format = "json" restart_delay = "10s" +# A starlark script that fixes bytes->bits to be more engineer friendly. +# This also provides an in telegraf "mapping" between the ifTable and ipTables +# since they use a slightly different indexing scheme. There might be a +# better way to do this. [[processors.starlark]] -namepass = ["interface"] -# Reads the Starlark script embedded +namepass = ["interface", "interface_address"] source = ''' +state = {} + def apply(metric): - # Convert from bytes to bits - metric.fields['ifHCInOctets'] *= 8 - metric.fields['ifHCOutOctets'] *= 8 - return metric + #print("metric = %s" % metric) + + # Convert from bytes to bits + if "ifHCInOctets" in metric.fields: + metric.fields["ifHCInOctets"] *= 8 + if "ifHCOutOctets" in metric.fields: + metric.fields["ifHCOutOctets"] *= 8 + + # Store the ifIndex out of the ifXTable, we can use + # use this to associate the interface name with the IP info + if "ifIndex" in metric.tags: + ipInfo = state.get(metric.tags["ifIndex"], []) + ipInfo = sorted(ipInfo, key = lambda x: x.tags["ipAdEntAddr"]) + + # We cannot store "arrays" here so we instead are going to create + # multiple fields. This is very ugly but we can fix this in the + # TSDS driver + i = 0 + for result in ipInfo: + value = result.tags["ipAdEntAddr"] + metric.tags["ip_address_%s" % i] = value + i += 1 + + # If we are handling an ipAddr metric, see if we can + # grab the ifTable info to associate it with an interface + if "ipAdEntIfIndex" in metric.tags: + ifIndex = metric.tags["ipAdEntIfIndex"] + ip_list = state.setdefault(ifIndex, []) + + # Filter out any that might have "timed out" since this is done + # asynchronously and one at a time, we never have the full picture. + # TODO: The expire time really should not be hardcoded? + # Apparently there is no "filter" in starlark either + ip_list = [x for x in ip_list if x.time > metric.time - (60*10)] + + # If we have not seen this IP before, store it + if not [x for x in ip_list if x.tags["ipAdEntAddr"] == metric.tags["ipAdEntAddr"]]: + ip_list.append(deepcopy(metric)) + + #print("STATE = %s" % ip_list) + state[ifIndex] = ip_list + + return metric ''' @@ -111,7 +156,7 @@ def apply(metric): ## Host addresses to retrieve values for (hostname or IP address) agents = [ - "udp://yourhost:161" + "udp://yourhost:161" ] ## Timeout for each request. @@ -132,9 +177,8 @@ def apply(metric): ## Add fields and tables defining the variables you wish to collect. This ## example collects the system uptime and interface variables. Reference the ## full plugin documentation for configuration details. - ## Add fields and tables defining the variables you wish to collect. This - ## example collects the system uptime and interface variables. Reference the - ## full plugin documentation for configuration details. + + # Walk the ifXTable for counters, names, etc. [[inputs.snmp.table]] oid = "IF-MIB::ifXTable" name = "interface" @@ -145,6 +189,11 @@ def apply(metric): name = "ifName" is_tag = true + [[inputs.snmp.table.field]] + oid = "IF-MIB::ifIndex" + name = "ifIndex" + is_tag = true + [[inputs.snmp.table.field]] oid = "IF-MIB::ifAlias" name = "ifAlias" @@ -166,5 +215,44 @@ def apply(metric): oid = "IF-MIB::ifOutDiscards" name = "ifOutDiscards" + [[inputs.snmp.table.field]] + oid = "IF-MIB::ifOperStatus" + name = "ifOperStatus" + [inputs.snmp.tagpass] ifName = ["et*", "xe*", "lo*"] + + +[[inputs.snmp]] + agents = [ + "udp://yourhost:161" + ] + + ## Timeout for each request. + timeout = "15s" + + ## SNMP version; can be 1, 2, or 3. + version = 2 + + ## SNMP community string. + community = "public" + + + # Walk ipAddrTable separately. It doesn't use the same + # mapping schema so we'll handle that in the Starlark configuration + [[inputs.snmp.table]] + oid = "IP-MIB::ipAddrTable" + name = "interface_address" + + [[inputs.snmp.table.field]] + oid = "IP-MIB::ipAdEntIfIndex" + name = "ipAdEntIfIndex" + is_tag = true + + [[inputs.snmp.table.field]] + oid = "IP-MIB::ipAdEntAddr" + name = "ipAdEntAddr" + is_tag = true + + [inputs.snmp.tagdrop] + ipAdEntAddr = ["192.168.*", "127.*", "10.*"]