Merge pull request #68 from alexandertgtalbot/master

Update to enable generating unique filenames

README.md

@@ -21,6 +21,7 @@ By adding additional env variables the container can send an HTTP request to spe
 - Extract files from config maps
 - Filter based on label
 - Update/Delete on change of configmap
+- Enforce unique filenames
 
 # Usage
 
@@ -129,6 +130,12 @@ If the filename ends with `.url` suffix, the content will be processed as an URL
   - required: false
   - type: boolean
 
+- `UNIQUE_FILENAMES`
+  - description: Set to true to produce unique filenames where duplicate data keys exist between ConfigMaps and/or Secrets within the same or multiple Namespaces.
+  - required: false
+  - default: false
+  - type: boolean
+
 - `DEFAULT_FILE_MODE`
   - description: The default file system permission for every file. Use three digits (e.g. '500', '440', ...)
   - required: false

sidecar/helpers.py

@@ -83,3 +83,19 @@ def request(url, method, payload=None):
 def timestamp():
     """Get a timestamp of the current time for logging."""
     return datetime.now().strftime("[%Y-%m-%d %X]")
+
+
+def uniqueFilename(filename, namespace, resource, resource_name):
+    """Return a unique filename derived from the arguments provided, e.g.
+    "namespace_{namespace}.{configmap|secret}_{resource_name}.{filename}".
+
+    This is used where duplicate data keys may exist between ConfigMaps
+    and/or Secrets within the same or multiple Namespaces.
+
+    Keyword arguments:
+    filename -- the filename derived from a data key present in a ConfigMap or Secret.
+    namespace -- the Namespace from which data is sourced.
+    resource -- the resource type, e.g. "configmap" or "secret".
+    resource_name -- the name of the "configmap" or "secret" resource instance.
+    """
+    return "namespace_" + namespace + "." + resource + "_" + resource_name + "." + filename

sidecar/resources.py

@@ -12,7 +12,7 @@
 from kubernetes.client.rest import ApiException
 from urllib3.exceptions import ProtocolError
 
-from helpers import request, writeTextToFile, removeFile, timestamp
+from helpers import request, writeTextToFile, removeFile, timestamp, uniqueFilename
 
 _list_namespaced = {
     "secret": "list_namespaced_secret",
@@ -56,7 +56,8 @@ def _get_destination_folder(metadata, defaultFolder, folderAnnotation):
     return defaultFolder
 
 
-def listResources(label, labelValue, targetFolder, url, method, payload, currentNamespace, folderAnnotation, resource):
+def listResources(label, labelValue, targetFolder, url, method, payload,
+                  currentNamespace, folderAnnotation, resource, uniqueFilenames):
     v1 = client.CoreV1Api()
     namespace = os.getenv("NAMESPACE", currentNamespace)
     # Filter resources based on label and value or just label
@@ -86,14 +87,20 @@ def listResources(label, labelValue, targetFolder, url, method, payload, current
         for data_key in dataMap.keys():
             filename, filedata = _get_file_data_and_name(data_key, dataMap[data_key],
                                                             resource)
+            if uniqueFilenames:
+                filename = uniqueFilename(filename      = filename,
+                                          namespace     = metadata.namespace,
+                                          resource      = resource,
+                                          resource_name = metadata.name)
+
             writeTextToFile(destFolder, filename, filedata)
 
             if url:
                 request(url, method, payload)
 
 
 def _watch_resource_iterator(label, labelValue, targetFolder, url, method, payload,
-                             currentNamespace, folderAnnotation, resource):
+                             currentNamespace, folderAnnotation, resource, uniqueFilenames):
     v1 = client.CoreV1Api()
     namespace = os.getenv("NAMESPACE", currentNamespace)
     # Filter resources based on label and value or just label
@@ -127,13 +134,26 @@ def _watch_resource_iterator(label, labelValue, targetFolder, url, method, paylo
             if (eventType == "ADDED") or (eventType == "MODIFIED"):
                 filename, filedata = _get_file_data_and_name(data_key, dataMap[data_key],
                                                                 resource)
+                if uniqueFilenames:
+                    filename = uniqueFilename(filename      = filename,
+                                              namespace     = metadata.namespace,
+                                              resource      = resource,
+                                              resource_name = metadata.name)
+
                 writeTextToFile(destFolder, filename, filedata)
 
                 if url:
                     request(url, method, payload)
             else:
                 # Get filename from event
                 filename = data_key[:-4] if data_key.endswith(".url") else data_key
+
+                if uniqueFilenames:
+                    filename = uniqueFilename(filename      = filename,
+                                              namespace     = metadata.namespace,
+                                              resource      = resource,
+                                              resource_name = metadata.name)
+
                 removeFile(destFolder, filename)
                 if url:
                     request(url, method, payload)
@@ -159,19 +179,19 @@ def _watch_resource_loop(mode, *args):
 
 
 def watchForChanges(mode, label, labelValue, targetFolder, url, method, payload,
-                    currentNamespace, folderAnnotation, resources):
+                    currentNamespace, folderAnnotation, resources, uniqueFilenames):
 
     firstProc = Process(target=_watch_resource_loop,
                         args=(mode, label, labelValue, targetFolder, url, method, payload,
-                              currentNamespace, folderAnnotation, resources[0])
+                              currentNamespace, folderAnnotation, resources[0], uniqueFilenames)
                         )
     firstProc.daemon=True
     firstProc.start()
 
     if len(resources) == 2:
         secProc = Process(target=_watch_resource_loop,
                           args=(mode, label, labelValue, targetFolder, url, method, payload,
-                                currentNamespace, folderAnnotation, resources[1])
+                                currentNamespace, folderAnnotation, resources[1], uniqueFilenames)
                           )
         secProc.daemon=True
         secProc.start()

sidecar/sidecar.py

@@ -49,13 +49,21 @@ def main():
         configuration.debug = False
         client.Configuration.set_default(configuration)
 
+    uniqueFilenames = os.getenv("UNIQUE_FILENAMES") 
+    if uniqueFilenames is not None and uniqueFilenames.lower() == "true":
+        print(f"{timestamp()} Unique filenames will be enforced.")
+        uniqueFilenames = True
+    else:
+        print(f"{timestamp()} Unique filenames will not be enforced.")
+        uniqueFilenames = False
+
     if os.getenv("METHOD") == "LIST":
         for res in resources:
             listResources(label, labelValue, targetFolder, url, method, payload,
-                          currentNamespace, folderAnnotation, res)
+                          currentNamespace, folderAnnotation, res, uniqueFilenames)
     else:
         watchForChanges(os.getenv("METHOD"), label, labelValue, targetFolder, url, method,
-                        payload, currentNamespace, folderAnnotation, resources)
+                        payload, currentNamespace, folderAnnotation, resources, uniqueFilenames)
 
 
 if __name__ == "__main__":