From 3b4047d472dce89863bce7c355656cd90b986045 Mon Sep 17 00:00:00 2001 From: Alex Talbot Date: Tue, 17 Mar 2020 14:21:00 +0000 Subject: [PATCH 1/6] Test requisite environment variable and updte to readme. --- README.md | 7 +++++++ sidecar/sidecar.py | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/README.md b/README.md index 382fbd7..30c7ac2 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ By adding additional env variables the container can send an HTTP request to spe - Extract files from config maps - Filter based on label - Update/Delete on change of configmap +- Enforce unique filenames # Usage @@ -119,6 +120,12 @@ If the filename ends with `.url` suffix, the content will be processed as an URL - required: false - type: boolean +- `UNIQUE_FILENAMES` + - description: Set to true to produce unique filenames where duplicate data keys exist between ConfigMaps and/or Secrets within the same or multiple Namespaces. + - required: false + - default: false + - type: boolean + - `DEFAULT_FILE_MODE` - description: The default file system permission for every file. Use three digits (e.g. '500', '440', ...) - required: false diff --git a/sidecar/sidecar.py b/sidecar/sidecar.py index 8ec6133..7199131 100755 --- a/sidecar/sidecar.py +++ b/sidecar/sidecar.py @@ -49,6 +49,14 @@ def main(): configuration.debug = False client.Configuration.set_default(configuration) + uniqueFilenames = os.getenv("UNIQUE_FILENAMES") + if uniqueFilenames is not None and uniqueFilenames.lower() == "true": + print(f"{timestamp()} Unique filenames will enforced.") + uniqueFilenames = True + else: + print(f"{timestamp()} Unique filenames will not be enforced.") + uniqueFilenames = False + if os.getenv("METHOD") == "LIST": for res in resources: listResources(label, labelValue, targetFolder, url, method, payload, From 904b94be632c5e03281d3d389230f895e4f51b3c Mon Sep 17 00:00:00 2001 From: Alex Talbot Date: Tue, 17 Mar 2020 15:19:28 +0000 Subject: [PATCH 2/6] Helper function to generate unique filenames. --- sidecar/helpers.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/sidecar/helpers.py b/sidecar/helpers.py index 25981f3..7ea68a8 100755 --- a/sidecar/helpers.py +++ b/sidecar/helpers.py @@ -76,3 +76,19 @@ def request(url, method, payload=None): def timestamp(): """Get a timestamp of the current time for logging.""" return datetime.now().strftime("[%Y-%m-%d %X]") + + +def uniqueFilename(filename, namespace, resource, resource_name): + """Return a unique filename derived from the arguments provided, e.g. + "namespace_{namespace}-{configmap|secret}_{resource_name}-{filename}". + + This is used where duplicate data keys may exist between ConfigMaps + and/or Secrets within the same or multiple Namespaces. + + Keyword arguments: + filename -- the filename derived from a data key present in a ConfigMap or Secret. + namespace -- the Namespace from which data is sourced. + resource -- the resource type, e.g. "configmap" or "secret". + resource_name -- the name of the "configmap" or "secret" resource instance. + """ + return "namespace_" + namespace + "-" + resource + "_" + resource_name + "-" + filename \ No newline at end of file From 318ed0067e9104c46d289674730e55914fbf2e6b Mon Sep 17 00:00:00 2001 From: Alex Talbot Date: Tue, 17 Mar 2020 15:32:00 +0000 Subject: [PATCH 3/6] Update to allow filename references to be updated to unique values. --- sidecar/resources.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/sidecar/resources.py b/sidecar/resources.py index 7253b7a..275427a 100755 --- a/sidecar/resources.py +++ b/sidecar/resources.py @@ -12,7 +12,7 @@ from kubernetes.client.rest import ApiException from urllib3.exceptions import ProtocolError -from helpers import request, writeTextToFile, removeFile, timestamp +from helpers import request, writeTextToFile, removeFile, timestamp, uniqueFilename _list_namespaced = { "secret": "list_namespaced_secret", @@ -56,7 +56,8 @@ def _get_destination_folder(metadata, defaultFolder, folderAnnotation): return defaultFolder -def listResources(label, labelValue, targetFolder, url, method, payload, currentNamespace, folderAnnotation, resource): +def listResources(label, labelValue, targetFolder, url, method, payload, + currentNamespace, folderAnnotation, resource, uniqueFilenames): v1 = client.CoreV1Api() namespace = os.getenv("NAMESPACE", currentNamespace) # Filter resources based on label and value or just label @@ -86,6 +87,12 @@ def listResources(label, labelValue, targetFolder, url, method, payload, current for data_key in dataMap.keys(): filename, filedata = _get_file_data_and_name(data_key, dataMap[data_key], resource) + if uniqueFilenames: + filename = uniqueFilename(filename = filename, + namespace = metadata.namespace, + resource = resource, + resource_name = metadata.name) + writeTextToFile(destFolder, filename, filedata) if url: @@ -93,7 +100,7 @@ def listResources(label, labelValue, targetFolder, url, method, payload, current def _watch_resource_iterator(label, labelValue, targetFolder, url, method, payload, - currentNamespace, folderAnnotation, resource): + currentNamespace, folderAnnotation, resource, uniqueFilenames): v1 = client.CoreV1Api() namespace = os.getenv("NAMESPACE", currentNamespace) # Filter resources based on label and value or just label @@ -127,6 +134,12 @@ def _watch_resource_iterator(label, labelValue, targetFolder, url, method, paylo if (eventType == "ADDED") or (eventType == "MODIFIED"): filename, filedata = _get_file_data_and_name(data_key, dataMap[data_key], resource) + if uniqueFilenames: + filename = uniqueFilename(filename = filename, + namespace = metadata.namespace, + resource = resource, + resource_name = metadata.name) + writeTextToFile(destFolder, filename, filedata) if url: @@ -134,6 +147,13 @@ def _watch_resource_iterator(label, labelValue, targetFolder, url, method, paylo else: # Get filename from event filename = data_key[:-4] if data_key.endswith(".url") else data_key + + if uniqueFilenames: + filename = uniqueFilename(filename = filename, + namespace = metadata.namespace, + resource = resource, + resource_name = metadata.name) + removeFile(destFolder, filename) if url: request(url, method, payload) From aaa32d049eab80291f458802863bcad5a5bc528b Mon Sep 17 00:00:00 2001 From: Alex Talbot Date: Tue, 17 Mar 2020 16:42:33 +0000 Subject: [PATCH 4/6] Changed unique filename's resource separator for improved readability. --- sidecar/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sidecar/helpers.py b/sidecar/helpers.py index 7ea68a8..a9612fd 100755 --- a/sidecar/helpers.py +++ b/sidecar/helpers.py @@ -80,7 +80,7 @@ def timestamp(): def uniqueFilename(filename, namespace, resource, resource_name): """Return a unique filename derived from the arguments provided, e.g. - "namespace_{namespace}-{configmap|secret}_{resource_name}-{filename}". + "namespace_{namespace}.{configmap|secret}_{resource_name}.{filename}". This is used where duplicate data keys may exist between ConfigMaps and/or Secrets within the same or multiple Namespaces. @@ -91,4 +91,4 @@ def uniqueFilename(filename, namespace, resource, resource_name): resource -- the resource type, e.g. "configmap" or "secret". resource_name -- the name of the "configmap" or "secret" resource instance. """ - return "namespace_" + namespace + "-" + resource + "_" + resource_name + "-" + filename \ No newline at end of file + return "namespace_" + namespace + "." + resource + "_" + resource_name + "." + filename \ No newline at end of file From 9646ce5a50f8c96edbfe6d10106eb961922523cb Mon Sep 17 00:00:00 2001 From: Alex Talbot Date: Tue, 17 Mar 2020 16:43:43 +0000 Subject: [PATCH 5/6] Added missing 'uniqueFilenames' parameter to higher level function calls. --- sidecar/resources.py | 6 +++--- sidecar/sidecar.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sidecar/resources.py b/sidecar/resources.py index 275427a..e3fff1a 100755 --- a/sidecar/resources.py +++ b/sidecar/resources.py @@ -179,11 +179,11 @@ def _watch_resource_loop(mode, *args): def watchForChanges(mode, label, labelValue, targetFolder, url, method, payload, - currentNamespace, folderAnnotation, resources): + currentNamespace, folderAnnotation, resources, uniqueFilenames): firstProc = Process(target=_watch_resource_loop, args=(mode, label, labelValue, targetFolder, url, method, payload, - currentNamespace, folderAnnotation, resources[0]) + currentNamespace, folderAnnotation, resources[0], uniqueFilenames) ) firstProc.daemon=True firstProc.start() @@ -191,7 +191,7 @@ def watchForChanges(mode, label, labelValue, targetFolder, url, method, payload, if len(resources) == 2: secProc = Process(target=_watch_resource_loop, args=(mode, label, labelValue, targetFolder, url, method, payload, - currentNamespace, folderAnnotation, resources[1]) + currentNamespace, folderAnnotation, resources[1], uniqueFilenames) ) secProc.daemon=True secProc.start() diff --git a/sidecar/sidecar.py b/sidecar/sidecar.py index 7199131..7448a61 100755 --- a/sidecar/sidecar.py +++ b/sidecar/sidecar.py @@ -60,10 +60,10 @@ def main(): if os.getenv("METHOD") == "LIST": for res in resources: listResources(label, labelValue, targetFolder, url, method, payload, - currentNamespace, folderAnnotation, res) + currentNamespace, folderAnnotation, res, uniqueFilenames) else: watchForChanges(os.getenv("METHOD"), label, labelValue, targetFolder, url, method, - payload, currentNamespace, folderAnnotation, resources) + payload, currentNamespace, folderAnnotation, resources, uniqueFilenames) if __name__ == "__main__": From 63b8d754e2ade09f7c86147307d67e3b30a1d3e9 Mon Sep 17 00:00:00 2001 From: Alex Talbot Date: Tue, 17 Mar 2020 18:06:43 +0000 Subject: [PATCH 6/6] Log message typo fix --- sidecar/sidecar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sidecar/sidecar.py b/sidecar/sidecar.py index 7448a61..ab8aeca 100755 --- a/sidecar/sidecar.py +++ b/sidecar/sidecar.py @@ -51,7 +51,7 @@ def main(): uniqueFilenames = os.getenv("UNIQUE_FILENAMES") if uniqueFilenames is not None and uniqueFilenames.lower() == "true": - print(f"{timestamp()} Unique filenames will enforced.") + print(f"{timestamp()} Unique filenames will be enforced.") uniqueFilenames = True else: print(f"{timestamp()} Unique filenames will not be enforced.")