Merge pull request #1836 from alicevision/dev/cgroups

[core] Add cgroups support to meshroom

meshroom/core/cgroup.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+# coding:utf-8
+
+import os
+
+#Try to retrieve limits of memory for the current process' cgroup
+def getCgroupMemorySize():
+
+    #first of all, get pid of process
+    pid = os.getpid()
+
+    #Get cgroup associated with pid
+    filename = f"/proc/{pid}/cgroup"
+
+    cgroup = None
+    try:
+        with open(filename, "r") as f :
+
+            #cgroup file is a ':' separated table
+            #lookup a line where the second field is "memory"
+            lines = f.readlines()
+            for line in lines:
+                tokens = line.rstrip("\r\n").split(":")
+                if len(tokens) < 3:
+                    continue
+                if tokens[1] == "memory":
+                    cgroup = tokens[2]
+    except OSError:
+        pass
+
+    if cgroup is None:
+        return -1
+
+    size = -1
+    filename = f"/sys/fs/cgroup/memory/{cgroup}/memory.limit_in_bytes"
+    try:
+        with open(filename, "r") as f :
+            value = f.read().rstrip("\r\n")
+            if value.isnumeric():
+                size = int(value)
+    except OSError:
+        pass
+
+    return size
+
+def parseNumericList(numericListString):
+
+    nList = []
+    for item in numericListString.split(','):
+        if '-' in item:
+            start, end = item.split('-')
+            start = int(start)
+            end = int(end)
+            nList.extend(range(start, end + 1))
+        else:
+            value = int(item)
+            nList.append(value)
+
+    return nList
+
+#Try to retrieve limits of cores for the current process' cgroup
+def getCgroupCpuCount():
+
+    #first of all, get pid of process
+    pid = os.getpid()
+
+    #Get cgroup associated with pid
+    filename = f"/proc/{pid}/cgroup"
+
+    cgroup = None
+    try:
+        with open(filename, "r") as f :
+
+            #cgroup file is a ':' separated table
+            #lookup a line where the second field is "memory"
+            lines = f.readlines()
+            for line in lines:
+                tokens = line.rstrip("\r\n").split(":")
+                if len(tokens) < 3:
+                    continue
+                if tokens[1] == "cpuset":
+                    cgroup = tokens[2]
+    except OSError:
+        pass
+
+    if cgroup is None:
+        return -1
+
+    size = -1
+    filename = f"/sys/fs/cgroup/cpuset/{cgroup}/cpuset.cpus"
+    try:
+        with open(filename, "r") as f :
+            value = f.read().rstrip("\r\n")
+            nlist = parseNumericList(value)
+            size = len(nlist)
+
+    except OSError:
+        pass
+
+    return size
+

meshroom/core/desc.py

@@ -1,4 +1,5 @@
 from meshroom.common import BaseObject, Property, Variant, VariantList, JSValue
+from meshroom.core import cgroup
 
 from collections.abc import Iterable
 from enum import Enum
@@ -534,16 +535,19 @@ class CommandLineNode(Node):
     commandLineRange = ''
 
     def buildCommandLine(self, chunk):
+
         cmdPrefix = ''
         # if rez available in env, we use it
         if 'REZ_ENV' in os.environ and chunk.node.packageVersion:
             # if the node package is already in the environment, we don't need a new dedicated rez environment
             alreadyInEnv = os.environ.get('REZ_{}_VERSION'.format(chunk.node.packageName.upper()), "").startswith(chunk.node.packageVersion)
             if not alreadyInEnv:
                 cmdPrefix = '{rez} {packageFullName} -- '.format(rez=os.environ.get('REZ_ENV'), packageFullName=chunk.node.packageFullName)
+
         cmdSuffix = ''
         if chunk.node.isParallelized and chunk.node.size > 1:
             cmdSuffix = ' ' + self.commandLineRange.format(**chunk.range.toDict())
+
         return cmdPrefix + chunk.node.nodeDesc.commandLine.format(**chunk.node._cmdVars) + cmdSuffix
 
     def stopProcess(self, chunk):
@@ -589,6 +593,34 @@ def processChunk(self, chunk):
         finally:
             chunk.subprocess = None
 
+#specific command line node for alicevision apps
+class AVCommandLineNode(CommandLineNode):
+
+    cgroupParsed = False
+    cmdMem = ''
+    cmdCore = ''
+
+    def __init__(self):
+
+        if AVCommandLineNode.cgroupParsed is False:
+
+            AVCommandLineNode.cmdMem = ''
+            memSize = cgroup.getCgroupMemorySize()
+            if memSize > 0:
+                AVCommandLineNode.cmdMem = ' --maxMemory={memSize}'.format(memSize=memSize)
+
+            AVCommandLineNode.cmdCore = ''
+            coresCount = cgroup.getCgroupCpuCount()
+            if coresCount > 0:
+                AVCommandLineNode.cmdCore = ' --maxCores={coresCount}'.format(coresCount=coresCount)
+
+            AVCommandLineNode.cgroupParsed = True
+
+    def buildCommandLine(self, chunk):
+
+        commandLineString = super(AVCommandLineNode, self).buildCommandLine(chunk)
+
+        return commandLineString + AVCommandLineNode.cmdMem + AVCommandLineNode.cmdCore
 
 # Test abstract node
 class InitNode:

meshroom/nodes/aliceVision/CameraCalibration.py

@@ -3,7 +3,7 @@
 from meshroom.core import desc
 
 
-class CameraCalibration(desc.CommandLineNode):
+class CameraCalibration(desc.AVCommandLineNode):
     commandLine = 'aliceVision_cameraCalibration {allParams}'
 
     category = 'Utils'

meshroom/nodes/aliceVision/CameraInit.py

@@ -119,7 +119,7 @@ def readSfMData(sfmFile):
     return views, intrinsics
 
 
-class CameraInit(desc.CommandLineNode, desc.InitNode):
+class CameraInit(desc.AVCommandLineNode, desc.InitNode):
     commandLine = 'aliceVision_cameraInit {allParams} --allowSingleView 1' # don't throw an error if there is only one image
 
     size = desc.DynamicNodeSize('viewpoints')

meshroom/nodes/aliceVision/CameraLocalization.py

@@ -4,7 +4,7 @@
 from meshroom.core import desc
 
 
-class CameraLocalization(desc.CommandLineNode):
+class CameraLocalization(desc.AVCommandLineNode):
     commandLine = 'aliceVision_cameraLocalization {allParams}'
 
     category = 'Utils'

meshroom/nodes/aliceVision/CameraRigCalibration.py

@@ -4,7 +4,7 @@
 from meshroom.core import desc
 
 
-class CameraRigCalibration(desc.CommandLineNode):
+class CameraRigCalibration(desc.AVCommandLineNode):
     commandLine = 'aliceVision_rigCalibration {allParams}'
 
     category = 'Utils'

meshroom/nodes/aliceVision/CameraRigLocalization.py

@@ -4,7 +4,7 @@
 from meshroom.core import desc
 
 
-class CameraRigLocalization(desc.CommandLineNode):
+class CameraRigLocalization(desc.AVCommandLineNode):
     commandLine = 'aliceVision_rigLocalization {allParams}'
 
     category = 'Utils'

meshroom/nodes/aliceVision/ColorCheckerCorrection.py

@@ -5,7 +5,7 @@
 import os.path
 
 
-class ColorCheckerCorrection(desc.CommandLineNode):
+class ColorCheckerCorrection(desc.AVCommandLineNode):
     commandLine = 'aliceVision_utils_colorCheckerCorrection {allParams}'
     size = desc.DynamicNodeSize('input')
     # parallelization = desc.Parallelization(blockSize=40)

meshroom/nodes/aliceVision/ColorCheckerDetection.py

@@ -5,7 +5,7 @@
 import os.path
 
 
-class ColorCheckerDetection(desc.CommandLineNode):
+class ColorCheckerDetection(desc.AVCommandLineNode):
     commandLine = 'aliceVision_utils_colorCheckerDetection {allParams}'
     size = desc.DynamicNodeSize('input')
     # parallelization = desc.Parallelization(blockSize=40)

meshroom/nodes/aliceVision/ConvertMesh.py

@@ -3,7 +3,7 @@
 from meshroom.core import desc
 
 
-class ConvertMesh(desc.CommandLineNode):
+class ConvertMesh(desc.AVCommandLineNode):
     commandLine = 'aliceVision_convertMesh {allParams}'
 
     category = 'Utils'

meshroom/nodes/aliceVision/ConvertSfMFormat.py

@@ -3,7 +3,7 @@
 from meshroom.core import desc
 
 
-class ConvertSfMFormat(desc.CommandLineNode):
+class ConvertSfMFormat(desc.AVCommandLineNode):
     commandLine = 'aliceVision_convertSfMFormat {allParams}'
     size = desc.DynamicNodeSize('input')
 

meshroom/nodes/aliceVision/DepthMap.py

@@ -3,7 +3,7 @@
 from meshroom.core import desc
 
 
-class DepthMap(desc.CommandLineNode):
+class DepthMap(desc.AVCommandLineNode):
     commandLine = 'aliceVision_depthMapEstimation {allParams}'
     gpu = desc.Level.INTENSIVE
     size = desc.DynamicNodeSize('input')

meshroom/nodes/aliceVision/DepthMapFilter.py

@@ -3,7 +3,7 @@
 from meshroom.core import desc
 
 
-class DepthMapFilter(desc.CommandLineNode):
+class DepthMapFilter(desc.AVCommandLineNode):
     commandLine = 'aliceVision_depthMapFiltering {allParams}'
     gpu = desc.Level.NORMAL
     size = desc.DynamicNodeSize('input')

meshroom/nodes/aliceVision/DistortionCalibration.py

@@ -3,7 +3,7 @@
 from meshroom.core import desc
 
 
-class DistortionCalibration(desc.CommandLineNode):
+class DistortionCalibration(desc.AVCommandLineNode):
     commandLine = 'aliceVision_distortionCalibration {allParams}'
     size = desc.DynamicNodeSize('input')
 

meshroom/nodes/aliceVision/ExportAnimatedCamera.py

@@ -3,7 +3,7 @@
 from meshroom.core import desc
 
 
-class ExportAnimatedCamera(desc.CommandLineNode):
+class ExportAnimatedCamera(desc.AVCommandLineNode):
     commandLine = 'aliceVision_exportAnimatedCamera {allParams}'
 
     category = 'Export'

meshroom/nodes/aliceVision/ExportColoredPointCloud.py

@@ -3,7 +3,7 @@
 from meshroom.core import desc
 
 
-class ExportColoredPointCloud(desc.CommandLineNode):
+class ExportColoredPointCloud(desc.AVCommandLineNode):
     commandLine = 'aliceVision_exportColoredPointCloud {allParams}'
 
     category = 'Export'

meshroom/nodes/aliceVision/ExportMatches.py

@@ -3,7 +3,7 @@
 from meshroom.core import desc
 
 
-class ExportMatches(desc.CommandLineNode):
+class ExportMatches(desc.AVCommandLineNode):
     commandLine = 'aliceVision_exportMatches {allParams}'
     size = desc.DynamicNodeSize('input')
 

meshroom/nodes/aliceVision/ExportMaya.py

@@ -3,7 +3,7 @@
 from meshroom.core import desc
 
 
-class ExportMaya(desc.CommandLineNode):
+class ExportMaya(desc.AVCommandLineNode):
     commandLine = 'aliceVision_exportMeshroomMaya {allParams}'
 
     category = 'Export'

meshroom/nodes/aliceVision/FeatureExtraction.py

@@ -3,7 +3,7 @@
 from meshroom.core import desc
 
 
-class FeatureExtraction(desc.CommandLineNode):
+class FeatureExtraction(desc.AVCommandLineNode):
     commandLine = 'aliceVision_featureExtraction {allParams}'
     size = desc.DynamicNodeSize('input')
     parallelization = desc.Parallelization(blockSize=40)

meshroom/nodes/aliceVision/FeatureMatching.py

@@ -3,7 +3,7 @@
 from meshroom.core import desc
 
 
-class FeatureMatching(desc.CommandLineNode):
+class FeatureMatching(desc.AVCommandLineNode):
     commandLine = 'aliceVision_featureMatching {allParams}'
     size = desc.DynamicNodeSize('input')
     parallelization = desc.Parallelization(blockSize=20)

meshroom/nodes/aliceVision/FeatureRepeatability.py

@@ -3,7 +3,7 @@
 from meshroom.core import desc
 
 
-class FeatureRepeatability(desc.CommandLineNode):
+class FeatureRepeatability(desc.AVCommandLineNode):
     commandLine = 'aliceVision_samples_repeatabilityDataset {allParams}'
     size = desc.DynamicNodeSize('input')
     # parallelization = desc.Parallelization(blockSize=40)

meshroom/nodes/aliceVision/GlobalSfM.py

@@ -6,7 +6,7 @@
 from meshroom.core import desc
 
 
-class GlobalSfM(desc.CommandLineNode):
+class GlobalSfM(desc.AVCommandLineNode):
     commandLine = 'aliceVision_globalSfM {allParams}'
     size = desc.DynamicNodeSize('input')
 

meshroom/nodes/aliceVision/ImageMasking.py

@@ -3,7 +3,7 @@
 from meshroom.core import desc
 
 
-class ImageMasking(desc.CommandLineNode):
+class ImageMasking(desc.AVCommandLineNode):
     commandLine = 'aliceVision_imageMasking {allParams}'
     size = desc.DynamicNodeSize('input')
     parallelization = desc.Parallelization(blockSize=40)

meshroom/nodes/aliceVision/ImageMatching.py

@@ -4,7 +4,7 @@
 from meshroom.core import desc
 
 
-class ImageMatching(desc.CommandLineNode):
+class ImageMatching(desc.AVCommandLineNode):
     commandLine = 'aliceVision_imageMatching {allParams}'
     size = desc.DynamicNodeSize('input')
 

meshroom/nodes/aliceVision/ImageMatchingMultiSfM.py

@@ -4,7 +4,7 @@
 from meshroom.core import desc
 
 
-class ImageMatchingMultiSfM(desc.CommandLineNode):
+class ImageMatchingMultiSfM(desc.AVCommandLineNode):
     commandLine = 'aliceVision_imageMatching {allParams}'
     # use both SfM inputs to define Node's size
     size = desc.MultiDynamicNodeSize(['input', 'inputB'])

meshroom/nodes/aliceVision/ImageProcessing.py

@@ -28,7 +28,7 @@ def outputImagesValueFunct(attr):
     return desc.Node.internalFolder + '*' + (outputExt or '.*')
 
 
-class ImageProcessing(desc.CommandLineNode):
+class ImageProcessing(desc.AVCommandLineNode):
     commandLine = 'aliceVision_utils_imageProcessing {allParams}'
     size = desc.DynamicNodeSize('input')
     # parallelization = desc.Parallelization(blockSize=40)

meshroom/nodes/aliceVision/ImportKnownPoses.py

@@ -3,7 +3,7 @@
 from meshroom.core import desc
 
 
-class ImportKnownPoses(desc.CommandLineNode):
+class ImportKnownPoses(desc.AVCommandLineNode):
     commandLine = 'aliceVision_importKnownPoses {allParams}'
     size = desc.DynamicNodeSize('sfmData')
 

meshroom/nodes/aliceVision/KeyframeSelection.py

@@ -4,7 +4,7 @@
 from meshroom.core import desc
 
 
-class KeyframeSelection(desc.CommandLineNode):
+class KeyframeSelection(desc.AVCommandLineNode):
     commandLine = 'aliceVision_utils_keyframeSelection {allParams}'
 
     category = 'Utils'