Release v2022.08.0 (#137)

# Updates 1. #136 2. #135 1. #134 2. #133
epam · Aug 2, 2022 · ff4a9b3 · ff4a9b3
1 parent f6ec947
commit ff4a9b3
Show file tree

Hide file tree

Showing 6 changed files with 15 additions and 2 deletions.
diff --git a/osci/__init__.py b/osci/__init__.py
@@ -1 +1 @@
-__version__ = '2022.06.0'
+__version__ = '2022.08.0'
diff --git a/osci/config/base.py b/osci/config/base.py
@@ -208,3 +208,7 @@ def github_token(self) -> str:
     @property
     def default_company(self) -> str:
         return self.__cfg.get('company', dict()).get('default', '')
+
+    @property
+    def spark_conf(self) -> Dict[str, str]:
+        return self.__cfg.get('spark', dict())
diff --git a/osci/config/files/default.yml b/osci/config/files/default.yml
@@ -1,5 +1,7 @@
 meta:
   config_source: 'yaml'
+spark:
+  spark.driver.memory: 6g
 file_system:
   type: 'local'
   base_path: '/data'

diff --git a/osci/jobs/session.py b/osci/jobs/session.py
@@ -26,6 +26,8 @@ def __init__(self, spark_session=None):
 
     def build_session(self):
         builder = SparkSession.builder
+        for param, value in Config().spark_conf.items():
+            builder = builder.config(param, value)
         self._ssc = builder.getOrCreate()
 
     @property

diff --git a/osci/preprocess/match_company/company_domain_match_list.yaml b/osci/preprocess/match_company/company_domain_match_list.yaml
@@ -58,6 +58,11 @@
     - agaric.com
   industry: Technology
   regex:
+- company: Aiven
+  domains:
+    - aiven.io
+  industry: Technology
+  regex: 
 - company: Alfresco
   domains:
     - alfresco.com

diff --git a/osci/requirements.txt b/osci/requirements.txt
@@ -12,5 +12,5 @@ six==1.13.0
 XlsxWriter==1.2.3
 Jinja2==2.11.3
 deepmerge==0.1.1
-numpy==1.19.5
+numpy==1.22.0
 python-dateutil==2.8.1