Draft lifecycle.

const.py

@@ -40,3 +40,14 @@
     'population',
     'gdp'
 ]
+
+LIFECYCLE_DISTRIBUTIONS = {
+    'consumptionAgricultureMT': {'mean': 2, 'std': 1},
+    'consumptionConstructionMT': {'mean': 35, 'std': 7},
+    'consumptionElectronicMT': {'mean': 8, 'std': 2},
+    'consumptionHouseholdLeisureSportsMT': {'mean': 3, 'std': 1},
+    'consumptionPackagingMT': {'mean': 0.5, 'std': 0.1},
+    'consumptionTransporationMT': {'mean': 13, 'std': 3},
+    'consumptionTextileMT': {'mean': 5, 'std': 1.5},
+    'consumptionOtherMT': {'mean': 5, 'std': 1.5}
+}

sql/08_project/build_model_table.sql

@@ -19,7 +19,8 @@ SELECT
     summary.netImportArticlesMT AS netImportArticlesMT,
     summary.netImportFibersMT AS netImportFibersMT,
     summary.netImportGoodsMT AS netImportGoodsMT,
-    summary.netImportResinMT AS netImportResinMT
+    summary.netImportResinMT AS netImportResinMT,
+    NULL AS newWasteMT
 FROM
     auxiliary
 LEFT OUTER JOIN

sql/08_project/normalize_eol.sql

@@ -0,0 +1,44 @@
+UPDATE
+    {table_name}
+SET
+    eolRecyclingPercent = updated.eolRecyclingPercent,
+    eolIncinerationPercent = updated.eolIncinerationPercent,
+    eolLandfillPercent = updated.eolLandfillPercent,
+    eolMismanagedPercent = updated.eolMismanagedPercent
+FROM
+    (
+        SELECT
+            year,
+            region,
+            eolRecyclingPercent / (
+                eolRecyclingPercent +
+                eolIncinerationPercent +
+                eolLandfillPercent +
+                eolMismanagedPercent
+            ) AS eolRecyclingPercent,
+            eolIncinerationPercent / (
+                eolRecyclingPercent +
+                eolIncinerationPercent +
+                eolLandfillPercent +
+                eolMismanagedPercent
+            ) AS eolIncinerationPercent,
+            eolLandfillPercent / (
+                eolRecyclingPercent +
+                eolIncinerationPercent +
+                eolLandfillPercent +
+                eolMismanagedPercent
+            ) AS eolLandfillPercent,
+            eolMismanagedPercent / (
+                eolRecyclingPercent +
+                eolIncinerationPercent +
+                eolLandfillPercent +
+                eolMismanagedPercent
+            ) AS eolMismanagedPercent
+        FROM
+            {table_name}
+        WHERE
+            year > 2020 OR year < 2005
+    ) updated
+WHERE
+    updated.year = {table_name}.year
+    AND updated.region = {table_name}.region

sql/08_project/normalize_trade.sql

@@ -0,0 +1,111 @@
+UPDATE
+    {table_name}
+SET
+    netImportArticlesMT = updated.netImportArticlesMT,
+    netImportFibersMT = updated.netImportFibersMT,
+    netImportGoodsMT = updated.netImportGoodsMT,
+    netImportResinMT = updated.netImportResinMT
+FROM
+    (
+        SELECT
+            {table_name}.year AS year,
+            {table_name}.region AS region,
+            (
+                CASE
+                    WHEN totalArticlesMT > 0 AND netImportArticlesMT > 0 THEN netImportArticlesMT - totalArticlesMT * netImportArticlesMT / totalImportArticlesMTPos
+                    WHEN totalArticlesMT < 0 AND netImportArticlesMT < 0 THEN netImportArticlesMT - totalArticlesMT * netImportArticlesMT / totalImportArticlesMTNeg
+                    ELSE netImportArticlesMT
+                END
+            ) AS netImportArticlesMT,
+            (
+                CASE
+                    WHEN totalFibersMT > 0 AND netImportFibersMT > 0 THEN netImportFibersMT - totalFibersMT * netImportFibersMT / totalImportFibersMTPos
+                    WHEN totalFibersMT < 0 AND netImportFibersMT < 0 THEN netImportFibersMT - totalFibersMT * netImportFibersMT / totalImportFibersMTNeg
+                    ELSE netImportFibersMT
+                END
+            ) AS netImportFibersMT,
+            (
+                CASE
+                    WHEN totalGoodsMT > 0 AND netImportGoodsMT > 0 THEN netImportGoodsMT - totalGoodsMT * netImportGoodsMT / totalImportGoodsMTPos
+                    WHEN totalGoodsMT < 0 AND netImportGoodsMT < 0 THEN netImportGoodsMT - totalGoodsMT * netImportGoodsMT / totalImportGoodsMTNeg
+                    ELSE netImportGoodsMT
+                END
+            ) AS netImportGoodsMT,
+            (
+                CASE
+                    WHEN totalResinMT > 0 AND netImportResinMT > 0 THEN netImportResinMT - totalResinMT * netImportResinMT / totalImportResinMTPos
+                    WHEN totalResinMT < 0 AND netImportResinMT < 0 THEN netImportResinMT - totalResinMT * netImportResinMT / totalImportResinMTNeg
+                    ELSE netImportResinMT
+                END
+            ) AS netImportResinMT
+        FROM
+            {table_name}
+        INNER JOIN
+            (
+                SELECT
+                    year,
+                    sum(
+                        CASE
+                            WHEN netImportArticlesMT > 0 THEN netImportArticlesMT
+                            ELSE 0
+                        END 
+                    ) AS totalImportArticlesMTPos,
+                    sum(
+                        CASE
+                            WHEN netImportArticlesMT < 0 THEN netImportArticlesMT
+                            ELSE 0
+                        END
+                    ) AS totalImportArticlesMTNeg,
+                    sum(
+                        CASE
+                            WHEN netImportFibersMT > 0 THEN netImportFibersMT
+                            ELSE 0
+                        END 
+                    ) AS totalImportFibersMTPos,
+                    sum(
+                        CASE
+                            WHEN netImportFibersMT < 0 THEN netImportFibersMT
+                            ELSE 0
+                        END
+                    ) AS totalImportFibersMTNeg,
+                    sum(
+                        CASE
+                            WHEN netImportGoodsMT > 0 THEN netImportGoodsMT
+                            ELSE 0
+                        END 
+                    ) AS totalImportGoodsMTPos,
+                    sum(
+                        CASE
+                            WHEN netImportGoodsMT < 0 THEN netImportGoodsMT
+                            ELSE 0
+                        END
+                    ) AS totalImportGoodsMTNeg,
+                    sum(
+                        CASE
+                            WHEN netImportResinMT > 0 THEN netImportResinMT
+                            ELSE 0
+                        END 
+                    ) AS totalImportResinMTPos,
+                    sum(
+                        CASE
+                            WHEN netImportResinMT < 0 THEN netImportResinMT
+                            ELSE 0
+                        END
+                    ) AS totalImportResinMTNeg,
+                    sum(netImportArticlesMT) AS totalArticlesMT,
+                    sum(netImportFibersMT) AS totalFibersMT,
+                    sum(netImportGoodsMT) AS totalGoodsMT,
+                    sum(netImportResinMT) AS totalResinMT
+                FROM
+                    {table_name}
+                GROUP BY
+                    year
+            ) totals
+        ON
+            {table_name}.year = totals.year
+        WHERE
+            {table_name}.year > 2020 OR {table_name}.year < 2005
+    ) updated
+WHERE
+    updated.year = {table_name}.year
+    AND updated.region = {table_name}.region

tasks_project.py

@@ -120,6 +120,14 @@ def get_trade_model_filename(self):
     def hot_encode(self, candidate, hot_value):
         return 1 if candidate == hot_value else 0
 
+    def get_year_selector(self, year):
+        if year > 2020:
+            selector = '{year} - year <= 5 AND {year} - year > 0'
+        else:
+            selector = 'year - {year} <= 5 AND year - {year} > 0'
+
+        return selector.format(year=year)
+
     def get_consumption_inputs_sql(self, year, region, sector):
         template_vals = {
             'table_name': self.get_table_name(),
@@ -138,6 +146,7 @@ def get_consumption_inputs_sql(self, year, region, sector):
             'flagPackaging': self.hot_encode(sector, 'consumptionPackagingMT'),
             'flagTextile': self.hot_encode(sector, 'consumptionTextileMT'),
             'flagTransporation': self.hot_encode(sector, 'consumptionTransporationMT'),
+            'yearSelector': self.get_year_selector(year)
         }
 
         return '''
@@ -180,8 +189,7 @@ def get_consumption_inputs_sql(self, year, region, sector):
                     FROM
                         {table_name}
                     WHERE
-                        {year} - year <= 5
-                        AND {year} - year > 0
+                        {yearSelector}
                         AND region = '{region}'
                 ) before
         '''.format(**template_vals)
@@ -199,7 +207,8 @@ def get_waste_inputs_sql(self, year, region, type_name):
             'flagRecycling': self.hot_encode(type_name, 'eolRecyclingPercent'),
             'flagIncineration': self.hot_encode(type_name, 'eolIncinerationPercent'),
             'flagLandfill': self.hot_encode(type_name, 'eolLandfillPercent'),
-            'flagMismanaged': self.hot_encode(type_name, 'eolMismanagedPercent')
+            'flagMismanaged': self.hot_encode(type_name, 'eolMismanagedPercent'),
+            'yearSelector': self.get_year_selector(year)
         }
 
         return '''
@@ -239,8 +248,7 @@ def get_waste_inputs_sql(self, year, region, type_name):
                     FROM
                         {table_name}
                     WHERE
-                        {year} - year <= 5
-                        AND {year} - year > 0
+                        {yearSelector}
                         AND region = '{region}'
                 ) before
         '''.format(**template_vals)
@@ -258,7 +266,8 @@ def get_trade_inputs_sql(self, year, region, type_name):
             'flagArticles': self.hot_encode(type_name, 'netImportArticlesMT'),
             'flagFibers': self.hot_encode(type_name, 'netImportFibersMT'),
             'flagGoods': self.hot_encode(type_name, 'netImportGoodsMT'),
-            'flagResin': self.hot_encode(type_name, 'netImportResinMT')
+            'flagResin': self.hot_encode(type_name, 'netImportResinMT'),
+            'yearSelector': self.get_year_selector(year)
         }
 
         return '''
@@ -297,8 +306,7 @@ def get_trade_inputs_sql(self, year, region, type_name):
                     FROM
                         {table_name}
                     WHERE
-                        {year} - year <= 5
-                        AND {year} - year > 0
+                        {yearSelector}
                         AND region = '{region}'
                 ) before
         '''.format(**template_vals)
@@ -580,7 +588,7 @@ def output(self):
         return luigi.LocalTarget(out_path)
 
     def get_table_name(self):
-        return 'project_curve'
+        return 'project_naive'
 
 
 class ProjectNaiveRawTask(ProjectCurveRawTask):
@@ -606,3 +614,107 @@ def get_waste_model_filename(self):
     def get_trade_model_filename(self):
         return 'trade_curve_naive.pickle'
 
+
+class NormalizeMlTask(tasks_project_template.NormalizeProjectionTask):
+
+    task_dir = luigi.Parameter(default=const.DEFAULT_TASK_DIR)
+
+    def requires(self):
+        return ProjectMlRawTask(task_dir=self.task_dir)
+
+    def output(self):
+        out_path = os.path.join(self.task_dir, '510_normalize_ml.json')
+        return luigi.LocalTarget(out_path)
+
+    def get_table_name(self):
+        return 'project_ml'
+
+
+class NormalizeCurveTask(tasks_project_template.NormalizeProjectionTask):
+
+    task_dir = luigi.Parameter(default=const.DEFAULT_TASK_DIR)
+
+    def requires(self):
+        return ProjectCurveRawTask(task_dir=self.task_dir)
+
+    def output(self):
+        out_path = os.path.join(self.task_dir, '511_normalize_curve.json')
+        return luigi.LocalTarget(out_path)
+
+    def get_table_name(self):
+        return 'project_curve'
+
+
+class NormalizeNaiveTask(tasks_project_template.NormalizeProjectionTask):
+
+    task_dir = luigi.Parameter(default=const.DEFAULT_TASK_DIR)
+
+    def requires(self):
+        return ProjectCurveRawTask(task_dir=self.task_dir)
+
+    def output(self):
+        out_path = os.path.join(self.task_dir, '512_normalize_naive.json')
+        return luigi.LocalTarget(out_path)
+
+    def get_table_name(self):
+        return 'project_naive'
+
+
+class CheckNormalizeMlTask(tasks_project_template.NormalizeCheckTask):
+
+    task_dir = luigi.Parameter(default=const.DEFAULT_TASK_DIR)
+
+    def requires(self):
+        return NormalizeMlTask(task_dir=self.task_dir)
+
+    def output(self):
+        out_path = os.path.join(self.task_dir, '513_check_normalize_ml.json')
+        return luigi.LocalTarget(out_path)
+
+    def get_table_name(self):
+        return 'project_ml'
+
+
+class CheckNormalizeCurveTask(tasks_project_template.NormalizeCheckTask):
+
+    task_dir = luigi.Parameter(default=const.DEFAULT_TASK_DIR)
+
+    def requires(self):
+        return NormalizeCurveTask(task_dir=self.task_dir)
+
+    def output(self):
+        out_path = os.path.join(self.task_dir, '514_check_normalize_curve.json')
+        return luigi.LocalTarget(out_path)
+
+    def get_table_name(self):
+        return 'project_curve'
+
+
+class CheckNormalizeNaiveTask(tasks_project_template.NormalizeCheckTask):
+
+    task_dir = luigi.Parameter(default=const.DEFAULT_TASK_DIR)
+
+    def requires(self):
+        return NormalizeNaiveTask(task_dir=self.task_dir)
+
+    def output(self):
+        out_path = os.path.join(self.task_dir, '515_check_normalize_naive.json')
+        return luigi.LocalTarget(out_path)
+
+    def get_table_name(self):
+        return 'project_naive'
+
+
+class ApplyLifecycleMLTask(tasks_project_template.ApplyLifecycleTask):
+
+    task_dir = luigi.Parameter(default=const.DEFAULT_TASK_DIR)
+
+    def requires(self):
+        return CheckNormalizeMlTask(task_dir=self.task_dir)
+
+    def output(self):
+        out_path = os.path.join(self.task_dir, '516_lifecycle_ml.json')
+        return luigi.LocalTarget(out_path)
+
+    def get_table_name(self):
+        return 'project_ml'