Merge pull request #68 from SchmidtDSE/expand-debugging

Expanded debugging.
SchmidtDSE · Oct 22, 2024 · c17302e · c17302e
2 parents 8ee90ff + 3e5647b
commit c17302e
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 11 deletions.
diff --git a/data/linearize_un_future.py b/data/linearize_un_future.py
@@ -50,7 +50,8 @@ def main():
 
             rows_nested = map(transform_row, reader)
             rows_flat = itertools.chain(*rows_nested)
-            writer.writerows(rows_flat)
+            rows_flat_allowed = filter(lambda x: x['ISO3 Alpha-code'] != '', rows_flat)
+            writer.writerows(rows_flat_allowed)
 
 
 if __name__ == '__main__':

diff --git a/data/raw_data_in_slim.zip b/data/raw_data_in_slim.zip
diff --git a/tasks_auxiliary.py b/tasks_auxiliary.py
@@ -120,6 +120,8 @@ def run(self):
         with open(os.path.join(workspace_dir, 'a2populationraw.csv')) as f:
             reader = csv.DictReader(f)
 
+            max_year_prior = 0
+
             for row in filter(lambda x: x['ISO3 Alpha-code'].strip() != '', reader):
                 iso_code = row['ISO3 Alpha-code']
                 year_str = row['Year']
@@ -129,16 +131,16 @@ def run(self):
                 year = int(year_str.strip())
                 population = float(population_str.replace(' ', '')) / 1000
 
-                if year < 2022:
-                    key = '{region}.{year}'.format(region=region, year=year)
-                    if key not in output_rows:
-                        output_rows[key] = {
-                            'region': region,
-                            'year': year,
-                            'population': 0
-                        }
+                max_year_prior = year if max_year_prior < year else max_year_prior
+                key = '{region}.{year}'.format(region=region, year=year)
+                if key not in output_rows:
+                    output_rows[key] = {
+                        'region': region,
+                        'year': year,
+                        'population': 0
+                    }
 
-                    output_rows[key]['population'] += population
+                output_rows[key]['population'] += population
 
         with open(os.path.join(workspace_dir, 'a4popprojection.csv')) as f:
             reader = csv.DictReader(f)
@@ -152,7 +154,7 @@ def run(self):
                 year = int(year_str.strip())
                 population = float(population_str.replace(' ', '')) / 1000
 
-                if year >= 2022 and year <= 2050:
+                if year > max_year_prior and year <= 2050:
                     key = '{region}.{year}'.format(region=region, year=year)
                     if key not in output_rows:
                         output_rows[key] = {

diff --git a/tasks_project_template.py b/tasks_project_template.py
@@ -268,6 +268,8 @@ def build_instances(self, connection, sql, cols):
         cursor.close()
 
         if (len(results_flat) == 0):
+            print('Failure details:')
+            print(sql)
             raise RuntimeError('No results.')
 
         results_keyed = [dict(zip(cols, result)) for result in results_flat]