diff --git a/resources/otoole/config.yaml b/resources/otoole.yaml similarity index 100% rename from resources/otoole/config.yaml rename to resources/otoole.yaml diff --git a/resources/otoole/data/AccumulatedAnnualDemand.csv b/resources/otoole/data/AccumulatedAnnualDemand.csv deleted file mode 100644 index 326b28c..0000000 --- a/resources/otoole/data/AccumulatedAnnualDemand.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,FUEL,YEAR,VALUE diff --git a/resources/otoole/data/AnnualEmissionLimit.csv b/resources/otoole/data/AnnualEmissionLimit.csv deleted file mode 100644 index 1fa535a..0000000 --- a/resources/otoole/data/AnnualEmissionLimit.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,EMISSION,YEAR,VALUE diff --git a/resources/otoole/data/AnnualExogenousEmission.csv b/resources/otoole/data/AnnualExogenousEmission.csv deleted file mode 100644 index 1fa535a..0000000 --- a/resources/otoole/data/AnnualExogenousEmission.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,EMISSION,YEAR,VALUE diff --git a/resources/otoole/data/AvailabilityFactor.csv b/resources/otoole/data/AvailabilityFactor.csv deleted file mode 100644 index 6a91e60..0000000 --- a/resources/otoole/data/AvailabilityFactor.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,YEAR,VALUE diff --git a/resources/otoole/data/CapacityFactor.csv b/resources/otoole/data/CapacityFactor.csv deleted file mode 100644 index ba3be6e..0000000 --- a/resources/otoole/data/CapacityFactor.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,TIMESLICE,YEAR,VALUE diff --git a/resources/otoole/data/CapacityOfOneTechnologyUnit.csv b/resources/otoole/data/CapacityOfOneTechnologyUnit.csv deleted file mode 100644 index 6a91e60..0000000 --- a/resources/otoole/data/CapacityOfOneTechnologyUnit.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,YEAR,VALUE diff --git a/resources/otoole/data/CapacityToActivityUnit.csv b/resources/otoole/data/CapacityToActivityUnit.csv deleted file mode 100644 index 1ca1a8e..0000000 --- a/resources/otoole/data/CapacityToActivityUnit.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,VALUE diff --git a/resources/otoole/data/CapitalCost.csv b/resources/otoole/data/CapitalCost.csv deleted file mode 100644 index 6a91e60..0000000 --- a/resources/otoole/data/CapitalCost.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,YEAR,VALUE diff --git a/resources/otoole/data/CapitalCostStorage.csv b/resources/otoole/data/CapitalCostStorage.csv deleted file mode 100644 index a7bcbd7..0000000 --- a/resources/otoole/data/CapitalCostStorage.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,STORAGE,YEAR,VALUE diff --git a/resources/otoole/data/Conversionld.csv b/resources/otoole/data/Conversionld.csv deleted file mode 100644 index 360887a..0000000 --- a/resources/otoole/data/Conversionld.csv +++ /dev/null @@ -1 +0,0 @@ -TIMESLICE,DAYTYPE,VALUE diff --git a/resources/otoole/data/Conversionlh.csv b/resources/otoole/data/Conversionlh.csv deleted file mode 100644 index 6fc0a29..0000000 --- a/resources/otoole/data/Conversionlh.csv +++ /dev/null @@ -1 +0,0 @@ -TIMESLICE,DAILYTIMEBRACKET,VALUE diff --git a/resources/otoole/data/Conversionls.csv b/resources/otoole/data/Conversionls.csv deleted file mode 100644 index 47b6ebd..0000000 --- a/resources/otoole/data/Conversionls.csv +++ /dev/null @@ -1 +0,0 @@ -TIMESLICE,SEASON,VALUE diff --git a/resources/otoole/data/DAILYTIMEBRACKET.csv b/resources/otoole/data/DAILYTIMEBRACKET.csv deleted file mode 100644 index 2dfe6a3..0000000 --- a/resources/otoole/data/DAILYTIMEBRACKET.csv +++ /dev/null @@ -1 +0,0 @@ -VALUE diff --git a/resources/otoole/data/DAYTYPE.csv b/resources/otoole/data/DAYTYPE.csv deleted file mode 100644 index 2dfe6a3..0000000 --- a/resources/otoole/data/DAYTYPE.csv +++ /dev/null @@ -1 +0,0 @@ -VALUE diff --git a/resources/otoole/data/DaySplit.csv b/resources/otoole/data/DaySplit.csv deleted file mode 100644 index 83dab5c..0000000 --- a/resources/otoole/data/DaySplit.csv +++ /dev/null @@ -1 +0,0 @@ -DAILYTIMEBRACKET,YEAR,VALUE diff --git a/resources/otoole/data/DaysInDayType.csv b/resources/otoole/data/DaysInDayType.csv deleted file mode 100644 index 7e5dd71..0000000 --- a/resources/otoole/data/DaysInDayType.csv +++ /dev/null @@ -1 +0,0 @@ -SEASON,DAYTYPE,YEAR,VALUE diff --git a/resources/otoole/data/DepreciationMethod.csv b/resources/otoole/data/DepreciationMethod.csv deleted file mode 100644 index 8f1fa36..0000000 --- a/resources/otoole/data/DepreciationMethod.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,VALUE diff --git a/resources/otoole/data/DiscountRate.csv b/resources/otoole/data/DiscountRate.csv deleted file mode 100644 index 8f1fa36..0000000 --- a/resources/otoole/data/DiscountRate.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,VALUE diff --git a/resources/otoole/data/DiscountRateStorage.csv b/resources/otoole/data/DiscountRateStorage.csv deleted file mode 100644 index 2176c14..0000000 --- a/resources/otoole/data/DiscountRateStorage.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,STORAGE,VALUE diff --git a/resources/otoole/data/EMISSION.csv b/resources/otoole/data/EMISSION.csv deleted file mode 100644 index 2dfe6a3..0000000 --- a/resources/otoole/data/EMISSION.csv +++ /dev/null @@ -1 +0,0 @@ -VALUE diff --git a/resources/otoole/data/EmissionActivityRatio.csv b/resources/otoole/data/EmissionActivityRatio.csv deleted file mode 100644 index 7c1c3ff..0000000 --- a/resources/otoole/data/EmissionActivityRatio.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,EMISSION,MODE_OF_OPERATION,YEAR,VALUE diff --git a/resources/otoole/data/EmissionsPenalty.csv b/resources/otoole/data/EmissionsPenalty.csv deleted file mode 100644 index 1fa535a..0000000 --- a/resources/otoole/data/EmissionsPenalty.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,EMISSION,YEAR,VALUE diff --git a/resources/otoole/data/FUEL.csv b/resources/otoole/data/FUEL.csv deleted file mode 100644 index 2dfe6a3..0000000 --- a/resources/otoole/data/FUEL.csv +++ /dev/null @@ -1 +0,0 @@ -VALUE diff --git a/resources/otoole/data/FixedCost.csv b/resources/otoole/data/FixedCost.csv deleted file mode 100644 index 6a91e60..0000000 --- a/resources/otoole/data/FixedCost.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,YEAR,VALUE diff --git a/resources/otoole/data/InputActivityRatio.csv b/resources/otoole/data/InputActivityRatio.csv deleted file mode 100644 index 7be9802..0000000 --- a/resources/otoole/data/InputActivityRatio.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,FUEL,MODE_OF_OPERATION,YEAR,VALUE diff --git a/resources/otoole/data/MODE_OF_OPERATION.csv b/resources/otoole/data/MODE_OF_OPERATION.csv deleted file mode 100644 index 2dfe6a3..0000000 --- a/resources/otoole/data/MODE_OF_OPERATION.csv +++ /dev/null @@ -1 +0,0 @@ -VALUE diff --git a/resources/otoole/data/MinStorageCharge.csv b/resources/otoole/data/MinStorageCharge.csv deleted file mode 100644 index a7bcbd7..0000000 --- a/resources/otoole/data/MinStorageCharge.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,STORAGE,YEAR,VALUE diff --git a/resources/otoole/data/ModelPeriodEmissionLimit.csv b/resources/otoole/data/ModelPeriodEmissionLimit.csv deleted file mode 100644 index ccd4bcb..0000000 --- a/resources/otoole/data/ModelPeriodEmissionLimit.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,EMISSION,VALUE diff --git a/resources/otoole/data/ModelPeriodExogenousEmission.csv b/resources/otoole/data/ModelPeriodExogenousEmission.csv deleted file mode 100644 index ccd4bcb..0000000 --- a/resources/otoole/data/ModelPeriodExogenousEmission.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,EMISSION,VALUE diff --git a/resources/otoole/data/OperationalLife.csv b/resources/otoole/data/OperationalLife.csv deleted file mode 100644 index 1ca1a8e..0000000 --- a/resources/otoole/data/OperationalLife.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,VALUE diff --git a/resources/otoole/data/OperationalLifeStorage.csv b/resources/otoole/data/OperationalLifeStorage.csv deleted file mode 100644 index 2176c14..0000000 --- a/resources/otoole/data/OperationalLifeStorage.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,STORAGE,VALUE diff --git a/resources/otoole/data/OutputActivityRatio.csv b/resources/otoole/data/OutputActivityRatio.csv deleted file mode 100644 index 7be9802..0000000 --- a/resources/otoole/data/OutputActivityRatio.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,FUEL,MODE_OF_OPERATION,YEAR,VALUE diff --git a/resources/otoole/data/REGION.csv b/resources/otoole/data/REGION.csv deleted file mode 100644 index 2dfe6a3..0000000 --- a/resources/otoole/data/REGION.csv +++ /dev/null @@ -1 +0,0 @@ -VALUE diff --git a/resources/otoole/data/REMinProductionTarget.csv b/resources/otoole/data/REMinProductionTarget.csv deleted file mode 100644 index b55c226..0000000 --- a/resources/otoole/data/REMinProductionTarget.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,YEAR,VALUE diff --git a/resources/otoole/data/RETagFuel.csv b/resources/otoole/data/RETagFuel.csv deleted file mode 100644 index 326b28c..0000000 --- a/resources/otoole/data/RETagFuel.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,FUEL,YEAR,VALUE diff --git a/resources/otoole/data/RETagTechnology.csv b/resources/otoole/data/RETagTechnology.csv deleted file mode 100644 index 6a91e60..0000000 --- a/resources/otoole/data/RETagTechnology.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,YEAR,VALUE diff --git a/resources/otoole/data/ReserveMargin.csv b/resources/otoole/data/ReserveMargin.csv deleted file mode 100644 index b55c226..0000000 --- a/resources/otoole/data/ReserveMargin.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,YEAR,VALUE diff --git a/resources/otoole/data/ReserveMarginTagFuel.csv b/resources/otoole/data/ReserveMarginTagFuel.csv deleted file mode 100644 index 326b28c..0000000 --- a/resources/otoole/data/ReserveMarginTagFuel.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,FUEL,YEAR,VALUE diff --git a/resources/otoole/data/ReserveMarginTagTechnology.csv b/resources/otoole/data/ReserveMarginTagTechnology.csv deleted file mode 100644 index 6a91e60..0000000 --- a/resources/otoole/data/ReserveMarginTagTechnology.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,YEAR,VALUE diff --git a/resources/otoole/data/ResidualCapacity.csv b/resources/otoole/data/ResidualCapacity.csv deleted file mode 100644 index 6a91e60..0000000 --- a/resources/otoole/data/ResidualCapacity.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,YEAR,VALUE diff --git a/resources/otoole/data/ResidualStorageCapacity.csv b/resources/otoole/data/ResidualStorageCapacity.csv deleted file mode 100644 index a7bcbd7..0000000 --- a/resources/otoole/data/ResidualStorageCapacity.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,STORAGE,YEAR,VALUE diff --git a/resources/otoole/data/SEASON.csv b/resources/otoole/data/SEASON.csv deleted file mode 100644 index 2dfe6a3..0000000 --- a/resources/otoole/data/SEASON.csv +++ /dev/null @@ -1 +0,0 @@ -VALUE diff --git a/resources/otoole/data/STORAGE.csv b/resources/otoole/data/STORAGE.csv deleted file mode 100644 index 2dfe6a3..0000000 --- a/resources/otoole/data/STORAGE.csv +++ /dev/null @@ -1 +0,0 @@ -VALUE diff --git a/resources/otoole/data/SpecifiedAnnualDemand.csv b/resources/otoole/data/SpecifiedAnnualDemand.csv deleted file mode 100644 index 326b28c..0000000 --- a/resources/otoole/data/SpecifiedAnnualDemand.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,FUEL,YEAR,VALUE diff --git a/resources/otoole/data/SpecifiedDemandProfile.csv b/resources/otoole/data/SpecifiedDemandProfile.csv deleted file mode 100644 index 9425f70..0000000 --- a/resources/otoole/data/SpecifiedDemandProfile.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,FUEL,TIMESLICE,YEAR,VALUE diff --git a/resources/otoole/data/StorageLevelStart.csv b/resources/otoole/data/StorageLevelStart.csv deleted file mode 100644 index 2176c14..0000000 --- a/resources/otoole/data/StorageLevelStart.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,STORAGE,VALUE diff --git a/resources/otoole/data/StorageMaxChargeRate.csv b/resources/otoole/data/StorageMaxChargeRate.csv deleted file mode 100644 index 2176c14..0000000 --- a/resources/otoole/data/StorageMaxChargeRate.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,STORAGE,VALUE diff --git a/resources/otoole/data/StorageMaxDischargeRate.csv b/resources/otoole/data/StorageMaxDischargeRate.csv deleted file mode 100644 index 2176c14..0000000 --- a/resources/otoole/data/StorageMaxDischargeRate.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,STORAGE,VALUE diff --git a/resources/otoole/data/TECHNOLOGY.csv b/resources/otoole/data/TECHNOLOGY.csv deleted file mode 100644 index 2dfe6a3..0000000 --- a/resources/otoole/data/TECHNOLOGY.csv +++ /dev/null @@ -1 +0,0 @@ -VALUE diff --git a/resources/otoole/data/TIMESLICE.csv b/resources/otoole/data/TIMESLICE.csv deleted file mode 100644 index 2dfe6a3..0000000 --- a/resources/otoole/data/TIMESLICE.csv +++ /dev/null @@ -1 +0,0 @@ -VALUE diff --git a/resources/otoole/data/TechnologyFromStorage.csv b/resources/otoole/data/TechnologyFromStorage.csv deleted file mode 100644 index 384c871..0000000 --- a/resources/otoole/data/TechnologyFromStorage.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,STORAGE,MODE_OF_OPERATION,VALUE diff --git a/resources/otoole/data/TechnologyToStorage.csv b/resources/otoole/data/TechnologyToStorage.csv deleted file mode 100644 index 384c871..0000000 --- a/resources/otoole/data/TechnologyToStorage.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,STORAGE,MODE_OF_OPERATION,VALUE diff --git a/resources/otoole/data/TotalAnnualMaxCapacity.csv b/resources/otoole/data/TotalAnnualMaxCapacity.csv deleted file mode 100644 index 6a91e60..0000000 --- a/resources/otoole/data/TotalAnnualMaxCapacity.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,YEAR,VALUE diff --git a/resources/otoole/data/TotalAnnualMaxCapacityInvestment.csv b/resources/otoole/data/TotalAnnualMaxCapacityInvestment.csv deleted file mode 100644 index 6a91e60..0000000 --- a/resources/otoole/data/TotalAnnualMaxCapacityInvestment.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,YEAR,VALUE diff --git a/resources/otoole/data/TotalAnnualMinCapacity.csv b/resources/otoole/data/TotalAnnualMinCapacity.csv deleted file mode 100644 index 6a91e60..0000000 --- a/resources/otoole/data/TotalAnnualMinCapacity.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,YEAR,VALUE diff --git a/resources/otoole/data/TotalAnnualMinCapacityInvestment.csv b/resources/otoole/data/TotalAnnualMinCapacityInvestment.csv deleted file mode 100644 index 6a91e60..0000000 --- a/resources/otoole/data/TotalAnnualMinCapacityInvestment.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,YEAR,VALUE diff --git a/resources/otoole/data/TotalTechnologyAnnualActivityLowerLimit.csv b/resources/otoole/data/TotalTechnologyAnnualActivityLowerLimit.csv deleted file mode 100644 index 6a91e60..0000000 --- a/resources/otoole/data/TotalTechnologyAnnualActivityLowerLimit.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,YEAR,VALUE diff --git a/resources/otoole/data/TotalTechnologyAnnualActivityUpperLimit.csv b/resources/otoole/data/TotalTechnologyAnnualActivityUpperLimit.csv deleted file mode 100644 index 6a91e60..0000000 --- a/resources/otoole/data/TotalTechnologyAnnualActivityUpperLimit.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,YEAR,VALUE diff --git a/resources/otoole/data/TotalTechnologyModelPeriodActivityLowerLimit.csv b/resources/otoole/data/TotalTechnologyModelPeriodActivityLowerLimit.csv deleted file mode 100644 index 1ca1a8e..0000000 --- a/resources/otoole/data/TotalTechnologyModelPeriodActivityLowerLimit.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,VALUE diff --git a/resources/otoole/data/TotalTechnologyModelPeriodActivityUpperLimit.csv b/resources/otoole/data/TotalTechnologyModelPeriodActivityUpperLimit.csv deleted file mode 100644 index 1ca1a8e..0000000 --- a/resources/otoole/data/TotalTechnologyModelPeriodActivityUpperLimit.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,VALUE diff --git a/resources/otoole/data/TradeRoute.csv b/resources/otoole/data/TradeRoute.csv deleted file mode 100644 index 326b28c..0000000 --- a/resources/otoole/data/TradeRoute.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,FUEL,YEAR,VALUE diff --git a/resources/otoole/data/VariableCost.csv b/resources/otoole/data/VariableCost.csv deleted file mode 100644 index 8563900..0000000 --- a/resources/otoole/data/VariableCost.csv +++ /dev/null @@ -1 +0,0 @@ -REGION,TECHNOLOGY,MODE_OF_OPERATION,YEAR,VALUE diff --git a/resources/otoole/data/YEAR.csv b/resources/otoole/data/YEAR.csv deleted file mode 100644 index 2dfe6a3..0000000 --- a/resources/otoole/data/YEAR.csv +++ /dev/null @@ -1 +0,0 @@ -VALUE diff --git a/resources/otoole/data/YearSplit.csv b/resources/otoole/data/YearSplit.csv deleted file mode 100644 index 54f1a2a..0000000 --- a/resources/otoole/data/YearSplit.csv +++ /dev/null @@ -1 +0,0 @@ -TIMESLICE,YEAR,VALUE diff --git a/workflow/rules/model.smk b/workflow/rules/model.smk index bb810b6..a21816a 100644 --- a/workflow/rules/model.smk +++ b/workflow/rules/model.smk @@ -1,25 +1,17 @@ import os import shutil -# REQUIRED - -configfile: 'config/config.yaml' - -# OUTPUT FILES - -osemosys_files = os.listdir('resources/otoole/data') - # RULES rule geographic_filter: message: 'Applying geographic filter...' input: - csv_files = expand('results/data/{osemosys_file}', osemosys_file = osemosys_files), + csv_files = expand('results/data/{csv}.csv', csv = OTOOLE_PARAMS), params: geographic_scope = config['geographic_scope'] output: - csv_files = expand('results/{{scenario}}/data/{osemosys_file}', osemosys_file = osemosys_files), + csv_files = expand('results/{{scenario}}/data/{csv}.csv', csv = OTOOLE_PARAMS), # conda: # '../envs/data_processing.yaml' log: @@ -31,7 +23,7 @@ rule copy_otoole_confg: message: 'Copying otoole configuration file...' input: - config='resources/otoole/config.yaml' + config=OTOOLE_YAML output: config='results/{scenario}/otoole.yaml' run: @@ -54,7 +46,7 @@ rule otoole_convert: csv_dir = 'results/{scenario}/data/' input: otoole_config = 'results/{scenario}/otoole.yaml', - csv_files = expand('results/{{scenario}}/data/{osemosys_file}', osemosys_file = osemosys_files), + csv_files = expand('results/{{scenario}}/data/{csv}.csv', csv = OTOOLE_PARAMS), output: data_file = 'results/{scenario}/{scenario}.txt' log: diff --git a/workflow/rules/postprocess.smk b/workflow/rules/postprocess.smk index 2a16c49..abac135 100644 --- a/workflow/rules/postprocess.smk +++ b/workflow/rules/postprocess.smk @@ -1,36 +1,7 @@ import os -# REQUIRED - -configfile: 'config/config.yaml' - # OUTPUT FILES -result_files = [ - 'AccumulatedNewCapacity.csv', - 'AnnualEmissions.csv', - 'AnnualFixedOperatingCost.csv', - 'AnnualTechnologyEmission.csv', - 'AnnualTechnologyEmissionByMode.csv', - 'AnnualVariableOperatingCost.csv', - # 'CapitalInvestment.csv', - 'Demand.csv', - 'DiscountedTechnologyEmissionsPenalty.csv', - 'NewCapacity.csv', - 'ProductionByTechnology.csv', - 'ProductionByTechnologyAnnual.csv', - 'RateOfActivity.csv', - 'RateOfProductionByTechnology.csv', - 'RateOfProductionByTechnologyByMode.csv', - 'RateOfUseByTechnology.csv', - 'RateOfUseByTechnologyByMode.csv', - 'TotalAnnualTechnologyActivityByMode.csv', - 'TotalCapacityAnnual.csv', - 'TotalTechnologyAnnualActivity.csv', - 'TotalTechnologyModelPeriodActivity.csv', - 'UseByTechnology.csv' -] - result_figures = [ 'TotalCapacityAnnual', 'GenerationAnnual', @@ -50,7 +21,7 @@ rule otoole_results: datafile = 'results/{scenario}/{scenario}.txt', otoole_config = 'results/{scenario}/otoole.yaml', output: - expand('results/{{scenario}}/results/{result_file}', result_file = result_files), + expand('results/{{scenario}}/results/{result_file}.csv', result_file = OTOOLE_RESULTS), # conda: # '../envs/otoole.yaml' log: @@ -68,7 +39,7 @@ rule visualisation: message: 'Generating result figures...' input: - csv_files = expand('results/{{scenario}}/results/{result_file}', result_file = result_files), + csv_files = expand('results/{{scenario}}/results/{result_file}.csv', result_file = OTOOLE_RESULTS), params: input_data = "results/{scenario}/data/", result_data = "results/{scenario}/results/", @@ -88,7 +59,7 @@ rule summarise_results: message: 'Generating summary of results...' input: - csv_files = expand('results/{{scenario}}/results/{result_file}', result_file = result_files), + csv_files = expand('results/{{scenario}}/results/{result_file}.csv', result_file = OTOOLE_RESULTS), params: start_year = config['startYear'], end_year = config['endYear'], diff --git a/workflow/rules/preprocess.smk b/workflow/rules/preprocess.smk index f49b8b6..4de378e 100644 --- a/workflow/rules/preprocess.smk +++ b/workflow/rules/preprocess.smk @@ -1,13 +1,8 @@ import os - -# required files - -configfile: 'config/config.yaml' +import yaml # model and scenario output files -osemosys_files = os.listdir('resources/otoole/data') - demand_figures = [ 'South America', 'Oceania', @@ -20,82 +15,77 @@ demand_figures = [ # output script files power_plant_files = [ - 'CapitalCost.csv', - 'FixedCost.csv', - 'CapacityToActivityUnit.csv', - 'OperationalLife.csv', - 'TotalAnnualMaxCapacityInvestment.csv', - 'TotalAnnualMinCapacityInvestment.csv', - 'TotalTechnologyModelPeriodActivityUpperLimit.csv', - 'FUEL.csv', - 'InputActivityRatio.csv', - 'OutputActivityRatio.csv', - 'MODE_OF_OPERATION.csv', - 'REGION.csv', - 'ResidualCapacity.csv', - 'TECHNOLOGY.csv', - 'YEAR.csv', - 'AvailabilityFactor.csv' + 'CapitalCost', + 'FixedCost', + 'CapacityToActivityUnit', + 'OperationalLife', + 'TotalAnnualMaxCapacityInvestment', + 'TotalAnnualMinCapacityInvestment', + 'TotalTechnologyModelPeriodActivityUpperLimit', + 'FUEL', + 'InputActivityRatio', + 'OutputActivityRatio', + 'MODE_OF_OPERATION', + 'REGION', + 'ResidualCapacity', + 'TECHNOLOGY', + 'YEAR', + 'AvailabilityFactor' ] timeslice_files = [ - 'CapacityFactor.csv', - 'TIMESLICE.csv', - 'SpecifiedDemandProfile.csv', - 'YearSplit.csv', - 'STORAGE.csv', - 'TechnologyToStorage.csv', - 'TechnologyFromStorage.csv', - 'Conversionls.csv', - 'Conversionld.csv', - 'Conversionlh.csv', - 'SEASON.csv', - 'DAYTYPE.csv', - 'DAILYTIMEBRACKET.csv', - 'CapitalCostStorage.csv', - 'DaySplit.csv', - 'ReserveMargin.csv', - 'ReserveMarginTagTechnology.csv', - 'ReserveMarginTagFuel.csv' + 'CapacityFactor', + 'TIMESLICE', + 'SpecifiedDemandProfile', + 'YearSplit', + 'STORAGE', + 'TechnologyToStorage', + 'TechnologyFromStorage', + 'Conversionls', + 'Conversionld', + 'Conversionlh', + 'SEASON', + 'DAYTYPE', + 'DAILYTIMEBRACKET', + 'CapitalCostStorage', + 'DaySplit', + 'ReserveMargin', + 'ReserveMarginTagTechnology', + 'ReserveMarginTagFuel' ] variable_cost_files = [ - 'VariableCost.csv' + 'VariableCost' ] demand_files = [ - 'SpecifiedAnnualDemand.csv' + 'SpecifiedAnnualDemand' ] emission_files = [ - 'EmissionActivityRatio.csv', - 'EmissionsPenalty.csv', - 'EMISSION.csv', - 'AnnualEmissionLimit.csv' + 'EmissionActivityRatio', + 'EmissionsPenalty', + 'EMISSION', + 'AnnualEmissionLimit' ] max_capacity_files = [ - 'TotalAnnualMaxCapacity.csv', - 'TotalTechnologyAnnualActivityUpperLimit.csv', - 'AccumulatedAnnualDemand.csv', -# 'TotalTechnologyModelPeriodActivityUpperLimit.csv' + 'TotalAnnualMaxCapacity', + 'TotalTechnologyAnnualActivityUpperLimit', + 'AccumulatedAnnualDemand', +# 'TotalTechnologyModelPeriodActivityUpperLimit' ] user_capacity_files = [ - 'TotalAnnualMinCapacityInvestment.csv', - 'TotalAnnualMaxCapacityInvestment.csv' + 'TotalAnnualMinCapacityInvestment', + 'TotalAnnualMaxCapacityInvestment' ] -check_files = os.listdir('resources/otoole/data') -generated_files = [ - power_plant_files, - timeslice_files, - variable_cost_files, - demand_files, - emission_files, - max_capacity_files] -for file_list in generated_files: - [check_files.remove(csv) for csv in file_list] +GENERATED_CSVS = ( + power_plant_files + timeslice_files + variable_cost_files + demand_files \ + + emission_files + max_capacity_files +) +EMPTY_CSVS = [x for x in OTOOLE_PARAMS if x not in GENERATED_CSVS] # rules @@ -119,7 +109,7 @@ rule powerplant: end_year = config['endYear'], invest_techs = config['no_invest_technologies'] output: - csv_files = expand('results/data/{output_file}', output_file = power_plant_files) + csv_files = expand('results/data/{output_file}.csv', output_file = power_plant_files) log: log = 'results/logs/powerplant.log' shell: @@ -142,7 +132,7 @@ rule timeslice: daypart = config['dayparts'], seasons = config['seasons'], output: - csv_files = expand('results/data/{output_file}', output_file=timeslice_files), + csv_files = expand('results/data/{output_file}.csv', output_file=timeslice_files), log: log = 'results/logs/timeslice.log' shell: @@ -158,7 +148,7 @@ rule variable_costs: start_year = config['startYear'], end_year = config['endYear'], output: - csv_files = expand('results/data/{output_file}', output_file=variable_cost_files), + csv_files = expand('results/data/{output_file}.csv', output_file=variable_cost_files), log: log = 'results/logs/variable_costs.log' shell: @@ -223,7 +213,7 @@ rule emissions: end_year = config['endYear'], emission = config['emission_penalty'] output: - csv_files = expand('results/data/{output_file}', output_file = emission_files), + csv_files = expand('results/data/{output_file}.csv', output_file = emission_files), log: log = 'results/logs/emissions.log' shell: @@ -240,28 +230,21 @@ rule max_capacity: start_year = config['startYear'], end_year = config['endYear'], output: - csv_files = expand('results/data/{output_file}', output_file = max_capacity_files), + csv_files = expand('results/data/{output_file}.csv', output_file = max_capacity_files), log: log = 'results/logs/max_capacity.log' shell: 'python workflow/scripts/osemosys_global/max_capacity.py 2> {log}' -rule file_check: +rule create_missing_csv: message: - 'Generating missing files...' + "Creating empty parameter data" + params: + out_dir = "results/data/" input: - rules.powerplant.output.csv_files, - rules.timeslice.output.csv_files, - rules.variable_costs.output.csv_files, - rules.demand_projections.output.csv_files, - rules.emissions.output.csv_files, - rules.max_capacity.output.csv_files, - #'resources/data/default_values.csv' - output: - expand('results/data/{check_file}', check_file = check_files), - log: - log = 'results/logs/file_check.log' - shell: - 'python workflow/scripts/osemosys_global/file_check.py 2> {log}' - - + otoole_config = OTOOLE_YAML, + csvs = expand("results/data/{full}.csv", full=GENERATED_CSVS) + output: + csvs = expand("results/data/{empty}.csv", empty=EMPTY_CSVS) + script: + "../scripts/osemosys_global/create_missing_csvs.py" diff --git a/workflow/scripts/osemosys_global/create_missing_csvs.py b/workflow/scripts/osemosys_global/create_missing_csvs.py new file mode 100644 index 0000000..7254f2d --- /dev/null +++ b/workflow/scripts/osemosys_global/create_missing_csvs.py @@ -0,0 +1,39 @@ +"""Creates empty CSVs that are required for otoole processing""" + +import yaml +import pandas as pd +from typing import Any +from pathlib import Path + + +def get_otoole_params(otoole_config: str) -> dict[str, dict[str, Any]]: + """Gets parameter/result files to be created""" + + with open(otoole_config) as f: + otoole = yaml.safe_load(f) + + return {x: otoole[x] for x in otoole if otoole[x]["type"] == "param"} + + +def get_empty_df(data: dict[str, Any]) -> pd.DataFrame: + """Gets empty parameter dataframe with correct indices""" + + indices = data["indices"] + ["VALUE"] + return pd.DataFrame(columns=indices) + + +if __name__ == "__main__": + if "snakemake" in globals(): + otoole_yaml = snakemake.input.otoole_config + out_dir = str(snakemake.params.out_dir) + else: + otoole_yaml = "resources/otoole/config.yaml" + out_dir = "results/data" + + parameters = get_otoole_params(otoole_yaml) + + for param, data in parameters.items(): + p = Path(out_dir, f"{param}.csv") + if not p.exists(): + df = get_empty_df(data) + df.to_csv(str(p), index=False) diff --git a/workflow/scripts/osemosys_global/file_check.py b/workflow/scripts/osemosys_global/file_check.py deleted file mode 100644 index 4a423af..0000000 --- a/workflow/scripts/osemosys_global/file_check.py +++ /dev/null @@ -1,21 +0,0 @@ -import pandas as pd -import os -from configuration import ConfigPaths -import logging -logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO) - -# CONFIGURATION PARAMETERS -config_paths = ConfigPaths() -input_data_dir = config_paths.input_data_dir -output_data_dir = config_paths.output_data_dir -otoole_data = os.path.join(config_paths.otoole, "data") - -# File Check logic - -for each_csv in os.listdir(otoole_data): - if each_csv not in os.listdir(output_data_dir): - csv_df_in = pd.read_csv(os.path.join(otoole_data, each_csv)) - csv_df_out = pd.DataFrame(columns = list(csv_df_in.columns)) - csv_df_out.to_csv(os.path.join(output_data_dir, each_csv), index = None) - -logging.info('File Check Completed') \ No newline at end of file diff --git a/workflow/snakefile b/workflow/snakefile index cb17d22..aed6d03 100644 --- a/workflow/snakefile +++ b/workflow/snakefile @@ -1,4 +1,5 @@ import os +import yaml from snakemake.utils import min_version min_version("6.0") @@ -6,16 +7,50 @@ min_version("6.0") configfile: 'config/config.yaml' -# helper files +# helper functions -include: 'rules/preprocess.smk' -include: 'rules/model.smk' -include: 'rules/postprocess.smk' -include: 'rules/retrieve.smk' +def get_otoole_data(otoole_config: str, var: str) -> list[str]: + """Gets parameter/result files to be created""" + + assert var in ("param", "result", "set") + + with open(otoole_config) as f: + otoole = yaml.safe_load(f) + + results = [x for x in otoole if otoole[x]["type"] == var] + + # no result calcualtions + missing = [ + "NewStorageCapacity", + "NumberOfNewTechnologyUnits", + "SalvageValueStorage", + "StorageLevelDayTypeStart", + "StorageLevelDayTypeFinish", + "StorageLevelSeasonStart", + "StorageLevelSeasonFinish", + "StorageLevelYearStart", + "StorageLevelYearFinish", + "Trade", + "DiscountedOperatingCost", + "TotalDiscountedStorageCost", + "ModelPeriodEmissions", + "StorageLowerLimit", + ] + + return [x for x in results if x not in missing] # constants -OTOOLE_FILES = os.listdir('resources/otoole/data') +OTOOLE_YAML = "resources/otoole.yaml" +OTOOLE_PARAMS = get_otoole_data(OTOOLE_YAML, "param") +OTOOLE_RESULTS = get_otoole_data(OTOOLE_YAML, "result") + +# rules + +include: "rules/preprocess.smk" +include: "rules/model.smk" +include: "rules/postprocess.smk" +include: "rules/retrieve.smk" # handlers @@ -50,7 +85,7 @@ rule generate_input_data: message: "Generating input CSV data..." input: - csv_files = expand('results/{scenario}/data/{csv}', scenario=config['scenario'], csv=OTOOLE_FILES), + csv_files = expand('results/{scenario}/data/{csv}', scenario=config['scenario'], csv=OTOOLE_PARAMS), rule make_dag: message: