diff --git a/lib/atlas.rb b/lib/atlas.rb index f0fa7d6b..1f169afd 100644 --- a/lib/atlas.rb +++ b/lib/atlas.rb @@ -67,7 +67,9 @@ require_relative 'atlas/graph_deserializer' require_relative 'atlas/scaler' -require_relative 'atlas/scaled_attributes' +require_relative 'atlas/scaler/area_attributes_scaler' +require_relative 'atlas/scaler/graph_scaler' +require_relative 'atlas/scaler/time_curve_scaler' require_relative 'atlas/merit_order_details' require_relative 'atlas/storage_details' diff --git a/lib/atlas/csv_document.rb b/lib/atlas/csv_document.rb index 19685e20..7b96980e 100644 --- a/lib/atlas/csv_document.rb +++ b/lib/atlas/csv_document.rb @@ -37,22 +37,54 @@ def self.curve(path) end # Public: Creates a new CSV document instance which will read data from a - # CSV file on disk. Document are read-only. + # CSV file on disk. Documents are read-write. # # path - Path to the CSV file. # # Returns a CSVDocument. - def initialize(path, normalizer = KEY_NORMALIZER) + def initialize(path, headers = nil) @path = Pathname.new(path) - @table = CSV.table(@path.to_s, { - converters: [YEAR_NORMALIZER, :all], - header_converters: [normalizer], - }) + if headers + raise(ExistingCSVHeaderError, path) if @path.file? + @headers = headers.map(&KEY_NORMALIZER) + @table = CSV::Table.new([CSV::Row.new(@headers, @headers, true)]) + else + @table = CSV.table(@path.to_s, { + converters: [YEAR_NORMALIZER, :all], + header_converters: [KEY_NORMALIZER], + # Needed to retrieve the headers in case + # of an otherwise empty csv file + return_headers: true + }) + + @headers = table.headers + + # Delete the header row for the internal representation - + # will be dynamically (re-)created when outputting + table.delete(0) + + raise(BlankCSVHeaderError, path) if @headers.any?(&:nil?) + end + end - @headers = @table.headers + # Public: Saves the CSV document to disk + def save! + FileUtils.mkdir_p(path.dirname) + File.write(path, table.to_csv) + self + end - raise(BlankCSVHeaderError, path) if @headers.any?(&:nil?) + # Public: Sets the value of a cell identified by its row and column. + # Non-existing rows are created automatically. + # + # row - The unique row name. + # column - The name of the column in which the data shall be put. + # value - The value that shall be set. + # + # Returns the set cell contents. + def set(row, column, value) + set_cell(normalize_key(row), normalize_key(column), value) end # Public: Retrieves the value of a cell identified by its row and column. @@ -65,6 +97,14 @@ def get(row, column) cell(normalize_key(row), normalize_key(column)) end + def row_keys + table.map { |row| normalize_key(row[0]) } + end + + def column_keys + @headers.map(&method(:normalize_key)) + end + ####### private ####### @@ -74,11 +114,19 @@ def get(row, column) # # Returns the cell content. def cell(row_key, column_key) - unless header?(column_key) - fail UnknownCSVCellError.new(self, column_key) - end + assert_header(column_key) - (data = row(row_key)) && data[column_key] + (table_row = row(row_key)) && table_row[column_key] + end + + # Internal: Sets the value of a cell, raising an UnknownCSVCellError if no + # such column exists. Non-existing rows are created automatically. + # + # Returns the cell content. + def set_cell(row_key, column_key, value) + assert_header(column_key) + + get_or_create_row(row_key)[column_key] = value end # Internal: Finds the row by the given +key+. @@ -86,8 +134,26 @@ def cell(row_key, column_key) # Returns a CSV::Row or raises an UnknownCSVRowError if no such row exists # in the file. def row(key) - @table.find { |row| normalize_key(row[0]) == key } || - fail(UnknownCSVRowError.new(self, key)) + safe_row(key) || fail(UnknownCSVRowError.new(self, key)) + end + + # Internal: Finds the row by the given +key+. + # + # Returns a CSV::Row or nil. + def safe_row(key) + table.find { |row| normalize_key(row[0]) == key } + end + + # Internal: Finds the row by the given +key+ or creates it if no such + # row exists in the file. + # + # Returns a CSV::Row. + def get_or_create_row(key) + safe_row(key) || begin + row = CSV::Row.new(@headers, [key]) + table << row + safe_row(key) + end end # Internal: Converts the given key to a format which removes all special @@ -98,12 +164,14 @@ def normalize_key(key) KEY_NORMALIZER.call(key) end - # Internal: Determines if the named column exists in the file. This will - # always be true if the column is named by index (a number) + # Internal: Raises unless the named column exists in the file. + # Never raises if the column is named by index (a number) # # Returns true or false. - def header?(key) - key.is_a?(Numeric) || @headers.nil? || @headers.include?(key) + def assert_header(key) + unless key.is_a?(Numeric) || @headers.nil? || @headers.include?(key) + fail UnknownCSVCellError.new(self, key) + end end end # CSVDocument @@ -120,13 +188,4 @@ def get(row) cell(normalize_key(row), 1) end end # CSVDocument::OneDimensional - - # A CSVDocument which reads CSV files which are output by the Exporter. Each - # left-hand column is a node, edge, or slot key whose value needs to be - # preserved without removing special characters. - class CSVDocument::Production < CSVDocument - def initialize(path) - super(path, ->(value) { value.to_sym }) - end - end # CSVDocument::Production end # Atlas diff --git a/lib/atlas/dataset.rb b/lib/atlas/dataset.rb index 845aecc8..e74bfe22 100644 --- a/lib/atlas/dataset.rb +++ b/lib/atlas/dataset.rb @@ -149,6 +149,14 @@ def efficiencies(key) dataset_dir.join("efficiencies/#{ key }_efficiency.csv")) end + def time_curves_dir + dataset_dir.join('time_curves') + end + + def time_curve_path(key) + time_curves_dir.join("#{ key }_time_curve.csv") + end + # Public: Retrieves the time curve data for the file whose name matches # +key+. # @@ -160,15 +168,14 @@ def efficiencies(key) # Returns a CSVDocument. def time_curve(key) (@time_curves ||= {})[key.to_sym] ||= - CSVDocument.new( - dataset_dir.join("time_curves/#{ key }_time_curve.csv")) + CSVDocument.new(time_curve_path(key)) end # Public: Retrieves all the time curves for the dataset's region. # # Returns a hash of document keys, and the CSVDocuments. def time_curves - Pathname.glob(dataset_dir.join('time_curves/*.csv')).each do |csv_path| + Pathname.glob(time_curves_dir.join('*.csv')).each do |csv_path| time_curve(csv_path.basename('_time_curve.csv').to_s) end diff --git a/lib/atlas/dataset/derived_dataset.rb b/lib/atlas/dataset/derived_dataset.rb index 3d1b0418..0039ab07 100644 --- a/lib/atlas/dataset/derived_dataset.rb +++ b/lib/atlas/dataset/derived_dataset.rb @@ -23,6 +23,11 @@ def dataset_dir @dataset_dir ||= Atlas.data_dir.join(DIRECTORY, base_dataset) end + # Overwrite + def time_curves_dir + Atlas.data_dir.join(DIRECTORY, key.to_s, 'time_curves') + end + private def base_dataset_exists diff --git a/lib/atlas/errors.rb b/lib/atlas/errors.rb index 0de34666..4da3dbf1 100644 --- a/lib/atlas/errors.rb +++ b/lib/atlas/errors.rb @@ -210,6 +210,11 @@ def self.error_class(superclass = AtlasError, &block) "of the cells in the first row must contain a non-blank value." end + ExistingCSVHeaderError = error_class do |path| + "Column headers provided although CSV file #{path} already exists" + end + + # Parser Errors ------------------------------------------------------------ CannotIdentifyError = error_class(ParserError) do |string| diff --git a/lib/atlas/graph_persistor.rb b/lib/atlas/graph_persistor.rb index 4ce67439..12f043c7 100644 --- a/lib/atlas/graph_persistor.rb +++ b/lib/atlas/graph_persistor.rb @@ -1,24 +1,15 @@ module Atlas - class GraphPersistor - def initialize(dataset, path) - @dataset = dataset - @path = path - end - - def self.call(dataset, path) - new(dataset, path).persist! - end - - def persist! - File.open(@path, 'w') do |f| - f.write EssentialExporter.dump(refinery_graph).to_yaml - end - end - - private - - def refinery_graph - Runner.new(@dataset).refinery_graph(:export) - end + # Public: Builds the graph and exports it to a YAML file. + # + # dataset - This dataset's graph will be built and persisted + # path - File to which the graph will be exported + # export_modifier - Will be called on the graph's exported hash prior to saving it + # + # Returns a Hash + GraphPersistor = lambda do |dataset, path, export_modifier: nil| + data = EssentialExporter.dump(Runner.new(dataset).refinery_graph(:export)) + export_modifier.call(data) if export_modifier + File.write(path, data.to_yaml) + data end end diff --git a/lib/atlas/scaled_attributes.rb b/lib/atlas/scaled_attributes.rb deleted file mode 100644 index 6119a022..00000000 --- a/lib/atlas/scaled_attributes.rb +++ /dev/null @@ -1,37 +0,0 @@ -module Atlas - class Scaler::ScaledAttributes - # Only attributes common to FullDataset and DerivedDataset - # may be scaled - SCALEABLE_AREA_ATTRS = Atlas::Dataset.attribute_set - .select { |attr| attr.options[:proportional] }.map(&:name).freeze - - def initialize(dataset, number_of_residences) - @dataset = dataset - @number_of_residences = number_of_residences - end - - def scale - Hash[ - SCALEABLE_AREA_ATTRS.map do |attr| - if value = @dataset[attr] - [attr, Util::round_computation_errors(value * scaling_factor)] - end - end.compact - ] - end - - private - - def scaling_factor - value.to_f / base_value - end - - def base_value - @dataset.number_of_residences - end - - def value - @number_of_residences - end - end -end diff --git a/lib/atlas/scaler.rb b/lib/atlas/scaler.rb index a8529551..4a9ed6e5 100644 --- a/lib/atlas/scaler.rb +++ b/lib/atlas/scaler.rb @@ -8,17 +8,31 @@ def initialize(base_dataset_key, derived_dataset_name, number_of_residences) def create_scaled_dataset derived_dataset = Dataset::DerivedDataset.new( - @base_dataset.attributes - .merge(scaled_attributes) - .merge(new_attributes)) + @base_dataset.attributes. + merge(AreaAttributesScaler.call(@base_dataset, scaling_factor)). + merge(new_attributes)) derived_dataset.save! - GraphPersistor.call(@base_dataset, derived_dataset.graph_path) + GraphPersistor.call(@base_dataset, derived_dataset.graph_path, export_modifier: Scaler::GraphScaler.new(scaling_factor)) + + TimeCurveScaler.call(@base_dataset, scaling_factor, derived_dataset) end private + def value + @number_of_residences + end + + def base_value + @base_dataset.number_of_residences + end + + def scaling_factor + value.to_r / base_value.to_r + end + def new_attributes id = Dataset.all.map(&:id).max + 1 { @@ -28,20 +42,13 @@ def new_attributes key: @derived_dataset_name, area: @derived_dataset_name, base_dataset: @base_dataset.area, - scaling: scaling, + scaling: + { + value: value, + base_value: base_value, + area_attribute: 'number_of_residences', + }, } end - - def scaling - { - value: @number_of_residences, - base_value: @base_dataset.number_of_residences, - area_attribute: 'number_of_residences', - } - end - - def scaled_attributes - ScaledAttributes.new(@base_dataset, @number_of_residences).scale - end - end -end + end # Scaler +end # Atlas diff --git a/lib/atlas/scaler/area_attributes_scaler.rb b/lib/atlas/scaler/area_attributes_scaler.rb new file mode 100644 index 00000000..a56d8482 --- /dev/null +++ b/lib/atlas/scaler/area_attributes_scaler.rb @@ -0,0 +1,27 @@ +module Atlas + class Scaler::AreaAttributesScaler + # Only attributes common to FullDataset and DerivedDataset + # may be scaled + SCALEABLE_AREA_ATTRS = Atlas::Dataset.attribute_set + .select { |attr| attr.options[:proportional] }.map(&:name).freeze + + def self.call(*args) + new(*args).scale + end + + def initialize(base_dataset, scaling_factor) + @base_dataset = base_dataset + @scaling_factor = scaling_factor + end + + def scale + result = {} + SCALEABLE_AREA_ATTRS.map do |attr| + if value = @base_dataset[attr] + result[attr] = Util::round_computation_errors(value * @scaling_factor) + end + end + result + end + end # Scaler::AreaAttributesScaler +end # Atlas diff --git a/lib/atlas/scaler/graph_scaler.rb b/lib/atlas/scaler/graph_scaler.rb new file mode 100644 index 00000000..c481d2d7 --- /dev/null +++ b/lib/atlas/scaler/graph_scaler.rb @@ -0,0 +1,41 @@ +module Atlas + class Scaler::GraphScaler + # Edge attributes which must be scaled. + EDGE_ATTRIBUTES = [:demand].freeze + + # Node attributes which must be scaled. + NODE_ATTRIBUTES = [ + :demand, + :max_demand, + :typical_input_capacity, + :electricity_output_capacity + ].freeze + + # Maps top-level keys from the dumped graph to arrays of attributes which + # need to be scaled. + SCALED_ATTRIBUTES = { + edges: EDGE_ATTRIBUTES, + nodes: NODE_ATTRIBUTES + }.freeze + + def initialize(scaling_factor) + @scaling_factor = scaling_factor + end + + # Public: Scales the demands in the graph hash - modifying the original hash! + # + # graph - A Hash containing nodes and edges. + # + # Returns the modified graph hash itself. + def call(graph) + SCALED_ATTRIBUTES.each do |graph_key, attributes| + graph[graph_key].each_value do |record| + attributes.each do |attr| + record[attr] = @scaling_factor * record[attr] if record[attr] + end + end + end + graph + end + end # Scaler::GraphScaler +end # Atlas diff --git a/lib/atlas/scaler/time_curve_scaler.rb b/lib/atlas/scaler/time_curve_scaler.rb new file mode 100644 index 00000000..940da277 --- /dev/null +++ b/lib/atlas/scaler/time_curve_scaler.rb @@ -0,0 +1,56 @@ +module Atlas + # Scales (a copy of) the time curves of a base dataset and + # saves them to a derived dataset + # + # @base_dataset - An Atlas::Dataset, whose time curves will be used as base + # @scaling_factor - The time curves will be scaled down by this factor + # @derived_dataset - An Atlas::Dataset, into whose directory the new scaled + # time curves will be saved as csv files + class Scaler::TimeCurveScaler + # Public: Scales the curves and saves them to new csv files + # + # Returns nil + def self.call(*args) + new(*args).scale + end + + def initialize(base_dataset, scaling_factor, derived_dataset) + @base_dataset = base_dataset + @scaling_factor = scaling_factor + @derived_dataset = derived_dataset + end + + # Public: Scales the curves and saves them to new csv files + # + # Returns nil + def scale + @base_dataset.time_curves.each do |key, csv| + scale_time_curve(key, csv) + end + nil + end + + private + + def scale_time_curve(time_curve_key, csv) + scaled_csv = + CSVDocument.new( + @derived_dataset.time_curve_path(time_curve_key), + csv.column_keys + ) + copy_csv_content(csv, scaled_csv) { |val| val * @scaling_factor } + scaled_csv.save! + end + + def copy_csv_content(src, dest) + row_keys = src.row_keys + column_keys = src.column_keys + row_keys.each do |row_key| + column_keys.each do |column_key| + value = yield src.get(row_key, column_key) + dest.set(row_key, column_key, value) + end + end + end + end # Scaler::TimeCurveScaler +end # Atlas diff --git a/spec/atlas/csv_document_spec.rb b/spec/atlas/csv_document_spec.rb index a3c6e49d..c8750e13 100644 --- a/spec/atlas/csv_document_spec.rb +++ b/spec/atlas/csv_document_spec.rb @@ -19,16 +19,41 @@ module Atlas CSVDocument.new(path.to_s) end - it 'raises when the file does not exist' do - expect { CSVDocument.new('no') }.to raise_error(/no such file/i) - end + describe '.new' do + context 'without specified headers' do + it 'raises when the file does not exist' do + expect { CSVDocument.new('no') }.to raise_error(/no such file/i) + end + + it 'raises when a header cell contains no value' do + path = Atlas.data_dir.join('blank.csv') + path.open('w') { |f| f.puts(",yes\nyes,1") } + + expect { CSVDocument.new(path.to_s) }. + to raise_error(BlankCSVHeaderError) + end + end - it 'raises when a header cell contains no value' do - path = Atlas.data_dir.join('blank.csv') - path.open('w') { |f| f.puts(",yes\nyes,1") } + context 'with specified headers' do + let(:headers) { %i( yes no maybe ) } + let(:path) { Atlas.data_dir.join('new.csv') } + let(:doc) { CSVDocument.new(path.to_s, headers) } + + it 'does not save the new file to disk' do + expect(File.exist?(doc.path)).to be_false + end + + it 'raises when the file already exists' do + doc.save! + expect { CSVDocument.new(path.to_s, %i( hello world )) }. + to raise_error(ExistingCSVHeaderError) + end + + it 'sets the headers / column_keys' do + expect(doc.column_keys).to eq(headers) + end + end - expect { CSVDocument.new(path.to_s) }. - to raise_error(BlankCSVHeaderError) end describe '#get' do @@ -56,7 +81,7 @@ module Atlas expect(doc.get('oh_he_said', 'yes')).to eq(-1) end - it 'finds special character carriers with special characters' do + it 'finds column headers with special characters' do expect(doc.get('yes', 'maybe (possibly)')).to be(1) end @@ -68,7 +93,7 @@ module Atlas expect { doc.get('foo', 'yes') }.to raise_error(UnknownCSVRowError) end - it 'raises an error when carrier is not known' do + it 'raises an error when column header is not known' do expect { doc.get('yes', 'nope') }.to raise_error(UnknownCSVCellError) end @@ -80,6 +105,52 @@ module Atlas expect { doc.get('blank', 1) }.to_not raise_error end end # get + + describe '#set' do + it 'sets a given value for given row and column' do + expect { doc.set('yes', 'no', 42) }.to change { doc.get('yes', 'no') }.from(0).to(42) + end + + it 'creates non-existing rows on-the-fly' do + expect { doc.get('foo bar', 'yes') }.to raise_error(UnknownCSVRowError) + doc.set('foo bar', 'yes', 21) + expect(doc.get('foo bar', 'yes')).to eq(21) + end + + it 'raises an error when column header is not known' do + expect { doc.set('yes', 'nope', 99) }.to raise_error(UnknownCSVCellError) + end + end # set + + describe '#save!' do + it 'saves the CSVDocument content to disk' do + doc.set('yes', 'no', 42) + doc.save! + + expect(File.readlines(doc.path).map(&:strip)).to eq( + <<-EOF.lines.map(&:strip)) + "",yes,no,maybe_possibly + yes,1,42,1 + no,0,0,0 + maybe possibly,1,0,0.5 + oh_%^&/_he_said,-1,-1,-1 + blank,,, + EOF + end + + context 'when the file did not exist before' do + let(:doc) { CSVDocument.new(Atlas.data_dir.join('doesnotexistbefore.csv'), %i( yes no maybe\ baby )) } + it 'creates a new csv file' do + doc.save! + expect(File.file?(doc.path)).to be_true + end + + it 'creates a normalized header row in the csv file' do + doc.save! + expect(File.readlines(doc.path).first.strip).to eq('yes,no,maybe_baby') + end + end + end end # CSVDocument describe CSVDocument::OneDimensional do diff --git a/spec/atlas/scaler_spec.rb b/spec/atlas/scaler_spec.rb index aba2439a..86df9187 100644 --- a/spec/atlas/scaler_spec.rb +++ b/spec/atlas/scaler_spec.rb @@ -1,31 +1,36 @@ require 'spec_helper' module Atlas; describe Scaler do - describe '#create_scaled_dataset' do - include GraphHelper + include GraphHelper - # Graph: - # [ a (25) ]--< a_b (10) >--[ b (10) ] + # Graph: + # [ a (25) ]--< a_b (10) >--[ b (10) ] - let(:graph) { Turbine::Graph.new } + let(:graph) do + Turbine::Graph.new.tap do |graph| + a = graph.add(make_node(:a, demand: 25)) + b = graph.add(make_node(:b, demand: 10)) + make_edge(a, b, :a_b, child_share: 1.0, demand: 10) + end + end - let!(:a) { graph.add(make_node(:a, demand: 25)) } - let!(:b) { graph.add(make_node(:b, demand: 10)) } + before { allow(GraphBuilder).to receive(:build).and_return(graph) } - let!(:ab_edge) { make_edge(a, b, :a_b, child_share: 1.0) } + let(:base_dataset) { Atlas::Dataset.find('nl') } + let(:scaling_value) { 1000 } + let(:scaling_factor) { scaling_value.to_r / 7_349_500 } - let!(:mock_graph) { - allow(GraphBuilder).to receive(:build).and_return(graph) - } - let(:derived_dataset) { Atlas::Dataset::DerivedDataset.find('ameland') } + describe '#create_scaled_dataset' do - context 'with scaling value 1000' do - let(:scaler) { Atlas::Scaler.new('nl', 'ameland', 1000) } + context 'with scaling value #{ scaling_value }' do + let(:scaler) { Scaler.new('nl', 'ameland', scaling_value) } before { scaler.create_scaled_dataset } + let(:derived_dataset) { Atlas::Dataset::DerivedDataset.find('ameland') } + it 'creates a valid DerivedDataset' do derived_dataset.valid? @@ -42,17 +47,17 @@ module Atlas; describe Scaler do expect(derived_dataset.parent_id).to eq(derived_dataset.id) end - it 'sets the scaling value of the DerivedDataset to 1000' do - expect(derived_dataset.scaling[:value]).to eq(1000) + it 'sets the scaling value of the DerivedDataset to #{ scaling_value }' do + expect(derived_dataset.scaling[:value]).to eq(scaling_value) end it 'sets the scaling base_value of the DerivedDataset to the number_of_residences in nl' do expect(derived_dataset.scaling[:base_value]). - to eq(Atlas::Dataset.find('nl').number_of_residences) + to eq(base_dataset.number_of_residences) end it 'assigns the correctly scaled number of residences' do - expect(derived_dataset.number_of_residences).to eq(1000) + expect(derived_dataset.number_of_residences).to eq(scaling_value) end it 'assigns the correctly scaled number of inhabitants' do @@ -62,19 +67,46 @@ module Atlas; describe Scaler do it 'dumps a graph.yml' do expect(derived_dataset.graph).to_not be_blank end - - it 'exports the correct demand 25/1 for node :a' do - expect(derived_dataset.graph.node(:a).get(:demand)).to eq(25/1) - end end context 'with scaling value nil' do - let(:scaler) { Atlas::Scaler.new('nl', 'ameland', nil) } + let(:scaler) { Scaler.new('nl', 'ameland', nil) } it 'creates an invalid DerivedDataset' do expect { scaler.create_scaled_dataset }. to raise_error(Atlas::InvalidDocumentError, /Scaling Value/) end end - end -end; end + end # create_scaled_dataset + + + describe Scaler::GraphScaler do + let(:graph_data) { EssentialExporter.dump(graph) } + + before { Scaler::GraphScaler.new(scaling_factor).call(graph_data) } + + it 'exports the correct demand 25 * scaling_factor for node :a' do + expect(graph_data[:nodes][:a][:demand]). + to eql(25.to_r * scaling_factor) + end + + it 'exports the correct demand 10 * scaling_factor for edge :a->:b' do + expect(graph_data[:edges][:'a-b@a_b'][:demand]). + to eql(10.to_r * scaling_factor) + end + end # GraphScaler + + + describe Scaler::TimeCurveScaler do + let(:derived_dataset) do + Atlas::Dataset::DerivedDataset.new(key: 'rotterdam', area: 'rotterdam') + end + + before { Scaler::TimeCurveScaler.call(base_dataset, scaling_factor, derived_dataset) } + + it 'scales the time curves' do + expect(derived_dataset.time_curve(:woody_biomass).get(2030, :max_demand)). + to be_within(0.001).of(9.73488372100000E+04 * scaling_factor) + end + end # TimeCurveScaler +end; end # Atlas::Scaler