diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..8f1d818d --- /dev/null +++ b/.env.example @@ -0,0 +1,9 @@ +DATAGOUV_MERSEA_HOST= +DATAGOUV_HOST="" +DATAGOUV_API_KEY="" +DATAGOUV_DATASET_ID="" +DATAGOUV_REPORTS_JSON_ID="" +DATAGOUV_REPORTS_GEOJSON_ID="" +DATAGOUV_REPORTS_CSV_ID="" +DATAGOUV_TRACERS_JSON_ID="" +DATAGOUV_TRACERS_CSV_ID="" diff --git a/.gitignore b/.gitignore index 99f26ad3..43cf70e4 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,8 @@ rerun.txt pickle-email-*.html my.txt .envrc - +datagouv/* +!datagouv/.gitkeep # TODO Comment out these rules if you are OK with secrets being uploaded to the repo # config/initializers/secret_token.rb # config/secrets.yml diff --git a/Gemfile b/Gemfile index 5138c89c..b8c30a9f 100644 --- a/Gemfile +++ b/Gemfile @@ -40,6 +40,8 @@ group :development do gem 'spring' gem 'spring-watcher-listen' gem 'seed_dump' + gem 'dotenv-rails' + gem 'http', '~> 5.0.0.pre2' end # Windows does not include zoneinfo files, so bundle the tzinfo-data gem diff --git a/Gemfile.lock b/Gemfile.lock index 2fd35f4d..d82c5ab1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -108,6 +108,12 @@ GEM devise (~> 4.0) warden-jwt_auth (~> 0.6) diff-lcs (1.5.0) + domain_name (0.5.20190701) + unf (>= 0.0.5, < 1.0.0) + dotenv (2.8.1) + dotenv-rails (2.8.1) + dotenv (= 2.8.1) + railties (>= 3.2) dry-auto_inject (0.9.0) dry-container (>= 0.3.4) dry-configurable (0.16.1) @@ -115,7 +121,7 @@ GEM zeitwerk (~> 2.6) dry-container (0.11.0) concurrent-ruby (~> 1.0) - dry-core (0.9.0) + dry-core (0.9.1) concurrent-ruby (~> 1.0) zeitwerk (~> 2.6) erubi (1.11.0) @@ -128,11 +134,22 @@ GEM faker (1.9.6) i18n (>= 0.7) ffi (1.15.5) + ffi-compiler (1.0.1) + ffi (>= 1.0.0) + rake globalid (1.0.0) activesupport (>= 5.0) haml (5.2.2) temple (>= 0.8.0) tilt + http (5.0.4) + addressable (~> 2.8) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.4.0) + http-cookie (1.0.5) + domain_name (~> 0.5) + http-form_data (2.3.0) http_accept_language (2.1.1) i18n (1.12.0) concurrent-ruby (~> 1.0) @@ -141,7 +158,7 @@ GEM image_processing (1.12.2) mini_magick (>= 4.9.5, < 5) ruby-vips (>= 2.0.17, < 3) - jquery-rails (4.5.0) + jquery-rails (4.5.1) rails-dom-testing (>= 1, < 3) railties (>= 4.2.0) thor (>= 0.14, < 2.0) @@ -163,11 +180,14 @@ GEM activerecord kaminari-core (= 1.2.2) kaminari-core (1.2.2) - leaflet-rails (1.8.0) + leaflet-rails (1.9.2) rails (>= 4.2.0) listen (3.7.1) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) + llhttp-ffi (0.4.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) loofah (2.19.0) crass (~> 1.0.2) nokogiri (>= 1.5.9) @@ -177,26 +197,26 @@ GEM method_source (1.0.0) mini_magick (4.11.0) mini_mime (1.1.2) - mini_portile2 (2.8.0) minitest (5.16.3) nested_form (0.3.2) nio4r (2.5.8) - nokogiri (1.13.9) - mini_portile2 (~> 2.8.0) + nokogiri (1.13.9-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.13.9-x86_64-linux) racc (~> 1.4) orm_adapter (0.5.0) parallel (1.22.1) parser (3.1.2.1) ast (~> 2.4.1) - pg (1.4.4) - postmark (1.22.1) + pg (1.4.5) + postmark (1.22.2) json public_suffix (5.0.0) puma (5.6.5) nio4r (~> 2.0) pundit (1.1.0) activesupport (>= 3.0.0) - pundit-matchers (1.7.0) + pundit-matchers (1.8.4) rspec-rails (>= 3.0.0) racc (1.6.0) rack (2.2.4) @@ -265,7 +285,7 @@ GEM rb-readline (0.5.5) recaptcha (5.12.3) json - regexp_parser (2.6.0) + regexp_parser (2.6.1) remotipart (1.4.4) responders (3.0.1) actionpack (>= 5.0) @@ -288,17 +308,17 @@ GEM rspec-mocks (~> 3.9.0) rspec-support (~> 3.9.0) rspec-support (3.9.4) - rubocop (1.36.0) + rubocop (1.39.0) json (~> 2.3) parallel (~> 1.10) parser (>= 3.1.2.1) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.20.1, < 2.0) + rubocop-ast (>= 1.23.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.21.0) + rubocop-ast (1.23.0) parser (>= 3.1.1.0) ruby-progressbar (1.11.0) ruby-vips (2.1.4) @@ -349,7 +369,7 @@ GEM state_machines-activerecord (0.8.0) activerecord (>= 5.1) state_machines-activemodel (>= 0.8.0) - temple (0.8.2) + temple (0.9.1) thor (1.2.1) thread_safe (0.3.6) tilt (2.0.11) @@ -357,6 +377,9 @@ GEM concurrent-ruby (~> 1.0) uglifier (4.2.0) execjs (>= 0.3.0, < 3) + unf (0.1.4) + unf_ext + unf_ext (0.0.8.2) unicode-display_width (2.3.0) warden (1.2.9) rack (>= 2.0.9) @@ -370,10 +393,11 @@ GEM websocket-extensions (0.1.5) wysiwyg-rails (2.9.8) railties (>= 3.2, < 7.0) - zeitwerk (2.6.1) + zeitwerk (2.6.6) PLATFORMS - ruby + x86_64-darwin-20 + x86_64-linux DEPENDENCIES active_model_serializers (= 0.10) @@ -386,8 +410,10 @@ DEPENDENCIES devise (>= 4.6.0) devise-i18n (~> 1) devise-jwt (~> 0.8) + dotenv-rails factory_bot_rails (~> 4.8) faker (~> 1.8) + http (~> 5.0.0.pre2) http_accept_language (~> 2) i18n-js (~> 3) image_processing (~> 1.12) diff --git a/README.md b/README.md index ec30589f..acf43fbe 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,13 @@ To set any environment variable in the container, use one or more `-e` flags: - `RECAPTCHA_SECRET_KEY` → Google reCaptcha secret - `BUGSNAG_API_KEY` → Bugsnag key (leave empty to disable error reporting) +## Datagouv export + +```sh +# Configure .env file with datagouv credentials +bundle exec rake datagouv +``` + ## License MIT. See the [LICENSE](https://github.com/FranckKe/mersea/blob/master/LICENSE) for more details. diff --git a/datagouv/.gitkeep b/datagouv/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/lib/tasks/datagouv.rake b/lib/tasks/datagouv.rake new file mode 100644 index 00000000..4e7b4759 --- /dev/null +++ b/lib/tasks/datagouv.rake @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +require 'http' + +# See .env.example for env configuration + +desc 'Upload tracers and reports to datagouv' +task datagouv: :environment do + tracer_attr = %i[ + id + name + description + origin + kind + longitude + latitude + created_at + color + category + ] + + @tracers = Tracer.select(tracer_attr) + Rails.application.routes.default_url_options[:host] = ENV['DATAGOUV_MERSEA_HOST'] + + puts "#{@tracers.size} tracers found." + + tracers_csv = CSV.generate do |csv| + # Header + csv << tracer_attr + ['photo_url'] + # Values + @tracers.each do |tracer| + puts [Rails.application.routes.url_helpers.url_for(tracer.photo)] + csv << tracer.slice(*tracer_attr).values + [Rails.application.routes.url_helpers.url_for(tracer.photo)] + end + end + + File.open('./datagouv/tracers.csv', 'w+') do |f| + f.write(tracers_csv) + end + + puts 'tracers.csv created.' + + File.open('./datagouv/tracers.json', 'w+') do |f| + @tracers_with_photo_url = @tracers.map do |tracer| + tracer.as_json(only: tracer_attr).merge(photo: Rails.application.routes.url_helpers.url_for(tracer.photo)) + end + f.write(@tracers_with_photo_url.to_json) + end + + puts 'tracers.json created.' + + report_attr = %i[ + id + tracer_id + name + quantity + address + longitude + latitude + description + reported_at + created_at + shore_length + ] + + @reports = Report.where(status: 'accepted').select(report_attr) + + puts "#{@reports.size} accepted reports found." + + reports_csv = CSV.generate do |csv| + csv << report_attr + @reports.each do |report| + csv << report.slice(*report_attr).values + end + end + + File.open('./datagouv/reports.csv', 'w+') do |f| + f.write(reports_csv) + end + + puts 'reports.csv created.' + + File.open('./datagouv/reports.json', 'w+') do |f| + f.write(@reports.as_json(only: report_attr).to_json) + end + + puts 'reports.json created.' + + File.open('./datagouv/reports.geojson', 'w+') do |f| + f.write(GeojsonReportsSerializer.new(@reports).to_json) + end + + puts 'reports.geojson created.' + + datagouv_dataset_url = "#{ENV['DATAGOUV_HOST']}/datasets/#{ENV['DATAGOUV_DATASET_ID']}/resources" + + puts "datagouv_dataset_url #{datagouv_dataset_url}" + + tracers_csv_res = HTTP + .headers(accept: 'application/json') + .headers('x-api-key': ENV['DATAGOUV_API_KEY']) + .post("#{datagouv_dataset_url}/#{ENV['DATAGOUV_TRACERS_CSV_ID']}/upload/", + form: { + file: HTTP::FormData::File.new('./datagouv/tracers.csv') + }) + puts tracers_csv_res + + tracers_json_res = HTTP + .headers(accept: 'application/json') + .headers('x-api-key': ENV['DATAGOUV_API_KEY']) + .post("#{datagouv_dataset_url}/#{ENV['DATAGOUV_TRACERS_JSON_ID']}/upload/", + form: { + file: HTTP::FormData::File.new('./tracers.json') + }) + puts tracers_json_res + + reports_json_res = HTTP + .headers(accept: 'application/json') + .headers('x-api-key': ENV['DATAGOUV_API_KEY']) + .post("#{datagouv_dataset_url}/#{ENV['DATAGOUV_REPORTS_JSON_ID']}/upload/", + form: { + file: HTTP::FormData::File.new('./reports.json') + }) + puts reports_json_res + + reports_geojson_res = HTTP + .headers(accept: 'application/json') + .headers('x-api-key': ENV['DATAGOUV_API_KEY']) + .post("#{datagouv_dataset_url}/#{ENV['DATAGOUV_REPORTS_GEOJSON_ID']}/upload/", + form: { + file: HTTP::FormData::File.new('./reports.geojson') + }) + puts reports_geojson_res + + reports_csv_res = HTTP + .headers(accept: 'application/json') + .headers('x-api-key': ENV['DATAGOUV_API_KEY']) + .post("#{datagouv_dataset_url}/#{ENV['DATAGOUV_REPORTS_CSV_ID']}/upload/", + form: { + file: HTTP::FormData::File.new('./reports.csv') + }) + puts reports_csv_res +end