diff --git a/CONTRIB.md b/CONTRIB.md index 1ba60f67f..0121672db 100644 --- a/CONTRIB.md +++ b/CONTRIB.md @@ -63,22 +63,25 @@ In addition to running the development server, `manage.py` (OpenOversight's mana ``` (oovirtenv)vagrant@vagrant-ubuntu-trusty-64:/vagrant/OpenOversight$ python manage.py +-------------------------------------------------------------------------------- +INFO in __init__ [/vagrant/OpenOversight/app/__init__.py:57]: +OpenOversight startup +-------------------------------------------------------------------------------- usage: manage.py [-?] - {shell,makemigrations,migrate,downgrade_db,runserver,make_admin_user} + {runserver,db,shell,make_admin_user,link_images_to_department} ... positional arguments: - {shell,makemigrations,migrate,downgrade_db,runserver,make_admin_user} - shell Runs a Python shell inside Flask application context. - makemigrations Make database migrations - migrate Migrate/upgrade the database - downgrade_db Downgrade the database + {runserver,db,shell,make_admin_user,link_images_to_department} runserver Runs the Flask development server i.e. app.run() + db Perform database migrations + shell Runs a Python shell inside Flask application context. make_admin_user Add confirmed administrator account + link_images_to_department + Link existing images to first department optional arguments: -?, --help show this help message and exit - ``` In development, you can make an administrator account without having to confirm your email: @@ -109,17 +112,16 @@ If you e.g. add a new column or table, you'll need to migrate the database. You can use the management interface to first generate migrations: ``` -(oovirtenv)vagrant@vagrant-ubuntu-trusty-64:/vagrant/OpenOversight$ python manage.py makemigrations -New migration saved as /vagrant/OpenOversight/app/db_repository/versions/002_migration.py +(oovirtenv)vagrant@vagrant-ubuntu-trusty-64:/vagrant/OpenOversight$ python manage.py db migrate ``` And then you should inspect/edit the migrations. You can then apply the migrations: ``` -(oovirtenv)vagrant@vagrant-ubuntu-trusty-64:/vagrant/OpenOversight$ python manage.py migrate +(oovirtenv)vagrant@vagrant-ubuntu-trusty-64:/vagrant/OpenOversight$ python manage.py db upgrade ``` -You can also downgrade the database using `python manage.py downgrade_db`. +You can also downgrade the database using `python manage.py db downgrade`. ## Changing the Development Environment diff --git a/OpenOversight/app/config.py b/OpenOversight/app/config.py index 12ad70204..7f211df79 100644 --- a/OpenOversight/app/config.py +++ b/OpenOversight/app/config.py @@ -8,7 +8,6 @@ class BaseConfig(object): # DB SETUP - SQLALCHEMY_MIGRATE_REPO = os.path.join(basedir, 'db_repository') SQLALCHEMY_TRACK_MODIFICATIONS = False SQLALCHEMY_COMMIT_ON_TEARDOWN = True diff --git a/OpenOversight/app/db_repository/README b/OpenOversight/app/db_repository/README deleted file mode 100644 index 6218f8cac..000000000 --- a/OpenOversight/app/db_repository/README +++ /dev/null @@ -1,4 +0,0 @@ -This is a database migration repository. - -More information at -http://code.google.com/p/sqlalchemy-migrate/ diff --git a/OpenOversight/app/db_repository/__init__.py b/OpenOversight/app/db_repository/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/OpenOversight/app/db_repository/manage.py b/OpenOversight/app/db_repository/manage.py deleted file mode 100644 index a15815d34..000000000 --- a/OpenOversight/app/db_repository/manage.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python -from migrate.versioning.shell import main # pragma: no cover - -if __name__ == '__main__': # pragma: no cover - main() diff --git a/OpenOversight/app/db_repository/migrate.cfg b/OpenOversight/app/db_repository/migrate.cfg deleted file mode 100644 index ec2db66b4..000000000 --- a/OpenOversight/app/db_repository/migrate.cfg +++ /dev/null @@ -1,25 +0,0 @@ -[db_settings] -# Used to identify which repository this database is versioned under. -# You can use the name of your project. -repository_id=database repository - -# The name of the database table used to track the schema version. -# This name shouldn't already be used by your project. -# If this is changed once a database is under version control, you'll need to -# change the table name in each database too. -version_table=migrate_version - -# When committing a change script, Migrate will attempt to generate the -# sql for all supported databases; normally, if one of them fails - probably -# because you don't have that database installed - it is ignored and the -# commit continues, perhaps ending successfully. -# Databases in this list MUST compile successfully during a commit, or the -# entire commit will fail. List the databases your application will actually -# be using to ensure your updates to that database work properly. -# This must be a list; example: ['postgres','sqlite'] -required_dbs=[] - -# When creating new change scripts, Migrate will stamp the new script with -# a version number. By default this is latest_version + 1. You can set this -# to 'true' to tell Migrate to use the UTC timestamp instead. -use_timestamp_numbering=False diff --git a/OpenOversight/app/db_repository/versions/001_migration.py b/OpenOversight/app/db_repository/versions/001_migration.py deleted file mode 100644 index bf6dfb3c3..000000000 --- a/OpenOversight/app/db_repository/versions/001_migration.py +++ /dev/null @@ -1,76 +0,0 @@ -from sqlalchemy import * # pragma: no cover -from migrate import * # pragma: no cover - - -from migrate.changeset import schema # pragma: no cover -pre_meta = MetaData() # pragma: no cover -post_meta = MetaData() # pragma: no cover -users = Table('users', post_meta, # pragma: no cover - Column('id', Integer, primary_key=True, nullable=False), - Column('email', String(length=64)), - Column('username', String(length=64)), - Column('password_hash', String(length=128)), - Column('confirmed', Boolean, default=ColumnDefault(False)), - Column('is_administrator', Boolean, default=ColumnDefault(False)), - Column('is_disabled', Boolean, default=ColumnDefault(False)), -) - -raw_images = Table('raw_images', post_meta, # pragma: no cover - Column('id', Integer, primary_key=True, nullable=False), - Column('filepath', String(length=120)), - Column('hash_img', String(length=120)), - Column('date_image_inserted', DateTime), - Column('date_image_taken', DateTime), - Column('contains_cops', Boolean), - Column('user_id', Integer), - Column('is_tagged', Boolean, default=ColumnDefault(False)), -) - -faces = Table('faces', pre_meta, # pragma: no cover - Column('id', INTEGER, primary_key=True, nullable=False), - Column('officer_id', INTEGER), - Column('img_id', INTEGER), - Column('face_position', VARCHAR(length=120)), -) - -faces = Table('faces', post_meta, # pragma: no cover - Column('id', Integer, primary_key=True, nullable=False), - Column('officer_id', Integer), - Column('img_id', Integer), - Column('face_position_x', Integer), - Column('face_position_y', Integer), - Column('face_width', Integer), - Column('face_height', Integer), - Column('user_id', Integer), -) - - -def upgrade(migrate_engine): # pragma: no cover - # Upgrade operations go here. Don't create your own engine; bind - # migrate_engine to your metadata - pre_meta.bind = migrate_engine - post_meta.bind = migrate_engine - post_meta.tables['users'].create() - post_meta.tables['raw_images'].columns['contains_cops'].create() - post_meta.tables['raw_images'].columns['user_id'].create() - pre_meta.tables['faces'].columns['face_position'].drop() - post_meta.tables['faces'].columns['face_height'].create() - post_meta.tables['faces'].columns['face_position_x'].create() - post_meta.tables['faces'].columns['face_position_y'].create() - post_meta.tables['faces'].columns['face_width'].create() - post_meta.tables['faces'].columns['user_id'].create() - - -def downgrade(migrate_engine): # pragma: no cover - # Operations to reverse the above upgrade go here. - pre_meta.bind = migrate_engine - post_meta.bind = migrate_engine - post_meta.tables['users'].drop() - post_meta.tables['raw_images'].columns['contains_cops'].drop() - post_meta.tables['raw_images'].columns['user_id'].drop() - pre_meta.tables['faces'].columns['face_position'].create() - post_meta.tables['faces'].columns['face_height'].drop() - post_meta.tables['faces'].columns['face_position_x'].drop() - post_meta.tables['faces'].columns['face_position_y'].drop() - post_meta.tables['faces'].columns['face_width'].drop() - post_meta.tables['faces'].columns['user_id'].drop() diff --git a/OpenOversight/app/db_repository/versions/002_migration.py b/OpenOversight/app/db_repository/versions/002_migration.py deleted file mode 100644 index 173f0fcfb..000000000 --- a/OpenOversight/app/db_repository/versions/002_migration.py +++ /dev/null @@ -1,32 +0,0 @@ -from sqlalchemy import * # pragma: no cover -from migrate import * # pragma: no cover - - -from migrate.changeset import schema # pragma: no cover -pre_meta = MetaData() # pragma: no cover -post_meta = MetaData() # pragma: no cover -raw_images = Table('raw_images', post_meta, # pragma: no cover - Column('id', Integer, primary_key=True, nullable=False), - Column('filepath', String(length=255)), - Column('hash_img', String(length=120)), - Column('date_image_inserted', DateTime), - Column('date_image_taken', DateTime), - Column('contains_cops', Boolean), - Column('user_id', Integer), - Column('is_tagged', Boolean, default=ColumnDefault(False)), -) - - -def upgrade(migrate_engine): # pragma: no cover - # Upgrade operations go here. Don't create your own engine; bind - # migrate_engine to your metadata - pre_meta.bind = migrate_engine - post_meta.bind = migrate_engine - post_meta.tables['raw_images'].columns['filepath'].alter(type=String(255)) - - -def downgrade(migrate_engine): # pragma: no cover - # Operations to reverse the above upgrade go here. - pre_meta.bind = migrate_engine - post_meta.bind = migrate_engine - post_meta.tables['raw_images'].columns['filepath'].alter(type=String(120)) diff --git a/OpenOversight/app/db_repository/versions/003_migration.py b/OpenOversight/app/db_repository/versions/003_migration.py deleted file mode 100644 index 7918af664..000000000 --- a/OpenOversight/app/db_repository/versions/003_migration.py +++ /dev/null @@ -1,27 +0,0 @@ -from sqlalchemy import * -from migrate import * - - -from migrate.changeset import schema -pre_meta = MetaData() -post_meta = MetaData() -departments = Table('departments', post_meta, - Column('id', Integer, primary_key=True, nullable=False), - Column('name', String(length=255), index=True, nullable=False), - Column('short_name', String(length=100), nullable=False), -) - - -def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; bind - # migrate_engine to your metadata - pre_meta.bind = migrate_engine - post_meta.bind = migrate_engine - post_meta.tables['departments'].create() - - -def downgrade(migrate_engine): - # Operations to reverse the above upgrade go here. - pre_meta.bind = migrate_engine - post_meta.bind = migrate_engine - post_meta.tables['departments'].drop() diff --git a/OpenOversight/app/db_repository/versions/004_migration.py b/OpenOversight/app/db_repository/versions/004_migration.py deleted file mode 100644 index b72cd01ec..000000000 --- a/OpenOversight/app/db_repository/versions/004_migration.py +++ /dev/null @@ -1,35 +0,0 @@ -from sqlalchemy import * -from migrate import * - - -from migrate.changeset import schema -pre_meta = MetaData() -post_meta = MetaData() - - -raw_images = Table('raw_images', post_meta, - Column('id', Integer, primary_key=True, nullable=False), - Column('filepath', String(length=255)), - Column('hash_img', String(length=120)), - Column('date_image_inserted', DateTime), - Column('date_image_taken', DateTime), - Column('contains_cops', Boolean), - Column('user_id', Integer), - Column('is_tagged', Boolean, default=ColumnDefault(False)), - Column('department_id', Integer), -) - - -def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; bind - # migrate_engine to your metadata - pre_meta.bind = migrate_engine - post_meta.bind = migrate_engine - post_meta.tables['raw_images'].columns['department_id'].create() - - -def downgrade(migrate_engine): - # Operations to reverse the above upgrade go here. - pre_meta.bind = migrate_engine - post_meta.bind = migrate_engine - post_meta.tables['raw_images'].columns['department_id'].drop() diff --git a/OpenOversight/app/db_repository/versions/005_migration.py b/OpenOversight/app/db_repository/versions/005_migration.py deleted file mode 100644 index 313b8111b..000000000 --- a/OpenOversight/app/db_repository/versions/005_migration.py +++ /dev/null @@ -1,41 +0,0 @@ -from sqlalchemy import * -from migrate import * -from migrate.changeset.constraint import ForeignKeyConstraint - -from migrate.changeset import schema -pre_meta = MetaData() -post_meta = MetaData() - -departments = Table('departments', post_meta, - Column('id', Integer, primary_key=True, nullable=False), - Column('name', String(length=255), index=True, nullable=False), - Column('short_name', String(length=100), nullable=False), -) - -raw_images = Table('raw_images', post_meta, - Column('id', Integer, primary_key=True, nullable=False), - Column('filepath', String(length=255)), - Column('hash_img', String(length=120)), - Column('date_image_inserted', DateTime), - Column('date_image_taken', DateTime), - Column('contains_cops', Boolean), - Column('user_id', Integer), - Column('is_tagged', Boolean, default=ColumnDefault(False)), - Column('department_id', Integer), -) - -cons = ForeignKeyConstraint([raw_images.c.department_id], [departments.c.id]) - -def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; bind - # migrate_engine to your metadata - pre_meta.bind = migrate_engine - post_meta.bind = migrate_engine - cons.create() - - -def downgrade(migrate_engine): - # Operations to reverse the above upgrade go here. - pre_meta.bind = migrate_engine - post_meta.bind = migrate_engine - cons.drop() diff --git a/OpenOversight/app/db_repository/versions/__init__.py b/OpenOversight/app/db_repository/versions/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/OpenOversight/app/main/forms.py b/OpenOversight/app/main/forms.py index 33e5f2ac0..6c63a9e83 100644 --- a/OpenOversight/app/main/forms.py +++ b/OpenOversight/app/main/forms.py @@ -8,7 +8,7 @@ Length, Optional) from flask_wtf.file import FileField, FileAllowed, FileRequired -from ..utils import unit_choices +from ..utils import unit_choices, dept_choices # Choices are a list of (value, label) tuples @@ -98,7 +98,7 @@ class AssignmentForm(Form): rank = SelectField('rank', default='COMMANDER', choices=RANK_CHOICES, validators=[AnyOf(allowed_values(RANK_CHOICES))]) unit = QuerySelectField('unit', validators=[Optional()], - query_factory=unit_choices) + query_factory=unit_choices, get_label='descrip') star_date = DateField('star_date', validators=[Optional()]) @@ -112,3 +112,34 @@ class DepartmentForm(Form): default='', validators=[Regexp('\w*'), Length(max=100), DataRequired()] ) submit = SubmitField(label='Add') + + +class AddOfficerForm(Form): + first_name = StringField('First name', default='', validators=[ + Regexp('\w*'), Length(max=50), DataRequired()]) + last_name = StringField('Last name', default='', validators=[ + Regexp('\w*'), Length(max=50), DataRequired()]) + middle_initial = StringField('Middle initial', default='', validators=[ + Regexp('\w*'), Length(max=50), DataRequired()]) + race = SelectField('Race', default='WHITE', choices=RACE_CHOICES, + validators=[AnyOf(allowed_values(RACE_CHOICES))]) + gender = SelectField('Gender', default='M', choices=GENDER_CHOICES, + validators=[AnyOf(allowed_values(GENDER_CHOICES))]) + star_no = IntegerField('Badge Number') + rank = SelectField('Rank', default='COMMANDER', choices=RANK_CHOICES, + validators=[AnyOf(allowed_values(RANK_CHOICES))]) + unit = QuerySelectField('Unit', validators=[Optional()], + query_factory=unit_choices, get_label='descrip') + employment_date = DateField('Employment Date', validators=[Optional()]) + birth_year = IntegerField('Birth Year', validators=[Optional()]) + department = QuerySelectField('Department', validators=[Optional()], + query_factory=dept_choices, get_label='name') + submit = SubmitField(label='Add') + + +class AddUnitForm(Form): + descrip = StringField('Unit name or description', default='', validators=[ + Regexp('\w*'), Length(max=120), DataRequired()]) + department = QuerySelectField('Department', validators=[Optional()], + query_factory=dept_choices, get_label='name') + submit = SubmitField(label='Add') diff --git a/OpenOversight/app/main/views.py b/OpenOversight/app/main/views.py index 91231bede..620ec981f 100644 --- a/OpenOversight/app/main/views.py +++ b/OpenOversight/app/main/views.py @@ -18,9 +18,10 @@ from ..utils import (grab_officers, roster_lookup, upload_file, compute_hash, serve_image, compute_leaderboard_stats, get_random_image, allowed_file, add_new_assignment) -from .forms import (FindOfficerForm, FindOfficerIDForm, - FaceTag, AssignmentForm, DepartmentForm) -from ..models import db, Image, User, Face, Officer, Assignment, Department +from .forms import (FindOfficerForm, FindOfficerIDForm, AddUnitForm, + FaceTag, AssignmentForm, DepartmentForm, AddOfficerForm) +from ..models import (db, Image, User, Face, Officer, Assignment, Department, + Unit) # Ensure the file is read/write by the creator only SAVED_UMASK = os.umask(0o077) @@ -104,7 +105,7 @@ def officer_profile(officer_id): officer = Officer.query.filter_by(id=officer_id).one() except NoResultFound: abort(404) - except: + except: # noqa exception_type, value, full_tback = sys.exc_info() current_app.logger.error('Error finding officer: {}'.format( ' '.join([str(exception_type), str(value), @@ -117,7 +118,7 @@ def officer_profile(officer_id): face_paths = [] for face in faces: face_paths.append(serve_image(face.image.filepath)) - except: + except: # noqa exception_type, value, full_tback = sys.exc_info() current_app.logger.error('Error loading officer profile: {}'.format( ' '.join([str(exception_type), str(value), @@ -188,8 +189,13 @@ def classify_submission(image_id, contains_cops): image.contains_cops = False db.session.commit() flash('Updated image classification') - except: + except: # noqa flash('Unknown error occurred') + exception_type, value, full_tback = sys.exc_info() + current_app.logger.error('Error classifying image: {}'.format( + ' '.join([str(exception_type), str(value), + format_exc(full_tback)]) + )) return redirect(redirect_url()) # return redirect(url_for('main.display_submission', image_id=image_id)) @@ -221,6 +227,54 @@ def add_department(): return render_template('add_department.html', form=form) +@main.route('/officer/new', methods=['GET', 'POST']) +@login_required +@admin_required +def add_officer(): + first_department = Department.query.first() + first_unit = Unit.query.first() + form = AddOfficerForm(department=first_department, unit=first_unit) + if form.validate_on_submit(): + officer = Officer(first_name=form.first_name.data, + last_name=form.last_name.data, + middle_initial=form.middle_initial.data, + race=form.race.data, + gender=form.gender.data, + birth_year=form.birth_year.data, + employment_date=form.employment_date.data, + department_id=form.department.data.id) + db.session.add(officer) + assignment = Assignment(baseofficer=officer, + star_no=form.star_no.data, + rank=form.rank.data, + unit=form.unit.data.id, + star_date=form.employment_date.data) + db.session.add(assignment) + db.session.commit() + flash('New Officer {} added to OpenOversight'.format(officer.last_name)) + return redirect(url_for('main.officer_profile', officer_id=officer.id)) + else: + return render_template('add_officer.html', form=form) + + +@main.route('/unit/new', methods=['GET', 'POST']) +@login_required +@admin_required +def add_unit(): + first_department = Department.query.first() + form = AddUnitForm(department=first_department) + + if form.validate_on_submit(): + unit = Unit(descrip=form.descrip.data, + department_id=form.department.data.id) + db.session.add(unit) + db.session.commit() + flash('New unit {} added to OpenOversight'.format(unit.descrip)) + return redirect(url_for('main.department_overview')) + else: + return render_template('add_unit.html', form=form) + + @main.route('/tag/delete/', methods=['POST']) @login_required @admin_required @@ -229,8 +283,13 @@ def delete_tag(tag_id): Face.query.filter_by(id=tag_id).delete() db.session.commit() flash('Deleted this tag') - except: + except: # noqa flash('Unknown error occurred') + exception_type, value, full_tback = sys.exc_info() + current_app.logger.error('Error classifying image: {}'.format( + ' '.join([str(exception_type), str(value), + format_exc(full_tback)]) + )) return redirect(url_for('main.index')) @@ -390,7 +449,7 @@ def upload(department_id): db.session.add(new_image) db.session.commit() return jsonify(success="Success!"), 200 - except: + except: # noqa exception_type, value, full_tback = sys.exc_info() current_app.logger.error('Error uploading to S3: {}'.format( ' '.join([str(exception_type), str(value), diff --git a/OpenOversight/app/models.py b/OpenOversight/app/models.py index 644a45fc8..204194f53 100644 --- a/OpenOversight/app/models.py +++ b/OpenOversight/app/models.py @@ -2,6 +2,7 @@ from sqlalchemy import UniqueConstraint from werkzeug.security import generate_password_hash, check_password_hash from itsdangerous import TimedJSONWebSignatureSerializer as Serializer +from itsdangerous import BadSignature, BadData from flask_login import UserMixin from flask import current_app from . import login_manager @@ -30,9 +31,10 @@ class Officer(db.Model): gender = db.Column(db.String(120), index=True, unique=False) employment_date = db.Column(db.DateTime, index=True, unique=False, nullable=True) birth_year = db.Column(db.Integer, index=True, unique=False, nullable=True) - pd_id = db.Column(db.Integer, index=True, unique=False) assignments = db.relationship('Assignment', backref='officer', lazy='dynamic') face = db.relationship('Face', backref='officer', lazy='dynamic') + department_id = db.Column(db.Integer, db.ForeignKey('departments.id')) + department = db.relationship('Department', backref='officers') def __repr__(self): return ''.format(self.id, @@ -46,6 +48,7 @@ class Assignment(db.Model): id = db.Column(db.Integer, primary_key=True) officer_id = db.Column(db.Integer, db.ForeignKey('officers.id')) + baseofficer = db.relationship('Officer') star_no = db.Column(db.Integer, index=True, unique=False) rank = db.Column(db.String(120), index=True, unique=False) unit = db.Column(db.Integer, db.ForeignKey('unit_types.id'), nullable=True) @@ -62,6 +65,8 @@ class Unit(db.Model): id = db.Column(db.Integer, primary_key=True) descrip = db.Column(db.String(120), index=True, unique=False) + department_id = db.Column(db.Integer, db.ForeignKey('departments.id')) + department = db.relationship('Department', backref='unit_types') def __repr__(self): return 'Unit: {}'.format(self.descrip) @@ -146,7 +151,7 @@ def confirm(self, token): s = Serializer(current_app.config['SECRET_KEY']) try: data = s.loads(token) - except: + except (BadSignature, BadData): return False if data.get('confirm') != self.id: return False @@ -162,7 +167,7 @@ def reset_password(self, token, new_password): s = Serializer(current_app.config['SECRET_KEY']) try: data = s.loads(token) - except: + except (BadSignature, BadData): return False if data.get('reset') != self.id: return False @@ -178,7 +183,7 @@ def change_email(self, token): s = Serializer(current_app.config['SECRET_KEY']) try: data = s.loads(token) - except: + except (BadSignature, BadData): return False if data.get('change_email') != self.id: return False diff --git a/OpenOversight/app/templates/add_officer.html b/OpenOversight/app/templates/add_officer.html new file mode 100644 index 000000000..fc1668d85 --- /dev/null +++ b/OpenOversight/app/templates/add_officer.html @@ -0,0 +1,33 @@ +{% extends "base.html" %} +{% import "bootstrap/wtf.html" as wtf %} +{% block title %}OpenOversight Admin - Add New Officer{% endblock %} + +{% block content %} +
+ +
+

Add Officer

+
+
+
+ {{ form.hidden_tag() }} + {{ wtf.form_errors(form, hiddens="only") }} + {{ wtf.form_field(form.first_name, autofocus="autofocus") }} + {{ wtf.form_field(form.middle_initial) }} + {{ wtf.form_field(form.last_name) }} + {{ wtf.form_field(form.race) }} + {{ wtf.form_field(form.gender) }} + {{ wtf.form_field(form.star_no) }} + {{ wtf.form_field(form.rank) }} + {{ wtf.form_field(form.unit) }} +

Don't see your unit? Add one!

+ {{ wtf.form_field(form.employment_date) }} + {{ wtf.form_field(form.birth_year) }} + {{ wtf.form_field(form.department) }} + {{ wtf.form_field(form.submit, id="submit", button_map={'submit':'primary'}) }} +
+
+
+ +
+{% endblock %} diff --git a/OpenOversight/app/templates/add_unit.html b/OpenOversight/app/templates/add_unit.html new file mode 100644 index 000000000..6965806e0 --- /dev/null +++ b/OpenOversight/app/templates/add_unit.html @@ -0,0 +1,23 @@ +{% extends "base.html" %} +{% import "bootstrap/wtf.html" as wtf %} +{% block title %}OpenOversight Admin - Add New Unit{% endblock %} + +{% block content %} +
+ +
+

Add Unit

+
+
+
+ {{ form.hidden_tag() }} + {{ wtf.form_errors(form, hiddens="only") }} + {{ wtf.form_field(form.descrip, autofocus="autofocus") }} + {{ wtf.form_field(form.department) }} + {{ wtf.form_field(form.submit, id="submit", button_map={'submit':'primary'}) }} +
+
+
+ +
+{% endblock %} diff --git a/OpenOversight/app/templates/departments.html b/OpenOversight/app/templates/departments.html index a2aa279c1..f3796441d 100644 --- a/OpenOversight/app/templates/departments.html +++ b/OpenOversight/app/templates/departments.html @@ -21,6 +21,18 @@

{{ department.name }} ({{department.short_name}})

Add New Department + + + + Add New Officer + + + + + Add New Unit + {% endif %} diff --git a/OpenOversight/app/utils.py b/OpenOversight/app/utils.py index b8f0a311b..e7330e5e7 100644 --- a/OpenOversight/app/utils.py +++ b/OpenOversight/app/utils.py @@ -7,13 +7,17 @@ from flask import current_app, url_for -from .models import db, Officer, Assignment, Image, Face, User, Unit +from .models import db, Officer, Assignment, Image, Face, User, Unit, Department def unit_choices(): return db.session.query(Unit).all() +def dept_choices(): + return db.session.query(Department).all() + + def add_new_assignment(officer_id, form): # Resign date should be null diff --git a/OpenOversight/manage.py b/OpenOversight/manage.py index d99b34730..4123ca34a 100644 --- a/OpenOversight/manage.py +++ b/OpenOversight/manage.py @@ -1,17 +1,17 @@ from getpass import getpass -import imp -from migrate.versioning import api import sys from flask_script import Manager, Server, Shell +from flask_migrate import Migrate, MigrateCommand from app import app from app.models import db, User -from app.config import config +migrate = Migrate(app, db) manager = Manager(app) manager.add_command("runserver", Server(host="0.0.0.0", port=3000)) +manager.add_command("db", MigrateCommand) def make_shell_context(): @@ -21,49 +21,6 @@ def make_shell_context(): manager.add_command("shell", Shell(make_context=make_shell_context)) -@manager.command -def makemigrations(config_name="default"): - """Make database migrations""" - - SQLALCHEMY_MIGRATE_REPO = config['default'].SQLALCHEMY_MIGRATE_REPO - SQLALCHEMY_DATABASE_URI = config['default'].SQLALCHEMY_DATABASE_URI - - v = api.db_version(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO) - migration = SQLALCHEMY_MIGRATE_REPO + ('/versions/%03d_migration.py' % (v + 1)) - tmp_module = imp.new_module('old_model') - old_model = api.create_model(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO) - exec(old_model, tmp_module.__dict__) - script = api.make_update_script_for_model(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO, tmp_module.meta, db.metadata) - open(migration, "wt").write(script) - - print('New migration saved as ' + migration) - print('Run python manage.py upgrade_db to upgrade the database') - - -@manager.command -def migrate(config_name="default"): - """Migrate/upgrade the database""" - - SQLALCHEMY_MIGRATE_REPO = config['default'].SQLALCHEMY_MIGRATE_REPO - SQLALCHEMY_DATABASE_URI = config['default'].SQLALCHEMY_DATABASE_URI - api.upgrade(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO) - v = api.db_version(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO) - print('Current database version: ' + str(v)) - - -@manager.command -def downgrade_db(config_name="default"): - """Downgrade the database""" - - SQLALCHEMY_MIGRATE_REPO = config['default'].SQLALCHEMY_MIGRATE_REPO - SQLALCHEMY_DATABASE_URI = config['default'].SQLALCHEMY_DATABASE_URI - - v = api.db_version(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO) - api.downgrade(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO, v - 1) - v = api.db_version(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO) - print('Current database version: ' + str(v)) - - @manager.command def make_admin_user(): "Add confirmed administrator account" @@ -115,5 +72,23 @@ def link_images_to_department(): db.session.commit() +@manager.command +def link_officers_to_department(): + """Links officers and units to first department""" + from app.models import Officer, Unit, db + + officers = Officer.query.all() + units = Unit.query.all() + + print "Linking officers and units to first department:" + for item in officers + units: + if not item.department_id: + sys.stdout.write(".") + item.department_id = 1 + else: + print "Skipped! Object already assigned to department!" + db.session.commit() + + if __name__ == "__main__": manager.run() diff --git a/OpenOversight/migrations/README b/OpenOversight/migrations/README new file mode 100755 index 000000000..98e4f9c44 --- /dev/null +++ b/OpenOversight/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/OpenOversight/migrations/alembic.ini b/OpenOversight/migrations/alembic.ini new file mode 100644 index 000000000..f8ed4801f --- /dev/null +++ b/OpenOversight/migrations/alembic.ini @@ -0,0 +1,45 @@ +# A generic, single database configuration. + +[alembic] +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/OpenOversight/migrations/env.py b/OpenOversight/migrations/env.py new file mode 100755 index 000000000..01c81fb31 --- /dev/null +++ b/OpenOversight/migrations/env.py @@ -0,0 +1,88 @@ +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig +import logging + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) +logger = logging.getLogger('alembic.env') + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +from flask import current_app # noqa +config.set_main_option('sqlalchemy.url', + current_app.config.get('SQLALCHEMY_DATABASE_URI')) +target_metadata = current_app.extensions['migrate'].db.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure(url=url) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + # this callback is used to prevent an auto-migration from being generated + # when there are no changes to the schema + # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html + def process_revision_directives(context, revision, directives): + if getattr(config.cmd_opts, 'autogenerate', False): + script = directives[0] + if script.upgrade_ops.is_empty(): + directives[:] = [] + logger.info('No changes in schema detected.') + + engine = engine_from_config(config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + connection = engine.connect() + context.configure(connection=connection, + target_metadata=target_metadata, + process_revision_directives=process_revision_directives, + **current_app.extensions['migrate'].configure_args) + + try: + with context.begin_transaction(): + context.run_migrations() + finally: + connection.close() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/OpenOversight/migrations/script.py.mako b/OpenOversight/migrations/script.py.mako new file mode 100755 index 000000000..2c0156303 --- /dev/null +++ b/OpenOversight/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/OpenOversight/migrations/versions/114919b27a9f_.py b/OpenOversight/migrations/versions/114919b27a9f_.py new file mode 100644 index 000000000..ebc530ca6 --- /dev/null +++ b/OpenOversight/migrations/versions/114919b27a9f_.py @@ -0,0 +1,45 @@ +"""empty message + +Revision ID: 114919b27a9f +Revises: +Create Date: 2017-12-10 05:20:45.748342 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '114919b27a9f' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('migrate_version') + op.add_column('officers', sa.Column('department_id', sa.Integer(), nullable=True)) + op.drop_index('ix_officers_pd_id', table_name='officers') + op.create_foreign_key(None, 'officers', 'departments', ['department_id'], ['id']) + op.drop_column('officers', 'pd_id') + op.add_column('unit_types', sa.Column('department_id', sa.Integer(), nullable=True)) + op.create_foreign_key(None, 'unit_types', 'departments', ['department_id'], ['id']) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, 'unit_types', type_='foreignkey') + op.drop_column('unit_types', 'department_id') + op.add_column('officers', sa.Column('pd_id', sa.INTEGER(), autoincrement=False, nullable=True)) + op.drop_constraint(None, 'officers', type_='foreignkey') + op.create_index('ix_officers_pd_id', 'officers', ['pd_id'], unique=False) + op.drop_column('officers', 'department_id') + op.create_table('migrate_version', + sa.Column('repository_id', sa.VARCHAR(length=250), autoincrement=False, nullable=False), + sa.Column('repository_path', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('version', sa.INTEGER(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('repository_id', name=u'migrate_version_pkey') + ) # noqa + # ### end Alembic commands ### diff --git a/OpenOversight/tests/conftest.py b/OpenOversight/tests/conftest.py index 7d0a0f38a..7cd596c9f 100644 --- a/OpenOversight/tests/conftest.py +++ b/OpenOversight/tests/conftest.py @@ -66,7 +66,7 @@ def generate_officer(): race=pick_race(), gender=pick_gender(), birth_year=year_born, employment_date=datetime(year_born + 20, 4, 4, 1, 1, 1), - pd_id=1 + department_id=1 ) @@ -193,8 +193,9 @@ def mockdata(session, request): session.add(test_unconfirmed_user) session.commit() - test_units = [models.Unit(descrip='District 13'), - models.Unit(descrip='Bureau of Organized Crime')] + test_units = [models.Unit(descrip='District 13', department_id=1), + models.Unit(descrip='Bureau of Organized Crime', + department_id=1)] session.add_all(test_units) session.commit() diff --git a/OpenOversight/tests/test_routes.py b/OpenOversight/tests/test_routes.py index eb22cf2be..3ba7a6d96 100644 --- a/OpenOversight/tests/test_routes.py +++ b/OpenOversight/tests/test_routes.py @@ -5,12 +5,13 @@ from OpenOversight.app.main.forms import (FindOfficerIDForm, AssignmentForm, - FaceTag, DepartmentForm) + FaceTag, DepartmentForm, + AddOfficerForm, AddUnitForm) from OpenOversight.app.auth.forms import (LoginForm, RegistrationForm, ChangePasswordForm, PasswordResetForm, PasswordResetRequestForm, ChangeEmailForm) -from OpenOversight.app.models import User, Face, Department +from OpenOversight.app.models import User, Face, Department, Unit, Officer @pytest.mark.parametrize("route", [ @@ -46,6 +47,8 @@ def test_routes_ok(route, client, mockdata): ('/tag/1'), ('/leaderboard'), ('/department/new'), + ('/officer/new'), + ('/unit/new'), ('/auth/logout'), ('/auth/confirm/abcd1234'), ('/auth/confirm'), @@ -685,3 +688,54 @@ def test_expected_dept_appears_in_submission_dept_selection(mockdata, client, ) assert 'Springfield Police Department' in rv.data + + +def test_admin_can_add_new_officer(mockdata, client, session): + with current_app.test_request_context(): + login_admin(client) + + form = AddOfficerForm(first_name='Test', + last_name='McTesterson', + middle_initial='T', + race='WHITE', + gender='M', + star_no=666, + rank='COMMANDER', + birth_year=1990) + + rv = client.post( + url_for('main.add_officer'), + data=form.data, + follow_redirects=True + ) + + assert 'McTesterson' in rv.data + + # Check the officer was added to the database + officer = Officer.query.filter_by( + last_name='McTesterson').one() + assert officer.first_name == 'Test' + assert officer.race == 'WHITE' + assert officer.gender == 'M' + + +def test_admin_can_add_new_unit(mockdata, client, session): + with current_app.test_request_context(): + login_admin(client) + + department = Department.query.filter_by( + name='Springfield Police Department').first() + form = AddUnitForm(descrip='Test') + + rv = client.post( + url_for('main.add_unit'), + data=form.data, + follow_redirects=True + ) + + assert 'New unit' in rv.data + + # Check the unit was added to the database + unit = Unit.query.filter_by( + descrip='Test').one() + assert unit.department_id == department.id diff --git a/fabfile.py b/fabfile.py index dcca84738..3121b2744 100644 --- a/fabfile.py +++ b/fabfile.py @@ -62,7 +62,7 @@ def deploy(): def migrate(): with cd(env.code_dir): if confirm("Apply any outstanding database migrations?", default=False): - run('su %s -c "%s/bin/python OpenOversight/manage.py migrate"' % (env.unprivileged_user, env.venv_dir)) + run('su %s -c "%s/bin/python OpenOversight/manage.py db upgrade"' % (env.unprivileged_user, env.venv_dir)) run('sudo systemctl restart openoversight') diff --git a/requirements.txt b/requirements.txt index ae3af2c6d..9a3fbff38 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,9 +7,9 @@ Flask-Limiter Flask-Login Flask-Script Flask-Mail +Flask-Migrate psycopg2>=2.6.2 sqlalchemy -sqlalchemy-migrate flask-sqlalchemy>=2.1 flask-bootstrap gunicorn==17.5 diff --git a/test_data.py b/test_data.py index 7588c334f..50ed7997f 100755 --- a/test_data.py +++ b/test_data.py @@ -70,7 +70,7 @@ def generate_officer(): race=pick_race(), gender=pick_gender(), birth_year=year_born, employment_date=datetime(year_born + 20, 4, 4, 1, 1, 1), - pd_id=1 + department_id=1 ) @@ -99,15 +99,15 @@ def populate(): # Add images from Springfield Police Department image1 = models.Image(filepath='static/images/test_cop1.png', - department_id=1) + department_id=department.id) image2 = models.Image(filepath='static/images/test_cop2.png', - department_id=1) + department_id=department.id) image3 = models.Image(filepath='static/images/test_cop3.png', - department_id=1) + department_id=department.id) image4 = models.Image(filepath='static/images/test_cop4.png', - department_id=1) + department_id=department.id) image5 = models.Image(filepath='static/images/test_cop5.jpg', - department_id=1) + department_id=department.id) test_images = [image1, image2, image3, image4, image5] db.session.add_all(test_images) @@ -135,8 +135,8 @@ def populate(): db.session.add(test_user) db.session.commit() - test_units = [models.Unit(descrip='District 13'), - models.Unit(descrip='Bureau of Organized Crime')] + test_units = [models.Unit(descrip='District 13', department_id=1), + models.Unit(descrip='Bureau of Organized Crime', department_id=1)] db.session.add_all(test_units) db.session.commit()