diff --git a/MANIFEST.in b/MANIFEST.in index 92a464cb4..8b890d639 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -10,8 +10,13 @@ include openreview/agora/process/*.py include openreview/duplicate_domains.json include openreview/journal/process/*.py include openreview/journal/webfield/*.js +include openreview/profile/process/*.py +include openreview/profile/process/*.js +include openreview/profile/webfield/*.js include openreview/venue/process/*.py include openreview/venue/process/*.js +include openreview/venue/configuration/process/*.py +include openreview/venue/configuration/webfield/*.js include openreview/venue/webfield/*.js include openreview/arr/process/*.js include openreview/arr/process/*.py diff --git a/openreview/api/client.py b/openreview/api/client.py index 3914e5a25..c246a44e9 100644 --- a/openreview/api/client.py +++ b/openreview/api/client.py @@ -2019,7 +2019,7 @@ def post_institution(self, institution): response = self.__handle_response(response) return response.json() - def post_invitation_edit(self, invitations, readers=None, writers=None, signatures=None, invitation=None, content=None, replacement=None): + def post_invitation_edit(self, invitations, readers=None, writers=None, signatures=None, invitation=None, content=None, replacement=None, domain=None): """ """ edit_json = {} @@ -2045,6 +2045,9 @@ def post_invitation_edit(self, invitations, readers=None, writers=None, signatur if invitation is not None: edit_json['invitation'] = invitation.to_json() + if domain is not None: + edit_json['domain'] = domain + response = self.session.post(self.invitation_edits_url, json = edit_json, headers = self.headers) response = self.__handle_response(response) @@ -2323,6 +2326,7 @@ def __init__(self, readers = None, writers = None, signatures = None, + content = None, note = None, group = None, invitation = None, @@ -2331,8 +2335,7 @@ def __init__(self, tcdate = None, tmdate = None, ddate = None, - tauthor = None, - content = None): + tauthor = None): self.id = id self.domain = domain @@ -2345,11 +2348,11 @@ def __init__(self, self.nonreaders = nonreaders self.writers = writers self.signatures = signatures + self.content = content self.note = note self.group = group self.invitation = invitation self.tauthor = tauthor - self.content = content def __repr__(self): content = ','.join([("%s = %r" % (attr, value)) for attr, value in vars(self).items()]) @@ -2380,6 +2383,8 @@ def to_json(self): body['writers'] = self.writers if (self.signatures): body['signatures'] = self.signatures + if (self.content): + body['content'] = self.content if (self.note): body['note'] = self.note.to_json() if (self.group): @@ -2390,8 +2395,6 @@ def to_json(self): body['invitation'] = self.invitation if (self.ddate): body['ddate'] = self.ddate - if (self.content): - body['content'] = self.content return body @@ -2417,11 +2420,11 @@ def from_json(Edit,e): nonreaders = e.get('nonreaders'), writers = e.get('writers'), signatures = e.get('signatures'), + content = e.get('content'), note = Note.from_json(e['note']) if 'note' in e else None, group = Group.from_json(e['group']) if 'group' in e else None, invitation = e.get('invitation'), - tauthor = e.get('tauthor'), - content = e.get('content') + tauthor = e.get('tauthor') ) if isinstance(edit.invitation, dict): @@ -2643,7 +2646,7 @@ def is_active(self): def get_content_value(self, field_name, default_value=None): if self.content: - return self.content.get(field_name, {}).get('value') + return self.content.get(field_name, {}).get('value', default_value) return default_value def pretty_id(self): @@ -2937,7 +2940,7 @@ def __init__(self, id=None, content=None, readers=None, writers=None, signatorie def get_content_value(self, field_name, default_value=None): if self.content: - return self.content.get(field_name, {}).get('value') + return self.content.get(field_name, {}).get('value', default_value) return default_value def __repr__(self): diff --git a/openreview/conference/helpers.py b/openreview/conference/helpers.py index 89eb2bf25..ca3c1e453 100644 --- a/openreview/conference/helpers.py +++ b/openreview/conference/helpers.py @@ -3,6 +3,110 @@ import datetime import json +def get_venue(client, venue_note_id, support_user='OpenReview.net/Support', setup=False): + + note = client.get_note(venue_note_id) + venue = openreview.venue.Venue(client, note.content['venue_id']['value'], support_user) + venue.name = note.content['official_venue_name']['value'] + venue.short_name = note.content['abbreviated_venue_name']['value'] + venue.website = note.content['venue_website_url']['value'] + venue.contact = note.content['contact_email']['value'] + venue.location = note.content['location']['value'] + set_start_date(note, venue) + venue.request_form_id = venue_note_id + venue.use_area_chairs = 'Yes' in note.content.get('area_chairs_and_senior_area_chairs', {}).get('value','') + venue.use_senior_area_chairs = note.content.get('area_chairs_and_senior_area_chairs', {}).get('value','') == 'Yes, our venue has Area Chairs and Senior Area Chairs' + venue.use_secondary_area_chairs = note.content.get('secondary_area_chairs', {}).get('value','') == 'Yes, our venue has Secondary Area Chairs' + venue.use_ethics_chairs = venue.use_ethics_reviewers = note.content.get('ethics_chairs_and_reviewers', {}).get('value', '') == 'Yes, our venue has Ethics Chairs and Reviewers' + venue.use_publication_chairs = note.content.get('publication_chairs', {}).get('value', '') == 'Yes, our venue has Publication Chairs' + + set_initial_stages_v2(note, venue) + venue.expertise_selection_stage = openreview.stages.ExpertiseSelectionStage(due_date = venue.submission_stage.due_date) + if setup: + venue.setup(note.content.get('program_chair_emails',{}).get('value')) + return venue + +def set_start_date(request_forum, venue): + + venue_start_date_str = 'TBD' + venue_start_date = None + start_date = request_forum.content.get('venue_start_date', {}).get('value', '').strip() + if start_date: + try: + venue_start_date = datetime.datetime.strptime(start_date, '%Y/%m/%d %H:%M') + except ValueError: + venue_start_date = datetime.datetime.strptime(start_date, '%Y/%m/%d') + venue_start_date_str = venue_start_date.strftime('%b %d %Y') + + venue.start_date = venue_start_date_str + +def set_initial_stages_v2(request_forum, venue): + + readers_map = { + 'All program committee (all reviewers, all area chairs, all senior area chairs if applicable)': [openreview.stages.SubmissionStage.Readers.SENIOR_AREA_CHAIRS, openreview.stages.SubmissionStage.Readers.AREA_CHAIRS, openreview.stages.SubmissionStage.Readers.REVIEWERS], + 'All area chairs only': [openreview.stages.SubmissionStage.Readers.SENIOR_AREA_CHAIRS, openreview.stages.SubmissionStage.Readers.AREA_CHAIRS], + 'Assigned program committee (assigned reviewers, assigned area chairs, assigned senior area chairs if applicable)': [openreview.stages.SubmissionStage.Readers.SENIOR_AREA_CHAIRS_ASSIGNED, openreview.stages.SubmissionStage.Readers.AREA_CHAIRS_ASSIGNED, openreview.stages.SubmissionStage.Readers.REVIEWERS_ASSIGNED], + 'Program chairs and paper authors only': [], + 'Everyone (submissions are public)': [openreview.stages.SubmissionStage.Readers.EVERYONE], + 'Make accepted submissions public and hide rejected submissions': [openreview.stages.SubmissionStage.Readers.EVERYONE_BUT_REJECTED] + } + #readers = readers_map[request_forum.content.get('submission_readers', {}).get('value', [])] + + submission_start_date = request_forum.content.get('submission_start_date', {}).get('value', '').strip() + + submission_start_date_str = '' + if submission_start_date: + try: + submission_start_date = datetime.datetime.strptime(submission_start_date, '%Y/%m/%d %H:%M') + except ValueError: + submission_start_date = datetime.datetime.strptime(submission_start_date, '%Y/%m/%d') + submission_start_date_str = submission_start_date.strftime('%b %d %Y %I:%M%p') + ' UTC-0' + else: + submission_start_date = None + + submission_deadline_str = 'TBD' + abstract_due_date_str = '' + submission_second_due_date = request_forum.content.get('submission_deadline', {}).get('value', '').strip() + if submission_second_due_date: + try: + submission_second_due_date = datetime.datetime.strptime(submission_second_due_date, '%Y/%m/%d %H:%M') + except ValueError: + submission_second_due_date = datetime.datetime.strptime(submission_second_due_date, '%Y/%m/%d') + submission_deadline_str = submission_second_due_date.strftime('%b %d %Y %I:%M%p') + ' UTC-0' + submission_due_date = request_forum.content.get('abstract_registration_deadline', {}).get('value', '').strip() + if submission_due_date: + try: + submission_due_date = datetime.datetime.strptime(submission_due_date, '%Y/%m/%d %H:%M') + except ValueError: + submission_due_date = datetime.datetime.strptime(submission_due_date, '%Y/%m/%d') + abstract_due_date_str = submission_due_date.strftime('%b %d %Y %I:%M%p') + ' UTC-0' + else: + submission_due_date = submission_second_due_date + submission_second_due_date = None + else: + submission_second_due_date = submission_due_date = None + + date = 'Submission Start: ' + submission_start_date_str + ', ' if submission_start_date_str else '' + if abstract_due_date_str: + date += 'Abstract Registration: ' + abstract_due_date_str + ', ' + date += 'Submission Deadline: ' + submission_deadline_str + venue.date = date + + venue.submission_stage = openreview.stages.SubmissionStage( + start_date=submission_start_date, + due_date=submission_due_date, + second_due_date=submission_second_due_date, + #readers=readers, + double_blind=request_forum.content.get('author_and_reviewer_anonymity', {}).get('value', '') == 'Double-blind', + #email_pcs='Yes' in request_forum.content.get('email_pcs_for_new_submissions', {}).get('value', ''), + #force_profiles='Yes' in request_forum.content.get('force_profiles_only', {}).get('value', '') + ) + + venue.review_stage = openreview.stages.ReviewStage( + start_date = (submission_second_due_date if submission_second_due_date else submission_due_date) + datetime.timedelta(weeks=1), + allow_de_anonymization = (request_forum.content.get('author_and_reviewer_anonymity', {}).get('value', 'No anonymity') == 'No anonymity'), + ) + def get_conference(client, request_form_id, support_user='OpenReview.net/Support', setup=False): note = client.get_note(request_form_id) diff --git a/openreview/stages/venue_stages.py b/openreview/stages/venue_stages.py index 22678dba6..721266371 100644 --- a/openreview/stages/venue_stages.py +++ b/openreview/stages/venue_stages.py @@ -541,7 +541,7 @@ def get_content(self, api_version='2', conference=None): invitation_id = conference.get_invitation_id(self.name) invitation = openreview.tools.get_invitation(conference.client, invitation_id) if invitation: - for field, value in invitation.edit['invitation']['edit']['note']['content'].items(): + for field, value in invitation.edit.get('invitation', {}).get('edit', {}).get('note', {}).get('content', {}).items() if invitation.edit else {}: if field not in content: content[field] = { 'delete': True } @@ -670,7 +670,7 @@ def get_content(self, api_version='2', conference=None): invitation_id = conference.get_invitation_id(self.name) invitation = openreview.tools.get_invitation(conference.client, invitation_id) if invitation: - for field, value in invitation.edit['invitation']['edit']['note']['content'].items(): + for field, value in invitation.edit.get('invitation', {}).get('edit', {}).get('note', {}).get('content', {}).items() if invitation.edit else {}: if field not in content: content[field] = { 'delete': True } @@ -813,7 +813,7 @@ def get_content(self, api_version='2', conference=None): invitation_id = conference.get_invitation_id(self.name) invitation = openreview.tools.get_invitation(conference.client, invitation_id) if invitation: - for field, value in invitation.edit['invitation']['edit']['note']['content'].items(): + for field, value in invitation.edit.get('invitation', {}).get('edit', {}).get('note', {}).get('content', {}).items() if invitation.edit else {}: if field not in content: content[field] = { 'delete': True } @@ -896,7 +896,7 @@ def get_content(self, api_version='2', conference=None): invitation_id = conference.get_invitation_id(self.name) invitation = openreview.tools.get_invitation(conference.client, invitation_id) if invitation: - for field, value in invitation.edit['invitation']['edit']['note']['content'].items(): + for field, value in invitation.edit.get('invitation', {}).get('edit', {}).get('note', {}).get('content', {}).items() if invitation.edit else {}: if field not in content: content[field] = { 'delete': True } @@ -1019,7 +1019,8 @@ def get_readers(self, conference, number, api_version='1'): if self.Readers.REVIEWERS_SUBMITTED in self.readers: readers.append({ 'value': conference.get_reviewers_id(number) + '/Submitted', 'optional': True }) - readers.append({ 'prefix': conference.get_anon_reviewer_id(number=number, anon_id='.*'), 'optional': True }) + if self.Readers.REVIEWERS_ASSIGNED in self.readers or self.Readers.REVIEWERS_SUBMITTED in self.readers: + readers.append({ 'prefix': conference.get_anon_reviewer_id(number=number, anon_id='.*'), 'optional': True }) if self.Readers.AUTHORS in self.readers: readers.append({ 'value': conference.get_authors_id(number), 'optional': True }) @@ -1160,7 +1161,7 @@ class Readers(Enum): REVIEWERS_SUBMITTED = 2 NO_REVIEWERS = 3 - def __init__(self, name='Meta_Review', start_date = None, due_date = None, exp_date = None, public = False, release_to_authors = False, release_to_reviewers = Readers.NO_REVIEWERS, additional_fields = {}, remove_fields=[], process = None, recommendation_field_name = 'recommendation', source_submissions_query = {}, child_invitations_name = 'Meta_Review'): + def __init__(self, name='Meta_Review', start_date = None, due_date = None, exp_date = None, public = False, release_to_authors = False, release_to_reviewers = Readers.NO_REVIEWERS, additional_fields = {}, remove_fields=[], process = None, recommendation_field_name = 'recommendation', source_submissions_query = {}, child_invitations_name = 'Meta_Review', content=None): self.start_date = start_date self.due_date = due_date @@ -1177,6 +1178,7 @@ def __init__(self, name='Meta_Review', start_date = None, due_date = None, exp_d self.preprocess_path = None self.source_submissions_query = source_submissions_query self.child_invitations_name = child_invitations_name + self.content = content def _get_reviewer_readers(self, conference, number): if self.release_to_reviewers is MetaReviewStage.Readers.REVIEWERS: @@ -1240,6 +1242,9 @@ def get_signatures(self, conference, number): return committee def get_content(self, api_version='2', conference=None): + + if self.content: + return self.content content = default_content.meta_review_v2.copy() @@ -1256,7 +1261,7 @@ def get_content(self, api_version='2', conference=None): invitation_id = conference.get_invitation_id(self.name) invitation = openreview.tools.get_invitation(conference.client, invitation_id) if invitation: - for field, value in invitation.edit['invitation']['edit']['note']['content'].items(): + for field, value in invitation.edit.get('invitation', {}).get('edit', {}).get('note', {}).get('content', {}).items() if invitation.edit else {}: if field not in content: content[field] = { 'delete': True } @@ -1273,14 +1278,14 @@ def __init__(self, start_date = None, due_date = None, name = 'Meta_Review_Revis class DecisionStage(object): - def __init__(self, options = None, accept_options = None, start_date = None, due_date = None, public = False, release_to_authors = False, release_to_reviewers = False, release_to_area_chairs = False, email_authors = False, additional_fields = {}, decisions_file=None): + def __init__(self, name = 'Decision', options = None, accept_options = None, start_date = None, due_date = None, public = False, release_to_authors = False, release_to_reviewers = False, release_to_area_chairs = False, email_authors = False, additional_fields = {}, decisions_file=None, content=None): if not options: options = ['Accept (Oral)', 'Accept (Poster)', 'Reject'] self.options = options self.accept_options = accept_options self.start_date = start_date self.due_date = due_date - self.name = 'Decision' + self.name = name self.public = public self.release_to_authors = release_to_authors self.release_to_reviewers = release_to_reviewers @@ -1290,6 +1295,7 @@ def __init__(self, options = None, accept_options = None, start_date = None, due self.decisions_file = decisions_file self.decision_field_name = 'decision' self.remove_fields = [] + self.content = content def get_readers(self, conference, number): @@ -1322,6 +1328,11 @@ def get_nonreaders(self, conference, number): return [conference.get_authors_id(number = number)] def get_content(self, api_version='2', conference=None): + + if self.content: + if 'decision' in self.content: + self.content['decision']['value']['param']['enum'] = self.options + return self.content content = default_content.decision_v2.copy() @@ -1338,7 +1349,7 @@ def get_content(self, api_version='2', conference=None): invitation_id = conference.get_invitation_id(self.name) invitation = openreview.tools.get_invitation(conference.client, invitation_id) if invitation: - for field, value in invitation.edit['invitation']['edit']['note']['content'].items(): + for field, value in invitation.edit.get('invitation', {}).get('edit', {}).get('note', {}).get('content', {}).items() if invitation.edit else {}: if field not in content: content[field] = { 'delete': True } @@ -1400,7 +1411,7 @@ def get_content(self, api_version='2', conference=None): invitation_id = conference.get_invitation_id(self.name) invitation = openreview.tools.get_invitation(conference.client, invitation_id) if invitation: - for field, value in invitation.edit['invitation']['edit']['note']['content'].items(): + for field, value in invitation.edit.get('invitation', {}).get('edit', {}).get('note', {}).get('content', {}).items() if invitation.edit else {}: if field not in content: content[field] = { 'delete': True } @@ -1609,7 +1620,7 @@ def get_content(self, api_version='2', conference=None): invitation_id = conference.get_invitation_id(self.name) invitation = openreview.tools.get_invitation(conference.client, invitation_id) if invitation: - for field, value in invitation.edit['invitation']['edit']['note']['content'].items(): + for field, value in invitation.edit.get('invitation', {}).get('edit', {}).get('note', {}).get('content', {}).items() if invitation.edit else {}: if field not in content: content[field] = { 'delete': True } diff --git a/openreview/venue/__init__.py b/openreview/venue/__init__.py index da3e857dd..b95c97921 100644 --- a/openreview/venue/__init__.py +++ b/openreview/venue/__init__.py @@ -1 +1,2 @@ from .venue import * +from .configuration import * \ No newline at end of file diff --git a/openreview/venue/configuration/__init__.py b/openreview/venue/configuration/__init__.py new file mode 100644 index 000000000..c41f1a850 --- /dev/null +++ b/openreview/venue/configuration/__init__.py @@ -0,0 +1 @@ +from .invitations import * \ No newline at end of file diff --git a/openreview/venue/configuration/invitations.py b/openreview/venue/configuration/invitations.py new file mode 100644 index 000000000..eea51db00 --- /dev/null +++ b/openreview/venue/configuration/invitations.py @@ -0,0 +1,992 @@ + +import openreview.api +from ... import openreview +from openreview.api import Invitation +import os +from ...stages import default_content + +class VenueConfiguration(): + + def __init__(self, client, support_group_id, super_user): + self.support_group_id = support_group_id + self.client = client + self.super_user = super_user + self.meta_invitation_id = f'{super_user}/-/Edit' + + def setup(self): + self.set_meta_invitation() + self.set_conference_venue_invitation() + self.set_comment_invitation() + self.set_deploy_invitation() + self.set_venues_homepage() + + #setup stage templates + workflow_invitations = WorkflowInvitations(self.client, self.support_group_id, self.super_user) + workflow_invitations.setup_metareview_template_invitation() + workflow_invitations.setup_comment_template_invitation() + workflow_invitations.setup_decision_template_invitation() + + def get_process_content(self, file_path): + process = None + with open(os.path.join(os.path.dirname(__file__), file_path)) as f: + process = f.read() + return process + + def post_invitation_edit(self, invitation): + return self.client.post_invitation_edit(invitations=self.meta_invitation_id, + readers=['~Super_User1'], + writers=['~Super_User1'], + signatures=['~Super_User1'], + invitation=invitation, + replacement=True + ) + + def set_meta_invitation(self): + + self.client.post_invitation_edit(invitations=None, + readers=['~Super_User1'], + writers=['~Super_User1'], + signatures=['~Super_User1'], + invitation=Invitation(id=self.meta_invitation_id, + invitees=['~Super_User1'], + readers=['~Super_User1'], + signatures=['~Super_User1'], + edit=True + ) + ) + + def set_venues_homepage(self): + + with open(os.path.join(os.path.dirname(__file__), 'webfield/venuepageWebfield.js')) as f: + content = f.read() + self.client.post_group_edit( + invitation=self.meta_invitation_id, + signatures=['~Super_User1'], + group=openreview.api.Group( + id='venues', + web=content + ) + ) + + def set_conference_venue_invitation(self): + + super_user = self.super_user + conference_venue_invitation_id = f'{super_user}/-/Venue_Configuration_Request' + + invitation_content = { + 'title': { + 'description': 'Used for display purposes. This is copied from the Official Venue Name', + 'order': 1, + 'value': { + 'param': { + 'type': 'string', + 'const': '${2/official_venue_name/value}', + 'hidden': True + } + } + }, + 'official_venue_name': { + 'order': 2, + 'description': 'The official name of your venue. This will appear on your venue\'s OpenReview page. Example: "Seventh International Conference on Learning Representations', + 'value': { + 'param': { + 'type': 'string', + 'regex': '.{0,500}' + } + } + }, + 'abbreviated_venue_name': { + 'order': 3, + 'description': 'The abbreviated name for your venue. Please include the year as well. This will be used to identify your venue on OpenReview and in email subject lines. Example: "ICLR 2019"', + 'value': { + 'param': { + 'type': 'string', + 'regex': '.{0,500}' + } + } + }, + 'venue_website_url': { + 'order': 4, + 'description': 'The URL of the official website for your venue.', + 'value': { + 'param': { + 'type': 'string', + 'regex': '.{0,500}' + } + } + }, + 'location': { + 'order': 5, + 'description': 'Where will your venue be held? This will be displayed on your venue\'s OpenReview page. Example: "Vancouver, Canada"', + 'value': { + 'param': { + 'type': 'string', + 'regex': '.{0,500}' + } + } + }, + 'venue_start_date': { + 'order': 6, + 'description': 'What date does the venue start? Please enter a time and date in GMT using the following format: YYYY/MM/DD (e.g. 2019/01/31)', + 'value': { + 'param': { + 'type': 'string', + 'regex': r'^[0-9]{4}\/([1-9]|0[1-9]|1[0-2])\/([1-9]|0[1-9]|[1-2][0-9]|3[0-1])(\s+)?((2[0-3]|[01][0-9]|[0-9]):[0-5][0-9])?(\s+)?$', + 'optional': True + } + } + }, + 'program_chair_emails': { + 'order': 7, + 'description': 'A comma separated list of *lower-cased* email addresses for the program chairs of your venue, including the PC submitting this request', + 'value': { + 'param': { + 'type': 'string[]', + 'regex': r"^[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$" + } + } + }, + 'contact_email': { + 'order': 8, + 'description': 'Single point of contact *lower-cased* email address which will be displayed on the venue page. For example: pc@venue.org', + 'value': { + 'param': { + 'type': 'string', + 'regex': r"^[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$" + } + } + }, + 'publication_chairs': { + 'order': 9, + 'description': 'Will your venue have Publication Chairs? The Publication Chairs will only have access to accepted submissions (including author names) and the author accepted group in order to email authors of accepted submissions.', + 'value': { + 'param': { + 'type': 'string', + 'enum': [ + 'Yes, our venue has Publication Chairs', + 'No, our venue does not have Publication Chairs' + ], + 'input': 'radio' + } + } + }, + 'area_chairs_and_senior_area_chairs' : { + 'order': 10, + 'description': 'Will your venue have Area Chairs and Senior Area Chairs?', + 'value': { + 'param': { + 'type': 'string', + 'enum': [ + 'Yes, our venue has Area Chairs and Senior Area Chairs', + 'Yes, our venue has Area Chairs, but no Senior Area Chairs', + 'No, our venue does not have Area Chairs nor Senior Area Chairs' + ], + 'input': 'radio' + } + } + }, + 'ethics_chairs_and_reviewers' : { + 'order': 11, + 'description': 'Will your venue have Ethics Chairs and Ethics Reviewers?', + 'value': { + 'param': { + 'type': 'string', + 'enum': [ + 'Yes, our venue has Ethics Chairs and Reviewers', + 'No, our venue does not have Ethics Chairs and Reviewers' + ], + 'input': 'radio' + } + } + }, + 'secondary_area_chairs': { + 'order': 12, + 'description': 'Will your venue have Secondary Area Chairs?', + 'value': { + 'param': { + 'type': 'string', + 'enum': [ + 'Yes, our venue has Secondary Area Chairs', + 'No, our venue does not have Secondary Area Chairs' + ], + 'input': 'radio' + } + } + }, + 'submission_start_date': { + 'order': 13, + 'description': 'When should your OpenReview submission portal open? Please specify the date and time in GMT using the following format: YYYY/MM/DD HH:MM(e.g. 2019/01/31 23:59). (Leave blank if you would like the portal to open for submissions as soon as possible)', + 'value': { + 'param': { + 'type': 'string', + 'regex': r'^[0-9]{4}\/([1-9]|0[1-9]|1[0-2])\/([1-9]|0[1-9]|[1-2][0-9]|3[0-1])(\s+)?((2[0-3]|[01][0-9]|[0-9]):[0-5][0-9])?(\s+)?$', + 'optional': True + } + } + }, + 'abstract_registration_deadline': { + 'order': 14, + 'description': 'By when do authors need to register their manuscripts? Please specify the due date in GMT using the following format: YYYY/MM/DD HH:MM(e.g. 2019/01/31 23:59) (Skip this if there is no abstract registration deadline)', + 'value': { + 'param': { + 'type': 'string', + 'regex': r'^[0-9]{4}\/([1-9]|0[1-9]|1[0-2])\/([1-9]|0[1-9]|[1-2][0-9]|3[0-1])(\s+)?((2[0-3]|[01][0-9]|[0-9]):[0-5][0-9])?(\s+)?$', + 'optional': True + } + } + }, + 'submission_deadline': { + 'order': 15, + 'description': 'By when do authors need to submit their manuscripts? Please specify the due date in GMT using the following format: YYYY/MM/DD HH:MM(e.g. 2019/01/31 23:59)', + 'value': { + 'param': { + 'type': 'string', + 'regex': r'^[0-9]{4}\/([1-9]|0[1-9]|1[0-2])\/([1-9]|0[1-9]|[1-2][0-9]|3[0-1])(\s+)?((2[0-3]|[01][0-9]|[0-9]):[0-5][0-9])?(\s+)?$', + 'optional': True + } + } + }, + 'author_and_reviewer_anonymity': { + 'order': 16, + 'description': 'Whick best describes your anonymity policy?', + 'value': { + 'param': { + 'type': 'string', + 'enum': [ + 'Double-blind', + 'Single-blind (Reviewers are anonymous)', + 'No anonymity' + ], + 'input': 'radio' + } + } + }, + # 'force_profiles_only': { + # 'order': 17, + # 'description': 'Submitting authors must have an OpenReview profile, however, should all co-authors be required to have profiles?', + # 'value': { + # 'param': { + # 'type': 'string', + # 'enum': [ + # 'Yes, require all authors to have an OpenReview profile', + # 'No, allow submissions with email addresses' + # ], + # 'input': 'radio' + # } + # } + # }, + # 'submission_readers': { + # 'order': 18, + # 'description': 'Please select who should have access to the submissions after the abstract deadline (if your venue has one) or the submission deadline. Note that program chairs and paper authors are always readers of submissions.', + # 'value': { + # 'param': { + # 'type': 'string', + # 'enum': [ + # 'All program committee (all reviewers, all area chairs, all senior area chairs if applicable)', + # 'All area chairs only', + # 'Assigned program committee (assigned reviewers, assigned area chairs, assigned senior area chairs if applicable)', + # 'Program chairs and paper authors only', + # 'Everyone (submissions are public)' + # ], + # 'input': 'radio' + # } + # } + # }, + 'submission_license': { + 'order': 19, + 'description': 'Which license should be applied to each submission? We recommend "CC BY 4.0". If you select multiple licenses, you allow authors to choose their license upon submission. If your license is not listed, please contact us. Refer to https://openreview.net/legal/terms for more information.', + 'value': { + 'param': { + 'type': 'string[]', + 'enum': [ + 'CC BY 4.0', + 'CC BY-SA 4.0', + 'CC BY-NC 4.0', + 'CC BY-ND 4.0', + 'CC BY-NC-SA 4.0', + 'CC BY-NC-ND 4.0', + 'CC0 1.0' + ], + 'input': 'checkbox' + } + } + }, + # 'email_pcs_for_new_submissions': { + # 'order': 20, + # 'description': 'Should PCs receive an email for every new submission?', + # 'value': { + # 'param': { + # 'type': 'string', + # 'enum': [ + # 'Yes, email PCs for every new submission.', + # 'No, do not email PCs.' + # ], + # 'input': 'radio' + # } + # } + # }, + 'other_important_information': { + 'order': 21, + 'description': 'Please provide any other important information about your venue that you would like to share with OpenReview. Please use this space to clarify any questions for which you could not use any of the provided options, and to clarify any other information that you think we may need.', + 'value': { + 'param': { + 'type': 'string', + 'maxLength': 5000, + 'optional': True, + 'deletable': True, + 'input': 'textarea' + } + } + }, + 'commitments_venue': { + 'order': 22, + 'value': { + 'param': { + 'type': 'string', + 'enum': ['Yes', 'No'], + 'input': 'radio', + 'hidden': True, + 'optional': True + } + } + } + } + + invitation = Invitation( + id = conference_venue_invitation_id, + invitees = ['everyone'], + readers = ['everyone'], + writers = [], + signatures = [super_user], + edit = { + 'signatures': { 'param': { 'regex': '~.*' } }, + 'writers': ['${2/note/writers}'], + 'readers': ['${2/note/readers}'], + 'note': { + 'signatures': ['${3/signatures}'], + 'readers': [ self.support_group_id, '${2/content/program_chair_emails/value}', '${3/signatures}' ], + 'writers': [ self.support_group_id, '${2/content/program_chair_emails/value}', '${3/signatures}'], + 'content': invitation_content, + 'id' : { + 'param': { + 'withInvitation': conference_venue_invitation_id, + 'optional': True + } + } + } + }, + process=self.get_process_content('process/support_process.py') + ) + + self.post_invitation_edit(invitation) + + def set_comment_invitation(self): + + super_user = self.super_user + comment_invitation_id = f'{super_user}/-/Comment' + support_group_id = self.support_group_id + + invitation = Invitation( + id = comment_invitation_id, + invitees = [support_group_id], + readers = ['everyone'], + writers = [support_group_id], + signatures = [support_group_id], + edit = { + 'signatures': [support_group_id], + 'readers': [support_group_id], + 'content': { + 'noteNumber': { + 'value': { + 'param': { + 'regex': '.*', 'type': 'integer' + } + } + }, + 'noteId': { + 'value': { + 'param': { + 'regex': '.*', 'type': 'string' + } + } + } + }, + 'replacement': True, + 'invitation': { + 'id': f'{super_user}/Venue_Configuration_Request' + '${2/content/noteNumber/value}' + '/-/Comment', + 'signatures': [ support_group_id ], + 'readers': ['everyone'], + 'writers': [support_group_id], + 'invitees': ['everyone'], + 'edit': { + 'signatures': { + 'param': { + 'items': [ + { 'value': support_group_id, 'optional': True }, + { 'prefix': '~.*', 'optional': True } ] + } + }, + 'readers': ['${2/note/readers}'], + 'writers': [support_group_id], + 'note': { + 'id': { + 'param': { + 'withInvitation': f'{super_user}/Venue_Configuration_Request' + '${6/content/noteNumber/value}' + '/-/Comment', + 'optional': True + } + }, + 'forum': '${4/content/noteId/value}', + 'replyto': { + 'param': { + 'withForum': '${6/content/noteId/value}', + } + }, + 'ddate': { + 'param': { + 'range': [ 0, 9999999999999 ], + 'optional': True, + 'deletable': True + } + }, + 'readers': [support_group_id, '${{3/note/forum}/content/program_chair_emails/value}'], + 'writers': [support_group_id, '${3/signatures}'], + 'signatures': ['${3/signatures}'], + 'content': { + 'title': { + 'order': 1, + 'description': 'Brief summary of your comment.', + 'value': { + 'param': { + 'type': 'string', + 'maxLength': 500, + 'optional': True, + 'deletable': True + } + } + }, + 'comment': { + 'order': 2, + 'description': 'Your comment or reply (max 200000 characters).', + 'value': { + 'param': { + 'type': 'string', + 'maxLength': 200000, + 'markdown': True, + 'input': 'textarea' + } + } + } + } + } + } + } + } + ) + + self.post_invitation_edit(invitation) + + def set_deploy_invitation(self): + + deploy_invitation_id = f'{self.super_user}/-/Deploy' + support_group_id = self.support_group_id + super_user = self.super_user + + invitation = Invitation( + id = deploy_invitation_id, + invitees = [support_group_id], + readers = ['everyone'], + writers = [support_group_id], + signatures = [support_group_id], + content={ + 'deploy_process_script': { + 'value': self.get_process_content('process/deploy_process.py') + } + }, + edit = { + 'signatures': [support_group_id], + 'readers': [support_group_id], + 'content': { + 'noteNumber': { + 'value': { + 'param': { + 'regex': '.*', 'type': 'integer' + } + } + }, + 'noteId': { + 'value': { + 'param': { + 'regex': '.*', 'type': 'string' + } + } + } + }, + 'replacement': True, + 'invitation': { + 'id': f'{super_user}/Venue_Configuration_Request' + '${2/content/noteNumber/value}' + '/-/Deploy', + 'signatures': [ '~Super_User1' ], + 'readers': ['everyone'], + 'writers': [support_group_id], + 'invitees': [support_group_id], + 'maxReplies': 1, + 'process': '''def process(client, edit, invitation): + meta_invitation = client.get_invitation(invitation.invitations[0]) + script = meta_invitation.content['deploy_process_script']['value'] + funcs = { + 'openreview': openreview + } + exec(script, funcs) + funcs['process'](client, edit, invitation) +''', + 'edit': { + 'signatures': { + 'param': { + 'items': [ { 'value': support_group_id, 'optional': True } ] + } + }, + 'readers': ['${{2/note/id}/readers}'], + 'writers': [support_group_id], + 'note': { + 'id': '${4/content/noteId/value}', + 'forum': '${4/content/noteId/value}', + 'ddate': { + 'param': { + 'range': [ 0, 9999999999999 ], + 'optional': True, + 'deletable': True + } + }, + 'signatures': ['${3/signatures}'], + 'content': { + 'venue_id': { + 'value': { + 'param': { + 'type': 'string', + 'regex': '.*' + } + } + } + } + } + } + } + } + ) + + self.post_invitation_edit(invitation) + +class WorkflowInvitations(): + + def __init__(self, client, support_group_id, super_user): + self.support_group_id = support_group_id + self.client = client + self.super_user = super_user + self.meta_invitation_id = f'{super_user}/-/Edit' + + def post_invitation_edit(self, invitation): + return self.client.post_invitation_edit(invitations=self.meta_invitation_id, + readers=['~Super_User1'], + writers=['~Super_User1'], + signatures=['~Super_User1'], + invitation=invitation, + replacement=True + ) + + def get_process_content(self, file_path): + process = None + with open(os.path.join(os.path.dirname(__file__), file_path)) as f: + process = f.read() + return process + + def setup_metareview_template_invitation(self): + + support_group_id = self.support_group_id + invitation_id = f'{support_group_id}/-/Meta_Review_Template' + + invitation = Invitation(id=invitation_id, + invitees=['active_venues'], + readers=['everyone'], + writers=[support_group_id], + signatures=[self.super_user], + process=self.get_process_content('process/templates_process.py'), + edit = { + 'signatures': { + 'param': { + 'items': [ + { 'prefix': '~.*', 'optional': True } + ] + } + }, + 'readers': [support_group_id], + 'writers': [support_group_id], + 'content' :{ + 'venue_id': { + 'order': 1, + 'description': 'Venue Id', + 'value': { + 'param': { + 'type': 'string', + 'maxLength': 100, + 'regex': '.*', + 'hidden': True + } + } + }, + 'name': { + 'order': 2, + 'description': 'Name for this step, use underscores to represent spaces. Default is Meta_Review. This name will be shown in the button users will click to perform this step.', + 'value': { + 'param': { + 'type': 'string', + 'maxLength': 100, + 'regex': '^[a-zA-Z0-9_]*$', + 'default': 'Meta_Review' + } + } + }, + 'activation_date': { + 'order': 3, + 'description': 'When should the meta reviewing of submissions begin?', + 'value': { + 'param': { + 'type': 'date', + 'range': [ 0, 9999999999999 ], + 'deletable': True + } + } + }, + 'due_date': { + 'order': 4, + 'description': 'By when should the meta-reviews be in the system? This is the official, soft deadline area chairs will see.', + 'value': { + 'param': { + 'type': 'date', + 'range': [ 0, 9999999999999 ], + 'optional': True, + 'deletable': True + } + } + }, + 'expiration_date': { + 'order': 5, + 'description': 'After this date, no more meta reviews can be submitted. This is the hard deadline area chairs will not be able to see.', + 'value': { + 'param': { + 'type': 'date', + 'range': [ 0, 9999999999999 ], + 'optional': True, + 'deletable': True + } + } + }, + 'readers': { + 'order': 6, + 'description': 'Select who should be able to read the meta reviews as soon as they are posted.', + 'value': { + 'param': { + 'type': 'string[]', + 'input': 'select', + 'items': [ + {'value': 'Program_Chairs', 'optional': False, 'description': 'Program Chairs'}, + {'value': 'Senior_Area_Chairs', 'optional': False, 'description': 'Assigned Senior Area Chairs (if applicable)'}, + {'value': 'Area_Chairs', 'optional': False, 'description': 'Assigned Area Chairs'}, + {'value': 'Reviewers', 'optional': True, 'description': 'Assigned Reviewers'}, + {'value': 'Reviewers/Submitted', 'optional': True, 'description': 'Assigned reviewers who have submitted a review'}, + {'value': 'Authors', 'optional': True, 'description': 'Paper Authors'}, + {'value': 'Everyone', 'optional': True, 'description': 'Public'} + ] + } + } + }, + 'content': { + 'order': 7, + 'description': 'Configure what fields the meta review should contain.', + 'value': { + 'param': { + 'type': 'json', + 'default': default_content.meta_review_v2 + } + } + }, + 'recommendation_field_name': { + 'order': 8, + 'description': 'Name of the field that will store the recommendation. Default is recommendation. This field should be defined in the content field above.', + 'value': { + 'param': { + 'type': 'string', + 'regex': '.*', + 'default': 'recommendation' + } + } + } + }, + 'domain': '${1/content/venue_id/value}', + 'invitation': { + 'id': '${2/content/venue_id/value}/-/${2/content/name/value}', + 'signatures': ['${3/content/venue_id/value}'], + } + } + ) + + self.post_invitation_edit(invitation) + + def setup_comment_template_invitation(self): + + support_group_id = self.support_group_id + invitation_id = f'{support_group_id}/-/Comment_Template' + + invitation = Invitation(id=invitation_id, + invitees=['active_venues'], + readers=['everyone'], + writers=[support_group_id], + signatures=[self.super_user], + process = self.get_process_content('process/templates_process.py'), + edit = { + 'signatures': { + 'param': { + 'items': [ + { 'prefix': '~.*', 'optional': True } + ] + } + }, + 'readers': [support_group_id], + 'writers': [support_group_id], + 'content' :{ + 'venue_id': { + 'order': 1, + 'description': 'Venue Id', + 'value': { + 'param': { + 'type': 'string', + 'maxLength': 100, + 'regex': '.*', + 'hidden': True + } + } + }, + 'name': { + 'order': 2, + 'description': 'Name for this step, use underscores to represent spaces. Default is Official_Comment. This name will be shown in the button users will click to perform this step.', + 'value': { + 'param': { + 'type': 'string', + 'maxLength': 100, + 'regex': '^[a-zA-Z0-9_]*$', + 'default': 'Official_Comment' + } + } + }, + 'activation_date': { + 'order': 3, + 'description': 'When does official and/or public commentary begin?', + 'value': { + 'param': { + 'type': 'date', + 'range': [ 0, 9999999999999 ], + 'optional': True, + 'deletable': True + } + } + }, + 'expiration_date': { + 'order': 4, + 'description': 'When does official and/or public commentary end?', + 'value': { + 'param': { + 'type': 'date', + 'range': [ 0, 9999999999999 ], + 'optional': True, + 'deletable': True + } + } + }, + 'participants': { + 'order': 5, + 'description': 'Select who should be allowed to post comments on submissions. These will be added as readers automatically.', + 'value': { + 'param': { + 'type': 'string[]', + 'input': 'select', + 'items': [ + {'value': 'Program Chairs', 'optional': False, 'description': 'Program Chairs'}, + {'value': 'Assigned Senior Area Chairs', 'optional': False, 'description': 'Assigned Senior Area Chairs'}, + {'value': 'Assigned Area Chairs', 'optional': False, 'description': 'Assigned Area Chairs'}, + {'value': 'Assigned Reviewers', 'optional': True, 'description': 'Assigned Reviewers'}, + {'value': 'Assigned Reviewers Submitted', 'optional': True, 'description': 'Assigned reviewers who have submitted a review'}, + {'value': 'Paper Authors', 'optional': True, 'description': 'Paper Authors'}, + {'value': 'Everyone (anonymously)', 'optional': True, 'description': 'Public (anonymously)'}, + {'value': 'Everyone (non-anonymously)', 'optional': True, 'description': 'Public (non-anonymously)'} + ], + 'default': ['Program Chairs'] + } + } + }, + 'additional_readers': { + 'order': 6, + 'description': 'Select who should only be allowed to view the comments on submissions (other than the participants). ', + 'value': { + 'param': { + 'type': 'string[]', + 'input': 'select', + 'items': [ + {'value': 'Program Chairs', 'optional': False, 'description': 'Program Chairs'}, + {'value': 'Assigned Senior Area Chairs', 'optional': False, 'description': 'Assigned Senior Area Chairs'}, + {'value': 'Assigned Area Chairs', 'optional': False, 'description': 'Assigned Area Chairs'}, + {'value': 'Assigned Reviewers', 'optional': True, 'description': 'Assigned Reviewers'}, + {'value': 'Assigned Reviewers Submitted', 'optional': True, 'description': 'Assigned reviewers who have submitted a review'}, + {'value': 'Paper Authors', 'optional': True, 'description': 'Paper Authors'}, + {'value': 'Everyone', 'optional': True, 'description': 'Public'} + ], + 'optional': True + } + } + }, + 'email_program_chairs_about_official_comments': { + 'order': 7, + 'description': 'Should the PCs receive an email for each official comment made in the venue? Default is no.', + 'value': { + 'param': { + 'type': 'string', + 'enum': [ + 'Yes, email PCs for every new comment.', + 'No, do not email PCs.' + ], + 'input': 'radio', + 'default': 'No, do not email PCs.' + } + } + }, + 'email_senior_area_chairs_about_official_comments': { + 'order': 8, + 'description': 'Should the SACs (if applicable) receive an email for each official comment made in the venue? Default is no.', + 'value': { + 'param': { + 'type': 'string', + 'enum': [ + 'Yes, email SACs for every new comment.', + 'No, do not email SACs.' + ], + 'input': 'radio', + 'default': 'No, do not email SACs.' + } + } + } + }, + 'domain': '${1/content/venue_id/value}', + 'invitation': { + 'id': '${2/content/venue_id/value}/-/${2/content/stage_name/value}', + 'signatures': ['${3/content/venue_id/value}'], + } + } + ) + + self.post_invitation_edit(invitation) + + def setup_decision_template_invitation(self): + + support_group_id = self.support_group_id + invitation_id = f'{support_group_id}/-/Decision_Template' + + invitation = Invitation(id=invitation_id, + invitees=['active_venues'], + readers=['everyone'], + writers=[support_group_id], + signatures=[self.super_user], + process = self.get_process_content('process/templates_process.py'), + edit = { + 'signatures': { + 'param': { + 'items': [ + { 'prefix': '~.*', 'optional': True } + ] + } + }, + 'readers': [support_group_id], + 'writers': [support_group_id], + 'content' :{ + 'venue_id': { + 'order': 1, + 'description': 'Venue Id', + 'value': { + 'param': { + 'type': 'string', + 'maxLength': 100, + 'regex': '.*', + 'hidden': True + } + } + }, + 'name': { + 'order': 2, + 'description': 'Name for this step, use underscores to represent spaces. Default is Decision. This name will be shown in the button users will click to perform this step.', + 'value': { + 'param': { + 'type': 'string', + 'maxLength': 100, + 'regex': '^[a-zA-Z0-9_]*$', + 'default': 'Decision' + } + } + }, + 'activation_date': { + 'order': 3, + 'description': 'When will the program chairs start posting decisions?', + 'value': { + 'param': { + 'type': 'date', + 'range': [ 0, 9999999999999 ], + 'optional': True, + 'deletable': True + } + } + }, + 'due_date': { + 'order': 4, + 'description': 'By when should all the decisions be in the system?', + 'value': { + 'param': { + 'type': 'date', + 'range': [ 0, 9999999999999 ], + 'deletable': True + } + } + }, + 'decision_options': { + 'order': 5, + 'description': 'List all decision options. For example: Accept, Reject, Invite to Workshop. Default options are: Accept (Oral), Accept (Poster), Reject', + 'value': { + 'param': { + 'type': 'string[]', + 'regex': '.+', + 'default': ['Accept (Oral)', 'Accept (Poster)', 'Reject'] + } + } + }, + 'readers': { + 'order': 6, + 'description': 'Select who should be able to read the decisions as soon as they are posted.', + 'value': { + 'param': { + 'type': 'string[]', + 'input': 'select', + 'items': [ + {'value': 'Program Chairs', 'optional': False, 'description': 'Program Chairs'}, + {'value': 'Assigned Senior Area Chairs', 'optional': True, 'description': 'Assigned Senior Area Chairs'}, + {'value': 'Assigned Area Chairs', 'optional': True, 'description': 'Assigned Area Chairs'}, + {'value': 'Assigned Reviewers', 'optional': True, 'description': 'Assigned Reviewers'}, + {'value': 'Paper Authors', 'optional': True, 'description': 'Paper Authors'}, + {'value': 'Everyone', 'optional': True, 'description': 'Public'} + ] + } + } + } + }, + 'domain': '${1/content/venue_id/value}', + 'invitation': { + 'id': '${2/content/venue_id/value}/-/${2/content/stage_name/value}', + 'signatures': ['${3/content/venue_id/value}'], + } + } + ) + + self.post_invitation_edit(invitation) \ No newline at end of file diff --git a/openreview/venue/configuration/process/deploy_process.py b/openreview/venue/configuration/process/deploy_process.py new file mode 100644 index 000000000..6e958d788 --- /dev/null +++ b/openreview/venue/configuration/process/deploy_process.py @@ -0,0 +1,33 @@ +def process(client, edit, invitation): + + support_user = 'openreview.net/Support' + domain = invitation.domain + + note = client.get_note(edit.note.id) + venue = openreview.helpers.get_venue(client, note.id, support_user, setup=True) + venue.create_submission_stage() + venue.create_submission_edit_invitations() + venue.create_review_stage() + venue.create_review_edit_invitations() + venue.edit_invitation_builder.set_edit_stage_invitation() + venue.invitation_builder.set_group_matching_setup_invitations(venue.get_reviewers_id()) + if venue.use_area_chairs: + venue.invitation_builder.set_group_matching_setup_invitations(venue.get_area_chairs_id()) + if venue.use_senior_area_chairs: + venue.invitation_builder.set_group_matching_setup_invitations(venue.get_senior_area_chairs_id()) + + # remove PC access to editing the note + client.post_note_edit( + invitation=f'{domain}/-/Edit', + signatures=[venue.id], + note = openreview.api.Note( + id = note.id, + writers = [], + readers = [support_user, + venue.id + ], + content = { + 'title': { 'value': openreview.tools.pretty_id(venue.id) }, + } + ) + ) \ No newline at end of file diff --git a/openreview/venue/configuration/process/stage_process.py b/openreview/venue/configuration/process/stage_process.py new file mode 100644 index 000000000..b97020bd7 --- /dev/null +++ b/openreview/venue/configuration/process/stage_process.py @@ -0,0 +1,92 @@ +def process(client, edit, invitation): + + support_user = 'openreview.net/Support' + domain = client.get_group(edit.domain) + request_form_id = domain.get_content_value('request_form_id') + + venue = openreview.helpers.get_venue(client, request_form_id, support_user) + + stage_name = edit.content['stage_name']['value'] + stage_type = edit.content['stage_type']['value'] + + activation_date = datetime.datetime.fromtimestamp(edit.content['activation_date']['value']/1000) + expiration_date = datetime.datetime.fromtimestamp(edit.content['expiration_date']['value']/1000) + + if stage_type == 'Official_Comment': + + print('Create Official Comment Stage') + venue.comment_stage = openreview.stages.CommentStage( + official_comment_name=stage_name, + start_date=activation_date, + end_date=expiration_date, + ) + venue.create_comment_stage() + venue.edit_invitation_builder.set_edit_deadlines_invitation(venue.get_invitation_id(stage_name)) + venue.edit_invitation_builder.set_edit_content_invitation(venue.get_invitation_id(stage_name)) + venue.edit_invitation_builder.set_edit_reply_readers_invitation(venue.get_invitation_id(stage_name)) + + if stage_type == 'Meta_Review': + + print('Create Meta Review Stage') + venue.meta_review_stage = openreview.stages.MetaReviewStage( + name = stage_name, + start_date = activation_date, + due_date = expiration_date, + exp_date = expiration_date, + ) + venue.create_meta_review_stage() + venue.edit_invitation_builder.set_edit_deadlines_invitation(venue.get_invitation_id(stage_name)) + venue.edit_invitation_builder.set_edit_content_invitation(venue.get_invitation_id(stage_name)) + venue.edit_invitation_builder.set_edit_reply_readers_invitation(venue.get_invitation_id(stage_name)) + + if stage_type == 'Decision': + + print('Create Decision Stage') + venue.decision_stage = openreview.stages.DecisionStage( + name = stage_name, + start_date = activation_date, + due_date = expiration_date, + ) + venue.create_decision_stage() + venue.edit_invitation_builder.set_edit_deadlines_invitation(venue.get_invitation_id(stage_name)) + venue.edit_invitation_builder.set_edit_content_invitation(venue.get_invitation_id(stage_name)) + venue.edit_invitation_builder.set_edit_reply_readers_invitation(venue.get_invitation_id(stage_name)) + + if stage_type == 'Submission_Revision': + + print('Create Submission Revision Stage') + venue.submission_revision_stage = openreview.stages.SubmissionRevisionStage( + name= stage_name, + start_date= activation_date, + due_date= expiration_date, + ) + venue.create_submission_revision_stage() + venue.edit_invitation_builder.set_edit_deadlines_invitation(venue.get_invitation_id(stage_name)) + venue.edit_invitation_builder.set_edit_content_invitation(venue.get_invitation_id(stage_name)) + + if stage_type == 'Custom': + + print('Create Custom Stage') + venue.custom_stage = openreview.stages.CustomStage( + name=stage_name, + reply_to=openreview.stages.CustomStage.ReplyTo.FORUM, + source=openreview.stages.CustomStage.Source.ALL_SUBMISSIONS, + reply_type=openreview.stages.CustomStage.ReplyType.REPLY, + start_date = activation_date, + due_date = expiration_date, + exp_date = expiration_date, + content = { + 'field_name': { + 'order': 1, + 'description': 'Field Description', + 'value': { + 'param': { + 'type': 'string', + } + } + } + } + ) + venue.create_custom_stage() + venue.edit_invitation_builder.set_edit_deadlines_invitation(venue.get_invitation_id(stage_name)) + venue.edit_invitation_builder.set_edit_content_invitation(venue.get_invitation_id(stage_name)) diff --git a/openreview/venue/configuration/process/support_process.py b/openreview/venue/configuration/process/support_process.py new file mode 100644 index 000000000..7ef47482d --- /dev/null +++ b/openreview/venue/configuration/process/support_process.py @@ -0,0 +1,26 @@ +def process(client, edit, invitation): + + note = client.get_note(edit.note.id) + print(note.forum) + + # post deploy invitation + inv = client.post_invitation_edit( + invitations='openreview.net/-/Deploy', + signatures=['openreview.net/Support'], + content = { + 'noteNumber': { 'value': note.number}, + 'noteId': { 'value': note.id } + } + ) + print(inv['invitation']['id']) + + # post comment invitation + inv = client.post_invitation_edit( + invitations='openreview.net/-/Comment', + signatures=['openreview.net/Support'], + content = { + 'noteNumber': { 'value': note.number}, + 'noteId': { 'value': note.id } + } + ) + print(inv['invitation']['id']) \ No newline at end of file diff --git a/openreview/venue/configuration/process/templates_process.py b/openreview/venue/configuration/process/templates_process.py new file mode 100644 index 000000000..f9f1ddcff --- /dev/null +++ b/openreview/venue/configuration/process/templates_process.py @@ -0,0 +1,152 @@ +def process(client, edit, invitation): + + support_user = 'openreview.net/Support' + venue_id = edit.content['venue_id']['value'] + venue_group = client.get_group(venue_id) + request_form_id = venue_group.get_content_value('request_form_id') + + venue = openreview.helpers.get_venue(client, request_form_id, support_user) + stage_name = edit.content['name']['value'] + activation_date = datetime.datetime.fromtimestamp(edit.content['activation_date']['value']/1000) if 'activation_date' in edit.content else None + due_date = datetime.datetime.fromtimestamp(edit.content['due_date']['value']/1000) if 'due_date' in edit.content else None + expiration_date = datetime.datetime.fromtimestamp(edit.content['expiration_date']['value']/1000) if 'expiration_date' in edit.content else None + + + if invitation.id.endswith('Meta_Review_Template'): + + release_to_reviewers = openreview.stages.MetaReviewStage.Readers.NO_REVIEWERS + reply_readers = edit.content['readers']['value'] + if 'Reviewers' in reply_readers: + release_to_reviewers = openreview.stages.MetaReviewStage.Readers.REVIEWERS_ASSIGNED + elif 'Reviewers/Submitted' in reply_readers: + release_to_reviewers = openreview.stages.MetaReviewStage.Readers.REVIEWERS_SUBMITTED + + recommendation_field_name = edit.content['recommendation_field_name']['value'] + + print('Create Meta Review Stage') + venue.meta_review_stage = openreview.stages.MetaReviewStage( + name = stage_name, + start_date = activation_date, + due_date = due_date, + exp_date = expiration_date, + public = 'Everyone' in edit.content['readers']['value'], + release_to_authors = 'Authors' in edit.content['readers']['value'], + release_to_reviewers = release_to_reviewers, + content = edit.content['content']['value'], + recommendation_field_name = recommendation_field_name, + child_invitations_name = stage_name + ) + venue.create_meta_review_stage() + venue.edit_invitation_builder.set_edit_deadlines_invitation(venue.get_invitation_id(stage_name)) + content = { + 'recommendation_field_name': { + 'value': { + 'param': { + 'type': 'string', + 'regex': '.*', + 'default': 'recommendation' + } + } + } + } + venue.edit_invitation_builder.set_edit_content_invitation(venue.get_invitation_id(stage_name), content, 'edit_metareview_recommendation.py') + venue.edit_invitation_builder.set_edit_reply_readers_invitation(venue.get_invitation_id(stage_name)) + + # edit group content + group_content = venue_group.content + group_content['meta_review_name'] = {'value': stage_name } + group_content['meta_review_recommendation'] = {'value': recommendation_field_name } + + client.post_group_edit( + invitation = venue.get_meta_invitation_id(), + signatures = [venue_id], + group = openreview.api.Group( + id = venue_id, + content = group_content + ) + ) + + elif invitation.id.endswith('Comment_Template'): + + participants = edit.content['participants']['value'] + additional_readers = edit.content.get('additional_readers', []) + anonymous = 'Everyone (anonymously)' in participants + allow_public_comments = anonymous or 'Everyone (non-anonymously)' in participants + + invitees = [] + readers = [] + if 'Assigned Reviewers' in participants: + invitees.append(openreview.stages.CommentStage.Readers.REVIEWERS_ASSIGNED) + readers.append(openreview.stages.CommentStage.Readers.REVIEWERS_ASSIGNED) + elif 'Assigned Reviewers' in additional_readers: + readers.append(openreview.stages.CommentStage.Readers.REVIEWERS_ASSIGNED) + + if 'Assigned Submitted Reviewers' in participants: + invitees.append(openreview.stages.CommentStage.Readers.REVIEWERS_SUBMITTED) + readers.append(openreview.stages.CommentStage.Readers.REVIEWERS_SUBMITTED) + elif 'Assigned Submitted Reviewers' in additional_readers: + readers.append(openreview.stages.CommentStage.Readers.REVIEWERS_SUBMITTED) + + if 'Assigned Area Chairs' in participants: + invitees.append(openreview.stages.CommentStage.Readers.AREA_CHAIRS_ASSIGNED) + readers.append(openreview.stages.CommentStage.Readers.AREA_CHAIRS_ASSIGNED) + elif 'Assigned Area Chairs' in additional_readers: + readers.append(openreview.stages.CommentStage.Readers.AREA_CHAIRS_ASSIGNED) + + if 'Assigned Senior Area Chairs' in participants: + invitees.append(openreview.stages.CommentStage.Readers.SENIOR_AREA_CHAIRS_ASSIGNED) + readers.append(openreview.stages.CommentStage.Readers.SENIOR_AREA_CHAIRS_ASSIGNED) + elif 'Assigned Senior Area Chairs' in additional_readers: + readers.append(openreview.stages.CommentStage.Readers.SENIOR_AREA_CHAIRS_ASSIGNED) + + if 'Paper Authors' in participants: + invitees.append(openreview.stages.CommentStage.Readers.AUTHORS) + readers.append(openreview.stages.CommentStage.Readers.AUTHORS) + elif 'Paper Authors' in additional_readers: + readers.append(openreview.stages.CommentStage.Readers.AUTHORS) + + if 'Everyone' in additional_readers: + readers.append(openreview.stages.CommentStage.Readers.EVERYONE) + + print('Create Official Comment Stage') + venue.comment_stage = openreview.stages.CommentStage( + official_comment_name=stage_name, + start_date=activation_date, + end_date=expiration_date, + allow_public_comments=allow_public_comments, + reader_selection=True, + email_pcs=True if 'Yes' in edit.content['email_program_chairs_about_official_comments']['value'] else False, + email_sacs=True if 'Yes' in edit.content['email_senior_area_chairs_about_official_comments']['value'] else False, + check_mandatory_readers=True, + readers=readers, + invitees=invitees + ) + + invitation_id = venue.get_invitation_id(stage_name) + venue.create_comment_stage() + venue.edit_invitation_builder.set_edit_deadlines_invitation(invitation_id, include_due_date=False) + venue.edit_invitation_builder.set_edit_content_invitation(invitation_id) + venue.edit_invitation_builder.set_edit_invitees_and_readers_selection_invitation(invitation_id) + venue.edit_invitation_builder.set_edit_comment_notification_invitation(invitation_id) + if allow_public_comments: + invitation_id = venue.get_invitation_id(venue.comment_stage.public_name) + venue.edit_invitation_builder.set_edit_deadlines_invitation(invitation_id) + venue.edit_invitation_builder.set_edit_content_invitation(invitation_id) + + elif invitation.id.endswith('Decision_Template'): + + venue.decision_stage = openreview.stages.DecisionStage( + options = edit.content['decision_options']['value'], + start_date = activation_date, + due_date = due_date, + name = stage_name, + public = 'Everyone' in edit.content['readers']['value'], + release_to_authors = 'Paper Authors' in edit.content['readers']['value'], + release_to_reviewers = 'Assigned Reviewers' in edit.content['readers']['value'], + release_to_area_chairs = 'Assigned Area Chairs' in edit.content['readers']['value'] + ) + + invitation_id = venue.get_invitation_id(stage_name) + venue.create_decision_stage() + venue.edit_invitation_builder.set_edit_deadlines_invitation(invitation_id) + venue.edit_invitation_builder.set_edit_reply_readers_invitation(invitation_id) \ No newline at end of file diff --git a/openreview/venue/configuration/webfield/venuepageWebfield.js b/openreview/venue/configuration/webfield/venuepageWebfield.js new file mode 100644 index 000000000..e205dcbdd --- /dev/null +++ b/openreview/venue/configuration/webfield/venuepageWebfield.js @@ -0,0 +1,34 @@ +// Webfield component +const tabs = [{ + name: 'Venue Configuration Requests', + query: { + 'invitation': 'openreview.net/-/Venue_Configuration_Request' + }, + options: { + enableSearch: true + } + }] + + +return { + component: 'VenueHomepage', + version: 1, + properties: { + header: { + title: 'OpenReview Venue Configuration Requests', + subtitle: '', + website: 'https://openreview.net', + contact: 'info@openreview.net', + //location: domain.content.location.value, + instructions: ` +**Instructions:** This page displays all venue configuration requests. Click on a request to view more details and to make a decision. +`, + //date: domain.content.start_date.value, + //deadline: domain.content.date.value + }, + submissionId: 'openreview.net/-/Venue_Configuration_Request', + //parentGroupId: domain.parent, + tabs: tabs + } + } + \ No newline at end of file diff --git a/openreview/venue/edit_invitation.py b/openreview/venue/edit_invitation.py new file mode 100644 index 000000000..96a0557c7 --- /dev/null +++ b/openreview/venue/edit_invitation.py @@ -0,0 +1,814 @@ +import os +import time +import datetime +from .. import tools +from openreview.api import Invitation + +class EditInvitationBuilder(object): + + def __init__(self, venue): + self.client = venue.client + self.venue = venue + self.venue_id = venue.venue_id + + def save_invitation(self, invitation, replacement=None): + self.client.post_invitation_edit(invitations=self.venue.get_meta_invitation_id(), + readers=[self.venue_id], + writers=[self.venue_id], + signatures=[self.venue_id], + replacement=replacement, + invitation=invitation + ) + invitation = self.client.get_invitation(invitation.id) + + if invitation.date_processes and len(invitation.date_processes[0]['dates']) > 1 and self.update_date_string == invitation.date_processes[0]['dates'][1]: + process_logs = self.client.get_process_logs(id=invitation.id + '-0-1', min_sdate = invitation.tmdate + self.update_wait_time - 1000) + count = 0 + while len(process_logs) == 0 and count < 180: ## wait up to 30 minutes + time.sleep(10) + process_logs = self.client.get_process_logs(id=invitation.id + '-0-1', min_sdate = invitation.tmdate + self.update_wait_time - 1000) + count += 1 + + if len(process_logs) == 0: + raise openreview.OpenReviewException('Time out waiting for invitation to complete: ' + invitation.id) + + if process_logs[0]['status'] == 'error': + raise openreview.OpenReviewException('Error saving invitation: ' + invitation.id) + + return invitation + + def expire_invitation(self, invitation_id): + invitation = tools.get_invitation(self.client, id = invitation_id) + + if invitation: + self.save_invitation(invitation=Invitation(id=invitation.id, + expdate=tools.datetime_millis(datetime.datetime.utcnow()), + signatures=[self.venue_id] + ) + ) + + def get_process_content(self, file_path): + process = None + with open(os.path.join(os.path.dirname(__file__), file_path)) as f: + process = f.read() + return process + + def set_edit_submission_deadlines_invitation(self, invitation_id, process_file=None): + + venue_id = self.venue_id + venue = self.venue + deadline_invitation_id = invitation_id + '/Deadlines' + + invitation = Invitation( + id = deadline_invitation_id, + invitees = [venue_id], + signatures = [venue_id], + readers = [venue_id], + writers = [venue_id], + edit = { + 'content': { + 'activation_date': { + 'value': { + 'param': { + 'type': 'date', + 'range': [ 0, 9999999999999 ], + 'optional': True, + 'deletable': True + } + } + }, + 'deadline': { + 'value': { + 'param': { + 'type': 'date', + 'range': [ 0, 9999999999999 ], + 'optional': True, + 'deletable': True + } + } + } + }, + 'signatures': [venue.get_program_chairs_id()], + 'readers': [venue_id], + 'writers': [venue_id], + 'invitation': { + 'id': invitation_id, + 'signatures': [venue_id], + 'cdate': '${2/content/activation_date/value}', + 'duedate': '${2/content/deadline/value}', + 'expdate': '${2/content/deadline/value}+1800000' ## 30 minutes buffer period + } + } + ) + + if process_file: + invitation.process = self.get_process_content(f'process/{process_file}') + + self.save_invitation(invitation, replacement=True) + return invitation + + def set_edit_deadlines_invitation(self, invitation_id, process_file=None, include_due_date=True): + + venue_id = self.venue_id + venue = self.venue + deadline_invitation_id = invitation_id + '/Deadlines' + + invitation = Invitation( + id = deadline_invitation_id, + invitees = [venue_id], + signatures = [venue_id], + readers = [venue_id], + writers = [venue_id], + edit = { + 'content': { + 'activation_date': { + 'value': { + 'param': { + 'type': 'date', + 'range': [ 0, 9999999999999 ], + 'optional': True, + 'deletable': True + } + } + }, + 'expiration_date': { + 'value': { + 'param': { + 'type': 'date', + 'range': [ 0, 9999999999999 ], + 'optional': True, + 'deletable': True + } + } + } + }, + 'signatures': [venue.get_program_chairs_id()], + 'readers': [venue_id], + 'writers': [venue_id], + 'invitation': { + 'id': invitation_id, + 'signatures': [venue_id], + 'cdate': '${2/content/activation_date/value}', + 'edit': { + 'invitation': { + 'cdate': '${4/content/activation_date/value}', + 'expdate': '${4/content/expiration_date/value}' + } + } + } + } + ) + + if process_file: + invitation.process = self.get_process_content(f'process/{process_file}') + + if include_due_date: + invitation.edit['content']['deadline'] = { + 'value': { + 'param': { + 'type': 'date', + 'range': [ 0, 9999999999999 ], + 'optional': True, + 'deletable': True + } + } + } + invitation.edit['invitation']['edit']['invitation']['duedate'] = '${4/content/deadline/value}' + + self.save_invitation(invitation, replacement=True) + return invitation + + def set_edit_submission_content_invitation(self, invitation_id): + + venue_id = self.venue_id + content_invitation_id = invitation_id + '/Form_Fields' + + invitation = Invitation( + id = content_invitation_id, + invitees = [venue_id], + signatures = [venue_id], + readers = [venue_id], + writers = [venue_id], + edit = { + 'signatures': [venue_id], + 'readers': [venue_id], + 'writers': [venue_id], + 'content' :{ + 'note_content': { + 'value': { + 'param': { + 'type': 'content' + } + } + }, + 'note_license': { + 'value': { + 'param': { + 'type': 'object[]', + 'input': 'select', + 'items': [ + {'value': {'value': 'CC BY 4.0', 'optional': True, 'description': 'CC BY 4.0'}, 'optional': True, 'description': 'CC BY 4.0'}, + {'value': {'value': 'CC BY-SA 4.0', 'optional': True, 'description': 'CC BY-SA 4.0'}, 'optional': True, 'description': 'CC BY-SA 4.0'}, + {'value': {'value': 'CC BY-NC 4.0', 'optional': True, 'description': 'CC BY-NC 4.0'}, 'optional': True, 'description': 'CC BY-NC 4.0'}, + {'value': {'value': 'CC BY-ND 4.0', 'optional': True, 'description': 'CC BY-ND 4.0'}, 'optional': True, 'description': 'CC BY-ND 4.0'}, + {'value': {'value': 'CC BY-NC-SA 4.0', 'optional': True, 'description': 'CC BY-NC-SA 4.0'}, 'optional': True, 'description': 'CC BY-NC-SA 4.0'}, + {'value': {'value': 'CC BY-NC-ND 4.0', 'optional': True, 'description': 'CC BY-NC-ND 4.0'}, 'optional': True, 'description': 'CC BY-NC-ND 4.0'}, + {'value': {'value': 'CC0 1.0', 'optional': True, 'description': 'CC0 1.0'}, 'optional': True, 'description': 'CC0 1.0'} + ] + } + } + } + }, + 'invitation': { + 'id': invitation_id, + 'signatures': [venue_id], + 'edit': { + 'note': { + 'content': '${4/content/note_content/value}', + 'license': { + 'param': { + 'enum': ['${7/content/note_license/value}'] + } + } + } + } + } + } + ) + + self.save_invitation(invitation, replacement=False) + return invitation + + def set_edit_content_invitation(self, invitation_id, content={}, process_file=None): + + venue_id = self.venue_id + content_invitation_id = invitation_id + '/Form_Fields' + + invitation = Invitation( + id = content_invitation_id, + invitees = [venue_id], + signatures = [venue_id], + readers = [venue_id], + writers = [venue_id], + edit = { + 'signatures': [venue_id], + 'readers': [venue_id], + 'writers': [venue_id], + 'content' :{ + 'note_content': { + 'value': { + 'param': { + 'type': 'content' + } + } + } + }, + 'invitation': { + 'id': invitation_id, + 'signatures': [venue_id], + 'edit': { + 'invitation': { + 'edit': { + 'note': { + 'content': '${6/content/note_content/value}' + } + } + } + } + } + } + ) + + if content: + invitation.edit['content'].update(content) + + if process_file: + invitation.process = self.get_process_content(f'process/{process_file}') + + self.save_invitation(invitation, replacement=False) + return invitation + + def set_edit_submission_notification_invitation(self): + + venue_id = self.venue_id + venue = self.venue + notifications_invitation_id = venue.get_submission_id() + '/Notifications' + + invitation = Invitation( + id = notifications_invitation_id, + invitees = [venue_id], + signatures = [venue_id], + readers = [venue_id], + writers = [venue_id], + edit = { + 'signatures': [venue_id], + 'readers': [venue_id], + 'writers': [venue_id], + 'content' :{ + 'email_authors': { + 'value': { + 'param': { + 'type': 'boolean', + 'enum': [True, False], + 'input': 'radio' + } + } + }, + 'email_pcs': { + 'value': { + 'param': { + 'type': 'boolean', + 'enum': [True, False], + 'input': 'radio' + } + } + } + }, + 'invitation': { + 'id': venue.get_submission_id(), + 'signatures': [venue_id], + 'content': { + 'email_authors': { + 'value': '${4/content/email_authors/value}' + }, + 'email_pcs': { + 'value': '${4/content/email_pcs/value}' + } + } + } + } + ) + + self.save_invitation(invitation, replacement=False) + return invitation + + def set_edit_submission_readers_invitation(self): + + venue_id = self.venue_id + venue = self.venue + post_submission_id = f'{venue_id}/-/Post_{venue.submission_stage.name}' + content_invitation_id = post_submission_id + '/Submission_Readers' + + invitation = Invitation( + id = content_invitation_id, + invitees = [venue_id], + signatures = [venue_id], + readers = [venue_id], + writers = [venue_id], + edit = { + 'signatures': [venue_id], + 'readers': [venue_id], + 'writers': [venue_id], + 'content' :{ + 'readers': { + 'value': { + 'param': { + 'type': 'string[]', + 'input': 'select', + 'items': [ + {'value': venue_id, 'optional': False, 'description': 'Program Chairs'}, + {'value': venue.get_authors_id(), 'optional': True, 'description': 'All Authors'}, + {'value': venue.get_reviewers_id(), 'optional': True, 'description': 'All Reviewers'}, + {'value': venue.get_area_chairs_id(), 'optional': True, 'description': 'All Area Chairs'}, + {'value': venue.get_senior_area_chairs_id(), 'optional': True, 'description': 'All Senior Area Chairs'}, + {'value': venue.get_authors_id('${{2/id}/number}'), 'optional': True, 'description': 'Submission Authors'}, + {'value': venue.get_reviewers_id('${{2/id}/number}'), 'optional': True, 'description': 'Assigned Reviewers'}, + {'value': venue.get_area_chairs_id('${{2/id}/number}'), 'optional': True, 'description': 'Assigned Area Chairs'}, + {'value': venue.get_senior_area_chairs_id('${{2/id}/number}'), 'optional': True, 'description': 'Assigned Senior Area Chairs'} + ] + } + } + } + }, + 'invitation': { + 'id': post_submission_id, + 'signatures': [venue_id], + 'edit': { + 'note': { + 'readers': ['${5/content/readers/value}'] + } + } + } + } + ) + + self.save_invitation(invitation, replacement=False) + return invitation + + def set_edit_submission_field_readers_invitation(self): + + venue_id = self.venue_id + venue = self.venue + post_submission_id = f'{venue_id}/-/Post_{venue.submission_stage.name}' + content_invitation_id = post_submission_id + '/Restrict_Field_Visibility' + + invitation = Invitation( + id = content_invitation_id, + invitees = [venue_id], + signatures = [venue_id], + readers = [venue_id], + writers = [venue_id], + edit = { + 'signatures': [venue_id], + 'readers': [venue_id], + 'writers': [venue_id], + 'content' :{ + 'author_readers': { + 'value': { + 'param': { + 'type': 'string[]', + 'input': 'select', + 'items': [ + {'value': venue_id, 'optional': False, 'description': 'Program Chairs'}, + {'value': venue.get_authors_id(), 'optional': True, 'description': 'All Authors'}, + {'value': venue.get_reviewers_id(), 'optional': True, 'description': 'All Reviewers'}, + {'value': venue.get_area_chairs_id(), 'optional': True, 'description': 'All Area Chairs'}, + {'value': venue.get_senior_area_chairs_id(), 'optional': True, 'description': 'All Senior Area Chairs'}, + {'value': venue.get_authors_id('${{4/id}/number}'), 'optional': True, 'description': 'Submission Authors'}, + {'value': venue.get_reviewers_id('${{4/id}/number}'), 'optional': True, 'description': 'Assigned Reviewers'}, + {'value': venue.get_area_chairs_id('${{4/id}/number}'), 'optional': True, 'description': 'Assigned Area Chairs'}, + {'value': venue.get_senior_area_chairs_id('${{4/id}/number}'), 'optional': True, 'description': 'Assigned Senior Area Chairs'}, + ] + } + } + }, + 'pdf_readers': { + 'value': { + 'param': { + 'type': 'string[]', + 'input': 'select', + 'items': [ + {'value': venue_id, 'optional': False, 'description': 'Program Chairs'}, + {'value': venue.get_authors_id(), 'optional': True, 'description': 'All Authors'}, + {'value': venue.get_reviewers_id(), 'optional': True, 'description': 'All Reviewers'}, + {'value': venue.get_area_chairs_id(), 'optional': True, 'description': 'All Area Chairs'}, + {'value': venue.get_senior_area_chairs_id(), 'optional': True, 'description': 'All Senior Area Chairs'}, + {'value': venue.get_authors_id('${{4/id}/number}'), 'optional': True, 'description': 'Submission Authors'}, + {'value': venue.get_reviewers_id('${{4/id}/number}'), 'optional': True, 'description': 'Assigned Reviewers'}, + {'value': venue.get_area_chairs_id('${{4/id}/number}'), 'optional': True, 'description': 'Assigned Area Chairs'}, + {'value': venue.get_senior_area_chairs_id('${{4/id}/number}'), 'optional': True, 'description': 'Assigned Senior Area Chairs'}, + ] + } + } + } + }, + 'invitation': { + 'id': post_submission_id, + 'signatures': [venue_id], + 'edit': { + 'note': { + 'content': { + 'authors': { + 'readers': ['${7/content/author_readers/value}'] + }, + 'authorids': { + 'readers': ['${7/content/author_readers/value}'] + }, + 'pdf': { + 'readers': ['${7/content/pdf_readers/value}'] + } + } + } + } + } + } + ) + + self.save_invitation(invitation, replacement=False) + return invitation + + def set_edit_reply_readers_invitation(self, invitation_id): + + venue_id = self.venue_id + venue = self.venue + reply_readers_invitation_id = invitation_id + '/Readers' + + reply_readers = [ + {'value': venue.get_program_chairs_id(), 'optional': False, 'description': 'Program Chairs.'} + ] + if venue.use_senior_area_chairs: + reply_readers.extend([ + {'value': venue.get_senior_area_chairs_id(), 'optional': True, 'description': 'All Senior Area Chairs'}, + {'value': venue.get_senior_area_chairs_id('${5/content/noteNumber/value}'), 'optional': True, 'description': 'Assigned Senior Area Chairs'} + ]) + if venue.use_area_chairs: + reply_readers.extend([ + {'value': venue.get_area_chairs_id(), 'optional': True, 'description': 'All Area Chairs'}, + {'value': venue.get_area_chairs_id('${5/content/noteNumber/value}'), 'optional': True, 'description': 'Assigned Area Chairs'} + ]) + reply_readers.extend([ + {'value': venue.get_reviewers_id(), 'optional': True, 'description': 'All Reviewers'}, + {'value': venue.get_reviewers_id('${5/content/noteNumber/value}'), 'optional': True, 'description': 'Assigned Reviewers'}, + {'value': venue.get_reviewers_id('${5/content/noteNumber/value}', submitted=True), 'optional': True, 'description': 'Assigned Reviewers who already submitted their review'}, + {'value': '${3/signatures}', 'optional': True, 'description': 'Reviewer who submitted the review'}, + {'value': venue.get_authors_id('${5/content/noteNumber/value}'), 'optional': True, 'description': 'Paper authors'} + ]) + + invitation = Invitation( + id = reply_readers_invitation_id, + invitees = [venue_id], + signatures = [venue_id], + readers = [venue_id], + writers = [venue_id], + edit = { + 'signatures': [venue_id], + 'readers': [venue_id], + 'writers': [venue_id], + 'content' :{ + 'reply_readers': { + 'value': { + 'param': { + 'type': 'string[]', + 'input': 'select', + 'items': reply_readers + } + } + } + }, + 'invitation': { + 'id': invitation_id, + 'signatures': [venue_id], + 'edit': { + 'invitation': { + 'edit': { + 'note': { + 'readers': ['${7/content/reply_readers/value}'] + } + } + } + } + } + } + ) + + self.save_invitation(invitation, replacement=False) + return invitation + + def set_edit_invitees_and_readers_selection_invitation(self, invitation_id): + + venue_id = self.venue_id + venue = self.venue + edit_invitation_id = invitation_id + '/Participants_and_Readers' + + reply_readers = [ + {'value': {'value': venue.get_program_chairs_id(), 'optional': False, 'description': 'Program Chairs.'}, 'optional': False, 'description': 'Program Chairs.'} + ] + participants = [ + {'value': venue.get_program_chairs_id(), 'optional': False, 'description': 'Program Chairs.'} + ] + if venue.use_senior_area_chairs: + reply_readers.extend([ + {'value': {'value': venue.get_senior_area_chairs_id(), 'optional': False, 'description': 'All Senior Area Chairs'}, 'optional': True, 'description': 'All Senior Area Chairs'}, + {'value': {'value': venue.get_senior_area_chairs_id('${8/content/noteNumber/value}'), 'optional': False, 'description': 'Assigned Senior Area Chairs'}, 'optional': True, 'description': 'Assigned Senior Area Chairs'} + ]) + participants.append( + {'value': venue.get_senior_area_chairs_id('${3/content/noteNumber/value}'), 'optional': True, 'description': 'Assigned Senior Area Chairs'} + ) + if venue.use_area_chairs: + reply_readers.extend([ + {'value': {'value': venue.get_area_chairs_id(), 'optional': True, 'description': 'All Area Chairs'}, 'optional': True, 'description': 'All Area Chairs'}, + {'value': {'value': venue.get_area_chairs_id('${8/content/noteNumber/value}'), 'optional': True, 'description': 'Assigned Area Chairs'}, 'optional': True, 'description': 'Assigned Area Chairs'} + ]) + participants.append( + {'value': venue.get_area_chairs_id('${3/content/noteNumber/value}'), 'optional': True, 'description': 'Assigned Area Chairs'} + ) + reply_readers.extend([ + {'value': {'value': venue.get_reviewers_id(), 'optional': True, 'description': 'All Reviewers'}, 'optional': True, 'description': 'All Reviewers'}, + {'value': {'value': venue.get_reviewers_id('${8/content/noteNumber/value}'), 'optional': True, 'description': 'Assigned Reviewers'}, 'optional': True, 'description': 'Assigned Reviewers'}, + {'value': {'value': venue.get_reviewers_id('${8/content/noteNumber/value}', submitted=True), 'optional': True, 'description': 'Assigned Reviewers who already submitted their review'}, 'optional': True, 'description': 'Assigned Reviewers who already submitted their review'}, + {'value': {'value': venue.get_authors_id(), 'optional': True, 'description': 'Paper authors'}, 'optional': True, 'description': 'Paper authors'} + ]) + participants.extend([ + {'value': venue.get_reviewers_id('${3/content/noteNumber/value}'), 'optional': True, 'description': 'Assigned Reviewers'}, + {'value': venue.get_reviewers_id('${3/content/noteNumber/value}', submitted=True), 'optional': True, 'description': 'Assigned Reviewers who already submitted their review'}, + {'value': venue.get_authors_id('${3/content/noteNumber/value}'), 'optional': True, 'description': 'Paper authors'} + ]) + + invitation = Invitation( + id = edit_invitation_id, + invitees = [venue_id], + signatures = [venue_id], + readers = [venue_id], + writers = [venue_id], + edit = { + 'signatures': [venue_id], + 'readers': [venue_id], + 'writers': [venue_id], + 'content' :{ + 'participants': { + 'order': 1, + 'description': 'Who should be able to participate in this stage (read and write comments)?', + 'value': { + 'param': { + 'type': 'string[]', + 'input': 'select', + 'items': participants + } + } + }, + 'reply_readers': { + 'order': 2, + 'description': 'Who should be able to only read comments?', + 'value': { + 'param': { + 'type': 'object[]', + 'input': 'select', + 'items': reply_readers + } + } + } + }, + 'invitation': { + 'id': invitation_id, + 'signatures': [venue_id], + 'edit': { + 'invitation': { + 'invitees': ['${5/content/participants/value}'], + 'edit': { + 'note': { + 'readers': { + 'param': { + 'items': '${8/content/reply_readers/value}' + } + } + } + } + } + } + } + } + ) + + self.save_invitation(invitation, replacement=False) + return invitation + + def set_edit_recommendation_field(self, invitation_id): + + venue_id = self.venue_id + + invitation = Invitation( + id = invitation_id + '/Recommendation_Field', + invitees = [venue_id], + signatures = [venue_id], + readers = [venue_id], + writers = [venue_id], + edit = { + 'signatures': [venue_id], + 'readers': [venue_id], + 'writers': [venue_id], + 'content': { + 'recommendation_field_name': { + 'value': { + 'param': { + 'type': 'string', + 'regex': '.*', + 'default': 'recommendation' + } + } + } + }, + 'group': { + 'id': self.venue_id, + 'content': { + 'meta_review_recommendation': { + 'value': '${4/content/recommendation_field_name/value}' + } + } + } + } + ) + + self.save_invitation(invitation, replacement=False) + return invitation + + def set_edit_comment_notification_invitation(self, invitation_id): + + venue_id = self.venue_id + venue = self.venue + invitation_content_invitation_id = invitation_id + '/Notifications' + + invitation = Invitation( + id = invitation_content_invitation_id, + invitees = [venue_id], + signatures = [venue_id], + readers = [venue_id], + writers = [venue_id], + edit = { + 'signatures': [venue_id], + 'readers': [venue_id], + 'writers': [venue_id], + 'content': { + 'email_pcs': { + 'value': { + 'param': { + 'type': 'boolean', + 'enum': [True, False], + 'input': 'radio' + } + } + }, + 'email_sacs': { + 'value': { + 'param': { + 'type': 'boolean', + 'enum': [True, False], + 'input': 'radio' + } + } + } + }, + 'invitation': { + 'id': invitation_id, + 'signatures': [venue_id], + 'content': { + 'email_pcs': { + 'value': '${4/content/email_pcs/value}' + }, + 'email_sacs': { + 'value': '${4/content/email_sacs/value}' + } + } + } + } + ) + + self.save_invitation(invitation, replacement=False) + return invitation + + def set_edit_stage_invitation(self): + + venue_id = self.venue_id + venue = self.venue + + invitation = Invitation( + id = f'{venue_id}/-/Stage', + invitees = [venue_id], + signatures = ['~Super_User1'], + readers = [venue_id], + writers = [venue_id], + process = self.get_process_content('configuration/process/stage_process.py'), + edit = { + 'signatures': [venue_id], + 'readers': [venue_id], + 'writers': [venue_id], + 'content' :{ + 'stage_type': { + 'order': 1, + 'description': 'Stage Type, e.g. Official_Comment, Meta_Review, Decision, Submission Revision, Custom Stage', + 'value': { + 'param': { + 'type': 'string', + 'input': 'select', + 'enum': [ + {'value': 'Official_Comment', 'description': 'Official Comment'}, + {'value': 'Meta_Review', 'description': 'Meta Review'}, + {'value': 'Decision', 'description': 'Decision'}, + {'value': 'Submission_Revision', 'description': 'Submission Revision'}, + {'value': 'Custom', 'description': 'Custom Stage'}, + ] + } + } + }, + 'stage_name': { + 'order': 2, + 'description': 'Stage Name, use underscores to represent spaces', + 'value': { + 'param': { + 'type': 'string', + 'maxLength': 100, + 'regex': '^[a-zA-Z0-9_]*$', + } + } + }, + 'activation_date': { + 'order': 3, + 'value': { + 'param': { + 'type': 'date', + 'range': [ 0, 9999999999999 ], + 'optional': True, + 'deletable': True + } + } + }, + 'expiration_date': { + 'order': 4, + 'value': { + 'param': { + 'type': 'date', + 'range': [ 0, 9999999999999 ], + 'optional': True, + 'deletable': True + } + } + } + }, + 'invitation': { + 'id': f'{venue_id}/-/${{2/content/stage_name/value}}', + 'signatures': [venue_id], + } + } + ) + + self.save_invitation(invitation, replacement=False) + return invitation \ No newline at end of file diff --git a/openreview/venue/invitation.py b/openreview/venue/invitation.py index 7c3a955db..b793087ae 100644 --- a/openreview/venue/invitation.py +++ b/openreview/venue/invitation.py @@ -152,6 +152,10 @@ def set_submission_invitation(self): cdate=submission_cdate, duedate=tools.datetime_millis(submission_stage.due_date) if submission_stage.due_date else None, expdate = tools.datetime_millis(submission_stage.exp_date) if submission_stage.exp_date else None, + content = { + 'email_authors': { 'value': True }, + 'email_pcs': { 'value': self.venue.submission_stage.email_pcs } + }, edit = { 'signatures': { 'param': { @@ -351,10 +355,9 @@ def set_submission_deletion_invitation(self, submission_revision_stage): def set_post_submission_invitation(self): venue_id = self.venue_id submission_stage = self.venue.submission_stage - submission_name = submission_stage.name submission_id = submission_stage.get_submission_id(self.venue) - post_submission_id = f'{venue_id}/-/Post_{submission_name}' + post_submission_id = self.venue.get_post_submission_id() post_submission_cdate = tools.datetime_millis(submission_stage.exp_date) if submission_stage.exp_date else None hidden_field_names = submission_stage.get_hidden_field_names() @@ -495,7 +498,7 @@ def set_review_invitation(self): previous_query = {} invitation = tools.get_invitation(self.client, review_invitation_id) if invitation: - previous_query = invitation.content.get('source_submissions_query', {}).get('value', {}) + previous_query = invitation.content.get('source_submissions_query', {}).get('value', {}) if invitation.content else {} source_submissions_query = review_stage.source_submissions_query if review_stage.source_submissions_query else previous_query @@ -509,7 +512,11 @@ def set_review_invitation(self): 'dates': ["#{4/edit/invitation/cdate}", self.update_date_string], 'script': self.invitation_edit_process }], - content={}, + content={ + 'email_pcs': { + 'value': review_stage.email_pcs + }, + }, edit={ 'signatures': [venue_id], 'readers': [venue_id], @@ -860,7 +867,7 @@ def set_meta_review_invitation(self): previous_query = {} invitation = tools.get_invitation(self.client, meta_review_invitation_id) if invitation: - previous_query = invitation.content.get('source_submissions_query', {}).get('value', {}) + previous_query = invitation.content.get('source_submissions_query', {}).get('value', {}) if invitation.content else {} source_submissions_query = meta_review_stage.source_submissions_query if meta_review_stage.source_submissions_query else previous_query @@ -1338,6 +1345,12 @@ def set_official_comment_invitation(self): }, 'comment_process_script': { 'value': self.get_process_content('process/comment_process.py') + }, + 'email_pcs': { + 'value': comment_stage.email_pcs + }, + 'email_sacs': { + 'value': comment_stage.email_sacs } }, edit={ @@ -1375,7 +1388,7 @@ def set_official_comment_invitation(self): const script = metaInvitation.content.comment_preprocess_script.value; eval(`var process = ${script}`); await process(client, edit, invitation); -}''' if comment_stage.check_mandatory_readers and comment_stage.reader_selection else None, +}''' if comment_stage.check_mandatory_readers and comment_stage.reader_selection else '', 'process': '''def process(client, edit, invitation): meta_invitation = client.get_invitation(invitation.invitations[0]) script = meta_invitation.content['comment_process_script']['value'] @@ -2551,7 +2564,7 @@ def set_submission_revision_invitation(self, submission_revision_stage=None): hidden_field_names = self.venue.submission_stage.get_hidden_field_names() existing_invitation = tools.get_invitation(self.client, revision_invitation_id) - invitation_content = existing_invitation.edit['invitation']['edit']['note']['content'] if existing_invitation else {} + invitation_content = existing_invitation.edit.get('invitation', {}).get('edit', {}).get('note', {}).get('content', {}) if existing_invitation and existing_invitation.edit else {} for field in content: if field in hidden_field_names: @@ -3999,7 +4012,12 @@ def set_group_recruitment_invitations(self, committee_name): } }, 'group': { - 'id': venue.get_committee_id_invited(committee_name) + 'id': venue.get_committee_id_invited(committee_name), + 'content': { + 'last_recruitment': { + 'value': '${4/tmdate}' + } + } } }) @@ -4017,45 +4035,140 @@ def set_group_recruitment_invitations(self, committee_name): 'signatures': [venue_id], 'readers': [venue_id], 'writers': [venue_id], + 'content': { + 'reduced_load': { + 'value': { + 'param': { + 'type': 'integer[]', + 'optional': True + } + } + }, + 'recruitment_subject': { + 'value': { + 'param': { + 'type': 'string', + 'regex': '.+', + 'optional': True, + 'default': f'[{venue.short_name}] Invitation to serve as {pretty_role}' + } + } + }, + 'recruitment_template': { + 'value': { + 'param': { + 'type': 'string', + 'maxLength': 5000, + 'input': 'textarea', + 'optional': True + } + } + }, + 'allow_overlap': { + 'value': { + 'param': { + 'type': 'boolean', + 'enum': [True, False] + } + } + } + }, 'group': { 'id': venue.get_committee_id_invited(committee_name), 'content': { 'reduced_load': { + 'value': '${4/content/reduced_load/value}' + }, + 'recruitment_subject': { + 'value': '${4/content/recruitment_subject/value}' + }, + 'recruitment_template': { + 'value': '${4/content/recruitment_template/value}' + }, + 'allow_overlap': { + 'value': '${4/content/allow_overlap/value}' + } + } + } + }) + + self.save_invitation(invitation, replacement=False) + + def set_group_matching_setup_invitations(self, committee_id): + + venue_id = self.venue_id + committee_name = openreview.tools.pretty_id(committee_id.split('/')[-1]).lower() + cdate = tools.datetime_millis(self.venue.submission_stage.exp_date) if self.venue.submission_stage.exp_date else None + + invitation = Invitation(id=self.venue.get_matching_setup_id(committee_id), + invitees=[venue_id], + readers=[venue_id], + writers=[venue_id], + signatures=[venue_id], + cdate=cdate, + process=self.get_process_content('process/group_matching_setup_process.py'), + edit={ + 'signatures': [venue_id], + 'readers': [venue_id], + 'writers': [venue_id], + 'group': { + 'id': committee_id, + 'content': { + 'assignment_mode': { + 'order': 1, + 'description': f'How do you want to assign {committee_name} to submissions?. Automatic assignment will assign {committee_name} to submissions based on their expertise and/or bids. Manual assignment will allow you to assign reviewers to submissions manually.', 'value': { 'param': { - 'type': 'integer[]', - 'optional': True + 'type': 'string', + 'enum': ['Automatic', 'Manual'] } } }, - 'recruitment_subject': { + 'affinity_score_model': { + 'order': 2, + 'description': f'Select the model to use for calculating affinity scores between {committee_name} and submissions or leaving it blank to not compute affinity scores.', 'value': { 'param': { 'type': 'string', - 'regex': '.+', 'optional': True, - 'default': f'[{venue.short_name}] Invitation to serve as {pretty_role}' + 'enum': ['specter+mfr', 'specter2', 'scincl', 'specter2+scincl'] } } }, - 'recruitment_template': { + 'affinity_score_upload': { + 'order': 3, + 'description': f'If you would like to use your own affinity scores, upload a CSV file containing affinity scores for user-paper pairs (one user-paper pair per line in the format: submission_id, user_id, affinity_score)', + 'value': { + 'param': { + 'type': 'file', + 'optional': True, + 'maxSize': 50, + 'extensions': ['csv'] + } + } + }, + 'conflict_policy': { + 'order': 4, + 'description': f'Select the policy to compute conflicts between the submissions and the {committee_name}. Leaving it blank to not compute any conflicts.', 'value': { 'param': { 'type': 'string', - 'maxLength': 5000, - 'input': 'textarea', - 'optional': True + 'optional': True, + 'enum': ['Default', 'NeurIPS', 'Authors_Only'] ## TODO: Add the authors only policy } } }, - 'allow_overlap': { + 'conflict_n_years': { + 'order': 5, + 'description': 'If conflict policy was selected, enter the number of the years we should use to get the information from the OpenReview profile in order to detect conflicts. Leave it empty if you want to use all the available information.', 'value': { 'param': { - 'type': 'boolean', - 'enum': [True, False] + 'type': 'integer', + 'minimum': 1, + 'optional': True } } - } + } } } }) diff --git a/openreview/venue/process/comment_process.py b/openreview/venue/process/comment_process.py index 70d960295..5de3a3c15 100644 --- a/openreview/venue/process/comment_process.py +++ b/openreview/venue/process/comment_process.py @@ -13,6 +13,8 @@ def process(client, edit, invitation): sender = domain.get_content_value('message_sender') + + parent_invitation = client.get_invitation(invitation.invitations[0]) submission = client.get_note(edit.note.forum) comment = client.get_note(edit.note.id) paper_group_id=f'{venue_id}/{submission_name}{submission.number}' @@ -39,7 +41,8 @@ def process(client, edit, invitation): To view the comment, click here: https://openreview.net/forum?id={submission.id}¬eId={comment.id}''' program_chairs_id = domain.get_content_value('program_chairs_id') - if domain.get_content_value('comment_email_pcs') and (program_chairs_id in comment.readers or 'everyone' in comment.readers): + comment_email_pcs = parent_invitation.get_content_value('email_pcs', domain.get_content_value('comment_email_pcs')) + if comment_email_pcs and (program_chairs_id in comment.readers or 'everyone' in comment.readers): client.post_message( invitation=meta_invitation_id, recipients=[program_chairs_id], @@ -53,7 +56,9 @@ def process(client, edit, invitation): senior_area_chairs_name = domain.get_content_value('senior_area_chairs_name') paper_senior_area_chairs_id = f'{paper_group_id}/{senior_area_chairs_name}' paper_senior_area_chairs_group = openreview.tools.get_group(client, paper_senior_area_chairs_id) - email_SAC = ((len(comment.readers)==3 and paper_senior_area_chairs_id in comment.readers and program_chairs_id in comment.readers) or domain.get_content_value('comment_email_sacs')) + + comment_email_sacs = parent_invitation.get_content_value('email_sacs', domain.get_content_value('comment_email_sacs')) + email_SAC = ((len(comment.readers)==3 and paper_senior_area_chairs_id in comment.readers and program_chairs_id in comment.readers) or comment_email_sacs) if paper_senior_area_chairs_group and senior_area_chairs_name and email_SAC: client.post_message( invitation=meta_invitation_id, diff --git a/openreview/venue/process/edit_metareview_recommendation.py b/openreview/venue/process/edit_metareview_recommendation.py new file mode 100644 index 000000000..b3531a8eb --- /dev/null +++ b/openreview/venue/process/edit_metareview_recommendation.py @@ -0,0 +1,20 @@ +def process(client, edit, invitation): + + domain = client.get_group(edit.domain) + venue_id = domain.id + meta_invitation_id = domain.get_content_value('meta_invitation_id') + recommendation_field_name = edit.content['recommendation_field_name']['value'] + + client.post_group_edit( + invitation = meta_invitation_id, + signatures = [venue_id], + group = openreview.api.Group( + id = venue_id, + content = { + 'meta_review_recommendation': { + 'value': recommendation_field_name + } + } + ) + ) + diff --git a/openreview/venue/process/edit_review_field_names_process.py b/openreview/venue/process/edit_review_field_names_process.py new file mode 100644 index 000000000..2ea517317 --- /dev/null +++ b/openreview/venue/process/edit_review_field_names_process.py @@ -0,0 +1,24 @@ +def process(client, edit, invitation): + + domain = client.get_group(edit.domain) + venue_id = domain.id + meta_invitation_id = domain.get_content_value('meta_invitation_id') + rating_field_name = edit.content['rating_field_name']['value'] + confidence_field_name = edit.content['confidence_field_name']['value'] + + client.post_group_edit( + invitation = meta_invitation_id, + signatures = [venue_id], + group = openreview.api.Group( + id = venue_id, + content = { + 'review_rating': { + 'value': rating_field_name + }, + 'review_confidence': { + 'value': confidence_field_name + } + } + ) + ) + diff --git a/openreview/venue/process/edit_submission_deadline_process.py b/openreview/venue/process/edit_submission_deadline_process.py new file mode 100644 index 000000000..00051e6dc --- /dev/null +++ b/openreview/venue/process/edit_submission_deadline_process.py @@ -0,0 +1,53 @@ +def process(client, edit, invitation): + + support_user = 'openreview.net/Support' + domain = client.get_group(edit.domain) + venue_id = domain.id + meta_invitation_id = domain.get_content_value('meta_invitation_id') + request_form_id = domain.get_content_value('request_form_id') + submission_name = domain.get_content_value('submission_name') + expdate = edit.invitation.expdate + print('Submission deadline: ', edit.invitation.duedate) + print('Setting post submission cdate to: ', expdate) + + + venue = openreview.helpers.get_venue(client, request_form_id, support_user) + + # update post submission cdate + client.post_invitation_edit( + invitations=meta_invitation_id, + signatures=[venue_id], + invitation=openreview.api.Invitation( + id=venue.get_post_submission_id(), + cdate=expdate + ) + ) + + client.post_invitation_edit( + invitations=meta_invitation_id, + signatures=[venue_id], + invitation=openreview.api.Invitation( + id=venue.get_matching_setup_id(venue.get_reviewers_id()), + cdate=expdate + ) + ) + + if venue.use_area_chairs: + client.post_invitation_edit( + invitations=meta_invitation_id, + signatures=[venue_id], + invitation=openreview.api.Invitation( + id=venue.get_matching_setup_id(venue.get_area_chairs_id()), + cdate=expdate + ) + ) + + if venue.use_senior_area_chairs: + client.post_invitation_edit( + invitations=meta_invitation_id, + signatures=[venue_id], + invitation=openreview.api.Invitation( + id=venue.get_matching_setup_id(venue.get_senior_area_chairs_id()), + cdate=expdate + ) + ) \ No newline at end of file diff --git a/openreview/venue/process/group_matching_setup_process.py b/openreview/venue/process/group_matching_setup_process.py new file mode 100644 index 000000000..9a50213cf --- /dev/null +++ b/openreview/venue/process/group_matching_setup_process.py @@ -0,0 +1,47 @@ +def process(client, edit, invitation): + + support_user = 'openreview.net/Support' + domain = client.get_group(invitation.domain) + venue_id = domain.id + meta_invitation_id = domain.content['meta_invitation_id']['value'] + request_form_id = domain.get_content_value('request_form_id') + + venue = openreview.helpers.get_venue(client, request_form_id, support_user) + venue.automatic_reviewer_assignment = edit.group.content.get('assignment_mode', {}).get('value', 'Manual') == 'Automatic' + + ## syncronize the venue group settings + content = { + 'automatic_reviewer_assignment': { 'value': venue.automatic_reviewer_assignment }, + } + if edit.group.id == venue.get_reviewers_id(): + if 'conflict_policy' in edit.group.content: + content['reviewers_conflict_policy'] = { 'value': edit.group.content.get('conflict_policy', {}).get('value') } + if 'conflict_n_years' in edit.group.content: + content['reviewers_conflict_n_years'] = { 'value': edit.group.content.get('conflict_n_years', {}).get('value') } + elif edit.group.id == venue.get_area_chairs_id(): + if 'conflict_policy' in edit.group.content: + content['area_chairs_conflict_policy'] = { 'value': edit.group.content.get('conflict_policy', {}).get('value') } + if 'conflict_n_years' in edit.group.content: + content['area_chairs_conflict_n_years'] = { 'value': edit.group.content.get('conflict_n_years', {}).get('value') } + + client.post_group_edit( + invitation=meta_invitation_id, + signatures=[venue_id], + readers=[venue_id], + group=openreview.api.Group( + id=venue_id, + content=content + ) + ) + + ## run setup matching + result = venue.setup_committee_matching( + committee_id=edit.group.id, + compute_affinity_scores=edit.group.content.get('affinity_score_model', {}).get('value', edit.group.content.get('affinity_score_upload', {}).get('value')), + compute_conflicts=edit.group.content.get('conflict_policy', {}).get('value'), + compute_conflicts_n_years=edit.group.content.get('conflict_n_years', {}).get('value'), + alternate_matching_group=None, + submission_track=None + ) + + print('Result', result) \ No newline at end of file diff --git a/openreview/venue/process/group_recruitment_process.py b/openreview/venue/process/group_recruitment_process.py index c7f91c938..da9a73ee9 100644 --- a/openreview/venue/process/group_recruitment_process.py +++ b/openreview/venue/process/group_recruitment_process.py @@ -2,7 +2,6 @@ def process(client, edit, invitation): domain = client.get_group(invitation.domain) venue_id = domain.id - venue_id = domain.id meta_invitation_id = domain.content['meta_invitation_id']['value'] contact_email = domain.content['contact']['value'] invited_group = client.get_group(invitation.edit['group']['id']) diff --git a/openreview/venue/process/review_process.py b/openreview/venue/process/review_process.py index 4042ffee2..d920cd732 100644 --- a/openreview/venue/process/review_process.py +++ b/openreview/venue/process/review_process.py @@ -19,8 +19,9 @@ def process(client, edit, invitation): paper_reviewers_id = f'{paper_group_id}/{reviewers_name}' paper_reviewers_submitted_id = f'{paper_reviewers_id}/{reviewers_submitted_name}' paper_area_chairs_id = f'{paper_group_id}/{area_chairs_name}' - paper_senior_area_chairs_id = f'{paper_group_id}/{senior_area_chairs_name}' + paper_senior_area_chairs_id = f'{paper_group_id}/{senior_area_chairs_name}' + parent_invitation = client.get_invitation(invitation.invitations[0]) review = client.get_note(edit.note.id) ## run process function for the first edit only @@ -62,7 +63,8 @@ def create_group(group_id, members=[]): content = f'To view the {review_name}, click here: https://openreview.net/forum?id={submission.id}¬eId={edit.note.id}' - if domain.get_content_value('review_email_pcs'): + review_email_pcs = parent_invitation.get_content_value('email_pcs', domain.get_content_value('review_email_pcs')) + if review_email_pcs: client.post_message( invitation=meta_invitation_id, signature=venue_id, @@ -70,7 +72,7 @@ def create_group(group_id, members=[]): recipients=[domain.get_content_value('program_chairs_id')], ignoreRecipients=ignore_groups, subject=f'''[{short_name}] A {review_name} has been received on Paper number: {submission.number}, Paper title: "{submission.content['title']['value']}"''', - message=f''''We have received a review on a submission to {short_name}. + message=f'''We have received a review on a submission to {short_name}. {content} ''' @@ -83,7 +85,7 @@ def create_group(group_id, members=[]): recipients=review.signatures, replyTo=contact, subject=f'''[{short_name}] Your {review_name} has been received on your assigned Paper number: {submission.number}, Paper title: "{submission.content['title']['value']}"''', - message=f''''We have received a review on a submission to {short_name}. + message=f'''We have received a review on a submission to {short_name}. Paper number: {submission.number} @@ -101,7 +103,7 @@ def create_group(group_id, members=[]): ignoreRecipients=ignore_groups, replyTo=contact, subject=f'''[{short_name}] {capital_review_name} posted to your assigned Paper number: {submission.number}, Paper title: "{submission.content['title']['value']}"''', - message=f''''A submission to {short_name}, for which you are an official area chair, has received a review. + message=f'''A submission to {short_name}, for which you are an official area chair, has received a review. Paper number: {submission.number} @@ -120,7 +122,7 @@ def create_group(group_id, members=[]): ignoreRecipients=ignore_groups, replyTo=contact, subject=f'''[{short_name}] {capital_review_name} posted to your assigned Paper number: {submission.number}, Paper title: "{submission.content['title']['value']}"''', - message=f''''A submission to {short_name}, for which you are a reviewer, has received a review. + message=f'''A submission to {short_name}, for which you are a reviewer, has received a review. Paper number: {submission.number} @@ -138,7 +140,7 @@ def create_group(group_id, members=[]): ignoreRecipients=ignore_groups, replyTo=contact, subject=f'''[{short_name}] {capital_review_name} posted to your assigned Paper number: {submission.number}, Paper title: "{submission.content['title']['value']}"''', - message=f''''A submission to {short_name}, for which you are a reviewer, has received a review. + message=f'''A submission to {short_name}, for which you are a reviewer, has received a review. Paper number: {submission.number} @@ -158,7 +160,7 @@ def create_group(group_id, members=[]): ignoreRecipients=ignore_groups, replyTo=contact, subject=f'''[{short_name}] {capital_review_name} posted to your submission - Paper number: {submission.number}, Paper title: "{submission.content['title']['value']}"''', - message=f''''Your submission to {short_name} has received a review. + message=f'''Your submission to {short_name} has received a review. {content} ''' diff --git a/openreview/venue/process/submission_process.py b/openreview/venue/process/submission_process.py index 0976cc697..2e796373c 100644 --- a/openreview/venue/process/submission_process.py +++ b/openreview/venue/process/submission_process.py @@ -9,7 +9,8 @@ def process(client, edit, invitation): short_phrase = domain.content['subtitle']['value'] contact = domain.content['contact']['value'] submission_email = domain.content['submission_email_template']['value'] - email_pcs = domain.content['submission_email_pcs']['value'] + email_pcs = invitation.get_content_value('email_pcs', domain.get_content_value('submission_email_pcs')) + email_authors = invitation.get_content_value('email_authors', True) program_chairs_id = domain.content['program_chairs_id']['value'] sender = domain.get_content_value('message_sender') @@ -119,31 +120,32 @@ def process(client, edit, invitation): group=openreview.api.Group() ) - #send tauthor email - if edit.tauthor.lower() != 'openreview.net': - client.post_message( - invitation=meta_invitation_id, - subject=author_subject, - message=author_message, - recipients=[edit.tauthor], - replyTo=contact, - signature=venue_id, - sender=sender - ) + if email_authors: + #send tauthor email + if edit.tauthor.lower() != 'openreview.net': + client.post_message( + invitation=meta_invitation_id, + subject=author_subject, + message=author_message, + recipients=[edit.tauthor], + replyTo=contact, + signature=venue_id, + sender=sender + ) - # send co-author emails - if ('authorids' in note.content and len(note.content['authorids']['value'])): - author_message += f'''\n\nIf you are not an author of this submission and would like to be removed, please contact the author who added you at {edit.tauthor}''' - client.post_message( - invitation=meta_invitation_id, - subject=author_subject, - message=author_message, - recipients=note.content['authorids']['value'], - ignoreRecipients=[edit.tauthor], - replyTo=contact, - signature=venue_id, - sender=sender - ) + # send co-author emails + if ('authorids' in note.content and len(note.content['authorids']['value'])): + author_message += f'''\n\nIf you are not an author of this submission and would like to be removed, please contact the author who added you at {edit.tauthor}''' + client.post_message( + invitation=meta_invitation_id, + subject=author_subject, + message=author_message, + recipients=note.content['authorids']['value'], + ignoreRecipients=[edit.tauthor], + replyTo=contact, + signature=venue_id, + sender=sender + ) if email_pcs: client.post_message( diff --git a/openreview/venue/venue.py b/openreview/venue/venue.py index 7bc5ba5fe..adb0d3a42 100644 --- a/openreview/venue/venue.py +++ b/openreview/venue/venue.py @@ -11,6 +11,7 @@ from openreview import tools from .invitation import InvitationBuilder from .group import GroupBuilder +from .edit_invitation import EditInvitationBuilder from openreview.api import Group from openreview.api import Note from .recruitment import Recruitment @@ -67,13 +68,14 @@ def __init__(self, client, venue_id, support_user): self.invitation_builder = InvitationBuilder(self) self.group_builder = GroupBuilder(self) self.recruitment = Recruitment(self) + self.edit_invitation_builder = EditInvitationBuilder(self) self.reviewer_identity_readers = [] self.area_chair_identity_readers = [] self.senior_area_chair_identity_readers = [] self.automatic_reviewer_assignment = False self.decision_heading_map = {} self.allow_gurobi_solver = False - self.submission_license = None + self.submission_license = ['CC BY 4.0'] self.use_publication_chairs = False self.source_submissions_query_mapping = {} self.sac_paper_assignments = False @@ -151,6 +153,10 @@ def get_meta_invitation_id(self): def get_submission_id(self): return self.submission_stage.get_submission_id(self) + + def get_post_submission_id(self): + submission_name = self.submission_stage.name + return self.get_invitation_id(f'Post_{submission_name}') def get_pc_submission_revision_id(self): return self.get_invitation_id('PC_Revision') @@ -171,6 +177,9 @@ def get_assignment_id(self, committee_id, deployed=False, invite=False): return self.get_invitation_id('Invite_Assignment', prefix=committee_id) return self.get_invitation_id('Proposed_Assignment', prefix=committee_id) + def get_matching_setup_id(self, committee_id): + return self.get_invitation_id('Matching_Setup', prefix=committee_id) + def get_affinity_score_id(self, committee_id): return self.get_invitation_id('Affinity_Score', prefix=committee_id) @@ -506,6 +515,40 @@ def create_submission_stage(self): self.invitation_builder.set_submission_revision_invitation(submission_revision_stage) self.invitation_builder.set_submission_deletion_invitation(submission_revision_stage) + def create_submission_edit_invitations(self): + self.edit_invitation_builder.set_edit_submission_deadlines_invitation(self.get_submission_id(), 'edit_submission_deadline_process.py') + self.edit_invitation_builder.set_edit_submission_content_invitation(self.get_submission_id()) + self.edit_invitation_builder.set_edit_submission_notification_invitation() + self.edit_invitation_builder.set_edit_submission_readers_invitation() + self.edit_invitation_builder.set_edit_submission_field_readers_invitation() + + def create_review_edit_invitations(self): + review_stage = self.review_stage + review_invitation_id = self.get_invitation_id(review_stage.name) + self.edit_invitation_builder.set_edit_deadlines_invitation(review_invitation_id) + content = { + 'rating_field_name': { + 'value': { + 'param': { + 'type': 'string', + 'regex': '.*', + 'default': 'rating' + } + } + }, + 'confidence_field_name': { + 'value': { + 'param': { + 'type': 'string', + 'regex': '.*', + 'default': 'confidence' + } + } + } + } + self.edit_invitation_builder.set_edit_content_invitation(review_invitation_id, content, 'edit_review_field_names_process.py') + self.edit_invitation_builder.set_edit_reply_readers_invitation(review_invitation_id) + def create_post_submission_stage(self): self.invitation_builder.set_post_submission_invitation() diff --git a/setup.py b/setup.py index 13a44f82f..327dd208d 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ 'openreview/profile', 'openreview/agora', 'openreview/venue', + 'openreview/venue/configuration', 'openreview/venue_request', 'openreview/journal', 'openreview/journal/journal_request', diff --git a/tests/test_aaai_conference.py b/tests/test_aaai_conference.py index adac13915..8cb5e7107 100644 --- a/tests/test_aaai_conference.py +++ b/tests/test_aaai_conference.py @@ -8,19 +8,10 @@ import csv from selenium.webdriver.common.by import By from selenium.common.exceptions import NoSuchElementException -from openreview import ProfileManagement class TestAAAIConference(): - - @pytest.fixture(scope="class") - def profile_management(self, openreview_client): - profile_management = ProfileManagement(openreview_client, 'openreview.net') - profile_management.setup() - return profile_management - - - def test_create_conference(self, client, openreview_client, helpers, profile_management): + def test_create_conference(self, client, openreview_client, helpers): now = datetime.datetime.utcnow() due_date = now + datetime.timedelta(days=3) diff --git a/tests/test_acl_commitment_v2.py b/tests/test_acl_commitment_v2.py index d3c2f272b..146b286df 100644 --- a/tests/test_acl_commitment_v2.py +++ b/tests/test_acl_commitment_v2.py @@ -8,7 +8,6 @@ import csv from selenium.webdriver.common.by import By from selenium.common.exceptions import NoSuchElementException -from openreview import ProfileManagement class TestACLCommitment(): diff --git a/tests/test_arr_venue.py b/tests/test_arr_venue.py index 140e9b9cb..bd52973e6 100644 --- a/tests/test_arr_venue.py +++ b/tests/test_arr_venue.py @@ -9,16 +9,9 @@ import csv import random from selenium.webdriver.common.by import By -from openreview import ProfileManagement class TestARRVenue(): - @pytest.fixture(scope="class") - def profile_management(self, openreview_client): - profile_management = ProfileManagement(openreview_client, 'openreview.net') - profile_management.setup() - return profile_management - @pytest.fixture(scope="class") def venue(self, client, helpers): pc_client=openreview.Client(username='pc@aclrollingreview.org', password=helpers.strong_password) @@ -28,7 +21,7 @@ def venue(self, client, helpers): return conference - def test_create_venue(self, client, helpers, profile_management): + def test_create_venue(self, client, helpers): now = datetime.datetime.utcnow() due_date = now + datetime.timedelta(days=3) diff --git a/tests/test_arr_venue_v2.py b/tests/test_arr_venue_v2.py index 7909d8a82..9bcb96ea4 100644 --- a/tests/test_arr_venue_v2.py +++ b/tests/test_arr_venue_v2.py @@ -10,7 +10,6 @@ from copy import deepcopy from selenium.webdriver.common.by import By from selenium.common.exceptions import NoSuchElementException -from openreview import ProfileManagement from openreview.venue import matching from openreview.stages.arr_content import ( arr_submission_content, @@ -27,12 +26,8 @@ ) # API2 template from ICML class TestARRVenueV2(): - @pytest.fixture(scope="class") - def profile_management(self, openreview_client): - profile_management = ProfileManagement(openreview_client, 'openreview.net') - profile_management.setup() - return profile_management - def test_august_cycle(self, client, openreview_client, helpers, test_client, profile_management, request_page, selenium): + + def test_august_cycle(self, client, openreview_client, helpers, test_client, request_page, selenium): now = datetime.datetime.utcnow() due_date = now + datetime.timedelta(days=3) @@ -362,7 +357,7 @@ def test_august_cycle(self, client, openreview_client, helpers, test_client, pro ) ) - def test_june_cycle(self, client, openreview_client, helpers, test_client, profile_management): + def test_june_cycle(self, client, openreview_client, helpers, test_client): # Build the previous cycle pc_client=openreview.Client(username='pc@aclrollingreview.org', password=helpers.strong_password) pc_client_v2 = openreview.api.OpenReviewClient(username='pc@aclrollingreview.org', password=helpers.strong_password) diff --git a/tests/test_cvpr_conference_v2.py b/tests/test_cvpr_conference_v2.py index d0ff699bb..71050862b 100644 --- a/tests/test_cvpr_conference_v2.py +++ b/tests/test_cvpr_conference_v2.py @@ -3,7 +3,6 @@ import datetime from selenium.webdriver.common.by import By from selenium.common.exceptions import NoSuchElementException -from openreview import ProfileManagement import csv import os import random diff --git a/tests/test_group_recruitment.py b/tests/test_group_recruitment.py index a73655359..689a32977 100644 --- a/tests/test_group_recruitment.py +++ b/tests/test_group_recruitment.py @@ -67,10 +67,9 @@ def test_recruitment(self, venue, openreview_client, helpers): # use invitation to edit group content openreview_client.post_group_edit( invitation='Venue.cc/Reviewers/Invited/-/Recruitment_Settings', - group=openreview.api.Group( - content = { - 'reduced_load': { 'value': [1,2,3] }, - 'recruitment_template': { 'value': '''Dear {{fullname}}, + content = { + 'reduced_load': { 'value': [1,2,3] }, + 'recruitment_template': { 'value': '''Dear {{fullname}}, You have been nominated by the program chair committee of V 24 to serve as Reviewer. As a respected researcher in the area, we hope you will accept and help us make V 24 a success. @@ -92,10 +91,9 @@ def test_recruitment(self, venue, openreview_client, helpers): Program Chairs ''' }, - 'recruitment_subject': { 'value': '[V 24] Invitation to serve as Reviewer' }, - 'allow_overlap': { 'value': False } - } - ) + 'recruitment_subject': { 'value': '[V 24] Invitation to serve as Reviewer' }, + 'allow_overlap': { 'value': False } + } ) invitee_details = '''~Reviewer_VenueThree1\nreviewer1@venue.cc, Reviewer VenueOne\nreviewer2@venue.cc, Reviewer VenueTwo''' diff --git a/tests/test_iclr_conference_v2.py b/tests/test_iclr_conference_v2.py index 4eb61aa80..f6aa6ece6 100644 --- a/tests/test_iclr_conference_v2.py +++ b/tests/test_iclr_conference_v2.py @@ -8,19 +8,10 @@ import csv from selenium.webdriver.common.by import By from selenium.common.exceptions import NoSuchElementException -from openreview import ProfileManagement class TestICLRConference(): - - @pytest.fixture(scope="class") - def profile_management(self, openreview_client): - profile_management = ProfileManagement(openreview_client, 'openreview.net') - profile_management.setup() - return profile_management - - - def test_create_conference(self, client, openreview_client, helpers, profile_management): + def test_create_conference(self, client, openreview_client, helpers): now = datetime.datetime.utcnow() abstract_date = now + datetime.timedelta(days=1) diff --git a/tests/test_icml_conference.py b/tests/test_icml_conference.py index 84833db06..09d502e1f 100644 --- a/tests/test_icml_conference.py +++ b/tests/test_icml_conference.py @@ -8,19 +8,10 @@ import csv from selenium.webdriver.common.by import By from selenium.common.exceptions import NoSuchElementException -from openreview import ProfileManagement class TestICMLConference(): - - @pytest.fixture(scope="class") - def profile_management(self, openreview_client): - profile_management = ProfileManagement(openreview_client, 'openreview.net') - profile_management.setup() - return profile_management - - - def test_create_conference(self, client, openreview_client, helpers, profile_management): + def test_create_conference(self, client, openreview_client, helpers): now = datetime.datetime.utcnow() due_date = now + datetime.timedelta(days=3) diff --git a/tests/test_icml_conference_configuration.py b/tests/test_icml_conference_configuration.py new file mode 100644 index 000000000..2b9d9005e --- /dev/null +++ b/tests/test_icml_conference_configuration.py @@ -0,0 +1,5565 @@ +import time +import openreview +import pytest +import datetime +import re +import random +import os +import csv +from selenium.webdriver.common.by import By +from selenium.common.exceptions import NoSuchElementException + +class TestICMLConference(): + + def test_create_conference(self, client, openreview_client, helpers): + + super_id = 'openreview.net' + support_group_id = super_id + '/Support' + + now = datetime.datetime.utcnow() + due_date = now + datetime.timedelta(days=3) + + # Post the request form note + helpers.create_user('pc@icml.cc', 'Program', 'ICMLChair') + pc_client = openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) + + + sac_client = helpers.create_user('sac1@gmail.com', 'SAC', 'ICMLOne') + + helpers.create_user('sac2@icml.cc', 'SAC', 'ICMLTwo') + helpers.create_user('ac1@icml.cc', 'AC', 'ICMLOne') + helpers.create_user('ac2@icml.cc', 'AC', 'ICMLTwo') + helpers.create_user('reviewer1@icml.cc', 'Reviewer', 'ICMLOne') + helpers.create_user('reviewer2@icml.cc', 'Reviewer', 'ICMLTwo') + helpers.create_user('reviewer3@icml.cc', 'Reviewer', 'ICMLThree') + helpers.create_user('reviewer4@yahoo.com', 'Reviewer', 'ICMLFour') + helpers.create_user('reviewer5@yahoo.com', 'Reviewer', 'ICMLFive') + helpers.create_user('reviewer6@yahoo.com', 'Reviewer', 'ICMLSix') + helpers.create_user('reviewerethics@yahoo.com', 'Reviewer', 'ICMLSeven') + helpers.create_user('peter@mail.com', 'Peter', 'SomeLastName') # Author + + + now = datetime.datetime.utcnow() + start_date = now + datetime.timedelta(minutes=30) + due_date = now + datetime.timedelta(days=1) + + # post a conference request form + conference_request_edit = pc_client.post_note_edit(invitation='openreview.net/-/Venue_Configuration_Request', + signatures=['~Program_ICMLChair1'], + note=openreview.api.Note( + content={ + 'official_venue_name': { 'value': 'Thirty-ninth International Conference on Machine Learning' }, + 'abbreviated_venue_name': { 'value': 'ICML 2024' }, + 'venue_website_url': { 'value': 'https://icml.cc' }, + 'location': { 'value': 'Vienna, Austria' }, + 'venue_start_date': { 'value': now.strftime('%Y/%m/%d') }, + 'program_chair_emails': { 'value': ['pc@icml.cc'] }, + 'contact_email': { 'value': 'pc@icml.cc' }, + 'publication_chairs': { 'value': 'No, our venue does not have Publication Chairs' }, + 'area_chairs_and_senior_area_chairs': { 'value': 'Yes, our venue has Area Chairs and Senior Area Chairs' }, + 'ethics_chairs_and_reviewers': { 'value': 'Yes, our venue has Ethics Chairs and Reviewers' }, + 'secondary_area_chairs': { 'value': 'No, our venue does not have Secondary Area Chairs' }, + 'submission_start_date': { 'value': start_date.strftime('%Y/%m/%d %H:%M') }, + 'submission_deadline': { 'value': due_date.strftime('%Y/%m/%d %H:%M') }, + 'author_and_reviewer_anonymity': { 'value': 'Double-blind' }, + 'submission_license': { 'value': ['CC BY-NC 4.0'] }, + } + )) + + helpers.await_queue_edit(openreview_client, edit_id=conference_request_edit['id']) + + request = openreview_client.get_note(conference_request_edit['note']['id']) + + edit = openreview_client.post_note_edit(invitation=f'openreview.net/Venue_Configuration_Request{request.number}/-/Deploy', + signatures=[support_group_id], + note=openreview.api.Note( + id=request.id, + content={ + 'venue_id': { 'value': 'ICML.cc/2024/Conference' } + } + )) + + helpers.await_queue_edit(openreview_client, edit_id=edit['id']) + + assert openreview_client.get_group('ICML.cc/2024/Conference') + assert openreview_client.get_group('ICML.cc/2024/Conference/Senior_Area_Chairs') + assert openreview_client.get_group('ICML.cc/2024/Conference/Area_Chairs') + assert openreview_client.get_group('ICML.cc/2024/Conference/Reviewers') + assert openreview_client.get_group('ICML.cc/2024/Conference/Authors') + + submission_invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/-/Submission') + assert submission_invitation + assert submission_invitation.duedate + + # assert openreview_client.get_invitation('ICML.cc/2024/Conference/Reviewers/-/Expertise_Selection') + # assert openreview_client.get_invitation('ICML.cc/2024/Conference/Area_Chairs/-/Expertise_Selection') + # assert openreview_client.get_invitation('ICML.cc/2024/Conference/Senior_Area_Chairs/-/Expertise_Selection') + + # sac_client.post_note_edit( + # invitation='openreview.net/Archive/-/Direct_Upload', + # signatures=['~SAC_ICMLOne1'], + # note = openreview.api.Note( + # pdate = openreview.tools.datetime_millis(datetime.datetime(2019, 4, 30)), + # content = { + # 'title': { 'value': 'Paper title 1' }, + # 'abstract': { 'value': 'Paper abstract 1' }, + # 'authors': { 'value': ['SAC ICML', 'Test2 Client'] }, + # 'authorids': { 'value': ['~SAC_ICMLOne1', 'test2@mail.com'] }, + # 'venue': { 'value': 'Arxiv' } + # }, + # license = 'CC BY-SA 4.0' + # )) + +# sac_client.post_note_edit( +# invitation='openreview.net/Archive/-/Direct_Upload', +# signatures=['~SAC_ICMLOne1'], +# note = openreview.api.Note( +# pdate = openreview.tools.datetime_millis(datetime.datetime(2019, 4, 30)), +# content = { +# 'title': { 'value': 'Paper title 2' }, +# 'abstract': { 'value': 'Paper abstract 2' }, +# 'authors': { 'value': ['SAC ICML', 'Test2 Client'] }, +# 'authorids': { 'value': ['~SAC_ICMLOne1', 'test2@mail.com'] } +# }, +# license = 'CC BY-SA 4.0' +# )) + +# pc_client.post_note(openreview.Note( +# invitation=f'openreview.net/Support/-/Request{request_form_note.number}/Revision', +# forum=request_form_note.id, +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# referent=request_form_note.id, +# replyto=request_form_note.id, +# signatures=['~Program_ICMLChair1'], +# writers=[], +# content={ +# 'title': 'Thirty-ninth International Conference on Machine Learning', +# 'Official Venue Name': 'Thirty-ninth International Conference on Machine Learning', +# 'Abbreviated Venue Name': 'ICML 2023', +# 'Official Website URL': 'https://icml.cc', +# 'program_chair_emails': ['pc@icml.cc'], +# 'contact_email': 'pc@icml.cc', +# 'publication_chairs':'No, our venue does not have Publication Chairs', +# 'Venue Start Date': '2023/07/01', +# 'Submission Deadline': due_date.strftime('%Y/%m/%d'), +# 'Location': 'Virtual', +# 'submission_reviewer_assignment': 'Automatic', +# 'How did you hear about us?': 'ML conferences', +# 'Expected Submissions': '100', +# 'use_recruitment_template': 'Yes', +# 'Additional Submission Options': { +# "supplementary_material": { +# "value": { +# "param": { +# "type": "file", +# "extensions": [ +# "zip", +# "pdf", +# "tgz", +# "gz" +# ], +# "maxSize": 100, +# "optional": True, +# "deletable": True +# } +# }, +# "description": "All supplementary material must be self-contained and zipped into a single file. Note that supplementary material will be visible to reviewers and the public throughout and after the review period, and ensure all material is anonymized. The maximum file size is 100MB.", +# "order": 8 +# }, +# "financial_aid": { +# "order": 9, +# "description": "Each paper may designate up to one (1) icml.cc account email address of a corresponding student author who confirms that they would need the support to attend the conference, and agrees to volunteer if they get selected.", +# "value": { +# "param": { +# "type": "string", +# "maxLength": 100, +# "optional": True +# } +# } +# }, +# "subject_areas": { +# "order": 19, +# "description": "Enter subject areas.", +# "value": { +# "param": { +# "type": "string[]", +# "enum": [ +# 'Algorithms: Approximate Inference', +# 'Algorithms: Belief Propagation', +# 'Learning: Deep Learning', +# 'Learning: General', +# 'Learning: Nonparametric Bayes', +# 'Methodology: Bayesian Methods', +# 'Methodology: Calibration', +# 'Principles: Causality', +# 'Principles: Cognitive Models', +# 'Representation: Constraints', +# 'Representation: Dempster-Shafer', +# 'Representation: Other' +# ], +# "input": "select" +# } +# } +# } +# }, +# 'remove_submission_options': ['TL;DR'], +# 'homepage_override': { +# 'location': 'Hawaii, USA', +# 'instructions': 'For author guidelines, please click [here](https://icml.cc/Conferences/2023/StyleAuthorInstructions)' +# }, +# 'source_submissions_query_mapping': { +# 'Official_Review': { +# 'position_paper_track': 'No' +# }, +# 'Position_Paper_Review': { +# 'position_paper_track': 'Yes' +# } +# } +# } +# )) +# helpers.await_queue() + +# submission_invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/-/Submission') +# assert submission_invitation +# assert 'supplementary_material' in submission_invitation.edit['note']['content'] +# assert 'financial_aid' in submission_invitation.edit['note']['content'] +# assert 'subject_areas' in submission_invitation.edit['note']['content'] +# assert 'TLDR' not in submission_invitation.edit['note']['content'] + +# domain = openreview_client.get_group('ICML.cc/2024/Conference') +# assert 'recommendation' == domain.content['meta_review_recommendation']['value'] + +# def test_add_pcs(self, client, openreview_client, helpers): + +# pc_client=openreview.Client(username='pc@icml.cc', password=helpers.strong_password) +# request_form=pc_client.get_notes(invitation='openreview.net/Support/-/Request_Form')[0] + +# now = datetime.datetime.utcnow() +# due_date = now + datetime.timedelta(days=3) + +# pc_client.post_note(openreview.Note( +# content={ +# 'title': 'Thirty-ninth International Conference on Machine Learning', +# 'Official Venue Name': 'Thirty-ninth International Conference on Machine Learning', +# 'Abbreviated Venue Name': 'ICML 2023', +# 'Official Website URL': 'https://icml.cc', +# 'program_chair_emails': ['pc@icml.cc', 'pc2@icml.cc'], +# 'contact_email': 'pc@icml.cc', +# 'publication_chairs':'No, our venue does not have Publication Chairs', +# 'Venue Start Date': '2023/07/01', +# 'Submission Start Date': now.strftime('%Y/%m/%d %H:%M'), +# 'Submission Deadline': due_date.strftime('%Y/%m/%d %H:%M'), +# 'Location': 'Virtual', +# 'submission_reviewer_assignment': 'Automatic', +# 'How did you hear about us?': 'ML conferences', +# 'Expected Submissions': '100', +# 'Additional Submission Options': { +# "supplementary_material": { +# "value": { +# "param": { +# "type": "file", +# "extensions": [ +# "zip", +# "pdf", +# "tgz", +# "gz" +# ], +# "maxSize": 100, +# "optional": True, +# "deletable": True +# } +# }, +# "description": "All supplementary material must be self-contained and zipped into a single file. Note that supplementary material will be visible to reviewers and the public throughout and after the review period, and ensure all material is anonymized. The maximum file size is 100MB.", +# "order": 8 +# }, +# "financial_aid": { +# "order": 9, +# "description": "Each paper may designate up to one (1) icml.cc account email address of a corresponding student author who confirms that they would need the support to attend the conference, and agrees to volunteer if they get selected.", +# "value": { +# "param": { +# "type": "string", +# "maxLength": 100, +# "optional": True +# } +# } +# } +# } + +# }, +# forum=request_form.forum, +# invitation='openreview.net/Support/-/Request{}/Revision'.format(request_form.number), +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# referent=request_form.forum, +# replyto=request_form.forum, +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) + +# helpers.await_queue() + +# pc_group = pc_client.get_group('ICML.cc/2024/Conference/Program_Chairs') +# assert ['pc@icml.cc', 'pc2@icml.cc'] == pc_group.members + +# submission_start = now.strftime('%b %d %Y %I:%M%p') +# submission_end = due_date.strftime('%b %d %Y %I:%M%p') +# group_content = openreview_client.get_group('ICML.cc/2024/Conference').content +# assert 'date' in group_content +# assert group_content['date']['value'] == f"Submission Start: {submission_start} UTC-0, Submission Deadline: {submission_end} UTC-0" + +# pc_client.post_note(openreview.Note( +# content={ +# 'title': 'Thirty-ninth International Conference on Machine Learning', +# 'Official Venue Name': 'Thirty-ninth International Conference on Machine Learning', +# 'Abbreviated Venue Name': 'ICML 2023', +# 'Official Website URL': 'https://icml.cc', +# 'program_chair_emails': ['pc@icml.cc', 'pc3@icml.cc'], +# 'contact_email': 'pc@icml.cc', +# 'publication_chairs':'No, our venue does not have Publication Chairs', +# 'Venue Start Date': '2023/07/01', +# 'Submission Deadline': due_date.strftime('%Y/%m/%d'), +# 'Location': 'Virtual', +# 'submission_reviewer_assignment': 'Automatic', +# 'How did you hear about us?': 'ML conferences', +# 'Expected Submissions': '100', +# 'Additional Submission Options': { +# "supplementary_material": { +# "value": { +# "param": { +# "type": "file", +# "extensions": [ +# "zip", +# "pdf", +# "tgz", +# "gz" +# ], +# "maxSize": 100, +# "optional": True, +# "deletable": True +# } +# }, +# "description": "All supplementary material must be self-contained and zipped into a single file. Note that supplementary material will be visible to reviewers and the public throughout and after the review period, and ensure all material is anonymized. The maximum file size is 100MB.", +# "order": 8 +# }, +# "financial_aid": { +# "order": 9, +# "description": "Each paper may designate up to one (1) icml.cc account email address of a corresponding student author who confirms that they would need the support to attend the conference, and agrees to volunteer if they get selected.", +# "value": { +# "param": { +# "type": "string", +# "maxLength": 100, +# "optional": True +# } +# } +# }, +# "subject_areas": { +# "order": 19, +# "description": "Enter subject areas.", +# "value": { +# "param": { +# "type": "string[]", +# "enum": [ +# 'Algorithms: Approximate Inference', +# 'Algorithms: Belief Propagation', +# 'Learning: Deep Learning', +# 'Learning: General', +# 'Learning: Nonparametric Bayes', +# 'Methodology: Bayesian Methods', +# 'Methodology: Calibration', +# 'Principles: Causality', +# 'Principles: Cognitive Models', +# 'Representation: Constraints', +# 'Representation: Dempster-Shafer', +# 'Representation: Other' +# ], +# "input": "select" +# } +# } +# }, +# "position_paper_track": { +# "order": 20, +# "description": "Is this a submission to the position paper track? See Call for Position Papers (https://icml.cc/Conferences/2024/CallForPositionPapers).", +# "value": { +# "param": { +# "type": "string", +# "enum": [ +# "Yes", +# "No" +# ], +# "input": "radio" +# } +# } +# } +# } +# }, +# forum=request_form.forum, +# invitation='openreview.net/Support/-/Request{}/Revision'.format(request_form.number), +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# referent=request_form.forum, +# replyto=request_form.forum, +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) + +# helpers.await_queue() + +# pc_group = pc_client.get_group('ICML.cc/2024/Conference/Program_Chairs') +# assert ['pc@icml.cc', 'pc3@icml.cc'] == pc_group.members + + def test_sac_recruitment(self, client, openreview_client, helpers, request_page, selenium): + + pc_client = openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) + request_form=pc_client.get_notes(invitation='openreview.net/-/Venue_Configuration_Request')[0] + + venue = openreview.helpers.get_venue(pc_client, request_form.id, 'openreview.net/Support') + venue.group_builder.create_recruitment_committee_groups('Senior_Area_Chairs') + venue.invitation_builder.set_group_recruitment_invitations('Senior_Area_Chairs') + + invitee_details = '''sac1@gmail.com, SAC ICMLOne\nsac2@icml.cc, SAC ICMLTwoo''' + + # use invitation to recruit reviewers + pc_client.post_group_edit( + invitation='ICML.cc/2024/Conference/Senior_Area_Chairs/Invited/-/Recruitment_Settings', + content = { + 'reduced_load': { 'value': [1,2,3] }, + 'recruitment_template': { 'value': '''Dear {{fullname}}, + +You have been nominated by the program chair committee of V 24 to serve as Reviewer. As a respected researcher in the area, we hope you will accept and help us make V 24 a success. + +You are also welcome to submit papers, so please also consider submitting to V 24. + +We will be using OpenReview.net and a reviewing process that we hope will be engaging and inclusive of the whole community. + +To respond the invitation, please click on the following link: + +{{invitation_url}} + +Please answer within 10 days. + +If you accept, please make sure that your OpenReview account is updated and lists all the emails you are using. Visit http://openreview.net/profile after logging in. + +If you have any questions, please contact us at {{contact_info}}. + +Cheers! + +Program Chairs +''' }, + 'recruitment_subject': { 'value': '[ICML 2024] Invitation to serve as Senior Area Chair' }, + 'allow_overlap': { 'value': False } + } + ) + + edit = pc_client.post_group_edit( + invitation='ICML.cc/2024/Conference/Senior_Area_Chairs/Invited/-/Members', + content={ + 'inviteeDetails': { 'value': invitee_details } + }, + group=openreview.api.Group() + ) + helpers.await_queue_edit(openreview_client, edit_id=edit['id']) + + invited_group = openreview_client.get_group('ICML.cc/2024/Conference/Senior_Area_Chairs/Invited') + assert len(invited_group.members) == 2 + assert 'sac1@gmail.com' in invited_group.members + assert 'sac2@icml.cc' in invited_group.members + + messages = openreview_client.get_messages(subject = '[ICML 2024] Invitation to serve as Senior Area Chair') + assert len(messages) == 2 + + invitee_details = '''sac3@gmail.com''' + edit = pc_client.post_group_edit( + invitation='ICML.cc/2024/Conference/Senior_Area_Chairs/Invited/-/Members', + content={ + 'inviteeDetails': { 'value': invitee_details } + }, + group=openreview.api.Group() + ) + helpers.await_queue_edit(openreview_client, edit_id=edit['id']) + + invited_group = openreview_client.get_group('ICML.cc/2024/Conference/Senior_Area_Chairs/Invited') + assert len(invited_group.members) == 3 + assert 'sac1@gmail.com' in invited_group.members + assert 'sac2@icml.cc' in invited_group.members + assert 'sac3@gmail.com' in invited_group.members + + messages = openreview_client.get_messages(subject = '[ICML 2024] Invitation to serve as Senior Area Chair') + assert len(messages) == 3 + +# assert len(openreview_client.get_group('ICML.cc/2024/Conference/Senior_Area_Chairs').members) == 0 +# group = openreview_client.get_group('ICML.cc/2024/Conference/Senior_Area_Chairs/Invited') +# assert len(group.members) == 2 +# assert group.readers == ['ICML.cc/2024/Conference', 'ICML.cc/2024/Conference/Senior_Area_Chairs/Invited'] + +# messages = openreview_client.get_messages(subject = '[ICML 2023] Invitation to serve as Senior Area Chair') +# assert len(messages) == 2 + +# for message in messages: +# text = message['content']['text'] + +# invitation_url = re.search('https://.*\n', text).group(0).replace('https://openreview.net', 'http://localhost:3030').replace('&', '&')[:-1] +# helpers.respond_invitation(selenium, request_page, invitation_url, accept=True) + +# helpers.await_queue_edit(openreview_client, invitation='ICML.cc/2024/Conference/Senior_Area_Chairs/-/Recruitment', count=2) + +# messages = openreview_client.get_messages(subject='[ICML 2023] Senior Area Chair Invitation accepted') +# assert len(messages) == 2 + +# assert len(openreview_client.get_group('ICML.cc/2024/Conference/Senior_Area_Chairs').members) == 2 +# assert len(openreview_client.get_group('ICML.cc/2024/Conference/Senior_Area_Chairs/Invited').members) == 2 + +# def test_ac_recruitment(self, client, openreview_client, helpers, request_page, selenium): + +# pc_client=openreview.Client(username='pc@icml.cc', password=helpers.strong_password) +# request_form=pc_client.get_notes(invitation='openreview.net/Support/-/Request_Form')[0] + +# reviewer_details = '''ac1@icml.cc, AC ICMLOne\nac2@icml.cc, AC ICMLTwo''' +# pc_client.post_note(openreview.Note( +# content={ +# 'title': 'Recruitment', +# 'invitee_role': 'Area_Chairs', +# 'invitee_details': reviewer_details, +# 'invitation_email_subject': '[ICML 2023] Invitation to serve as {{invitee_role}}', +# 'invitation_email_content': 'Dear {{fullname}},\n\nYou have been nominated by the program chair committee of Theoretical Foundations of RL Workshop @ ICML 2020 to serve as {{invitee_role}}.\n\n{{invitation_url}}\n\nCheers!\n\nProgram Chairs' +# }, +# forum=request_form.forum, +# replyto=request_form.forum, +# invitation='openreview.net/Support/-/Request{}/Recruitment'.format(request_form.number), +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) + +# helpers.await_queue() + +# assert len(openreview_client.get_group('ICML.cc/2024/Conference/Area_Chairs').members) == 0 +# assert len(openreview_client.get_group('ICML.cc/2024/Conference/Area_Chairs/Invited').members) == 2 + +# messages = openreview_client.get_messages(subject = '[ICML 2023] Invitation to serve as Area Chair') +# assert len(messages) == 2 + +# for message in messages: +# text = message['content']['text'] + +# invitation_url = re.search('https://.*\n', text).group(0).replace('https://openreview.net', 'http://localhost:3030').replace('&', '&')[:-1] +# helpers.respond_invitation(selenium, request_page, invitation_url, accept=True) + +# helpers.await_queue_edit(openreview_client, invitation='ICML.cc/2024/Conference/Area_Chairs/-/Recruitment', count=2) + +# messages = openreview_client.get_messages(subject='[ICML 2023] Area Chair Invitation accepted') +# assert len(messages) == 2 + +# assert len(openreview_client.get_group('ICML.cc/2024/Conference/Area_Chairs').members) == 2 +# assert len(openreview_client.get_group('ICML.cc/2024/Conference/Area_Chairs/Invited').members) == 2 + +# def test_reviewer_recruitment(self, client, openreview_client, helpers, request_page, selenium): + +# pc_client=openreview.Client(username='pc@icml.cc', password=helpers.strong_password) +# request_form=pc_client.get_notes(invitation='openreview.net/Support/-/Request_Form')[0] + +# reviewer_details = '''reviewer1@icml.cc, Reviewer ICMLOne +# reviewer2@icml.cc, Reviewer ICMLTwo +# reviewer3@icml.cc, Reviewer ICMLThree +# reviewer4@yahoo.com, Reviewer ICMLFour +# reviewer5@yahoo.com, Reviewer ICMLFive +# reviewer6@yahoo.com, Reviewer ICMLSix +# ''' +# pc_client.post_note(openreview.Note( +# content={ +# 'title': 'Recruitment', +# 'invitee_role': 'Reviewers', +# 'invitee_details': reviewer_details, +# 'invitee_reduced_load': ["1", "2", "3"], +# 'invitation_email_subject': '[ICML 2023] Invitation to serve as {{invitee_role}}', +# 'invitation_email_content': 'Dear {{fullname}},\n\nYou have been nominated by the program chair committee of Theoretical Foundations of RL Workshop @ ICML 2020 to serve as {{invitee_role}}.\n\n{{invitation_url}}\n\nCheers!\n\nProgram Chairs' +# }, +# forum=request_form.forum, +# replyto=request_form.forum, +# invitation='openreview.net/Support/-/Request{}/Recruitment'.format(request_form.number), +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) + +# helpers.await_queue() + +# assert len(openreview_client.get_group('ICML.cc/2024/Conference/Reviewers').members) == 0 +# assert len(openreview_client.get_group('ICML.cc/2024/Conference/Reviewers/Invited').members) == 6 +# assert len(openreview_client.get_group('ICML.cc/2024/Conference/Reviewers/Declined').members) == 0 + +# messages = openreview_client.get_messages(subject = '[ICML 2023] Invitation to serve as Reviewer') +# assert len(messages) == 6 + +# for message in messages: +# text = message['content']['text'] + +# invitation_url = re.search('https://.*\n', text).group(0).replace('https://openreview.net', 'http://localhost:3030').replace('&', '&')[:-1] +# helpers.respond_invitation(selenium, request_page, invitation_url, accept=True, quota=3) + +# helpers.await_queue_edit(openreview_client, invitation='ICML.cc/2024/Conference/Reviewers/-/Recruitment', count=12) + +# messages = openreview_client.get_messages(subject='[ICML 2023] Reviewer Invitation accepted with reduced load') +# assert len(messages) == 6 + +# assert len(openreview_client.get_group('ICML.cc/2024/Conference/Reviewers').members) == 6 +# assert len(openreview_client.get_group('ICML.cc/2024/Conference/Reviewers/Invited').members) == 6 +# assert len(openreview_client.get_group('ICML.cc/2024/Conference/Reviewers/Declined').members) == 0 + +# messages = openreview_client.get_messages(to = 'reviewer6@yahoo.com', subject = '[ICML 2023] Invitation to serve as Reviewer') +# invitation_url = re.search('https://.*\n', messages[0]['content']['text']).group(0).replace('https://openreview.net', 'http://localhost:3030').replace('&', '&')[:-1] +# helpers.respond_invitation(selenium, request_page, invitation_url, accept=False) + +# helpers.await_queue_edit(openreview_client, invitation='ICML.cc/2024/Conference/Reviewers/-/Recruitment', count=13) + +# assert len(openreview_client.get_group('ICML.cc/2024/Conference/Reviewers').members) == 5 +# assert len(openreview_client.get_group('ICML.cc/2024/Conference/Reviewers/Invited').members) == 6 +# assert len(openreview_client.get_group('ICML.cc/2024/Conference/Reviewers/Declined').members) == 1 + +# reviewer_client = openreview.api.OpenReviewClient(username='reviewer1@icml.cc', password=helpers.strong_password) + +# request_page(selenium, "http://localhost:3030/group?id=ICML.cc/2024/Conference/Reviewers", reviewer_client.token, wait_for_element='header') +# header = selenium.find_element(By.ID, 'header') +# assert 'You have agreed to review up to 1 submission' in header.text + +# def test_registrations(self, client, openreview_client, helpers, test_client, request_page, selenium): + +# pc_client=openreview.Client(username='pc@icml.cc', password=helpers.strong_password) +# request_form=pc_client.get_notes(invitation='openreview.net/Support/-/Request_Form')[0] +# venue = openreview.get_conference(client, request_form.id, support_user='openreview.net/Support') + +# now = datetime.datetime.utcnow() +# due_date = now + datetime.timedelta(days=3) +# venue.registration_stages.append(openreview.stages.RegistrationStage(committee_id = venue.get_senior_area_chairs_id(), +# name = 'Registration', +# start_date = None, +# due_date = due_date, +# instructions = 'TODO: instructions', +# title = 'ICML 2023 Conference - Senior Area Chair registration')) + +# venue.registration_stages.append(openreview.stages.RegistrationStage(committee_id = venue.get_area_chairs_id(), +# name = 'Registration', +# start_date = None, +# due_date = due_date, +# instructions = 'TODO: instructions', +# title = 'ICML 2023 Conference - Area Chair registration', +# additional_fields = { +# 'statement': { +# 'description': 'Please write a short (1-2 sentence) statement about why you think peer review is important to the advancement of science.', +# 'value': { +# 'param': { +# 'type': 'string', +# 'input': 'textarea', +# 'maxLength': 200000 +# } +# }, +# 'order': 3 +# } +# })) + +# venue.registration_stages.append(openreview.stages.RegistrationStage(committee_id = venue.get_reviewers_id(), +# name = 'Registration', +# start_date = None, +# due_date = due_date, +# instructions = 'TODO: instructions', +# title = 'ICML 2023 Conference - Reviewer registration', +# additional_fields = { +# 'statement': { +# 'description': 'Please write a short (1-2 sentence) statement about why you think peer review is important to the advancement of science.', +# 'value': { +# 'param': { +# 'type': 'string', +# 'input': 'textarea', +# 'maxLength': 200000 +# } +# }, +# 'order': 3 +# } +# }, +# remove_fields = ['profile_confirmed', 'expertise_confirmed'])) + +# venue.create_registration_stages() + +# sac_client = openreview.api.OpenReviewClient(username = 'sac1@gmail.com', password=helpers.strong_password) + +# request_page(selenium, 'http://localhost:3030/group?id=ICML.cc/2024/Conference/Senior_Area_Chairs', sac_client.token, by=By.CLASS_NAME, wait_for_element='tabs-container') +# tabs = selenium.find_element(By.CLASS_NAME, 'tabs-container') +# assert tabs +# assert tabs.find_element(By.LINK_TEXT, "Paper Status") +# assert tabs.find_element(By.LINK_TEXT, "Area Chair Status") +# assert tabs.find_element(By.LINK_TEXT, "Senior Area Chair Tasks") + +# registration_forum = sac_client.get_notes(invitation='ICML.cc/2024/Conference/Senior_Area_Chairs/-/Registration_Form') +# assert len(registration_forum) == 1 + +# sac_client.post_note_edit(invitation='ICML.cc/2024/Conference/Senior_Area_Chairs/-/Registration', +# signatures=['~SAC_ICMLOne1'], +# note=openreview.api.Note( +# content = { +# 'profile_confirmed': { 'value': 'Yes' }, +# 'expertise_confirmed': { 'value': 'Yes' } +# } +# )) + +# ac_client = openreview.api.OpenReviewClient(username = 'ac1@icml.cc', password=helpers.strong_password) + +# invitation = ac_client.get_invitation('ICML.cc/2024/Conference/Area_Chairs/-/Registration') +# assert 'statement' in invitation.edit['note']['content'] +# assert 'profile_confirmed' in invitation.edit['note']['content'] +# assert 'expertise_confirmed' in invitation.edit['note']['content'] + +# reviewer_client = openreview.api.OpenReviewClient(username = 'reviewer1@icml.cc', password=helpers.strong_password) + +# invitation = reviewer_client.get_invitation('ICML.cc/2024/Conference/Reviewers/-/Registration') +# assert 'statement' in invitation.edit['note']['content'] +# assert 'profile_confirmed' not in invitation.edit['note']['content'] +# assert 'expertise_confirmed' not in invitation.edit['note']['content'] + +# def test_submissions(self, client, openreview_client, helpers, test_client, request_page, selenium): + +# test_client = openreview.api.OpenReviewClient(token=test_client.token) + +# domains = ['umass.edu', 'amazon.com', 'fb.com', 'cs.umass.edu', 'google.com', 'mit.edu', 'deepmind.com', 'co.ux', 'apple.com', 'nvidia.com'] +# subject_areas = ['Algorithms: Approximate Inference', 'Algorithms: Belief Propagation', 'Learning: Deep Learning', 'Learning: General', 'Learning: Nonparametric Bayes', 'Methodology: Bayesian Methods', 'Methodology: Calibration', 'Principles: Causality', 'Principles: Cognitive Models', 'Representation: Constraints', 'Representation: Dempster-Shafer', 'Representation: Other'] +# for i in range(1,102): +# note = openreview.api.Note( +# content = { +# 'title': { 'value': 'Paper title ' + str(i) }, +# 'abstract': { 'value': 'This is an abstract ' + str(i) }, +# 'authorids': { 'value': ['~SomeFirstName_User1', 'peter@mail.com', 'andrew@' + domains[i % 10]] }, +# 'authors': { 'value': ['SomeFirstName User', 'Peter SomeLastName', 'Andrew Mc'] }, +# 'keywords': { 'value': ['machine learning', 'nlp'] }, +# 'pdf': {'value': '/pdf/' + 'p' * 40 +'.pdf' }, +# 'supplementary_material': { 'value': '/attachment/' + 's' * 40 +'.zip'}, +# 'financial_aid': { 'value': 'Yes' }, +# 'subject_areas': { 'value': [subject_areas[random.randint(0, 11)], subject_areas[random.randint(0, 11)]] }, +# 'position_paper_track': { 'value': 'Yes' if i % 2 == 0 else 'No' } +# } +# ) +# if i == 1 or i == 101: +# note.content['authors']['value'].append('SAC ICMLOne') +# note.content['authorids']['value'].append('~SAC_ICMLOne1') + +# test_client.post_note_edit(invitation='ICML.cc/2024/Conference/-/Submission', +# signatures=['~SomeFirstName_User1'], +# note=note) + +# helpers.await_queue_edit(openreview_client, invitation='ICML.cc/2024/Conference/-/Submission', count=101) + +# submissions = openreview_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', sort='number:asc') +# assert len(submissions) == 101 +# assert ['ICML.cc/2024/Conference', '~SomeFirstName_User1', 'peter@mail.com', 'andrew@amazon.com', '~SAC_ICMLOne1'] == submissions[0].readers +# assert ['~SomeFirstName_User1', 'peter@mail.com', 'andrew@amazon.com', '~SAC_ICMLOne1'] == submissions[0].content['authorids']['value'] + +# authors_group = openreview_client.get_group(id='ICML.cc/2024/Conference/Authors') + +# for i in range(1,102): +# assert f'ICML.cc/2024/Conference/Submission{i}/Authors' in authors_group.members + +# ## delete a submission and update authors group +# submission = submissions[0] +# test_client.post_note_edit(invitation='ICML.cc/2024/Conference/-/Submission', +# signatures=['~SomeFirstName_User1'], +# note=openreview.api.Note( +# id = submission.id, +# ddate = openreview.tools.datetime_millis(datetime.datetime.utcnow()), +# content = { +# 'title': submission.content['title'], +# 'abstract': submission.content['abstract'], +# 'authorids': submission.content['authorids'], +# 'authors': submission.content['authors'], +# 'keywords': submission.content['keywords'], +# 'pdf': submission.content['pdf'], +# 'supplementary_material': submission.content['supplementary_material'], +# 'financial_aid': submission.content['financial_aid'], +# 'subject_areas': submission.content['subject_areas'], +# 'position_paper_track': submission.content['position_paper_track'] +# } +# )) + +# helpers.await_queue_edit(openreview_client, invitation='ICML.cc/2024/Conference/-/Submission', count=102) + +# authors_group = openreview_client.get_group(id='ICML.cc/2024/Conference/Authors') + +# assert f'ICML.cc/2024/Conference/Submission1/Authors' not in authors_group.members +# for i in range(2,101): +# assert f'ICML.cc/2024/Conference/Submission{i}/Authors' in authors_group.members + +# ## restore the submission and update the authors group +# submission = submissions[0] +# test_client.post_note_edit(invitation='ICML.cc/2024/Conference/-/Submission', +# signatures=['~SomeFirstName_User1'], +# note=openreview.api.Note( +# id = submission.id, +# ddate = { 'delete': True }, +# content = { +# 'title': submission.content['title'], +# 'abstract': submission.content['abstract'], +# 'authorids': submission.content['authorids'], +# 'authors': submission.content['authors'], +# 'keywords': submission.content['keywords'], +# 'pdf': submission.content['pdf'], +# 'supplementary_material': submission.content['supplementary_material'], +# 'financial_aid': submission.content['financial_aid'], +# 'subject_areas': submission.content['subject_areas'], +# 'position_paper_track': submission.content['position_paper_track'] +# } +# )) + +# helpers.await_queue_edit(openreview_client, invitation='ICML.cc/2024/Conference/-/Submission', count=103) + +# authors_group = openreview_client.get_group(id='ICML.cc/2024/Conference/Authors') + +# for i in range(1,101): +# assert f'ICML.cc/2024/Conference/Submission{i}/Authors' in authors_group.members + +# # assert authors see Submission button to edit their submissions +# request_page(selenium, 'http://localhost:3030/forum?id={}'.format(submission.id), test_client.token, by=By.CLASS_NAME, wait_for_element='forum-note') +# note_div = selenium.find_element(By.CLASS_NAME, 'forum-note') +# assert note_div +# button_row = note_div.find_element(By.CLASS_NAME, 'invitation-buttons') +# assert button_row +# buttons = button_row.find_elements(By.CLASS_NAME, 'btn-xs') +# assert buttons[0].text == 'Edit ' +# buttons[0].click() +# time.sleep(0.5) +# dropdown = button_row.find_element(By.CLASS_NAME, 'dropdown-menu') +# dropdown_values = dropdown.find_elements(By.TAG_NAME, "a") +# values = [value.text for value in dropdown_values] +# assert ['Submission'] == values + +# # assert PCs can also see Submission button to edit submissions +# pc_client_v2=openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) +# request_page(selenium, 'http://localhost:3030/forum?id={}'.format(submission.id), pc_client_v2.token, by=By.CLASS_NAME, wait_for_element='forum-note') +# note_div = selenium.find_element(By.CLASS_NAME, 'forum-note') +# assert note_div +# button_row = note_div.find_element(By.CLASS_NAME, 'invitation-buttons') +# assert button_row +# buttons = button_row.find_elements(By.CLASS_NAME, 'btn-xs') +# assert buttons[0].text == 'Edit ' +# buttons[0].click() +# time.sleep(0.5) +# dropdown = button_row.find_element(By.CLASS_NAME, 'dropdown-menu') +# dropdown_values = dropdown.find_elements(By.TAG_NAME,"a") +# values = [value.text for value in dropdown_values] +# assert ['Submission', 'Post Submission', 'PC Revision'] == values + +# def test_post_submission(self, client, openreview_client, test_client, helpers, request_page, selenium): + +# pc_client=openreview.Client(username='pc@icml.cc', password=helpers.strong_password) +# request_form=pc_client.get_notes(invitation='openreview.net/Support/-/Request_Form')[0] +# venue = openreview.get_conference(client, request_form.id, support_user='openreview.net/Support') + +# ## close the submissions +# now = datetime.datetime.utcnow() +# due_date = now - datetime.timedelta(days=1) +# exp_date = now + datetime.timedelta(days=10) +# pc_client.post_note(openreview.Note( +# content={ +# 'title': 'Thirty-ninth International Conference on Machine Learning', +# 'Official Venue Name': 'Thirty-ninth International Conference on Machine Learning', +# 'Abbreviated Venue Name': 'ICML 2023', +# 'Official Website URL': 'https://icml.cc', +# 'program_chair_emails': ['pc@icml.cc', 'pc3@icml.cc'], +# 'contact_email': 'pc@icml.cc', +# 'publication_chairs':'No, our venue does not have Publication Chairs', +# 'Venue Start Date': '2023/07/01', +# 'Submission Deadline': due_date.strftime('%Y/%m/%d'), +# 'Location': 'Virtual', +# 'submission_reviewer_assignment': 'Automatic', +# 'How did you hear about us?': 'ML conferences', +# 'Expected Submissions': '100', +# 'Additional Submission Options': { +# "supplementary_material": { +# "value": { +# "param": { +# "type": "file", +# "extensions": [ +# "zip", +# "pdf", +# "tgz", +# "gz" +# ], +# "maxSize": 100, +# "optional": True, +# "deletable": True +# } +# }, +# "description": "All supplementary material must be self-contained and zipped into a single file. Note that supplementary material will be visible to reviewers and the public throughout and after the review period, and ensure all material is anonymized. The maximum file size is 100MB.", +# "order": 8 +# }, +# "financial_aid": { +# "order": 9, +# "description": "Each paper may designate up to one (1) icml.cc account email address of a corresponding student author who confirms that they would need the support to attend the conference, and agrees to volunteer if they get selected.", +# "value": { +# "param": { +# "type": "string", +# "maxLength": 100, +# "optional": True +# } +# } +# }, +# "subject_areas": { +# "order": 19, +# "description": "Enter subject areas.", +# "value": { +# "param": { +# "type": "string[]", +# "enum": [ +# 'Algorithms: Approximate Inference', +# 'Algorithms: Belief Propagation', +# 'Learning: Deep Learning', +# 'Learning: General', +# 'Learning: Nonparametric Bayes', +# 'Methodology: Bayesian Methods', +# 'Methodology: Calibration', +# 'Principles: Causality', +# 'Principles: Cognitive Models', +# 'Representation: Constraints', +# 'Representation: Dempster-Shafer', +# 'Representation: Other' +# ], +# "input": "select" +# } +# } +# }, +# "position_paper_track": { +# "order": 20, +# "description": "Is this a submission to the position paper track? See Call for Position Papers (https://icml.cc/Conferences/2024/CallForPositionPapers).", +# "value": { +# "param": { +# "type": "string", +# "enum": [ +# "Yes", +# "No" +# ], +# "input": "radio" +# } +# } +# } +# }, +# 'withdraw_submission_expiration': exp_date.strftime('%Y/%m/%d') +# }, +# forum=request_form.forum, +# invitation='openreview.net/Support/-/Request{}/Revision'.format(request_form.number), +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# referent=request_form.forum, +# replyto=request_form.forum, +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) + +# helpers.await_queue() + +# submissions = openreview_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', sort='number:asc') +# submission = submissions[0] + +# # assert authors don't see Submission button anymore +# request_page(selenium, 'http://localhost:3030/forum?id={}'.format(submission.id), test_client.token, by=By.CLASS_NAME, wait_for_element='forum-note') +# note_div = selenium.find_element(By.CLASS_NAME, 'forum-note') +# assert note_div +# button_row = note_div.find_element(By.CLASS_NAME, 'invitation-buttons') +# assert button_row +# buttons = button_row.find_elements(By.CLASS_NAME, 'btn-xs') +# assert len(buttons) == 0 + +# pc_client_v2=openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) +# submission_invitation = pc_client_v2.get_invitation('ICML.cc/2024/Conference/-/Submission') +# assert submission_invitation.expdate < openreview.tools.datetime_millis(now) + +# assert len(pc_client_v2.get_all_invitations(invitation='ICML.cc/2024/Conference/-/Withdrawal')) == 101 +# withdrawal_inv = pc_client_v2.get_invitation('ICML.cc/2024/Conference/Submission1/-/Withdrawal') +# assert withdrawal_inv.expdate == openreview.tools.datetime_millis(exp_date.replace(hour=0, minute=0, second=0, microsecond=0)) +# assert len(pc_client_v2.get_all_invitations(invitation='ICML.cc/2024/Conference/-/Desk_Rejection')) == 101 +# desk_reject_inv = pc_client_v2.get_invitation('ICML.cc/2024/Conference/Submission1/-/Desk_Rejection') +# desk_reject_due_date = due_date + datetime.timedelta(days=90) +# assert desk_reject_inv.expdate == openreview.tools.datetime_millis(desk_reject_due_date.replace(hour=0, minute=0, second=0, microsecond=0)) +# assert pc_client_v2.get_invitation('ICML.cc/2024/Conference/-/PC_Revision') + +# ## make submissions visible to ACs only +# pc_client.post_note(openreview.Note( +# content= { +# 'force': 'Yes', +# 'submission_readers': 'All area chairs only', +# 'hide_fields': ['financial_aid'] +# }, +# forum= request_form.id, +# invitation= f'openreview.net/Support/-/Request{request_form.number}/Post_Submission', +# readers= ['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# referent= request_form.id, +# replyto= request_form.id, +# signatures= ['~Program_ICMLChair1'], +# writers= [], +# )) + +# helpers.await_queue() + +# ac_client = openreview.api.OpenReviewClient(username = 'ac1@icml.cc', password=helpers.strong_password) +# submissions = ac_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', sort='number:asc') +# assert len(submissions) == 101 +# assert ['ICML.cc/2024/Conference', +# 'ICML.cc/2024/Conference/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Authors'] == submissions[0].readers +# assert ['ICML.cc/2024/Conference', +# 'ICML.cc/2024/Conference/Submission1/Authors'] == submissions[0].writers +# assert ['ICML.cc/2024/Conference/Submission1/Authors'] == submissions[0].signatures +# assert 'authorids' not in submissions[0].content +# assert 'authors' not in submissions[0].content +# assert 'financial_aid'not in submissions[0].content +# assert not submissions[0].odate + +# ## make submissions visible to the committee +# pc_client.post_note(openreview.Note( +# content= { +# 'force': 'Yes', +# 'submission_readers': 'All program committee (all reviewers, all area chairs, all senior area chairs if applicable)', +# 'hide_fields': ['financial_aid'] +# }, +# forum= request_form.id, +# invitation= f'openreview.net/Support/-/Request{request_form.number}/Post_Submission', +# readers= ['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# referent= request_form.id, +# replyto= request_form.id, +# signatures= ['~Program_ICMLChair1'], +# writers= [], +# )) + +# helpers.await_queue() + +# submissions = venue.get_submissions(sort='number:asc') +# assert len(submissions) == 101 + +# #desk-reject paper +# pc_openreview_client = openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) + +# submission = submissions[-1] +# desk_reject_note = pc_openreview_client.post_note_edit(invitation=f'ICML.cc/2024/Conference/Submission{submission.number}/-/Desk_Rejection', +# signatures=['ICML.cc/2024/Conference/Program_Chairs'], +# note=openreview.api.Note( +# content={ +# 'desk_reject_comments': { 'value': 'Out of scope' }, +# } +# )) + +# helpers.await_queue_edit(openreview_client, edit_id=desk_reject_note['id']) +# helpers.await_queue_edit(openreview_client, invitation='ICML.cc/2024/Conference/-/Desk_Rejected_Submission') + +# note = pc_openreview_client.get_note(desk_reject_note['note']['forum']) +# assert note +# assert note.invitations == ['ICML.cc/2024/Conference/-/Submission', 'ICML.cc/2024/Conference/-/Post_Submission', 'ICML.cc/2024/Conference/-/Desk_Rejected_Submission'] + +# assert desk_reject_note['readers'] == [ +# "ICML.cc/2024/Conference/Program_Chairs", +# f"ICML.cc/2024/Conference/Submission{submission.number}/Senior_Area_Chairs", +# f"ICML.cc/2024/Conference/Submission{submission.number}/Area_Chairs", +# f"ICML.cc/2024/Conference/Submission{submission.number}/Reviewers", +# f"ICML.cc/2024/Conference/Submission{submission.number}/Authors" +# ] + +# # reverse desk-rejection and withdraw paper +# desk_rejection_reversion_note = openreview_client.post_note_edit(invitation=f'ICML.cc/2024/Conference/Submission{submission.number}/-/Desk_Rejection_Reversion', +# signatures=['ICML.cc/2024/Conference/Program_Chairs'], +# note=openreview.api.Note( +# content={ +# 'revert_desk_rejection_confirmation': { 'value': 'We approve the reversion of desk-rejected submission.' }, +# } +# )) + +# helpers.await_queue_edit(openreview_client, edit_id=desk_rejection_reversion_note['id']) + +# withdrawal_note = pc_openreview_client.post_note_edit(invitation=f'ICML.cc/2024/Conference/Submission{submission.number}/-/Withdrawal', +# signatures=[f'ICML.cc/2024/Conference/Submission{submission.number}/Authors'], +# note=openreview.api.Note( +# content={ +# 'withdrawal_confirmation': { 'value': 'I have read and agree with the venue\'s withdrawal policy on behalf of myself and my co-authors.' }, +# } +# )) + +# helpers.await_queue_edit(openreview_client, edit_id=withdrawal_note['id']) +# helpers.await_queue_edit(openreview_client, invitation='ICML.cc/2024/Conference/-/Withdrawn_Submission') + +# withdrawn_submission = openreview_client.get_note(withdrawal_note['note']['forum']) +# assert withdrawn_submission.readers == ['ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission101/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Submission101/Area_Chairs', +# 'ICML.cc/2024/Conference/Submission101/Reviewers', +# 'ICML.cc/2024/Conference/Submission101/Authors'] + +# assert withdrawal_note['readers'] == [ +# "ICML.cc/2024/Conference/Program_Chairs", +# f"ICML.cc/2024/Conference/Submission{submission.number}/Senior_Area_Chairs", +# f"ICML.cc/2024/Conference/Submission{submission.number}/Area_Chairs", +# f"ICML.cc/2024/Conference/Submission{submission.number}/Reviewers", +# f"ICML.cc/2024/Conference/Submission{submission.number}/Authors" +# ] + +# submissions = venue.get_submissions(sort='number:asc') +# assert len(submissions) == 100 + +# ac_client = openreview.api.OpenReviewClient(username = 'ac1@icml.cc', password=helpers.strong_password) +# submissions = ac_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', sort='number:asc') +# assert len(submissions) == 100 #withdrawn papers are no longer visible to ACs because ACs have not been assigned yet +# assert ['ICML.cc/2024/Conference', +# 'ICML.cc/2024/Conference/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Area_Chairs', +# 'ICML.cc/2024/Conference/Reviewers', +# 'ICML.cc/2024/Conference/Submission1/Authors'] == submissions[0].readers +# assert ['ICML.cc/2024/Conference', +# 'ICML.cc/2024/Conference/Submission1/Authors'] == submissions[0].writers +# assert ['ICML.cc/2024/Conference/Submission1/Authors'] == submissions[0].signatures +# assert 'authorids' not in submissions[0].content +# assert 'authors' not in submissions[0].content +# assert 'financial_aid'not in submissions[0].content + +# assert client.get_group('ICML.cc/2024/Conference/Submission1/Reviewers') +# assert client.get_group('ICML.cc/2024/Conference/Submission1/Area_Chairs') +# assert client.get_group('ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs') + +# active_venues = pc_client.get_group('active_venues') +# assert 'ICML.cc/2024/Conference' in active_venues.members + +# ## try to edit a submission as a PC +# submissions = pc_client_v2.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', sort='number:asc') +# submission = submissions[0] +# edit_note = pc_client_v2.post_note_edit(invitation='ICML.cc/2024/Conference/-/PC_Revision', +# signatures=['ICML.cc/2024/Conference/Program_Chairs'], +# note=openreview.api.Note( +# id = submission.id, +# content = { +# 'title': { 'value': submission.content['title']['value'] + ' Version 2' }, +# 'abstract': submission.content['abstract'], +# 'authorids': { 'value': submission.content['authorids']['value'] + ['melisa@yahoo.com'] }, +# 'authors': { 'value': submission.content['authors']['value'] + ['Melisa ICML'] }, +# 'keywords': submission.content['keywords'], +# 'pdf': submission.content['pdf'], +# 'supplementary_material': { 'value': { 'delete': True } }, +# 'financial_aid': { 'value': submission.content['financial_aid']['value'] }, +# 'subject_areas': { 'value': submission.content['subject_areas']['value'] }, +# 'position_paper_track': { 'value': submission.content['position_paper_track']['value'] } +# } +# )) + +# helpers.await_queue_edit(openreview_client, edit_id=edit_note['id']) + +# submission = ac_client.get_note(submission.id) +# assert ['ICML.cc/2024/Conference', +# 'ICML.cc/2024/Conference/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Area_Chairs', +# 'ICML.cc/2024/Conference/Reviewers', +# 'ICML.cc/2024/Conference/Submission1/Authors'] == submission.readers +# assert ['ICML.cc/2024/Conference', +# 'ICML.cc/2024/Conference/Submission1/Authors'] == submission.writers +# assert ['ICML.cc/2024/Conference/Submission1/Authors'] == submission.signatures +# assert 'authorids' not in submission.content +# assert 'authors' not in submission.content +# assert 'financial_aid'not in submission.content +# assert 'supplementary_material'not in submission.content + +# author_group = pc_client_v2.get_group('ICML.cc/2024/Conference/Submission1/Authors') +# assert ['~SomeFirstName_User1', 'peter@mail.com', 'andrew@amazon.com', '~SAC_ICMLOne1', 'melisa@yahoo.com'] == author_group.members + +# messages = openreview_client.get_messages(to = 'melisa@yahoo.com', subject = 'ICML 2023 has received a new revision of your submission titled Paper title 1 Version 2') +# assert len(messages) == 1 +# assert messages[0]['content']['replyTo'] == 'pc@icml.cc' +# assert messages[0]['content']['text'] == f'''Your new revision of the submission to ICML 2023 has been posted. + +# Title: Paper title 1 Version 2 + +# Abstract: This is an abstract 1 + +# To view your submission, click here: https://openreview.net/forum?id={submission.id} + +# Please note that responding to this email will direct your reply to pc@icml.cc. +# ''' + +# def test_ac_bidding(self, client, openreview_client, helpers, test_client): + +# pc_client=openreview.Client(username='pc@icml.cc', password=helpers.strong_password) +# pc_client_v2=openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) +# request_form=pc_client.get_notes(invitation='openreview.net/Support/-/Request_Form')[0] + +# with pytest.raises(openreview.OpenReviewException, match=r'Please deploy SAC-AC assignments first. SAC-submission conflicts must be transferred to assigned ACs before computing AC-submission conflicts.'): +# client.post_note(openreview.Note( +# content={ +# 'title': 'Paper Matching Setup', +# 'matching_group': 'ICML.cc/2024/Conference/Area_Chairs', +# 'compute_conflicts': 'NeurIPS', +# 'compute_conflicts_N_years': '3', +# 'compute_affinity_scores': 'No' + +# }, +# forum=request_form.id, +# replyto=request_form.id, +# invitation=f'openreview.net/Support/-/Request{request_form.number}/Paper_Matching_Setup', +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) + +# openreview.tools.replace_members_with_ids(openreview_client, openreview_client.get_group('ICML.cc/2024/Conference/Senior_Area_Chairs')) + +# with open(os.path.join(os.path.dirname(__file__), 'data/rev_scores_venue.csv'), 'w') as file_handle: +# writer = csv.writer(file_handle) +# for sac in openreview_client.get_group('ICML.cc/2024/Conference/Senior_Area_Chairs').members: +# for ac in openreview_client.get_group('ICML.cc/2024/Conference/Area_Chairs').members: +# writer.writerow([ac, sac, round(random.random(), 2)]) + +# affinity_scores_url = client.put_attachment(os.path.join(os.path.dirname(__file__), 'data/rev_scores_venue.csv'), f'openreview.net/Support/-/Request{request_form.number}/Paper_Matching_Setup', 'upload_affinity_scores') + +# ## setup matching to assign SAC to each AC +# with pytest.raises(openreview.OpenReviewException, match=r'Conflicts are not computed between SACs and ACs. Please select "No" for Compute Conflicts.'): +# client.post_note(openreview.Note( +# content={ +# 'title': 'Paper Matching Setup', +# 'matching_group': 'ICML.cc/2024/Conference/Senior_Area_Chairs', +# 'compute_conflicts': 'Default', +# 'compute_affinity_scores': 'No', +# 'upload_affinity_scores': affinity_scores_url +# }, +# forum=request_form.id, +# replyto=request_form.id, +# invitation=f'openreview.net/Support/-/Request{request_form.number}/Paper_Matching_Setup', +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) + +# client.post_note(openreview.Note( +# content={ +# 'title': 'Paper Matching Setup', +# 'matching_group': 'ICML.cc/2024/Conference/Senior_Area_Chairs', +# 'compute_conflicts': 'No', +# 'compute_affinity_scores': 'No', +# 'upload_affinity_scores': affinity_scores_url +# }, +# forum=request_form.id, +# replyto=request_form.id, +# invitation=f'openreview.net/Support/-/Request{request_form.number}/Paper_Matching_Setup', +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) +# helpers.await_queue() + +# assert pc_client_v2.get_edges_count(invitation='ICML.cc/2024/Conference/Senior_Area_Chairs/-/Affinity_Score') == 4 + +# openreview_client.post_edge(openreview.api.Edge( +# invitation = 'ICML.cc/2024/Conference/Senior_Area_Chairs/-/Proposed_Assignment', +# head = '~AC_ICMLOne1', +# tail = '~SAC_ICMLOne1', +# signatures = ['ICML.cc/2024/Conference/Program_Chairs'], +# weight = 1, +# label = 'sac-matching' +# )) + +# openreview_client.post_edge(openreview.api.Edge( +# invitation = 'ICML.cc/2024/Conference/Senior_Area_Chairs/-/Proposed_Assignment', +# head = '~AC_ICMLTwo1', +# tail = '~SAC_ICMLOne1', +# signatures = ['ICML.cc/2024/Conference/Program_Chairs'], +# weight = 1, +# label = 'sac-matching' +# )) + +# venue = openreview.helpers.get_conference(pc_client, request_form.id, setup=False) + +# venue.set_assignments(assignment_title='sac-matching', committee_id='ICML.cc/2024/Conference/Senior_Area_Chairs') + +# sac_assignment_count = pc_client_v2.get_edges_count(invitation='ICML.cc/2024/Conference/Senior_Area_Chairs/-/Assignment') +# assert sac_assignment_count == 2 + +# submissions = pc_client_v2.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', sort='number:asc') + +# openreview.tools.replace_members_with_ids(openreview_client, openreview_client.get_group('ICML.cc/2024/Conference/Area_Chairs')) + +# with open(os.path.join(os.path.dirname(__file__), 'data/rev_scores_venue.csv'), 'w') as file_handle: +# writer = csv.writer(file_handle) +# for submission in submissions: +# for ac in openreview_client.get_group('ICML.cc/2024/Conference/Area_Chairs').members: +# writer.writerow([submission.id, ac, round(random.random(), 2)]) + +# affinity_scores_url = client.put_attachment(os.path.join(os.path.dirname(__file__), 'data/rev_scores_venue.csv'), f'openreview.net/Support/-/Request{request_form.number}/Paper_Matching_Setup', 'upload_affinity_scores') + +# ## setup matching data before starting bidding +# client.post_note(openreview.Note( +# content={ +# 'title': 'Paper Matching Setup', +# 'matching_group': 'ICML.cc/2024/Conference/Area_Chairs', +# 'compute_conflicts': 'NeurIPS', +# 'compute_conflicts_N_years': '3', +# 'compute_affinity_scores': 'No', +# 'upload_affinity_scores': affinity_scores_url +# }, +# forum=request_form.id, +# replyto=request_form.id, +# invitation=f'openreview.net/Support/-/Request{request_form.number}/Paper_Matching_Setup', +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) +# helpers.await_queue() + +# assert openreview_client.get_invitation('ICML.cc/2024/Conference/Area_Chairs/-/Conflict') +# assert openreview_client.get_invitation('ICML.cc/2024/Conference/Area_Chairs/-/Affinity_Score') + +# affinity_score_count = openreview_client.get_edges_count(invitation='ICML.cc/2024/Conference/Area_Chairs/-/Affinity_Score') +# assert affinity_score_count == 100 * 2 ## submissions * ACs +# assert pc_client_v2.get_edges_count(invitation='ICML.cc/2024/Conference/Area_Chairs/-/Conflict') == 200 ## assigned SAC is an author of paper 1 + +# openreview.tools.replace_members_with_ids(openreview_client, openreview_client.get_group('ICML.cc/2024/Conference/Reviewers')) + +# with open(os.path.join(os.path.dirname(__file__), 'data/rev_scores_venue.csv'), 'w') as file_handle: +# writer = csv.writer(file_handle) +# for submission in submissions: +# for ac in openreview_client.get_group('ICML.cc/2024/Conference/Reviewers').members: +# writer.writerow([submission.id, ac, round(random.random(), 2)]) + +# affinity_scores_url = client.put_attachment(os.path.join(os.path.dirname(__file__), 'data/rev_scores_venue.csv'), f'openreview.net/Support/-/Request{request_form.number}/Paper_Matching_Setup', 'upload_affinity_scores') + +# client.post_note(openreview.Note( +# content={ +# 'title': 'Paper Matching Setup', +# 'matching_group': 'ICML.cc/2024/Conference/Reviewers', +# 'compute_conflicts': 'NeurIPS', +# 'compute_conflicts_N_years': '3', +# 'compute_affinity_scores': 'No', +# 'upload_affinity_scores': affinity_scores_url +# }, +# forum=request_form.id, +# replyto=request_form.id, +# invitation=f'openreview.net/Support/-/Request{request_form.number}/Paper_Matching_Setup', +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) + +# with pytest.raises(openreview.OpenReviewException, match=r'Paper matching is already being run for this group. Please wait for a status reply in the forum.'): +# client.post_note(openreview.Note( +# content={ +# 'title': 'Paper Matching Setup', +# 'matching_group': 'ICML.cc/2024/Conference/Reviewers', +# 'compute_conflicts': 'NeurIPS', +# 'compute_conflicts_N_years': '3', +# 'compute_affinity_scores': 'No', +# 'upload_affinity_scores': affinity_scores_url +# }, +# forum=request_form.id, +# replyto=request_form.id, +# invitation=f'openreview.net/Support/-/Request{request_form.number}/Paper_Matching_Setup', +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) + +# helpers.await_queue() + +# # Only 1 reviewer matching note was posted +# matching_notes = client.get_all_notes(invitation=f'openreview.net/Support/-/Request{request_form.number}/Paper_Matching_Setup') +# rev_matching_notes = [note for note in matching_notes if note.content['matching_group'] == 'ICML.cc/2024/Conference/Reviewers'] +# assert len(rev_matching_notes) == 1 + +# assert openreview_client.get_invitation('ICML.cc/2024/Conference/Reviewers/-/Conflict') + +# assert openreview_client.get_edges_count(invitation='ICML.cc/2024/Conference/Reviewers/-/Conflict') == 0 + +# affinity_scores = openreview_client.get_grouped_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Affinity_Score', groupby='id') +# assert affinity_scores +# assert len(affinity_scores) == 100 * 5 ## submissions * reviewers + +# now = datetime.datetime.utcnow() +# due_date = now + datetime.timedelta(days=3) + +# ## Hide the pdf and supplementary material +# pc_client.post_note(openreview.Note( +# content= { +# 'force': 'Yes', +# 'submission_readers': 'All program committee (all reviewers, all area chairs, all senior area chairs if applicable)', +# 'hide_fields': ['financial_aid', 'pdf', 'supplementary_material'] +# }, +# forum= request_form.id, +# invitation= f'openreview.net/Support/-/Request{request_form.number}/Post_Submission', +# readers= ['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# referent= request_form.id, +# replyto= request_form.id, +# signatures= ['~Program_ICMLChair1'], +# writers= [], +# )) + +# helpers.await_queue() + +# ac_client = openreview.api.OpenReviewClient(username = 'ac1@icml.cc', password=helpers.strong_password) +# submissions = ac_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', sort='number:asc') +# assert len(submissions) == 100 +# assert ['ICML.cc/2024/Conference', +# 'ICML.cc/2024/Conference/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Area_Chairs', +# 'ICML.cc/2024/Conference/Reviewers', +# 'ICML.cc/2024/Conference/Submission1/Authors'] == submissions[0].readers +# assert ['ICML.cc/2024/Conference', +# 'ICML.cc/2024/Conference/Submission1/Authors'] == submissions[0].writers +# assert ['ICML.cc/2024/Conference/Submission1/Authors'] == submissions[0].signatures +# assert 'authorids' not in submissions[0].content +# assert 'authors' not in submissions[0].content +# assert 'financial_aid'not in submissions[0].content +# assert 'pdf' not in submissions[0].content +# assert 'supplementary_material' not in submissions[0].content + +# bid_stage_note = pc_client.post_note(openreview.Note( +# content={ +# 'bid_start_date': now.strftime('%Y/%m/%d'), +# 'bid_due_date': due_date.strftime('%Y/%m/%d'), +# 'bid_count': 5 +# }, +# forum=request_form.forum, +# replyto=request_form.forum, +# referent=request_form.forum, +# invitation=f'openreview.net/Support/-/Request{request_form.number}/Bid_Stage', +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) + +# helpers.await_queue() + +# invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Area_Chairs/-/Bid') +# assert invitation.edit['tail']['param']['options']['group'] == 'ICML.cc/2024/Conference/Area_Chairs' +# invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Reviewers/-/Bid') +# assert invitation.edit['tail']['param']['options']['group'] == 'ICML.cc/2024/Conference/Reviewers' + +# ## Hide the pdf and supplementary material +# pc_client.post_note(openreview.Note( +# content= { +# 'force': 'Yes', +# 'submission_readers': 'All program committee (all reviewers, all area chairs, all senior area chairs if applicable)', +# 'hide_fields': ['financial_aid', 'pdf', 'supplementary_material'] +# }, +# forum= request_form.id, +# invitation= f'openreview.net/Support/-/Request{request_form.number}/Post_Submission', +# readers= ['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# referent= request_form.id, +# replyto= request_form.id, +# signatures= ['~Program_ICMLChair1'], +# writers= [], +# )) + +# helpers.await_queue() + +# ac_client = openreview.api.OpenReviewClient(username = 'ac1@icml.cc', password=helpers.strong_password) +# submissions = ac_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', sort='number:asc') +# assert len(submissions) == 100 +# assert ['ICML.cc/2024/Conference', +# 'ICML.cc/2024/Conference/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Area_Chairs', +# 'ICML.cc/2024/Conference/Reviewers', +# 'ICML.cc/2024/Conference/Submission1/Authors'] == submissions[0].readers +# assert ['ICML.cc/2024/Conference', +# 'ICML.cc/2024/Conference/Submission1/Authors'] == submissions[0].writers +# assert ['ICML.cc/2024/Conference/Submission1/Authors'] == submissions[0].signatures +# assert 'authorids' not in submissions[0].content +# assert 'authors' not in submissions[0].content +# assert 'financial_aid'not in submissions[0].content +# assert 'pdf' not in submissions[0].content +# assert 'supplementary_material' not in submissions[0].content + +# def test_assignment(self, client, openreview_client, helpers, request_page, selenium): + +# pc_client=openreview.Client(username='pc@icml.cc', password=helpers.strong_password) +# pc_client_v2=openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) +# request_form=pc_client.get_notes(invitation='openreview.net/Support/-/Request_Form')[0] +# venue = openreview.helpers.get_conference(pc_client, request_form.id, setup=False) +# submissions = pc_client_v2.get_notes(content= { 'venueid': 'ICML.cc/2024/Conference/Submission'}, sort='number:asc') + +# reviewers_proposed_edges = [] +# for i in range(0,20): +# for r in ['~Reviewer_ICMLOne1', '~Reviewer_ICMLTwo1', '~Reviewer_ICMLThree1']: +# reviewers_proposed_edges.append(openreview.api.Edge( +# invitation = 'ICML.cc/2024/Conference/Reviewers/-/Proposed_Assignment', +# head = submissions[i].id, +# tail = r, +# signatures = ['ICML.cc/2024/Conference/Program_Chairs'], +# weight = 1, +# label = 'reviewer-matching', +# readers = ["ICML.cc/2024/Conference", f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Senior_Area_Chairs", f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Area_Chairs", r], +# nonreaders = [f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Authors"], +# writers = ["ICML.cc/2024/Conference", f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Senior_Area_Chairs", f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Area_Chairs"] +# )) + +# openreview_client.post_edge(openreview.api.Edge( +# invitation = 'ICML.cc/2024/Conference/Area_Chairs/-/Proposed_Assignment', +# head = submissions[i].id, +# tail = '~AC_ICMLOne1', +# signatures = ['ICML.cc/2024/Conference/Program_Chairs'], +# weight = 1, +# label = 'ac-matching' +# )) + +# # post duplicate AC Proposed_Assignment edge +# openreview_client.post_edge(openreview.api.Edge( +# invitation = 'ICML.cc/2024/Conference/Area_Chairs/-/Proposed_Assignment', +# head = submissions[0].id, +# tail = '~AC_ICMLOne1', +# signatures = ['ICML.cc/2024/Conference/Program_Chairs'], +# weight = 1, +# label = 'ac-matching' +# )) + +# for i in range(20,40): +# for r in ['~Reviewer_ICMLTwo1', '~Reviewer_ICMLThree1', '~Reviewer_ICMLFour1']: +# reviewers_proposed_edges.append(openreview.api.Edge( +# invitation = 'ICML.cc/2024/Conference/Reviewers/-/Proposed_Assignment', +# head = submissions[i].id, +# tail = r, +# signatures = ['ICML.cc/2024/Conference/Program_Chairs'], +# weight = 1, +# label = 'reviewer-matching', +# readers = ["ICML.cc/2024/Conference", f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Senior_Area_Chairs", f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Area_Chairs", r], +# nonreaders = [f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Authors"], +# writers = ["ICML.cc/2024/Conference", f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Senior_Area_Chairs", f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Area_Chairs"] +# )) + +# openreview_client.post_edge(openreview.api.Edge( +# invitation = 'ICML.cc/2024/Conference/Area_Chairs/-/Proposed_Assignment', +# head = submissions[i].id, +# tail = '~AC_ICMLOne1', +# signatures = ['ICML.cc/2024/Conference/Program_Chairs'], +# weight = 1, +# label = 'ac-matching' +# )) + +# for i in range(40,60): +# for r in ['~Reviewer_ICMLThree1', '~Reviewer_ICMLFour1', '~Reviewer_ICMLFive1']: +# reviewers_proposed_edges.append(openreview.api.Edge( +# invitation = 'ICML.cc/2024/Conference/Reviewers/-/Proposed_Assignment', +# head = submissions[i].id, +# tail = r, +# signatures = ['ICML.cc/2024/Conference/Program_Chairs'], +# weight = 1, +# label = 'reviewer-matching', +# readers = ["ICML.cc/2024/Conference", f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Senior_Area_Chairs", f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Area_Chairs", r], +# nonreaders = [f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Authors"], +# writers = ["ICML.cc/2024/Conference", f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Senior_Area_Chairs", f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Area_Chairs"] +# )) + +# openreview_client.post_edge(openreview.api.Edge( +# invitation = 'ICML.cc/2024/Conference/Area_Chairs/-/Proposed_Assignment', +# head = submissions[i].id, +# tail = '~AC_ICMLOne1', +# signatures = ['ICML.cc/2024/Conference/Program_Chairs'], +# weight = 1, +# label = 'ac-matching' +# )) + + +# for i in range(60,80): +# for r in ['~Reviewer_ICMLFour1', '~Reviewer_ICMLFive1', '~Reviewer_ICMLOne1']: +# reviewers_proposed_edges.append(openreview.api.Edge( +# invitation = 'ICML.cc/2024/Conference/Reviewers/-/Proposed_Assignment', +# head = submissions[i].id, +# tail = r, +# signatures = ['ICML.cc/2024/Conference/Program_Chairs'], +# weight = 1, +# label = 'reviewer-matching', +# readers = ["ICML.cc/2024/Conference", f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Senior_Area_Chairs", f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Area_Chairs", r], +# nonreaders = [f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Authors"], +# writers = ["ICML.cc/2024/Conference", f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Senior_Area_Chairs", f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Area_Chairs"] +# )) + +# openreview_client.post_edge(openreview.api.Edge( +# invitation = 'ICML.cc/2024/Conference/Area_Chairs/-/Proposed_Assignment', +# head = submissions[i].id, +# tail = '~AC_ICMLTwo1', +# signatures = ['ICML.cc/2024/Conference/Program_Chairs'], +# weight = 1, +# label = 'ac-matching' +# )) + +# for i in range(80,100): +# for r in ['~Reviewer_ICMLFive1', '~Reviewer_ICMLOne1', '~Reviewer_ICMLTwo1']: +# reviewers_proposed_edges.append(openreview.api.Edge( +# invitation = 'ICML.cc/2024/Conference/Reviewers/-/Proposed_Assignment', +# head = submissions[i].id, +# tail = r, +# signatures = ['ICML.cc/2024/Conference/Program_Chairs'], +# weight = 1, +# label = 'reviewer-matching', +# readers = ["ICML.cc/2024/Conference", f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Senior_Area_Chairs", f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Area_Chairs", r], +# nonreaders = [f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Authors"], +# writers = ["ICML.cc/2024/Conference", f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Senior_Area_Chairs", f"ICML.cc/2024/Conference/Submission{submissions[i].number}/Area_Chairs"] +# )) + +# openreview_client.post_edge(openreview.api.Edge( +# invitation = 'ICML.cc/2024/Conference/Area_Chairs/-/Proposed_Assignment', +# head = submissions[i].id, +# tail = '~AC_ICMLTwo1', +# signatures = ['ICML.cc/2024/Conference/Program_Chairs'], +# weight = 1, +# label = 'ac-matching' +# )) + +# openreview.tools.post_bulk_edges(client=openreview_client, edges=reviewers_proposed_edges) + +# venue.set_assignments(assignment_title='ac-matching', committee_id='ICML.cc/2024/Conference/Area_Chairs') + +# ac_group = pc_client_v2.get_group('ICML.cc/2024/Conference/Submission1/Area_Chairs') +# assert ['~AC_ICMLOne1'] == ac_group.members + +# ac_group = pc_client_v2.get_group('ICML.cc/2024/Conference/Submission100/Area_Chairs') +# assert ['~AC_ICMLTwo1'] == ac_group.members + +# sac_group = pc_client_v2.get_group('ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs') +# assert ['~SAC_ICMLOne1'] == sac_group.members + +# sac_group = pc_client_v2.get_group('ICML.cc/2024/Conference/Submission100/Senior_Area_Chairs') +# assert ['~SAC_ICMLOne1'] == sac_group.members + +# assignment_edges = pc_client_v2.get_edges(invitation='ICML.cc/2024/Conference/Area_Chairs/-/Assignment', head=submissions[0].id, tail='~AC_ICMLOne1') +# assert assignment_edges and len(assignment_edges) == 2 + +# # remove duplicate edge and make sure assignment still remains +# assignment_edge = assignment_edges[0] +# assignment_edge.ddate = openreview.tools.datetime_millis(datetime.datetime.utcnow()) +# assignment_edge.cdate = None +# edge = pc_client_v2.post_edge(assignment_edge) + +# helpers.await_queue_edit(openreview_client, edit_id=edge.id) + +# ac_group = pc_client_v2.get_group('ICML.cc/2024/Conference/Submission1/Area_Chairs') +# assert ['~AC_ICMLOne1'] == ac_group.members + +# sac_group = pc_client_v2.get_group('ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs') +# assert ['~SAC_ICMLOne1'] == sac_group.members + +# assignment_edges = pc_client_v2.get_edges(invitation='ICML.cc/2024/Conference/Area_Chairs/-/Assignment', head=submissions[0].id, tail='~AC_ICMLOne1') +# assert assignment_edges and len(assignment_edges) == 1 + +# ### Reviewers reassignment of proposed assignments + +# now = datetime.datetime.utcnow() +# due_date = now + datetime.timedelta(days=3) +# venue.setup_assignment_recruitment(committee_id='ICML.cc/2024/Conference/Reviewers', assignment_title='reviewer-matching', hash_seed='1234', due_date=due_date) + +# venue_group = pc_client_v2.get_group('ICML.cc/2024/Conference') +# 'NeurIPS' == venue_group.content['reviewers_conflict_policy']['value'] + +# pc_client_v2.post_group_edit(invitation='ICML.cc/2024/Conference/-/Edit', +# readers = ['ICML.cc/2024/Conference'], +# writers = ['ICML.cc/2024/Conference'], +# signatures = ['ICML.cc/2024/Conference'], +# group = openreview.api.Group( +# id = 'ICML.cc/2024/Conference', +# content = { +# 'enable_reviewers_reassignment': { 'value': True }, +# 'reviewers_proposed_assignment_title': { 'value': 'reviewer-matching' } +# } +# ) +# ) + +# ## increse quota for reviewer 4 +# quota_edge = pc_client_v2.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Custom_Max_Papers', tail='~Reviewer_ICMLFour1')[0] +# quota_edge.weight = 15 +# quota_edge.cdate = None +# pc_client_v2.post_edge(quota_edge) + +# ac_client = openreview.api.OpenReviewClient(username='ac1@icml.cc', password=helpers.strong_password) +# request_page(selenium, "http://localhost:3030/group?id=ICML.cc/2024/Conference/Area_Chairs", ac_client.token, wait_for_element='header') +# header = selenium.find_element(By.ID, 'header') +# assert 'Reviewer Assignment Browser:' in header.text + +# url = header.find_element(By.ID, 'edge_browser_url') +# assert url +# assert url.get_attribute('href') == 'http://localhost:3030/edges/browse?start=ICML.cc/2024/Conference/Area_Chairs/-/Assignment,tail:~AC_ICMLOne1&traverse=ICML.cc/2024/Conference/Reviewers/-/Proposed_Assignment,label:reviewer-matching&edit=ICML.cc/2024/Conference/Reviewers/-/Proposed_Assignment,label:reviewer-matching;ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment&browse=ICML.cc/2024/Conference/Reviewers/-/Aggregate_Score,label:reviewer-matching;ICML.cc/2024/Conference/Reviewers/-/Affinity_Score;ICML.cc/2024/Conference/Reviewers/-/Bid;ICML.cc/2024/Conference/Reviewers/-/Custom_Max_Papers,head:ignore&hide=ICML.cc/2024/Conference/Reviewers/-/Conflict&maxColumns=2&version=2&referrer=[AC%20Console](/group?id=ICML.cc/2024/Conference/Area_Chairs)' + +# anon_group_id = ac_client.get_groups(prefix='ICML.cc/2024/Conference/Submission1/Area_Chair_', signatory='~AC_ICMLOne1')[0].id + +# ## add a reviewer with max quota an get an error +# with pytest.raises(openreview.OpenReviewException, match=r'Max Papers allowed reached for Reviewer ICMLFive'): +# ac_client.post_edge( +# openreview.api.Edge(invitation='ICML.cc/2024/Conference/Reviewers/-/Proposed_Assignment', +# signatures=[anon_group_id], +# head=submissions[0].id, +# tail='~Reviewer_ICMLFive1', +# label='reviewer-matching', +# weight=1 +# )) + +# ## recruit external reviewer +# with pytest.raises(openreview.OpenReviewException, match=r'the user has a conflict'): +# ac_client.post_edge( +# openreview.api.Edge(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', +# signatures=[anon_group_id], +# head=submissions[0].id, +# tail='danielle@mail.com', +# label='Invitation Sent', +# weight=1 +# )) + +# with pytest.raises(openreview.OpenReviewException, match=r'the user is already assigned'): +# ac_client.post_edge( +# openreview.api.Edge(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', +# signatures=[anon_group_id], +# head=submissions[0].id, +# tail='~Reviewer_ICMLOne1', +# label='Invitation Sent', +# weight=1 +# )) + +# with pytest.raises(openreview.OpenReviewException, match=r'the user is an official reviewer'): +# ac_client.post_edge( +# openreview.api.Edge(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', +# signatures=[anon_group_id], +# head=submissions[0].id, +# tail='~Reviewer_ICMLFive1', +# label='Invitation Sent', +# weight=1 +# )) + +# edge = ac_client.post_edge( +# openreview.api.Edge(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', +# signatures=[anon_group_id], +# head=submissions[0].id, +# tail='melisa@icml.cc', +# label='Invitation Sent', +# weight=1 +# )) +# helpers.await_queue_edit(openreview_client, edge.id) + +# helpers.create_user('javier@icml.cc', 'Javier', 'ICML') + +# edge = ac_client.post_edge( +# openreview.api.Edge(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', +# signatures=[anon_group_id], +# head=submissions[0].id, +# tail='~Javier_ICML1', +# label='Invitation Sent', +# weight=1 +# )) +# helpers.await_queue_edit(openreview_client, edge.id) + +# helpers.create_user('emilia@icml.cc', 'Emilia', 'ICML') +# edge = ac_client.post_edge( +# openreview.api.Edge(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', +# signatures=[anon_group_id], +# head=submissions[0].id, +# tail='~Emilia_ICML1', +# label='Invitation Sent', +# weight=1 +# )) +# helpers.await_queue_edit(openreview_client, edge.id) + +# # delete Invitation Sent edge +# invite_edge=ac_client.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', head=submissions[0].id, tail='~Emilia_ICML1')[0] +# invite_edge.ddate = openreview.tools.datetime_millis(datetime.datetime.utcnow()) +# edge = ac_client.post_edge(invite_edge) + +# time.sleep(5) ## wait until the process function runs + +# messages = openreview_client.get_messages(to='emilia@icml.cc', subject='[ICML 2023] Invitation canceled to review paper titled "Paper title 1 Version 2"') +# assert messages and len(messages) == 1 + +# with pytest.raises(openreview.OpenReviewException, match=r'the user is already invited'): +# ac_client.post_edge( +# openreview.api.Edge(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', +# signatures=[anon_group_id], +# head=submissions[0].id, +# tail='javier@icml.cc', +# label='Invitation Sent', +# weight=1 +# )) + +# assert openreview_client.get_groups('ICML.cc/2024/Conference/Submission1/External_Reviewers/Invited', member='melisa@icml.cc') +# assert openreview_client.get_groups('ICML.cc/2024/Conference/External_Reviewers/Invited', member='melisa@icml.cc') + +# assert not openreview_client.get_groups('ICML.cc/2024/Conference/Submission1/External_Reviewers', member='melisa@icml.cc') +# assert not openreview_client.get_groups('ICML.cc/2024/Conference/External_Reviewers', member='melisa@icml.cc') +# assert not openreview_client.get_groups('ICML.cc/2024/Conference/Reviewers', member='melisa@icml.cc') + +# messages = openreview_client.get_messages(to='melisa@icml.cc', subject='[ICML 2023] Invitation to review paper titled "Paper title 1 Version 2"') +# assert messages and len(messages) == 1 +# invitation_url = re.search('https://.*\n', messages[0]['content']['text']).group(0).replace('https://openreview.net', 'http://localhost:3030').replace('&', '&')[:-1] +# helpers.respond_invitation(selenium, request_page, invitation_url, accept=True) + +# helpers.await_queue_edit(openreview_client, invitation='ICML.cc/2024/Conference/Reviewers/-/Proposed_Assignment_Recruitment') + +# ## External reviewer is set pending profile creation +# invite_edges=pc_client_v2.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', head=submissions[0].id, tail='melisa@icml.cc') +# assert len(invite_edges) == 1 +# assert invite_edges[0].label == 'Pending Sign Up' + +# assignment_edges=pc_client_v2.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Proposed_Assignment', label='reviewer-matching', head=submissions[0].id) +# assert len(assignment_edges) == 3 + +# messages = openreview_client.get_messages(to='melisa@icml.cc', subject='[ICML 2023] Reviewer Invitation accepted for paper 1, assignment pending') +# assert messages and len(messages) == 1 +# assert messages[0]['content']['text'] == '''Hi melisa@icml.cc, +# Thank you for accepting the invitation to review the paper number: 1, title: Paper title 1 Version 2. + +# Please signup in OpenReview using the email address melisa@icml.cc and complete your profile. +# Confirmation of the assignment is pending until your profile is active and no conflicts of interest are detected. + +# If you would like to change your decision, please follow the link in the previous invitation email and click on the "Decline" button. + +# OpenReview Team + +# Please note that responding to this email will direct your reply to pc@icml.cc. +# ''' + +# messages = openreview_client.get_messages(to='ac1@icml.cc', subject='[ICML 2023] Reviewer melisa@icml.cc accepted to review paper 1, assignment pending') +# assert messages and len(messages) == 1 +# assert messages[0]['content']['text'] == '''Hi AC ICMLOne, +# The Reviewer melisa@icml.cc that you invited to review paper 1 has accepted the invitation. + +# Confirmation of the assignment is pending until the invited reviewer creates a profile in OpenReview and no conflicts of interest are detected. + +# OpenReview Team + +# Please note that responding to this email will direct your reply to pc@icml.cc. +# ''' + +# # try to remove Invite_Assignment edge with label == 'Pending Sign Up' +# with pytest.raises(openreview.OpenReviewException, match=r'Cannot cancel the invitation since it has status: "Pending Sign Up"'): +# invite_edge=pc_client_v2.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', head=submissions[0].id, tail='melisa@icml.cc')[0] +# invite_edge.ddate = openreview.tools.datetime_millis(datetime.datetime.utcnow()) +# pc_client_v2.post_edge(invite_edge) + +# ## Run Job +# openreview.venue.Venue.check_new_profiles(openreview_client) + +# ## External reviewer creates a profile and accepts the invitation again +# helpers.create_user('melisa@icml.cc', 'Melisa', 'ICML') + +# ## Run Job +# openreview.venue.Venue.check_new_profiles(openreview_client) + +# invite_edges=pc_client.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', head=submissions[0].id, tail='melisa@icml.cc') +# assert len(invite_edges) == 0 + +# invite_edges=pc_client.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', head=submissions[0].id, tail='~Melisa_ICML1') +# assert len(invite_edges) == 1 +# assert invite_edges[0].label == 'Accepted' + +# assignment_edges=pc_client.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Proposed_Assignment', label='reviewer-matching', head=submissions[0].id) +# assert len(assignment_edges) == 4 + +# messages = openreview_client.get_messages(to='melisa@icml.cc', subject='[ICML 2023] Reviewer Assignment confirmed for paper 1') +# assert messages and len(messages) == 1 +# assert messages[0]['content']['text'] == '''Hi Melisa ICML, +# Thank you for accepting the invitation to review the paper number: 1, title: Paper title 1 Version 2. + +# The ICML 2023 program chairs will be contacting you with more information regarding next steps soon. In the meantime, please add noreply@openreview.net to your email contacts to ensure that you receive all communications. + +# If you would like to change your decision, please click the Decline link in the previous invitation email. + +# OpenReview Team + +# Please note that responding to this email will direct your reply to pc@icml.cc. +# ''' + +# messages = openreview_client.get_messages(to='ac1@icml.cc', subject='[ICML 2023] Reviewer Melisa ICML signed up and is assigned to paper 1') +# assert messages and len(messages) == 1 +# assert messages[0]['content']['text'] == '''Hi AC ICMLOne, +# The Reviewer Melisa ICML(melisa@icml.cc) that you invited to review paper 1 has accepted the invitation, signed up and is now assigned to the paper 1. + +# OpenReview Team + +# Please note that responding to this email will direct your reply to pc@icml.cc. +# ''' + +# assert openreview_client.get_groups('ICML.cc/2024/Conference/Submission1/External_Reviewers', member='melisa@icml.cc') +# assert openreview_client.get_groups('ICML.cc/2024/Conference/External_Reviewers', member='melisa@icml.cc') +# assert not openreview_client.get_groups('ICML.cc/2024/Conference/Reviewers', member='melisa@icml.cc') + +# venue.set_assignments(assignment_title='reviewer-matching', committee_id='ICML.cc/2024/Conference/Reviewers', enable_reviewer_reassignment=True) + +# # Check that deploying assignments removes reviewers_proposed_assignment_title +# venue_group = pc_client_v2.get_group('ICML.cc/2024/Conference') +# assert 'reviewers_proposed_assignment_title' not in venue_group.content + +# proposed_recruitment_inv = openreview_client.get_invitation('ICML.cc/2024/Conference/Reviewers/-/Proposed_Assignment_Recruitment') +# assert proposed_recruitment_inv.expdate and proposed_recruitment_inv.expdate < openreview.tools.datetime_millis(datetime.datetime.utcnow()) + +# invite_edges=pc_client.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', head=submissions[0].id, tail='~Javier_ICML1') +# assert len(invite_edges) == 1 + +# messages = openreview_client.get_messages(to='javier@icml.cc', subject='[ICML 2023] Invitation to review paper titled "Paper title 1 Version 2"') +# assert messages and len(messages) == 1 +# invitation_url = re.search('https://.*\n', messages[0]['content']['text']).group(0).replace('https://openreview.net', 'http://localhost:3030').replace('&', '&')[:-1] +# with pytest.raises(NoSuchElementException): +# helpers.respond_invitation(selenium, request_page, invitation_url, accept=True) + +# reviewers_group = pc_client_v2.get_group('ICML.cc/2024/Conference/Submission1/Reviewers') +# assert len(reviewers_group.members) == 4 +# assert '~Reviewer_ICMLOne1' in reviewers_group.members +# assert '~Reviewer_ICMLTwo1' in reviewers_group.members +# assert '~Reviewer_ICMLThree1' in reviewers_group.members +# assert '~Melisa_ICML1' in reviewers_group.members + +# reviewers_group = pc_client_v2.get_group('ICML.cc/2024/Conference/Submission100/Reviewers') +# assert len(reviewers_group.members) == 3 +# assert '~Reviewer_ICMLOne1' in reviewers_group.members +# assert '~Reviewer_ICMLTwo1' in reviewers_group.members +# assert '~Reviewer_ICMLFive1' in reviewers_group.members + +# assert pc_client_v2.get_invitation('ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment') + +# ## Change assigned SAC +# assignment_edge = pc_client_v2.get_edges(invitation='ICML.cc/2024/Conference/Senior_Area_Chairs/-/Assignment', head='~AC_ICMLTwo1', tail='~SAC_ICMLOne1')[0] +# assignment_edge.ddate = openreview.tools.datetime_millis(datetime.datetime.utcnow()) +# assignment_edge.cdate = None +# pc_client_v2.post_edge(assignment_edge) + +# helpers.await_queue_edit(openreview_client, edit_id=assignment_edge.id, count=1) + +# sac_group = pc_client_v2.get_group('ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs') +# assert ['~SAC_ICMLOne1'] == sac_group.members + +# sac_group = pc_client_v2.get_group('ICML.cc/2024/Conference/Submission100/Senior_Area_Chairs') +# assert [] == sac_group.members + +# assignment_edge = openreview_client.post_edge(openreview.api.Edge( +# invitation = 'ICML.cc/2024/Conference/Senior_Area_Chairs/-/Assignment', +# head = '~AC_ICMLTwo1', +# tail = '~SAC_ICMLTwo1', +# signatures = ['ICML.cc/2024/Conference/Program_Chairs'], +# weight = 1 +# )) + +# helpers.await_queue_edit(openreview_client, edit_id=assignment_edge.id) + +# sac_group = pc_client_v2.get_group('ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs') +# assert ['~SAC_ICMLOne1'] == sac_group.members + +# sac_group = pc_client_v2.get_group('ICML.cc/2024/Conference/Submission100/Senior_Area_Chairs') +# assert ['~SAC_ICMLTwo1'] == sac_group.members + +# ## Change assigned AC, add new AC first and then remove old AC +# edge = pc_client_v2.post_edge(openreview.api.Edge( +# invitation = 'ICML.cc/2024/Conference/Area_Chairs/-/Assignment', +# head = submissions[0].id, +# tail = '~AC_ICMLTwo1', +# signatures = ['ICML.cc/2024/Conference/Program_Chairs'], +# weight = 1 +# )) + +# helpers.await_queue_edit(openreview_client, edit_id=edge.id) + +# ac_group = pc_client_v2.get_group('ICML.cc/2024/Conference/Submission1/Area_Chairs') +# assert ['~AC_ICMLOne1', '~AC_ICMLTwo1'] == ac_group.members + +# sac_group = pc_client_v2.get_group('ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs') +# assert ['~SAC_ICMLOne1','~SAC_ICMLTwo1'] == sac_group.members + +# assignment_edge = pc_client_v2.get_edges(invitation='ICML.cc/2024/Conference/Area_Chairs/-/Assignment', head=submissions[0].id, tail='~AC_ICMLOne1')[0] +# assignment_edge.ddate = openreview.tools.datetime_millis(datetime.datetime.utcnow()) +# assignment_edge.cdate = None +# edge = pc_client_v2.post_edge(assignment_edge) + +# helpers.await_queue_edit(openreview_client, edit_id=edge.id) + +# ac_group = pc_client_v2.get_group('ICML.cc/2024/Conference/Submission1/Area_Chairs') +# assert ['~AC_ICMLTwo1'] == ac_group.members + +# sac_group = pc_client_v2.get_group('ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs') +# assert ['~SAC_ICMLTwo1'] == sac_group.members + +# def test_reviewer_reassignment(self, client, openreview_client, helpers, selenium, request_page): + +# pc_client = openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) +# ac_client = openreview.api.OpenReviewClient(username='ac2@icml.cc', password=helpers.strong_password) + +# pc_client.post_group_edit(invitation='ICML.cc/2024/Conference/-/Edit', +# readers = ['ICML.cc/2024/Conference'], +# writers = ['ICML.cc/2024/Conference'], +# signatures = ['ICML.cc/2024/Conference'], +# group = openreview.api.Group( +# id = 'ICML.cc/2024/Conference', +# content = { +# 'enable_reviewers_reassignment': { 'value': True } +# } +# ) +# ) + +# request_page(selenium, "http://localhost:3030/group?id=ICML.cc/2024/Conference/Area_Chairs", ac_client.token, wait_for_element='header') +# header = selenium.find_element(By.ID, 'header') +# assert 'Reviewer Assignment Browser:' in header.text + +# url = header.find_element(By.ID, 'edge_browser_url') +# assert url +# assert url.get_attribute('href') == 'http://localhost:3030/edges/browse?start=ICML.cc/2024/Conference/Area_Chairs/-/Assignment,tail:~AC_ICMLTwo1&traverse=ICML.cc/2024/Conference/Reviewers/-/Assignment&edit=ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment&browse=ICML.cc/2024/Conference/Reviewers/-/Affinity_Score;ICML.cc/2024/Conference/Reviewers/-/Bid;ICML.cc/2024/Conference/Reviewers/-/Custom_Max_Papers,head:ignore&hide=ICML.cc/2024/Conference/Reviewers/-/Conflict&maxColumns=2&version=2&referrer=[AC%20Console](/group?id=ICML.cc/2024/Conference/Area_Chairs)' + +# submissions = ac_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', sort='number:asc') +# anon_group_id = ac_client.get_groups(prefix='ICML.cc/2024/Conference/Submission1/Area_Chair_', signatory='~AC_ICMLTwo1')[0].id + +# invite_assignment_edge = ac_client.post_edge( +# openreview.api.Edge(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', +# signatures=[anon_group_id], +# head=submissions[0].id, +# tail='carlos@icml.cc', +# label='Invitation Sent', +# weight=1 +# )) + +# helpers.await_queue_edit(openreview_client, edit_id=invite_assignment_edge.id) + +# assert openreview_client.get_groups('ICML.cc/2024/Conference/Emergency_Reviewers/Invited', member='carlos@icml.cc') + +# assert not openreview_client.get_groups('ICML.cc/2024/Conference/Emergency_Reviewers', member='carlos@icml.cc') +# assert not openreview_client.get_groups('ICML.cc/2024/Conference/Reviewers', member='carlos@icml.cc') + +# messages = openreview_client.get_messages(to='carlos@icml.cc', subject='[ICML 2023] Invitation to review paper titled "Paper title 1 Version 2"') +# assert messages and len(messages) == 1 +# invitation_url = re.search('https://.*\n', messages[0]['content']['text']).group(0).replace('https://openreview.net', 'http://localhost:3030').replace('&', '&')[:-1] +# helpers.respond_invitation(selenium, request_page, invitation_url, accept=True) + +# helpers.await_queue_edit(openreview_client, invitation='ICML.cc/2024/Conference/Reviewers/-/Assignment_Recruitment') + +# ## External reviewer is set pending profile creation +# invite_edges=pc_client.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', head=submissions[0].id, tail='carlos@icml.cc') +# assert len(invite_edges) == 1 +# assert invite_edges[0].label == 'Pending Sign Up' + +# helpers.respond_invitation(selenium, request_page, invitation_url, accept=True) +# error_message = selenium.find_element(By.CLASS_NAME, 'important_message') +# assert 'You have already accepted this invitation, but the assignment is pending until you create a profile and no conflict are detected.' == error_message.text + +# assignment_edges=pc_client.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Assignment', head=submissions[0].id) +# assert len(assignment_edges) == 4 + +# messages = openreview_client.get_messages(to='carlos@icml.cc', subject='[ICML 2023] Reviewer Invitation accepted for paper 1, assignment pending') +# assert messages and len(messages) == 1 +# assert messages[0]['content']['text'] == '''Hi carlos@icml.cc, +# Thank you for accepting the invitation to review the paper number: 1, title: Paper title 1 Version 2. + +# Please signup in OpenReview using the email address carlos@icml.cc and complete your profile. +# Confirmation of the assignment is pending until your profile is active and no conflicts of interest are detected. + +# If you would like to change your decision, please follow the link in the previous invitation email and click on the "Decline" button. + +# OpenReview Team + +# Please note that responding to this email will direct your reply to pc@icml.cc. +# ''' + +# messages = openreview_client.get_messages(to='ac2@icml.cc', subject='[ICML 2023] Reviewer carlos@icml.cc accepted to review paper 1, assignment pending') +# assert messages and len(messages) == 1 +# assert messages[0]['content']['text'] == '''Hi AC ICMLTwo, +# The Reviewer carlos@icml.cc that you invited to review paper 1 has accepted the invitation. + +# Confirmation of the assignment is pending until the invited reviewer creates a profile in OpenReview and no conflicts of interest are detected. + +# OpenReview Team + +# Please note that responding to this email will direct your reply to pc@icml.cc. +# ''' + +# ## External reviewer creates a profile and accepts the invitation again +# helpers.create_user('carlos@icml.cc', 'Carlos', 'ICML', institution='amazon.com') + +# ## Run Job +# openreview.venue.Venue.check_new_profiles(openreview_client) + +# invite_edges=pc_client.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', head=submissions[0].id, tail='carlos@icml.cc') +# assert len(invite_edges) == 0 + +# invite_edges=pc_client.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', head=submissions[0].id, tail='~Carlos_ICML1') +# assert len(invite_edges) == 1 +# assert invite_edges[0].label == 'Conflict Detected' + +# assignment_edges=pc_client.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Assignment', head=submissions[0].id) +# assert len(assignment_edges) == 4 + +# messages = openreview_client.get_messages(to='carlos@icml.cc', subject='[ICML 2023] Conflict detected for paper 1') +# assert messages and len(messages) == 1 +# assert messages[0]['content']['text'] == '''Hi Carlos ICML, +# You have accepted the invitation to review the paper number: 1, title: Paper title 1 Version 2. + +# A conflict was detected between you and the submission authors and the assignment can not be done. + +# If you have any questions, please contact us as info@openreview.net. + +# OpenReview Team + +# Please note that responding to this email will direct your reply to pc@icml.cc. +# ''' + +# messages = openreview_client.get_messages(to='ac2@icml.cc', subject='[ICML 2023] Conflict detected between reviewer Carlos ICML and paper 1') +# assert messages and len(messages) == 1 +# assert messages[0]['content']['text'] == '''Hi AC ICMLTwo, +# A conflict was detected between Carlos ICML(carlos@icml.cc) and the paper 1 and the assignment can not be done. + +# If you have any questions, please contact us as info@openreview.net. + +# OpenReview Team + +# Please note that responding to this email will direct your reply to pc@icml.cc. +# ''' + +# assert not openreview_client.get_groups('ICML.cc/2024/Conference/Emergency_Reviewers', member='carlos@icml.cc') +# assert not openreview_client.get_groups('ICML.cc/2024/Conference/Reviewers', member='carlos@icml.cc') + +# helpers.respond_invitation(selenium, request_page, invitation_url, accept=True) +# error_message = selenium.find_element(By.CLASS_NAME, 'important_message') +# assert "You have already accepted this invitation, but a conflict was detected and the assignment cannot be made." == error_message.text + +# invite_assignment_edge = ac_client.post_edge( +# openreview.api.Edge(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', +# signatures=[anon_group_id], +# head=submissions[0].id, +# tail='celeste@icml.cc', +# label='Invitation Sent', +# weight=1 +# )) + +# helpers.await_queue_edit(openreview_client, edit_id=invite_assignment_edge.id) + +# messages = openreview_client.get_messages(to='celeste@icml.cc', subject='[ICML 2023] Invitation to review paper titled "Paper title 1 Version 2"') +# assert messages and len(messages) == 1 +# invitation_url = re.search('https://.*\n', messages[0]['content']['text']).group(0).replace('https://openreview.net', 'http://localhost:3030').replace('&', '&')[:-1] +# helpers.respond_invitation(selenium, request_page, invitation_url, accept=True) + +# helpers.await_queue_edit(openreview_client, invitation='ICML.cc/2024/Conference/Reviewers/-/Assignment_Recruitment', count=2) + +# ## External reviewer creates a profile and accepts the invitation again +# helpers.create_user('celeste@icml.cc', 'Celeste', 'ICML') + +# ## Run Job +# openreview.venue.Venue.check_new_profiles(openreview_client) + +# invite_edges=pc_client.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', head=submissions[0].id, tail='~Celeste_ICML1') +# assert len(invite_edges) == 1 +# assert invite_edges[0].label == 'Accepted' + +# assignment_edges=pc_client.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Assignment', head=submissions[0].id) +# assert len(assignment_edges) == 5 + +# messages = openreview_client.get_messages(to='celeste@icml.cc', subject='[ICML 2023] Reviewer Assignment confirmed for paper 1') +# assert messages and len(messages) == 1 +# assert messages[0]['content']['text'] == '''Hi Celeste ICML, +# Thank you for accepting the invitation to review the paper number: 1, title: Paper title 1 Version 2. + +# Please go to the ICML 2023 Reviewers Console and check your pending tasks: https://openreview.net/group?id=ICML.cc/2024/Conference/Reviewers + +# If you would like to change your decision, please click the Decline link in the previous invitation email. + +# OpenReview Team + +# Please note that responding to this email will direct your reply to pc@icml.cc. +# ''' + +# messages = openreview_client.get_messages(to='ac2@icml.cc', subject='[ICML 2023] Reviewer Celeste ICML signed up and is assigned to paper 1') +# assert messages and len(messages) == 1 +# assert messages[0]['content']['text'] == '''Hi AC ICMLTwo, +# The Reviewer Celeste ICML(celeste@icml.cc) that you invited to review paper 1 has accepted the invitation, signed up and is now assigned to the paper 1. + +# OpenReview Team + +# Please note that responding to this email will direct your reply to pc@icml.cc. +# ''' + +# assignment_edge = pc_client.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Assignment', head=submissions[0].id, tail='~Celeste_ICML1')[0] +# helpers.await_queue_edit(openreview_client, edit_id=assignment_edge.id) + +# messages = openreview_client.get_messages(to='celeste@icml.cc', subject='[ICML 2023] You have been assigned as a Reviewer for paper number 1') +# assert messages and len(messages) == 1 +# assert messages[0]['content']['replyTo'] == 'pc@icml.cc' +# assert messages[0]['content']['text'] == f'''This is to inform you that you have been assigned as a Reviewer for paper number 1 for ICML 2023. + +# To review this new assignment, please login to OpenReview and go to https://openreview.net/forum?id={submissions[0].id}. + +# To check all of your assigned papers, go to https://openreview.net/group?id=ICML.cc/2024/Conference/Reviewers. + +# Thank you, + +# ICML 2023 Conference Program Chairs + +# Please note that responding to this email will direct your reply to pc@icml.cc. +# ''' + +# assert openreview_client.get_groups('ICML.cc/2024/Conference/Emergency_Reviewers', member='celeste@icml.cc') +# assert openreview_client.get_groups('ICML.cc/2024/Conference/Reviewers', member='celeste@icml.cc') + +# helpers.respond_invitation(selenium, request_page, invitation_url, accept=True) +# error_message = selenium.find_element(By.CLASS_NAME, 'important_message') +# assert 'You have already accepted this invitation.' == error_message.text + +# reviewers_group = pc_client.get_group('ICML.cc/2024/Conference/Submission1/Reviewers') +# assert len(reviewers_group.members) == 5 +# assert '~Reviewer_ICMLOne1' in reviewers_group.members +# assert '~Reviewer_ICMLTwo1' in reviewers_group.members +# assert '~Reviewer_ICMLThree1' in reviewers_group.members +# assert '~Melisa_ICML1' in reviewers_group.members +# assert '~Celeste_ICML1' in reviewers_group.members + +# ac_group = pc_client.get_group('ICML.cc/2024/Conference/Submission1/Area_Chairs') +# assert len(ac_group.members) == 1 +# assert '~AC_ICMLTwo1' in ac_group.members + +# invite_assignment_edge = ac_client.post_edge( +# openreview.api.Edge(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', +# signatures=[anon_group_id], +# head=submissions[0].id, +# tail='~Reviewer_ICMLFour1', +# label='Invitation Sent', +# weight=1 +# )) + +# helpers.await_queue_edit(openreview_client, edit_id=invite_assignment_edge.id) + +# messages = openreview_client.get_messages(to='reviewer4@yahoo.com', subject='[ICML 2023] Invitation to review paper titled "Paper title 1 Version 2"') +# assert messages and len(messages) == 1 +# invitation_url = re.search('https://.*\n', messages[0]['content']['text']).group(0).replace('https://openreview.net', 'http://localhost:3030').replace('&', '&')[:-1] +# helpers.respond_invitation(selenium, request_page, invitation_url, accept=False) + +# helpers.await_queue_edit(openreview_client, invitation='ICML.cc/2024/Conference/Reviewers/-/Assignment_Recruitment', count=3) + +# helpers.respond_invitation(selenium, request_page, invitation_url, accept=False) +# error_message = selenium.find_element(By.CLASS_NAME, 'important_message') +# assert 'You have already declined this invitation.' == error_message.text + +# helpers.respond_invitation(selenium, request_page, invitation_url, accept=True) + +# helpers.await_queue_edit(openreview_client, invitation='ICML.cc/2024/Conference/Reviewers/-/Assignment_Recruitment', count=4) +# helpers.await_queue_edit(openreview_client, invitation='ICML.cc/2024/Conference/Reviewers/-/Assignment', count=2) + +# # try to delete Invite Assignment edge after reviewer Accepted +# with pytest.raises(openreview.OpenReviewException, match=r'Cannot cancel the invitation since it has status: "Accepted"'): +# invite_edge=ac_client.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', head=submissions[0].id, tail='~Reviewer_ICMLFour1')[0] +# invite_edge.ddate = openreview.tools.datetime_millis(datetime.datetime.utcnow()) +# ac_client.post_edge(invite_edge) + +# reviewers_group = pc_client.get_group('ICML.cc/2024/Conference/Submission1/Reviewers') +# assert len(reviewers_group.members) == 6 +# assert '~Reviewer_ICMLOne1' in reviewers_group.members +# assert '~Reviewer_ICMLTwo1' in reviewers_group.members +# assert '~Reviewer_ICMLThree1' in reviewers_group.members +# assert '~Melisa_ICML1' in reviewers_group.members +# assert '~Celeste_ICML1' in reviewers_group.members +# assert '~Reviewer_ICMLFour1' in reviewers_group.members + +# helpers.create_user('rachel@icml.cc', 'Rachel', 'ICML') + +# invite_assignment_edge = ac_client.post_edge( +# openreview.api.Edge(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', +# signatures=[anon_group_id], +# head=submissions[0].id, +# tail='~Rachel_ICML1', +# label='Invitation Sent', +# weight=1 +# )) + +# helpers.await_queue_edit(openreview_client, edit_id=invite_assignment_edge.id) + +# messages = openreview_client.get_messages(to='rachel@icml.cc', subject='[ICML 2023] Invitation to review paper titled "Paper title 1 Version 2"') +# assert messages and len(messages) == 1 +# invitation_url = re.search('https://.*\n', messages[0]['content']['text']).group(0).replace('https://openreview.net', 'http://localhost:3030').replace('&', '&')[:-1] + +# ## create another profile and merge +# helpers.create_user('rachel_bis@icml.cc', 'Rachel', 'ICML') + +# client.rename_edges(new_id='~Rachel_ICML2', current_id='~Rachel_ICML1') +# client.merge_profiles(profileTo='~Rachel_ICML2', profileFrom='~Rachel_ICML1') + +# helpers.respond_invitation(selenium, request_page, invitation_url, accept=False, comment='I am too busy.') + +# helpers.await_queue_edit(openreview_client, invitation='ICML.cc/2024/Conference/Reviewers/-/Assignment_Recruitment', count=5) + +# messages = openreview_client.get_messages(to='rachel_bis@icml.cc', subject='[ICML 2023] Reviewer Invitation declined for paper 1') +# assert len(messages) == 2 + +# invite_edges=openreview_client.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', head=submissions[0].id, tail='~Rachel_ICML2') +# assert len(invite_edges) == 1 +# assert invite_edges[0].label == 'Declined: I am too busy.' + +# # accept invitation after declining with comment +# helpers.respond_invitation(selenium, request_page, invitation_url, accept=True) + +# helpers.await_queue_edit(openreview_client, invitation='ICML.cc/2024/Conference/Reviewers/-/Assignment_Recruitment', count=6) + +# messages = openreview_client.get_messages(to='rachel_bis@icml.cc', subject='[ICML 2023] Reviewer Invitation accepted for paper 1') +# assert len(messages) == 1 + +# invite_edges=openreview_client.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', head=submissions[0].id, tail='~Rachel_ICML2') +# assert len(invite_edges) == 1 +# assert invite_edges[0].label == 'Accepted' + +# helpers.create_user('ana@icml.cc', 'Ana', 'ICML') + +# invite_assignment_edge = ac_client.post_edge( +# openreview.api.Edge(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', +# signatures=[anon_group_id], +# head=submissions[0].id, +# tail='~Ana_ICML1', +# label='Invitation Sent', +# weight=1 +# )) + +# helpers.await_queue_edit(openreview_client, edit_id=invite_assignment_edge.id) + +# # delete invite assignment edge +# invite_assignment = pc_client.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Invite_Assignment', head=submissions[0].id, tail='~Ana_ICML1')[0] +# invite_assignment.ddate = openreview.tools.datetime_millis(datetime.datetime.utcnow()) +# invite_assignment.cdate = None +# pc_client.post_edge(invite_assignment) + +# messages = openreview_client.get_messages(to='ana@icml.cc', subject='[ICML 2023] Invitation to review paper titled "Paper title 1 Version 2"') +# assert messages and len(messages) == 1 +# invitation_url = re.search('https://.*\n', messages[0]['content']['text']).group(0).replace('https://openreview.net', 'http://localhost:3030').replace('&', '&')[:-1] + +# #try to accept invitation that has been deleted +# helpers.respond_invitation(selenium, request_page, invitation_url, accept=True) +# error_message = selenium.find_element(By.CLASS_NAME, 'important_message') +# assert 'Invitation no longer exists. No action is required from your end.' == error_message.text + +# #delete assignments before review stage and not get key error +# assignment = pc_client.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Assignment', head=submissions[10].id, tail='~Reviewer_ICMLThree1')[0] +# assignment.ddate = openreview.tools.datetime_millis(datetime.datetime.utcnow()) +# assignment.cdate = None +# edge = pc_client.post_edge(assignment) + +# helpers.await_queue_edit(openreview_client, edit_id=edge.id) + +# reviewers_group = pc_client.get_group('ICML.cc/2024/Conference/Submission11/Reviewers') +# assert len(reviewers_group.members) == 2 +# assert '~Reviewer_ICMLThree1' not in reviewers_group.members + +# assignment = pc_client.get_edges(invitation='ICML.cc/2024/Conference/Area_Chairs/-/Assignment', head=submissions[10].id, tail='~AC_ICMLOne1')[0] +# assignment.ddate = openreview.tools.datetime_millis(datetime.datetime.utcnow()) +# assignment.cdate = None +# edge = pc_client.post_edge(assignment) + +# helpers.await_queue_edit(openreview_client, edit_id=edge.id) + +# ac_group = pc_client.get_group('ICML.cc/2024/Conference/Submission11/Area_Chairs') +# assert [] == ac_group.members + +# sac_group = pc_client.get_group('ICML.cc/2024/Conference/Submission11/Senior_Area_Chairs') +# assert [] == sac_group.members + +# def test_review_stage(self, client, openreview_client, helpers, selenium, request_page): + +# pc_client=openreview.Client(username='pc@icml.cc', password=helpers.strong_password) +# request_form=pc_client.get_notes(invitation='openreview.net/Support/-/Request_Form')[0] + +# ## Show the pdf and supplementary material to assigned reviewers +# pc_client.post_note(openreview.Note( +# content= { +# 'force': 'Yes', +# 'submission_readers': 'Assigned program committee (assigned reviewers, assigned area chairs, assigned senior area chairs if applicable)', +# 'hide_fields': ['financial_aid'] +# }, +# forum= request_form.id, +# invitation= f'openreview.net/Support/-/Request{request_form.number}/Post_Submission', +# readers= ['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# referent= request_form.id, +# replyto= request_form.id, +# signatures= ['~Program_ICMLChair1'], +# writers= [], +# )) + +# helpers.await_queue() + +# ac_client = openreview.api.OpenReviewClient(username='ac1@icml.cc', password=helpers.strong_password) +# submissions = ac_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', sort='number:asc') +# assert len(submissions) == 58 +# assert ['ICML.cc/2024/Conference', +# 'ICML.cc/2024/Conference/Submission2/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Submission2/Area_Chairs', +# 'ICML.cc/2024/Conference/Submission2/Reviewers', +# 'ICML.cc/2024/Conference/Submission2/Authors'] == submissions[0].readers +# assert ['ICML.cc/2024/Conference', +# 'ICML.cc/2024/Conference/Submission2/Authors'] == submissions[0].writers +# assert ['ICML.cc/2024/Conference/Submission2/Authors'] == submissions[0].signatures +# assert 'authorids' not in submissions[0].content +# assert 'authors' not in submissions[0].content +# assert 'financial_aid'not in submissions[0].content +# assert 'pdf' in submissions[0].content +# assert 'supplementary_material' in submissions[0].content + +# now = datetime.datetime.utcnow() +# start_date = now - datetime.timedelta(days=2) +# due_date = now + datetime.timedelta(days=3) + +# venue = openreview.helpers.get_conference(client, request_form.id, setup=False) +# venue.review_stage = openreview.stages.ReviewStage( +# start_date=start_date, +# due_date=due_date, +# additional_fields={ +# "summarry": { +# "order": 1, +# "description": "Briefly summarize the paper and its contributions. This is not the place to critique the paper; the authors should generally agree with a well-written summary.", +# "value": { +# "param": { +# "maxLength": 200000, +# "type": "string", +# "input": "textarea", +# "markdown": True +# } +# } +# }, +# "strengths_and_weaknesses": { +# "order": 2, +# "description": "Please provide a thorough assessment of the strengths and weaknesses of the paper, touching on each of the following dimensions: originality, quality, clarity, and significance. We encourage people to be broad in their definitions of originality and significance. For example, originality may arise from creative combinations of existing ideas, application to a new domain, or removing restrictive assumptions from prior theoretical results. You can incorporate Markdown and Latex into your review. See https://openreview.net/faq.", +# "value": { +# "param": { +# "maxLength": 200000, +# "type": "string", +# "input": "textarea", +# "markdown": True +# } +# } +# }, +# "questions": { +# "order": 3, +# "description": "Please list up and carefully describe any questions and suggestions for the authors. Think of the things where a response from the author can change your opinion, clarify a confusion or address a limitation. This can be very important for a productive rebuttal and discussion phase with the authors.", +# "value": { +# "param": { +# "maxLength": 200000, +# "type": "string", +# "input": "textarea", +# "markdown": True +# } +# } +# }, +# "limitations": { +# "order": 4, +# "description": "Have the authors adequately addressed the limitations and potential negative societal impact of their work? If not, please include constructive suggestions for improvement. Authors should be rewarded rather than punished for being up front about the limitations of their work and any potential negative societal impact.", +# "value": { +# "param": { +# "maxLength": 200000, +# "type": "string", +# "input": "textarea", +# "markdown": True +# } +# } +# }, +# "ethics_flag": { +# "order": 5, +# "description": "If there are ethical issues with this paper, please flag the paper for an ethics review. For guidance on when this is appropriate, please review the ethics guidelines (https://icml.cc/Conferences/2023/PublicationEthics).", +# "value": { +# "param": { +# "type": "string", +# "enum": [ +# "Yes", +# "No" +# ], +# "input": "radio" +# } +# } +# }, +# "ethics_review_area": { +# "order": 6, +# "description": "If you flagged this paper for ethics review, what area of expertise would it be most useful for the ethics reviewer to have? Please click all that apply.", +# "value": { +# "param": { +# "type": "string[]", +# "enum": [ +# "Discrimination / Bias / Fairness Concerns", +# "Inadequate Data and Algorithm Evaluation", +# "Inappropriate Potential Applications & Impact (e.g., human rights concerns)", +# "Privacy and Security (e.g., consent)", +# "Legal Compliance (e.g., GDPR, copyright, terms of use)", +# "Research Integrity Issues (e.g., plagiarism)", +# "Responsible Research Practice (e.g., IRB, documentation, research ethics)", +# "I don't know" +# ], +# "input": "checkbox", +# "optional": True, +# } +# } +# }, +# "soundness": { +# "order": 7, +# "description": "Please assign the paper a numerical rating on the following scale to indicate the soundness of the technical claims, experimental and research methodology and on whether the central claims of the paper are adequately supported with evidence.", +# "value": { +# "param": { +# "type": "string", +# "enum": [ +# "4 excellent", +# "3 good", +# "2 fair", +# "1 poor" +# ], +# "input": "radio" +# } +# } +# }, +# "presentation": { +# "order": 8, +# "description": "Please assign the paper a numerical rating on the following scale to indicate the quality of the presentation. This should take into account the writing style and clarity, as well as contextualization relative to prior work.", +# "value": { +# "param": { +# "type": "string", +# "enum": [ +# "4 excellent", +# "3 good", +# "2 fair", +# "1 poor" +# ], +# "input": "radio" +# } +# } +# }, +# "contribution": { +# "order": 9, +# "description": "Please assign the paper a numerical rating on the following scale to indicate the quality of the overall contribution this paper makes to the research area being studied. Are the questions being asked important? Does the paper bring a significant originality of ideas and/or execution? Are the results valuable to share with the broader ICML community?", +# "value": { +# "param": { +# "type": "string", +# "enum": [ +# "4 excellent", +# "3 good", +# "2 fair", +# "1 poor" +# ], +# "input": "radio" +# } +# } +# }, +# "rating": { +# "order": 10, +# "description": "Please provide an \"overall score\" for this submission.", +# "value": { +# "param": { +# "type": 'integer', +# "enum": [ +# { 'value': 10, 'description': "10: Award quality: Technically flawless paper with groundbreaking impact, with exceptionally strong evaluation, reproducibility, and resources, and no unaddressed ethical considerations." }, +# { 'value': 9, 'description': "9: Very Strong Accept: Technically flawless paper with groundbreaking impact on at least one area of AI/ML and excellent impact on multiple areas of AI/ML, with flawless evaluation, resources, and reproducibility, and no unaddressed ethical considerations." }, +# { 'value': 8, 'description': "8: Strong Accept: Technically strong paper, with novel ideas, excellent impact on at least one area, or high-to-excellent impact on multiple areas, with excellent evaluation, resources, and reproducibility, and no unaddressed ethical considerations." }, +# { 'value': 7, 'description': "7: Accept: Technically solid paper, with high impact on at least one sub-area, or moderate-to-high impact on more than one areas, with good-to-excellent evaluation, resources, reproducibility, and no unaddressed ethical considerations." }, +# { 'value': 6, 'description': "6: Weak Accept: Technically solid, moderate-to-high impact paper, with no major concerns with respect to evaluation, resources, reproducibility, ethical considerations." }, +# { 'value': 5, 'description': "5: Borderline accept: Technically solid paper where reasons to accept outweigh reasons to reject, e.g., limited evaluation. Please use sparingly." }, +# { 'value': 4, 'description': "4: Borderline reject: Technically solid paper where reasons to reject, e.g., limited evaluation, outweigh reasons to accept, e.g., good evaluation. Please use sparingly." }, +# { 'value': 3, 'description': "3: Reject: For instance, a paper with technical flaws, weak evaluation, inadequate reproducibility and incompletely addressed ethical considerations." }, +# { 'value': 2, 'description': "2: Strong Reject: For instance, a paper with major technical flaws, and/or poor evaluation, limited impact, poor reproducibility and mostly unaddressed ethical considerations." }, +# { 'value': 1, 'description': "1: Very Strong Reject: For instance, a paper with trivial results or unaddressed ethical considerations" } +# ], +# "input": "radio" + +# } +# } +# }, +# "confidence": { +# "order": 11, +# "description": "Please provide a \"confidence score\" for your assessment of this submission to indicate how confident you are in your evaluation.", +# "value": { +# "param": { +# "type": 'integer', +# "enum": [ +# { 'value': 5, 'description': "5: You are absolutely certain about your assessment. You are very familiar with the related work and checked the math/other details carefully." }, +# { 'value': 4, 'description': "4: You are confident in your assessment, but not absolutely certain. It is unlikely, but not impossible, that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work." }, +# { 'value': 3, 'description': "3: You are fairly confident in your assessment. It is possible that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked." }, +# { 'value': 2, 'description': "2: You are willing to defend your assessment, but it is quite likely that you did not understand the central parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked." }, +# { 'value': 1, 'description': "1: Your assessment is an educated guess. The submission is not in your area or the submission was difficult to understand. Math/other details were not carefully checked." } +# ], +# "input": "radio" +# } +# } +# }, +# "code_of_conduct": { +# "description": "While performing my duties as a reviewer (including writing reviews and participating in discussions), I have and will continue to abide by the ICML code of conduct (https://icml.cc/public/CodeOfConduct).", +# "order": 12, +# "value": { +# "param": { +# "type": "string", +# "enum": ["Yes"], +# "input": "checkbox" +# } +# } +# } +# }, +# remove_fields=['title', 'review'], +# source_submissions_query={ +# 'position_paper_track': 'No' +# } +# ) + +# venue.create_review_stage() + +# helpers.await_queue_edit(openreview_client, 'ICML.cc/2024/Conference/-/Official_Review-0-1', count=1) + +# assert len(openreview_client.get_invitations(invitation='ICML.cc/2024/Conference/-/Official_Review')) == 50 +# invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Submission1/-/Official_Review') +# assert 'summarry' in invitation.edit['note']['content'] +# assert invitation.cdate < openreview.tools.datetime_millis(now) +# # duedate + 30 min +# exp_date = invitation.duedate + (30*60*1000) +# assert invitation.expdate == exp_date + +# review_exp_date = due_date + datetime.timedelta(days=2) + +# venue.review_stage = openreview.stages.ReviewStage( +# start_date=start_date, +# due_date=due_date, +# exp_date=review_exp_date, +# additional_fields={ +# "summary": { +# "order": 1, +# "description": "Briefly summarize the paper and its contributions. This is not the place to critique the paper; the authors should generally agree with a well-written summary.", +# "value": { +# "param": { +# "maxLength": 200000, +# "type": "string", +# "input": "textarea", +# "markdown": True +# } +# } +# }, +# "strengths_and_weaknesses": { +# "order": 2, +# "description": "Please provide a thorough assessment of the strengths and weaknesses of the paper, touching on each of the following dimensions: originality, quality, clarity, and significance. We encourage people to be broad in their definitions of originality and significance. For example, originality may arise from creative combinations of existing ideas, application to a new domain, or removing restrictive assumptions from prior theoretical results. You can incorporate Markdown and Latex into your review. See https://openreview.net/faq.", +# "value": { +# "param": { +# "maxLength": 200000, +# "type": "string", +# "input": "textarea", +# "markdown": True +# } +# } +# }, +# "questions": { +# "order": 3, +# "description": "Please list up and carefully describe any questions and suggestions for the authors. Think of the things where a response from the author can change your opinion, clarify a confusion or address a limitation. This can be very important for a productive rebuttal and discussion phase with the authors.", +# "value": { +# "param": { +# "maxLength": 200000, +# "type": "string", +# "input": "textarea", +# "markdown": True +# } +# } +# }, +# "limitations": { +# "order": 4, +# "description": "Have the authors adequately addressed the limitations and potential negative societal impact of their work? If not, please include constructive suggestions for improvement. Authors should be rewarded rather than punished for being up front about the limitations of their work and any potential negative societal impact.", +# "value": { +# "param": { +# "maxLength": 200000, +# "type": "string", +# "input": "textarea", +# "markdown": True +# } +# } +# }, +# "ethics_flag": { +# "order": 5, +# "description": "If there are ethical issues with this paper, please flag the paper for an ethics review. For guidance on when this is appropriate, please review the ethics guidelines (https://icml.cc/Conferences/2023/PublicationEthics).", +# "value": { +# "param": { +# "type": "string", +# "enum": [ +# "Yes", +# "No" +# ], +# "input": "radio" +# } +# } +# }, +# "ethics_review_area": { +# "order": 6, +# "description": "If you flagged this paper for ethics review, what area of expertise would it be most useful for the ethics reviewer to have? Please click all that apply.", +# "value": { +# "param": { +# "type": "string[]", +# "enum": [ +# "Discrimination / Bias / Fairness Concerns", +# "Inadequate Data and Algorithm Evaluation", +# "Inappropriate Potential Applications & Impact (e.g., human rights concerns)", +# "Privacy and Security (e.g., consent)", +# "Legal Compliance (e.g., GDPR, copyright, terms of use)", +# "Research Integrity Issues (e.g., plagiarism)", +# "Responsible Research Practice (e.g., IRB, documentation, research ethics)", +# "I don't know" +# ], +# "input": "checkbox", +# "optional": True, +# } +# } +# }, +# "soundness": { +# "order": 7, +# "description": "Please assign the paper a numerical rating on the following scale to indicate the soundness of the technical claims, experimental and research methodology and on whether the central claims of the paper are adequately supported with evidence.", +# "value": { +# "param": { +# "type": "string", +# "enum": [ +# "4 excellent", +# "3 good", +# "2 fair", +# "1 poor" +# ], +# "input": "radio" +# } +# } +# }, +# "presentation": { +# "order": 8, +# "description": "Please assign the paper a numerical rating on the following scale to indicate the quality of the presentation. This should take into account the writing style and clarity, as well as contextualization relative to prior work.", +# "value": { +# "param": { +# "type": "string", +# "enum": [ +# "4 excellent", +# "3 good", +# "2 fair", +# "1 poor" +# ], +# "input": "radio" +# } +# } +# }, +# "contribution": { +# "order": 9, +# "description": "Please assign the paper a numerical rating on the following scale to indicate the quality of the overall contribution this paper makes to the research area being studied. Are the questions being asked important? Does the paper bring a significant originality of ideas and/or execution? Are the results valuable to share with the broader ICML community?", +# "value": { +# "param": { +# "type": "string", +# "enum": [ +# "4 excellent", +# "3 good", +# "2 fair", +# "1 poor" +# ], +# "input": "radio" +# } +# } +# }, +# "rating": { +# "order": 10, +# "description": "Please provide an \"overall score\" for this submission.", +# "value": { +# "param": { +# "type": 'integer', +# "enum": [ +# { 'value': 10, 'description': "10: Award quality: Technically flawless paper with groundbreaking impact, with exceptionally strong evaluation, reproducibility, and resources, and no unaddressed ethical considerations." }, +# { 'value': 9, 'description': "9: Very Strong Accept: Technically flawless paper with groundbreaking impact on at least one area of AI/ML and excellent impact on multiple areas of AI/ML, with flawless evaluation, resources, and reproducibility, and no unaddressed ethical considerations." }, +# { 'value': 8, 'description': "8: Strong Accept: Technically strong paper, with novel ideas, excellent impact on at least one area, or high-to-excellent impact on multiple areas, with excellent evaluation, resources, and reproducibility, and no unaddressed ethical considerations." }, +# { 'value': 7, 'description': "7: Accept: Technically solid paper, with high impact on at least one sub-area, or moderate-to-high impact on more than one areas, with good-to-excellent evaluation, resources, reproducibility, and no unaddressed ethical considerations." }, +# { 'value': 6, 'description': "6: Weak Accept: Technically solid, moderate-to-high impact paper, with no major concerns with respect to evaluation, resources, reproducibility, ethical considerations." }, +# { 'value': 5, 'description': "5: Borderline accept: Technically solid paper where reasons to accept outweigh reasons to reject, e.g., limited evaluation. Please use sparingly." }, +# { 'value': 4, 'description': "4: Borderline reject: Technically solid paper where reasons to reject, e.g., limited evaluation, outweigh reasons to accept, e.g., good evaluation. Please use sparingly." }, +# { 'value': 3, 'description': "3: Reject: For instance, a paper with technical flaws, weak evaluation, inadequate reproducibility and incompletely addressed ethical considerations." }, +# { 'value': 2, 'description': "2: Strong Reject: For instance, a paper with major technical flaws, and/or poor evaluation, limited impact, poor reproducibility and mostly unaddressed ethical considerations." }, +# { 'value': 1, 'description': "1: Very Strong Reject: For instance, a paper with trivial results or unaddressed ethical considerations" } +# ], +# "input": "radio" + +# } +# } +# }, +# "confidence": { +# "order": 11, +# "description": "Please provide a \"confidence score\" for your assessment of this submission to indicate how confident you are in your evaluation.", +# "value": { +# "param": { +# "type": 'integer', +# "enum": [ +# { 'value': 5, 'description': "5: You are absolutely certain about your assessment. You are very familiar with the related work and checked the math/other details carefully." }, +# { 'value': 4, 'description': "4: You are confident in your assessment, but not absolutely certain. It is unlikely, but not impossible, that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work." }, +# { 'value': 3, 'description': "3: You are fairly confident in your assessment. It is possible that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked." }, +# { 'value': 2, 'description': "2: You are willing to defend your assessment, but it is quite likely that you did not understand the central parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked." }, +# { 'value': 1, 'description': "1: Your assessment is an educated guess. The submission is not in your area or the submission was difficult to understand. Math/other details were not carefully checked." } +# ], +# "input": "radio" +# } +# } +# }, +# "code_of_conduct": { +# "description": "While performing my duties as a reviewer (including writing reviews and participating in discussions), I have and will continue to abide by the ICML code of conduct (https://icml.cc/public/CodeOfConduct).", +# "order": 12, +# "value": { +# "param": { +# "type": "string", +# "enum": ["Yes"], +# "input": "checkbox" +# } +# } +# } +# }, +# remove_fields=['title', 'review'], +# source_submissions_query={ +# 'position_paper_track': 'No' +# } +# ) + +# venue.create_review_stage() + +# helpers.await_queue_edit(openreview_client, 'ICML.cc/2024/Conference/-/Official_Review-0-1', count=2) + +# assert len(openreview_client.get_invitations(invitation='ICML.cc/2024/Conference/-/Official_Review')) == 50 +# invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Submission1/-/Official_Review') +# assert 'summarry' not in invitation.edit['note']['content'] +# assert 'summary' in invitation.edit['note']['content'] +# assert invitation.cdate < openreview.tools.datetime_millis(datetime.datetime.utcnow()) +# # duedate + 2 days +# exp_date = invitation.duedate + (2*24*60) + +# venue.review_stage = openreview.stages.ReviewStage( +# start_date=start_date, +# due_date=due_date, +# exp_date=review_exp_date, +# name='Position_Paper_Review', +# remove_fields=['title'], +# source_submissions_query={ +# 'position_paper_track': 'Yes' +# } +# ) + +# venue.create_review_stage() + +# helpers.await_queue_edit(openreview_client, 'ICML.cc/2024/Conference/-/Position_Paper_Review-0-1', count=1) + +# assert len(openreview_client.get_invitations(invitation='ICML.cc/2024/Conference/-/Position_Paper_Review')) == 50 +# invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Submission2/-/Official_Review') +# assert 'review' in invitation.edit['note']['content'] +# assert 'summary' not in invitation.edit['note']['content'] + +# reviewer_client = openreview.api.OpenReviewClient(username='reviewer1@icml.cc', password=helpers.strong_password) + +# anon_groups = reviewer_client.get_groups(prefix='ICML.cc/2024/Conference/Submission1/Reviewer_', signatory='~Reviewer_ICMLOne1') +# anon_group_id = anon_groups[0].id + +# review_edit = reviewer_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission1/-/Official_Review', +# signatures=[anon_group_id], +# note=openreview.api.Note( +# content={ +# 'summary': { 'value': 'good paper' }, +# 'strengths_and_weaknesses': { 'value': '7: Good paper, accept'}, +# 'questions': { 'value': '7: Good paper, accept'}, +# 'limitations': { 'value': '7: Good paper, accept'}, +# 'ethics_flag': { 'value': 'No'}, +# 'soundness': { 'value': '3 good'}, +# 'presentation': { 'value': '3 good'}, +# 'contribution': { 'value': '3 good'}, +# 'rating': { 'value': 10 }, +# 'confidence': { 'value': 5 }, +# 'code_of_conduct': { 'value': 'Yes'}, +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=review_edit['id']) + +# messages = openreview_client.get_messages(to='ac2@icml.cc', subject='[ICML 2023] Official Review posted to your assigned Paper number: 1, Paper title: "Paper title 1 Version 2"') +# assert messages and len(messages) == 1 + +# messages = openreview_client.get_messages(to='reviewer1@icml.cc', subject='[ICML 2023] Your official review has been received on your assigned Paper number: 1, Paper title: "Paper title 1 Version 2"') +# assert messages and len(messages) == 1 +# assert messages[0]['content']['replyTo'] == 'pc@icml.cc' + +# ## check how the description is rendered +# note = review_edit['note'] +# review_id = note['id'] +# request_page(selenium, "http://localhost:3030/forum?id=" + review_edit['note']['forum'], openreview_client.token, by=By.ID, wait_for_element='forum-replies') +# note_panel = selenium.find_element(By.XPATH, f'//div[@data-id="{review_id}"]') +# fields = note_panel.find_elements(By.CLASS_NAME, 'note-content-field') +# assert len(fields) == 11 +# assert fields[8].text == 'Rating:' +# assert fields[9].text == 'Confidence:' +# values = note_panel.find_elements(By.CLASS_NAME, 'note-content-value') +# assert len(values) == 11 +# assert values[8].text == '10: Award quality: Technically flawless paper with groundbreaking impact, with exceptionally strong evaluation, reproducibility, and resources, and no unaddressed ethical considerations.' +# assert values[9].text == '5: You are absolutely certain about your assessment. You are very familiar with the related work and checked the math/other details carefully.' + +# review_edit = reviewer_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission1/-/Official_Review', +# signatures=[anon_group_id], +# note=openreview.api.Note( +# id = review_edit['note']['id'], +# content={ +# 'summary': { 'value': 'good paper version 2' }, +# 'strengths_and_weaknesses': { 'value': '7: Good paper, accept'}, +# 'questions': { 'value': '7: Good paper, accept'}, +# 'limitations': { 'value': '7: Good paper, accept'}, +# 'ethics_flag': { 'value': 'No'}, +# 'soundness': { 'value': '3 good'}, +# 'presentation': { 'value': '3 good'}, +# 'contribution': { 'value': '3 good'}, +# 'rating': { 'value': 10 }, +# 'confidence': { 'value': 5 }, +# 'code_of_conduct': { 'value': 'Yes'}, +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=review_edit['id']) + +# messages = openreview_client.get_messages(to='ac2@icml.cc', subject='[ICML 2023] Official Review posted to your assigned Paper number: 1, Paper title: "Paper title 1 Version 2"') +# assert messages and len(messages) == 1 + +# messages = openreview_client.get_messages(to='reviewer1@icml.cc', subject='[ICML 2023] Your official review has been received on your assigned Paper number: 1, Paper title: "Paper title 1 Version 2"') +# assert messages and len(messages) == 1 + +# openreview_client.add_members_to_group(f'ICML.cc/2024/Conference/Submission1/Reviewers', '~Reviewer_ICMLOne1') +# openreview_client.add_members_to_group(f'ICML.cc/2024/Conference/Submission1/Reviewers', '~Reviewer_ICMLTwo1') +# openreview_client.add_members_to_group(f'ICML.cc/2024/Conference/Submission1/Reviewers', '~Reviewer_ICMLThree1') + +# reviewer_client_2 = openreview.api.OpenReviewClient(username='reviewer2@icml.cc', password=helpers.strong_password) +# anon_groups = reviewer_client.get_groups(prefix='ICML.cc/2024/Conference/Submission1/Reviewer_', signatory='~Reviewer_ICMLTwo1') +# anon_group_id = anon_groups[0].id +# review_edit = reviewer_client_2.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission1/-/Official_Review', +# signatures=[anon_group_id], +# note=openreview.api.Note( +# content={ +# 'summary': { 'value': 'bad paper' }, +# 'strengths_and_weaknesses': { 'value': '2: Bad paper, reject'}, +# 'questions': { 'value': '2: Bad paper, reject'}, +# 'limitations': { 'value': '2: Bad paper, reject'}, +# 'ethics_flag': { 'value': 'No'}, +# 'soundness': { 'value': '1 poor'}, +# 'presentation': { 'value': '1 poor'}, +# 'contribution': { 'value': '1 poor'}, +# 'rating': { 'value': 1 }, +# 'confidence': { 'value': 5 }, +# 'code_of_conduct': { 'value': 'Yes'}, +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=review_edit['id']) + +# anon_groups = reviewer_client.get_groups(prefix='ICML.cc/2024/Conference/Submission2/Reviewer_', signatory='~Reviewer_ICMLOne1') +# anon_group_id = anon_groups[0].id + +# review_edit = reviewer_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission2/-/Official_Review', +# signatures=[anon_group_id], +# note=openreview.api.Note( +# content={ +# 'review': { 'value': 'This is a good review for a good paper' }, +# 'rating': { 'value': 7 }, +# 'confidence': { 'value': 5 } +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=review_edit['id']) + +# pc_client_v2=openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) + +# # try post review note signing as PC +# with pytest.raises(openreview.OpenReviewException) as openReviewError: +# review_edit = pc_client_v2.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission2/-/Official_Review', +# signatures=['ICML.cc/2024/Conference/Program_Chairs'], +# note=openreview.api.Note( +# content={ +# 'review': { 'value': 'review by PC' }, +# 'rating': { 'value': 10 }, +# 'confidence': { 'value': 1 } +# } +# ) +# ) +# assert openReviewError.value.args[0].get('name') == 'ItemsError' + +# pc_client_v2=openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) + +# anon_groups = reviewer_client.get_groups(prefix='ICML.cc/2024/Conference/Submission1/Reviewer_', signatory='~Reviewer_ICMLOne1') +# anon_group_id = anon_groups[0].id + +# reviews = pc_client_v2.get_notes(invitation='ICML.cc/2024/Conference/Submission1/-/Official_Review', sort='number:asc') +# assert len(reviews) == 2 +# assert anon_group_id in reviews[0].readers + +# ## Extend deadline using a meta invitation and propagate the change to all the children +# new_due_date = openreview.tools.datetime_millis(now + datetime.timedelta(days=20)) +# new_exp_date = openreview.tools.datetime_millis(now + datetime.timedelta(days=25)) +# pc_client_v2.post_invitation_edit( +# invitations='ICML.cc/2024/Conference/-/Edit', +# readers=['ICML.cc/2024/Conference'], +# writers=['ICML.cc/2024/Conference'], +# signatures=['ICML.cc/2024/Conference'], +# invitation=openreview.api.Invitation( +# id='ICML.cc/2024/Conference/-/Official_Review', +# edit={ +# 'invitation': { +# 'duedate': new_due_date, +# 'expdate': new_exp_date +# } +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, 'ICML.cc/2024/Conference/-/Official_Review-0-1', count=3) + +# invitation = pc_client_v2.get_invitation('ICML.cc/2024/Conference/Submission1/-/Official_Review') +# assert invitation.duedate == new_due_date +# assert invitation.expdate == new_exp_date + +# ## Extend deadline using a meta invitation and propagate the change to all the children +# new_due_date = openreview.tools.datetime_millis(now + datetime.timedelta(days=10)) +# new_exp_date = openreview.tools.datetime_millis(now + datetime.timedelta(days=15)) +# pc_client_v2.post_invitation_edit( +# invitations='ICML.cc/2024/Conference/-/Edit', +# readers=['ICML.cc/2024/Conference'], +# writers=['ICML.cc/2024/Conference'], +# signatures=['ICML.cc/2024/Conference'], +# invitation=openreview.api.Invitation( +# id='ICML.cc/2024/Conference/-/Position_Paper_Review', +# edit={ +# 'invitation': { +# 'duedate': new_due_date, +# 'expdate': new_exp_date +# } +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, 'ICML.cc/2024/Conference/-/Position_Paper_Review-0-1', count=2) + +# #get rebuttal stage invitation +# with pytest.raises(openreview.OpenReviewException) as openReviewError: +# rebuttal_stage_invitation = pc_client.get_invitation(f'openreview.net/Support/-/Request{request_form.number}/Rebuttal_Stage') +# assert openReviewError.value.args[0].get('name') == 'NotFoundError' + +# def test_review_rating(self, client, openreview_client, helpers): + +# pc_client=openreview.Client(username='pc@icml.cc', password=helpers.strong_password) +# request_form=pc_client.get_notes(invitation='openreview.net/Support/-/Request_Form')[0] +# venue = openreview.get_conference(client, request_form.id, support_user='openreview.net/Support') + +# now = datetime.datetime.utcnow() +# due_date = now + datetime.timedelta(days=3) +# venue.custom_stage = openreview.stages.CustomStage(name='Rating', +# reply_to=openreview.stages.CustomStage.ReplyTo.REVIEWS, +# source=openreview.stages.CustomStage.Source.ALL_SUBMISSIONS, +# due_date=due_date, +# exp_date=due_date + datetime.timedelta(days=1), +# invitees=[openreview.stages.CustomStage.Participants.AREA_CHAIRS_ASSIGNED], +# readers=[openreview.stages.CustomStage.Participants.SENIOR_AREA_CHAIRS_ASSIGNED, openreview.stages.CustomStage.Participants.AREA_CHAIRS_ASSIGNED], +# content={ +# 'review_quality': { +# 'order': 1, +# 'description': 'How helpful is this review:', +# 'value': { +# 'param': { +# 'type': 'string', +# 'input': 'radio', +# 'enum': [ +# 'Poor - not very helpful', +# 'Good', +# 'Outstanding' +# ] +# } +# } +# } +# }, +# notify_readers=True, +# email_sacs=True) + +# venue.create_custom_stage() + +# submissions = venue.get_submissions(sort='number:asc', details='directReplies') +# first_submission = submissions[0] +# reviews = [reply for reply in first_submission.details['directReplies'] if f'ICML.cc/2024/Conference/Submission{first_submission.number}/-/Official_Review'] + +# reviewer_client = openreview.api.OpenReviewClient(username='reviewer1@icml.cc', password=helpers.strong_password) +# anon_groups = reviewer_client.get_groups(prefix='ICML.cc/2024/Conference/Submission1/Reviewer_', signatory='~Reviewer_ICMLOne1') +# anon_group_id = anon_groups[0].id + +# assert len(openreview_client.get_invitations(invitation='ICML.cc/2024/Conference/-/Rating')) == 3 + +# invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Submission1/Official_Review1/-/Rating') +# assert invitation.invitees == ['ICML.cc/2024/Conference/Program_Chairs', 'ICML.cc/2024/Conference/Submission1/Area_Chairs'] +# assert 'review_quality' in invitation.edit['note']['content'] +# assert invitation.edit['note']['forum'] == submissions[0].id +# assert invitation.edit['note']['replyto'] == reviews[0]['id'] +# assert invitation.edit['note']['readers'] == [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Area_Chairs' +# ] + +# ac_client = openreview.api.OpenReviewClient(username='ac2@icml.cc', password=helpers.strong_password) +# ac_anon_groups = ac_client.get_groups(prefix='ICML.cc/2024/Conference/Submission1/Area_Chair_', signatory='~AC_ICMLTwo1') +# ac_anon_group_id = ac_anon_groups[0].id + +# #post a review rating +# rating_edit = ac_client.post_note_edit( +# invitation=invitation.id, +# signatures=[ac_anon_group_id], +# note=openreview.api.Note( +# content={ +# 'review_quality': { 'value': 'Poor - not very helpful' }, +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=rating_edit['id']) + +# reviewer_client = openreview.api.OpenReviewClient(username='reviewer2@icml.cc', password=helpers.strong_password) +# anon_groups = reviewer_client.get_groups(prefix='ICML.cc/2024/Conference/Submission1/Reviewer_', signatory='~Reviewer_ICMLTwo1') +# anon_group_id = anon_groups[0].id +# invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Submission1/Official_Review2/-/Rating') + +# #post another review rating to same paper +# rating_edit = ac_client.post_note_edit( +# invitation=invitation.id, +# signatures=[ac_anon_group_id], +# note=openreview.api.Note( +# content={ +# 'review_quality': { 'value': 'Outstanding' }, +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=rating_edit['id']) + +# pc_client_v2=openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) + +# notes = pc_client_v2.get_notes(invitation=invitation.id) +# assert len(notes) == 1 +# assert notes[0].readers == [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Area_Chairs' +# ] +# assert notes[0].signatures == [ac_anon_group_id] + +# #hide review ratings from Senior Area Chairs +# venue.custom_stage = openreview.stages.CustomStage(name='Rating', +# reply_to=openreview.stages.CustomStage.ReplyTo.REVIEWS, +# source=openreview.stages.CustomStage.Source.ALL_SUBMISSIONS, +# due_date=due_date, +# exp_date=due_date + datetime.timedelta(days=1), +# invitees=[openreview.stages.CustomStage.Participants.AREA_CHAIRS_ASSIGNED], +# readers=[openreview.stages.CustomStage.Participants.AREA_CHAIRS_ASSIGNED], +# content={ +# 'review_quality': { +# 'order': 1, +# 'description': 'How helpful is this review:', +# 'value': { +# 'param': { +# 'type': 'string', +# 'input': 'radio', +# 'enum': [ +# 'Poor - not very helpful', +# 'Good', +# 'Outstanding' +# ] +# } +# } +# } +# }) + +# venue.create_custom_stage() + +# notes = pc_client_v2.get_notes(invitation=invitation.id) +# assert len(notes) == 1 +# assert notes[0].readers == [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Area_Chairs' +# ] +# assert notes[0].signatures == [ac_anon_group_id] + +# messages = openreview_client.get_messages(to='sac2@icml.cc', subject='[ICML 2023] A rating has been received on your assigned Paper Number: 1, Paper Title: "Paper title 1 Version 2"') +# assert len(messages) == 2 +# assert 'We have received a rating on a submission to ICML 2023 for which you are serving as Senior Area Chair.' in messages[0]['content']['text'] +# messages = openreview_client.get_messages(to='ac2@icml.cc', subject='[ICML 2023] Your rating has been received on Paper Number: 1, Paper Title: "Paper title 1 Version 2"') +# assert len(messages) == 2 +# assert 'We have received your rating on a submission to ICML 2023.' in messages[0]['content']['text'] + +# # post review and check review rating inv is created +# reviewer_client = openreview.api.OpenReviewClient(username='reviewer1@icml.cc', password=helpers.strong_password) +# anon_groups = reviewer_client.get_groups(prefix='ICML.cc/2024/Conference/Submission3/Reviewer_', signatory='~Reviewer_ICMLOne1') +# anon_group_id = anon_groups[0].id + +# review_edit = reviewer_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission3/-/Official_Review', +# signatures=[anon_group_id], +# note=openreview.api.Note( +# content={ +# 'summary': { 'value': 'good paper' }, +# 'strengths_and_weaknesses': { 'value': '7: Good paper, accept'}, +# 'questions': { 'value': '7: Good paper, accept'}, +# 'limitations': { 'value': '7: Good paper, accept'}, +# 'ethics_flag': { 'value': 'No'}, +# 'soundness': { 'value': '3 good'}, +# 'presentation': { 'value': '3 good'}, +# 'contribution': { 'value': '3 good'}, +# 'rating': { 'value': 10 }, +# 'confidence': { 'value': 5 }, +# 'code_of_conduct': { 'value': 'Yes'}, +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=review_edit['id']) + +# assert len(openreview_client.get_invitations(invitation='ICML.cc/2024/Conference/-/Rating')) == 4 + +# invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Submission3/Official_Review1/-/Rating') +# assert invitation.invitees == ['ICML.cc/2024/Conference/Program_Chairs', 'ICML.cc/2024/Conference/Submission3/Area_Chairs'] +# assert 'review_quality' in invitation.edit['note']['content'] +# assert invitation.edit['note']['forum'] == review_edit['note']['forum'] +# assert invitation.edit['note']['replyto'] == review_edit['note']['id'] +# assert invitation.edit['note']['readers'] == [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission3/Area_Chairs' +# ] + +# def test_delete_assignments(self, openreview_client, helpers): + +# ac_client = openreview.api.OpenReviewClient(username='ac2@icml.cc', password=helpers.strong_password) + +# submissions = ac_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', sort='number:asc') +# assignment = ac_client.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Assignment', head=submissions[0].id, tail='~Reviewer_ICMLOne1')[0] + +# anon_group_id = ac_client.get_groups(prefix='ICML.cc/2024/Conference/Submission1/Area_Chair_', signatory='~AC_ICMLTwo1')[0].id +# assignment.ddate = openreview.tools.datetime_millis(datetime.datetime.utcnow()) +# assignment.cdate = None +# assignment.signatures = [anon_group_id] + +# with pytest.raises(openreview.OpenReviewException, match=r'Can not remove assignment, the user ~Reviewer_ICMLOne1 already posted a Official Review.'): +# ac_client.post_edge(assignment) + +# assignment = ac_client.get_edges(invitation='ICML.cc/2024/Conference/Reviewers/-/Assignment', head=submissions[0].id, tail='~Celeste_ICML1')[0] +# assignment.ddate = openreview.tools.datetime_millis(datetime.datetime.utcnow()) +# assignment.signatures = [anon_group_id] +# assignment.cdate = None +# ac_client.post_edge(assignment) + +# #delete AC assignment of paper with a review with no error +# pc_client_v2=openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) + +# assignment = pc_client_v2.get_edges(invitation='ICML.cc/2024/Conference/Area_Chairs/-/Assignment', head=submissions[0].id, tail='~AC_ICMLTwo1')[0] +# assignment.ddate = openreview.tools.datetime_millis(datetime.datetime.utcnow()) +# assignment.cdate = None +# pc_client_v2.post_edge(assignment) + +# helpers.await_queue_edit(openreview_client, edit_id=assignment.id, count=2) + +# ac_group = pc_client_v2.get_group('ICML.cc/2024/Conference/Submission1/Area_Chairs') +# assert [] == ac_group.members + +# sac_group = pc_client_v2.get_group('ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs') +# assert [] == sac_group.members + +# #re-add AC to paper 1 +# assignment = pc_client_v2.get_edges(invitation='ICML.cc/2024/Conference/Area_Chairs/-/Assignment', head=submissions[0].id, tail='~AC_ICMLTwo1', trash=True)[0] +# assignment.ddate = { 'delete': True } +# assignment.cdate = None +# pc_client_v2.post_edge(assignment) + +# helpers.await_queue_edit(openreview_client, edit_id=assignment.id, count=3) + +# ac_group = pc_client_v2.get_group('ICML.cc/2024/Conference/Submission1/Area_Chairs') +# assert ['~AC_ICMLTwo1'] == ac_group.members + +# sac_group = pc_client_v2.get_group('ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs') +# assert ['~SAC_ICMLTwo1'] == sac_group.members + +# def test_ethics_review_stage(self, openreview_client, helpers, selenium, request_page): +# pc_client=openreview.Client(username='pc@icml.cc', password=helpers.strong_password) +# request_form=pc_client.get_notes(invitation='openreview.net/Support/-/Request_Form')[0] + +# reviewer_details = '''reviewerethics@yahoo.com, Reviewer ICMLSeven''' +# recruitment_note = pc_client.post_note(openreview.Note( +# content={ +# 'title': 'Recruitment', +# 'invitee_role': 'Ethics_Reviewers', +# 'invitee_reduced_load': ['2', '3', '4'], +# 'invitee_details': reviewer_details, +# 'invitation_email_subject': '[' + request_form.content['Abbreviated Venue Name'] + '] Invitation to serve as {{invitee_role}}', +# 'invitation_email_content': 'Dear {{fullname}},\n\nYou have been nominated by the program chair committee of ICML 2023 to serve as {{invitee_role}}.\n\n{{invitation_url}}\n\nIf you have any questions, please contact {{contact_info}}.\n\nCheers!\n\nProgram Chairs' +# }, +# forum=request_form.forum, +# replyto=request_form.forum, +# invitation='openreview.net/Support/-/Request{}/Recruitment'.format(request_form.number), +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) +# assert recruitment_note +# helpers.await_queue() + +# group = openreview_client.get_group('ICML.cc/2024/Conference/Ethics_Reviewers') +# assert group +# assert 'ICML.cc/2024/Conference/Ethics_Chairs' in group.readers +# assert openreview_client.get_group('ICML.cc/2024/Conference/Ethics_Reviewers/Declined') +# group = openreview_client.get_group('ICML.cc/2024/Conference/Ethics_Reviewers/Invited') +# assert group +# assert len(group.members) == 1 +# assert 'reviewerethics@yahoo.com' in group.members +# assert 'ICML.cc/2024/Conference/Ethics_Chairs' in group.readers + +# messages = openreview_client.get_messages(to='reviewerethics@yahoo.com', subject='[ICML 2023] Invitation to serve as Ethics Reviewer') +# assert messages and len(messages) == 1 +# invitation_url = re.search('https://.*\n', messages[0]['content']['text']).group(0).replace('https://openreview.net', 'http://localhost:3030').replace('&', '&')[:-1] +# helpers.respond_invitation(selenium, request_page, invitation_url, accept=True) + +# helpers.await_queue() + +# group = openreview_client.get_group('ICML.cc/2024/Conference/Ethics_Reviewers') +# assert group +# assert len(group.members) == 1 +# assert 'reviewerethics@yahoo.com' in group.members + +# now = datetime.datetime.utcnow() +# start_date = now - datetime.timedelta(days=2) +# due_date = now + datetime.timedelta(days=3) +# stage_note = pc_client.post_note(openreview.Note( +# content={ +# 'ethics_review_start_date': start_date.strftime('%Y/%m/%d'), +# 'ethics_review_deadline': due_date.strftime('%Y/%m/%d'), +# 'make_ethics_reviews_public': 'No, ethics reviews should NOT be revealed publicly when they are posted', +# 'release_ethics_reviews_to_authors': "No, ethics reviews should NOT be revealed when they are posted to the paper\'s authors", +# 'release_ethics_reviews_to_reviewers': 'Ethics Review should not be revealed to any reviewer, except to the author of the ethics review', +# 'remove_ethics_review_form_options': 'ethics_review', +# 'additional_ethics_review_form_options': { +# "ethics_concerns": { +# 'order': 1, +# 'description': 'Briefly summarize the ethics concerns.', +# 'value': { +# 'param': { +# 'type': 'string', +# 'maxLength': 200000, +# 'markdown': True, +# 'input': 'textarea' +# } +# } +# } +# }, +# 'release_submissions_to_ethics_reviewers': 'We confirm we want to release the submissions and reviews to the ethics reviewers' +# }, +# forum=request_form.forum, +# referent=request_form.forum, +# invitation='openreview.net/Support/-/Request{}/Ethics_Review_Stage'.format(request_form.number), +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) + +# helpers.await_queue() + +# configuration_invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Ethics_Reviewers/-/Assignment_Configuration') +# assert configuration_invitation.edit['note']['content']['paper_invitation']['value']['param']['default'] == 'ICML.cc/2024/Conference/-/Submission&content.venueid=ICML.cc/2024/Conference/Submission&content.flagged_for_ethics_review=true' + +# pc_client_v2=openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) +# notes = openreview_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', number=[1,5]) +# for note in notes: +# note_edit = pc_client_v2.post_note_edit( +# invitation='ICML.cc/2024/Conference/-/Ethics_Review_Flag', +# note=openreview.api.Note( +# id=note.id, +# content = { +# 'flagged_for_ethics_review': { 'value': True }, +# 'ethics_comments': { 'value': 'These are ethics comments visible to ethics chairs and ethics reviewers' } +# } +# ), +# signatures=['ICML.cc/2024/Conference'] +# ) + +# helpers.await_queue() +# helpers.await_queue_edit(openreview_client, edit_id=note_edit['id']) + +# openreview_client.add_members_to_group('ICML.cc/2024/Conference/Submission5/Ethics_Reviewers', '~Celeste_ICML1') + +# submissions = openreview_client.get_notes(content= { 'venueid': 'ICML.cc/2024/Conference/Submission'}, sort='number:asc') +# assert submissions and len(submissions) == 100 +# assert 'flagged_for_ethics_review' in submissions[0].content and submissions[0].content['flagged_for_ethics_review']['value'] +# assert 'ethics_comments' in submissions[0].content +# assert submissions[0].content['flagged_for_ethics_review']['readers'] == [ +# 'ICML.cc/2024/Conference', +# 'ICML.cc/2024/Conference/Ethics_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Ethics_Reviewers', +# 'ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Reviewers' +# ] +# assert 'flagged_for_ethics_review' in submissions[4].content and submissions[4].content['flagged_for_ethics_review']['value'] +# assert 'ethics_comments' in submissions[4].content +# assert submissions[4].content['flagged_for_ethics_review']['readers'] == [ +# 'ICML.cc/2024/Conference', +# 'ICML.cc/2024/Conference/Ethics_Chairs', +# 'ICML.cc/2024/Conference/Submission5/Ethics_Reviewers', +# 'ICML.cc/2024/Conference/Submission5/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Submission5/Area_Chairs', +# 'ICML.cc/2024/Conference/Submission5/Reviewers' +# ] +# ethics_group = openreview.tools.get_group(openreview_client, 'ICML.cc/2024/Conference/Submission1/Ethics_Reviewers') +# assert ethics_group +# ethics_group = openreview.tools.get_group(openreview_client, 'ICML.cc/2024/Conference/Submission2/Ethics_Reviewers') +# assert not ethics_group +# ethics_group = openreview.tools.get_group(openreview_client, 'ICML.cc/2024/Conference/Submission5/Ethics_Reviewers') +# assert ethics_group and '~Celeste_ICML1' in ethics_group.members +# assert submissions[0].readers == [ +# "ICML.cc/2024/Conference", +# "ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs", +# "ICML.cc/2024/Conference/Submission1/Area_Chairs", +# "ICML.cc/2024/Conference/Submission1/Reviewers", +# "ICML.cc/2024/Conference/Submission1/Authors", +# "ICML.cc/2024/Conference/Submission1/Ethics_Reviewers" +# ] +# assert submissions[1].readers == [ +# "ICML.cc/2024/Conference", +# "ICML.cc/2024/Conference/Submission2/Senior_Area_Chairs", +# "ICML.cc/2024/Conference/Submission2/Area_Chairs", +# "ICML.cc/2024/Conference/Submission2/Reviewers", +# "ICML.cc/2024/Conference/Submission2/Authors" ] +# assert submissions[4].readers == [ +# "ICML.cc/2024/Conference", +# "ICML.cc/2024/Conference/Submission5/Senior_Area_Chairs", +# "ICML.cc/2024/Conference/Submission5/Area_Chairs", +# "ICML.cc/2024/Conference/Submission5/Reviewers", +# "ICML.cc/2024/Conference/Submission5/Authors", +# "ICML.cc/2024/Conference/Submission5/Ethics_Reviewers" +# ] + +# reviews = openreview_client.get_notes(invitation='ICML.cc/2024/Conference/Submission1/-/Official_Review') +# assert reviews and len(reviews) == 2 +# for review in reviews: +# assert review.readers == [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Ethics_Reviewers', +# review.signatures[0] +# ] + +# invitations = openreview_client.get_invitations(invitation='ICML.cc/2024/Conference/-/Ethics_Review') +# assert len(invitations) == 2 +# invitation = openreview_client.get_invitations(id='ICML.cc/2024/Conference/Submission1/-/Ethics_Review')[0] +# assert invitation +# assert 'ICML.cc/2024/Conference/Submission1/Ethics_Reviewers' in invitation.invitees + +# # re-run ethics review stage +# now = datetime.datetime.utcnow() +# start_date = now - datetime.timedelta(days=1) +# stage_note = pc_client.post_note(openreview.Note( +# content={ +# 'ethics_review_start_date': start_date.strftime('%Y/%m/%d'), +# 'ethics_review_deadline': due_date.strftime('%Y/%m/%d'), +# 'make_ethics_reviews_public': 'No, ethics reviews should NOT be revealed publicly when they are posted', +# 'release_ethics_reviews_to_authors': "No, ethics reviews should NOT be revealed when they are posted to the paper\'s authors", +# 'release_ethics_reviews_to_reviewers': 'Ethics Review should not be revealed to any reviewer, except to the author of the ethics review', +# 'remove_ethics_review_form_options': 'ethics_review', +# 'additional_ethics_review_form_options': { +# "ethics_concerns": { +# 'order': 1, +# 'description': 'Briefly summarize the ethics concerns.', +# 'value': { +# 'param': { +# 'type': 'string', +# 'maxLength': 200000, +# 'markdown': True, +# 'input': 'textarea' +# } +# } +# } +# }, +# 'release_submissions_to_ethics_reviewers': 'We confirm we want to release the submissions and reviews to the ethics reviewers', +# 'enable_comments_for_ethics_reviewers': 'Yes, enable commenting for ethics reviewers.' +# }, +# forum=request_form.forum, +# referent=request_form.forum, +# invitation='openreview.net/Support/-/Request{}/Ethics_Review_Stage'.format(request_form.number), +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) + +# helpers.await_queue() + +# notes = openreview_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', number=[6,7,8,100]) +# for note in notes: +# note_edit = pc_client_v2.post_note_edit( +# invitation='ICML.cc/2024/Conference/-/Ethics_Review_Flag', +# note=openreview.api.Note( +# id=note.id, +# content = { +# 'flagged_for_ethics_review': { 'value': True }, +# } +# ), +# signatures=['ICML.cc/2024/Conference'] +# ) + +# helpers.await_queue() +# helpers.await_queue_edit(openreview_client, edit_id=note_edit['id']) + +# submissions = openreview_client.get_notes(content= { 'venueid': 'ICML.cc/2024/Conference/Submission'}, sort='number:asc') +# assert submissions and len(submissions) == 100 +# assert 'flagged_for_ethics_review' in submissions[-1].content and submissions[-1].content['flagged_for_ethics_review']['value'] +# ethics_group = openreview.tools.get_group(openreview_client, 'ICML.cc/2024/Conference/Submission7/Ethics_Reviewers') +# assert ethics_group +# ethics_group = openreview.tools.get_group(openreview_client, 'ICML.cc/2024/Conference/Submission8/Ethics_Reviewers') +# assert ethics_group +# ethics_group = openreview.tools.get_group(openreview_client, 'ICML.cc/2024/Conference/Submission100/Ethics_Reviewers') +# assert ethics_group +# assert submissions[0].readers == [ +# "ICML.cc/2024/Conference", +# "ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs", +# "ICML.cc/2024/Conference/Submission1/Area_Chairs", +# "ICML.cc/2024/Conference/Submission1/Reviewers", +# "ICML.cc/2024/Conference/Submission1/Authors", +# "ICML.cc/2024/Conference/Submission1/Ethics_Reviewers" +# ] +# assert submissions[1].readers == [ +# "ICML.cc/2024/Conference", +# "ICML.cc/2024/Conference/Submission2/Senior_Area_Chairs", +# "ICML.cc/2024/Conference/Submission2/Area_Chairs", +# "ICML.cc/2024/Conference/Submission2/Reviewers", +# "ICML.cc/2024/Conference/Submission2/Authors" ] +# assert submissions[4].readers == [ +# "ICML.cc/2024/Conference", +# "ICML.cc/2024/Conference/Submission5/Senior_Area_Chairs", +# "ICML.cc/2024/Conference/Submission5/Area_Chairs", +# "ICML.cc/2024/Conference/Submission5/Reviewers", +# "ICML.cc/2024/Conference/Submission5/Authors", +# "ICML.cc/2024/Conference/Submission5/Ethics_Reviewers" +# ] +# assert submissions[-1].readers == [ +# "ICML.cc/2024/Conference", +# "ICML.cc/2024/Conference/Submission100/Senior_Area_Chairs", +# "ICML.cc/2024/Conference/Submission100/Area_Chairs", +# "ICML.cc/2024/Conference/Submission100/Reviewers", +# "ICML.cc/2024/Conference/Submission100/Authors", +# "ICML.cc/2024/Conference/Submission100/Ethics_Reviewers" +# ] + +# reviews = openreview_client.get_notes(invitation='ICML.cc/2024/Conference/Submission1/-/Official_Review') +# assert reviews and len(reviews) == 2 +# for review in reviews: +# assert review.readers == [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Ethics_Reviewers', +# review.signatures[0] +# ] + +# invitations = openreview_client.get_invitations(invitation='ICML.cc/2024/Conference/-/Ethics_Review') +# assert len(invitations) == 6 +# invitation = openreview_client.get_invitations(id='ICML.cc/2024/Conference/Submission100/-/Ethics_Review')[0] +# assert invitation +# assert 'ICML.cc/2024/Conference/Submission100/Ethics_Reviewers' in invitation.invitees + +# # use invitation to flag paper +# pc_client_v2=openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) +# note = openreview_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', number=[52])[0] +# note_edit = pc_client_v2.post_note_edit( +# invitation='ICML.cc/2024/Conference/-/Ethics_Review_Flag', +# note=openreview.api.Note( +# id=note.id, +# content = { +# 'flagged_for_ethics_review': { 'value': True }, +# } +# ), +# signatures=['ICML.cc/2024/Conference'] +# ) + +# helpers.await_queue() +# helpers.await_queue_edit(openreview_client, edit_id=note_edit['id']) + +# submissions = openreview_client.get_notes(content= { 'venueid': 'ICML.cc/2024/Conference/Submission'}, sort='number:asc') +# assert submissions and len(submissions) == 100 +# assert 'flagged_for_ethics_review' in submissions[51].content and submissions[51].content['flagged_for_ethics_review']['value'] +# assert 'ICML.cc/2024/Conference/Submission52/Ethics_Reviewers' in submissions[51].readers +# ethics_group = openreview.tools.get_group(openreview_client, 'ICML.cc/2024/Conference/Submission52/Ethics_Reviewers') +# assert ethics_group +# invitation = openreview_client.get_invitations(id='ICML.cc/2024/Conference/Submission52/-/Ethics_Review')[0] +# assert invitation +# assert 'ICML.cc/2024/Conference/Submission52/Ethics_Reviewers' in invitation.invitees + +# # comment invitations are created for all papers, with only PCs and ethics reviewers as invitees +# invitations = openreview_client.get_all_invitations(invitation='ICML.cc/2024/Conference/-/Official_Comment') +# assert len(invitations) == 100 +# invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Submission1/-/Official_Comment') +# assert invitation.invitees == ['ICML.cc/2024/Conference', 'openreview.net/Support', 'ICML.cc/2024/Conference/Submission1/Ethics_Reviewers'] + +# # post ethics review +# openreview_client.add_members_to_group('ICML.cc/2024/Conference/Submission5/Ethics_Reviewers', '~Reviewer_ICMLOne1') +# reviewer_client = openreview.api.OpenReviewClient(username='reviewer1@icml.cc', password=helpers.strong_password) + +# anon_groups = reviewer_client.get_groups(prefix='ICML.cc/2024/Conference/Submission5/Ethics_Reviewer_', signatory='~Reviewer_ICMLOne1') +# anon_group_id = anon_groups[0].id + +# review_edit = reviewer_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission5/-/Ethics_Review', +# signatures=[anon_group_id], +# note=openreview.api.Note( +# content={ +# "recommendation": { +# "value": "1: No serious ethical issues" +# }, +# "ethics_concerns": { +# "value": "I have no concerns." +# } +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=review_edit['id']) + +# reviews = pc_client_v2.get_notes(invitation='ICML.cc/2024/Conference/Submission5/-/Ethics_Review') +# assert len(reviews) == 1 +# assert reviews[0].readers == [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Ethics_Chairs', +# reviews[0].signatures[0] +# ] + +# # Set expiration date +# exp_date = now +# stage_note = pc_client.post_note(openreview.Note( +# content={ +# 'ethics_review_start_date': start_date.strftime('%Y/%m/%d'), +# 'ethics_review_deadline': due_date.strftime('%Y/%m/%d'), +# 'ethics_review_expiration_date': exp_date.strftime('%Y/%m/%d'), +# 'make_ethics_reviews_public': 'No, ethics reviews should NOT be revealed publicly when they are posted', +# 'release_ethics_reviews_to_authors': "No, ethics reviews should NOT be revealed when they are posted to the paper\'s authors", +# 'release_ethics_reviews_to_reviewers': 'Ethics Review should not be revealed to any reviewer, except to the author of the ethics review', +# 'remove_ethics_review_form_options': 'ethics_review', +# 'additional_ethics_review_form_options': { +# "ethics_concerns": { +# 'order': 1, +# 'description': 'Briefly summarize the ethics concerns.', +# 'value': { +# 'param': { +# 'type': 'string', +# 'maxLength': 200000, +# 'markdown': True, +# 'input': 'textarea' +# } +# } +# } +# }, +# 'release_submissions_to_ethics_reviewers': 'We confirm we want to release the submissions and reviews to the ethics reviewers' +# }, +# forum=request_form.forum, +# referent=request_form.forum, +# invitation='openreview.net/Support/-/Request{}/Ethics_Review_Stage'.format(request_form.number), +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) + +# helpers.await_queue() + +# # attempt to post another note +# openreview_client.add_members_to_group('ICML.cc/2024/Conference/Submission5/Ethics_Reviewers', '~Reviewer_ICMLTwo1') +# reviewer_client = openreview.api.OpenReviewClient(username='reviewer2@icml.cc', password=helpers.strong_password) +# anon_groups = reviewer_client.get_groups(prefix='ICML.cc/2024/Conference/Submission5/Ethics_Reviewer_', signatory='~Reviewer_ICMLTwo1') +# anon_group_id = anon_groups[0].id + +# with pytest.raises(openreview.OpenReviewException, match=r'The Invitation ICML.cc/2024/Conference/Submission5/-/Ethics_Review has expired'): +# review_edit = reviewer_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission5/-/Ethics_Review', +# signatures=[anon_group_id], +# note=openreview.api.Note( +# content={ +# "recommendation": { +# "value": "1: No serious ethical issues" +# }, +# "ethics_concerns": { +# "value": "I have very serious concerns." +# } +# } +# ) +# ) + +# # assert number of Official_Review and Position_Paper_Review invitations has not changed after flagging papers for ethics reviews +# assert len(openreview_client.get_invitations(invitation='ICML.cc/2024/Conference/-/Official_Review')) == 50 +# assert len(openreview_client.get_invitations(invitation='ICML.cc/2024/Conference/-/Position_Paper_Review')) == 50 + +# def test_comment_stage(self, openreview_client, helpers): + +# pc_client=openreview.Client(username='pc@icml.cc', password=helpers.strong_password) +# request_form=pc_client.get_notes(invitation='openreview.net/Support/-/Request_Form')[0] + +# # Post an official comment stage note +# now = datetime.datetime.utcnow() +# start_date = now - datetime.timedelta(days=2) +# end_date = now + datetime.timedelta(days=3) +# comment_stage_note = pc_client.post_note(openreview.Note( +# content={ +# 'commentary_start_date': start_date.strftime('%Y/%m/%d'), +# 'commentary_end_date': end_date.strftime('%Y/%m/%d'), +# 'participants': ['Program Chairs', 'Assigned Senior Area Chairs', 'Assigned Area Chairs', 'Assigned Reviewers'], +# 'additional_readers': ['Program Chairs', 'Assigned Senior Area Chairs', 'Assigned Area Chairs', 'Assigned Reviewers', 'Assigned Submitted Reviewers'], +# 'email_program_chairs_about_official_comments': 'No, do not email PCs for each official comment made in the venue', +# 'enable_chat_between_committee_members': 'Yes, enable chat between committee members' +# }, +# forum=request_form.forum, +# invitation=f'openreview.net/Support/-/Request{request_form.number}/Comment_Stage', +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# replyto=request_form.forum, +# referent=request_form.forum, +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) + +# helpers.await_queue() +# helpers.await_queue_edit(openreview_client, 'ICML.cc/2024/Conference/-/Official_Comment-0-1', count=1) +# helpers.await_queue_edit(openreview_client, 'ICML.cc/2024/Conference/-/Chat-0-1', count=1) +# helpers.await_queue_edit(openreview_client, 'ICML.cc/2024/Conference/-/Chat_Reaction-0-1', count=1) + +# chat_invitations = openreview_client.get_invitations(invitation='ICML.cc/2024/Conference/-/Chat') +# assert len(chat_invitations) == 100 + +# chat_reaction_invitations = openreview_client.get_invitations(invitation='ICML.cc/2024/Conference/-/Chat_Reaction') +# assert len(chat_reaction_invitations) == 100 + +# invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Submission1/-/Official_Comment') +# assert invitation +# assert 'ICML.cc/2024/Conference/Submission1/Ethics_Reviewers' in invitation.invitees +# assert 'ICML.cc/2024/Conference/Submission1/Ethics_Reviewers' in invitation.edit['note']['readers']['param']['enum'] +# assert invitation.edit['signatures']['param']['items'] == [ +# { +# "value": "ICML.cc/2024/Conference/Program_Chairs", +# "optional": True +# }, +# { +# "value": "ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs", +# "optional": True +# }, +# { +# "prefix": "ICML.cc/2024/Conference/Submission1/Area_Chair_.*", +# "optional": True +# }, +# { +# "prefix": "ICML.cc/2024/Conference/Submission1/Reviewer_.*", +# "optional": True +# }, +# { +# "prefix": "ICML.cc/2024/Conference/Submission1/Ethics_Reviewer_.*", +# "optional": True +# }, +# { +# "value": "ICML.cc/2024/Conference/Ethics_Chairs", +# "optional": True +# } +# ] +# invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Submission2/-/Official_Comment') +# assert invitation +# assert 'ICML.cc/2024/Conference/Submission2/Ethics_Reviewers' not in invitation.edit['note']['readers']['param']['enum'] +# assert openreview_client.get_invitation('ICML.cc/2024/Conference/Submission3/-/Official_Comment') +# assert openreview_client.get_invitation('ICML.cc/2024/Conference/Submission4/-/Official_Comment') +# invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Submission5/-/Official_Comment') +# assert invitation +# assert 'ICML.cc/2024/Conference/Submission5/Ethics_Reviewers' in invitation.invitees +# assert 'ICML.cc/2024/Conference/Submission5/Ethics_Reviewers' in invitation.edit['note']['readers']['param']['enum'] + +# assert invitation.edit['signatures']['param']['items'] == [ +# { +# "value": "ICML.cc/2024/Conference/Program_Chairs", +# "optional": True +# }, +# { +# "value": "ICML.cc/2024/Conference/Submission5/Senior_Area_Chairs", +# "optional": True +# }, +# { +# "prefix": "ICML.cc/2024/Conference/Submission5/Area_Chair_.*", +# "optional": True +# }, +# { +# "prefix": "ICML.cc/2024/Conference/Submission5/Reviewer_.*", +# "optional": True +# }, +# { +# "prefix": "ICML.cc/2024/Conference/Submission5/Ethics_Reviewer_.*", +# "optional": True +# }, +# { +# "value": "ICML.cc/2024/Conference/Ethics_Chairs", +# "optional": True +# } +# ] + +# pc_client_v2=openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) +# # unflag a paper +# note = openreview_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', number=[5])[0] +# note_edit = pc_client_v2.post_note_edit( +# invitation='ICML.cc/2024/Conference/-/Ethics_Review_Flag', +# note=openreview.api.Note( +# id=note.id, +# content = { +# 'flagged_for_ethics_review': { 'value': False }, +# } +# ), +# signatures=['ICML.cc/2024/Conference'] +# ) + +# helpers.await_queue() +# helpers.await_queue_edit(openreview_client, edit_id=note_edit['id']) + +# invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Submission5/-/Official_Comment') +# assert invitation +# assert 'ICML.cc/2024/Conference/Submission5/Ethics_Reviewers' not in invitation.invitees +# assert 'ICML.cc/2024/Conference/Submission5/Ethics_Reviewers' in invitation.edit['note']['readers']['param']['enum'] +# assert invitation.edit['signatures']['param']['items'] == [ +# { +# "value": "ICML.cc/2024/Conference/Program_Chairs", +# "optional": True +# }, +# { +# "value": "ICML.cc/2024/Conference/Submission5/Senior_Area_Chairs", +# "optional": True +# }, +# { +# "prefix": "ICML.cc/2024/Conference/Submission5/Area_Chair_.*", +# "optional": True +# }, +# { +# "prefix": "ICML.cc/2024/Conference/Submission5/Reviewer_.*", +# "optional": True +# }, +# { +# "prefix": "ICML.cc/2024/Conference/Submission5/Ethics_Reviewer_.*", +# "optional": True +# }, +# { +# "value": "ICML.cc/2024/Conference/Ethics_Chairs", +# "optional": True +# } +# ] +# submissions = openreview_client.get_notes(content= { 'venueid': 'ICML.cc/2024/Conference/Submission'}, sort='number:asc') +# assert submissions and len(submissions) == 100 +# assert 'flagged_for_ethics_review' in submissions[4].content and not submissions[4].content['flagged_for_ethics_review']['value'] +# invitation = openreview_client.get_invitations(id='ICML.cc/2024/Conference/Submission5/-/Ethics_Review')[0] +# assert invitation.expdate < openreview.tools.datetime_millis(datetime.datetime.utcnow()) +# ethics_group = openreview_client.get_group('ICML.cc/2024/Conference/Submission5/Ethics_Reviewers') +# assert ethics_group and '~Celeste_ICML1' in ethics_group.members + +# reviewer_client = openreview.api.OpenReviewClient(username='reviewer1@icml.cc', password=helpers.strong_password) + +# submissions = reviewer_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', sort='number:asc') + +# anon_groups = reviewer_client.get_groups(prefix='ICML.cc/2024/Conference/Submission1/Reviewer_', signatory='~Reviewer_ICMLOne1') +# anon_group_id = anon_groups[0].id + +# comment_edit = reviewer_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission1/-/Official_Comment', +# signatures=[anon_group_id], +# note=openreview.api.Note( +# replyto = submissions[0].id, +# readers = [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Area_Chairs', +# anon_group_id, +# 'ICML.cc/2024/Conference/Submission1/Ethics_Reviewers' +# ], +# content={ +# 'comment': { 'value': 'I can not review this paper' }, +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=comment_edit['id']) + +# signature = anon_group_id.split('/')[-1] +# pretty_signature = openreview.tools.pretty_id(signature) +# messages = openreview_client.get_messages(to='ac2@icml.cc', subject=f'[ICML 2023] {pretty_signature} commented on a paper in your area. Paper Number: 1, Paper Title: "Paper title 1 Version 2"') +# assert messages and len(messages) == 1 +# assert messages[0]['content']['replyTo'] == 'pc@icml.cc' + +# messages = openreview_client.get_messages(to='reviewer1@icml.cc', subject='[ICML 2023] Your comment was received on Paper Number: 1, Paper Title: "Paper title 1 Version 2"') +# assert messages and len(messages) == 1 +# assert messages[0]['content']['replyTo'] == 'pc@icml.cc' + +# comment_edit = reviewer_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission1/-/Official_Comment', +# signatures=[anon_group_id], +# note=openreview.api.Note( +# id = comment_edit['note']['id'], +# replyto = submissions[0].id, +# readers = [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Area_Chairs', +# anon_group_id, +# 'ICML.cc/2024/Conference/Submission1/Ethics_Reviewers' +# ], +# content={ +# 'comment': { 'value': 'I can not review this paper, EDITED' }, +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=comment_edit['id']) + +# messages = openreview_client.get_messages(to='ac2@icml.cc', subject=f'[ICML 2023] {pretty_signature} commented on a paper in your area. Paper Number: 1, Paper Title: "Paper title 1 Version 2"') +# assert messages and len(messages) == 1 + +# messages = openreview_client.get_messages(to='reviewer1@icml.cc', subject='[ICML 2023] Your comment was received on Paper Number: 1, Paper Title: "Paper title 1 Version 2"') +# assert messages and len(messages) == 1 + +# comment_edit = reviewer_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission1/-/Official_Comment', +# signatures=[anon_group_id], +# note=openreview.api.Note( +# replyto = submissions[0].id, +# readers = [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs', +# anon_group_id +# ], +# content={ +# 'comment': { 'value': 'private message to SAC' }, +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=comment_edit['id']) + +# messages = openreview_client.get_messages(to='ac2@icml.cc', subject=f'[ICML 2023] {pretty_signature} commented on a paper in your area. Paper Number: 1, Paper Title: "Paper title 1 Version 2"') +# assert messages and len(messages) == 1 + +# messages = openreview_client.get_messages(to='sac2@icml.cc', subject=f'[ICML 2023] {pretty_signature} commented on a paper in your area. Paper Number: 1, Paper Title: "Paper title 1 Version 2"') +# assert messages and len(messages) == 1 + +# messages = openreview_client.get_messages(to='reviewer1@icml.cc', subject='[ICML 2023] Your comment was received on Paper Number: 1, Paper Title: "Paper title 1 Version 2"') +# assert messages and len(messages) == 2 + +# # Enable Author-AC confidential comments +# venue = openreview.helpers.get_conference(pc_client, request_form.id, setup=False) +# now = datetime.datetime.utcnow() +# start_date = now - datetime.timedelta(days=2) +# end_date = now + datetime.timedelta(days=3) + +# venue.custom_stage = openreview.stages.CustomStage(name='Author_AC_Confidential_Comment', +# notify_readers=True, +# reply_to=openreview.stages.CustomStage.ReplyTo.WITHFORUM, +# source=openreview.stages.CustomStage.Source.ALL_SUBMISSIONS, +# reply_type=openreview.stages.CustomStage.ReplyType.REPLY, +# invitees=[openreview.stages.CustomStage.Participants.AUTHORS, openreview.stages.CustomStage.Participants.AREA_CHAIRS_ASSIGNED], +# readers=[openreview.stages.CustomStage.Participants.SENIOR_AREA_CHAIRS_ASSIGNED,openreview.stages.CustomStage.Participants.AREA_CHAIRS_ASSIGNED,openreview.stages.CustomStage.Participants.AUTHORS], +# start_date=start_date, +# due_date=end_date, +# content={ +# 'title': { +# 'order': 1, +# 'description': '(Optional) Brief summary of your comment.', +# 'value': { +# 'param': { +# 'type': 'string', +# 'maxLength': 500, +# 'optional': True, +# 'deletable': True +# } +# } +# }, +# 'comment': { +# 'order': 2, +# 'description': 'Your comment or reply (max 5000 characters). Add formatting using Markdown and formulas using LaTeX. For more information see https://openreview.net/faq', +# 'value': { +# 'param': { +# 'type': 'string', +# 'maxLength': 5000, +# 'markdown': True, +# 'input': 'textarea' +# } +# } +# } +# }, +# multi_reply=True +# ) +# venue.create_custom_stage() +# helpers.await_queue_edit(openreview_client, 'ICML.cc/2024/Conference/-/Author_AC_Confidential_Comment-0-1', count=1) + +# assert len(openreview_client.get_invitations(invitation='ICML.cc/2024/Conference/-/Author_AC_Confidential_Comment')) == 100 +# invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Submission1/-/Author_AC_Confidential_Comment') +# assert invitation.invitees == [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Authors' +# ] + +# author_client = openreview.api.OpenReviewClient(username='peter@mail.com', password=helpers.strong_password) +# confidential_comment_edit = author_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission1/-/Author_AC_Confidential_Comment', +# signatures=['ICML.cc/2024/Conference/Submission1/Authors'], +# note=openreview.api.Note( +# replyto=submissions[0].id, +# content={ +# 'comment': { 'value': 'Author confidential comment to AC' }, +# } +# ) +# ) +# helpers.await_queue_edit(openreview_client, edit_id=confidential_comment_edit['id']) + +# confidential_comment = openreview_client.get_notes(invitation='ICML.cc/2024/Conference/Submission1/-/Author_AC_Confidential_Comment')[0] +# assert confidential_comment.readers == [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Authors' +# ] + +# # Check messages sent to readers +# messages = openreview_client.get_messages(subject = '[ICML 2023] An author ac confidential comment has been received on your.*') +# assert messages and len(messages) == 5 +# recipients = [msg['content']['to'] for msg in messages] +# assert 'test@mail.com'in recipients +# assert 'andrew@amazon.com' in recipients +# assert 'sac1@gmail.com' in recipients +# assert 'melisa@yahoo.com' in recipients +# assert 'ac2@icml.cc' in recipients +# assert 'peter@mail.com' not in recipients + +# ac_client = openreview.api.OpenReviewClient(username='ac2@icml.cc', password=helpers.strong_password) +# anon_groups = ac_client.get_groups(prefix='ICML.cc/2024/Conference/Submission1/Area_Chair_', signatory='~AC_ICMLTwo1') +# anon_group_id = anon_groups[0].id + +# confidential_comment_edit = ac_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission1/-/Author_AC_Confidential_Comment', +# signatures=[anon_group_id], +# note=openreview.api.Note( +# replyto=confidential_comment.id, +# content={ +# 'comment': { 'value': 'AC confidential reply to Author' }, +# } +# ) +# ) +# helpers.await_queue_edit(openreview_client, edit_id=confidential_comment_edit['id']) + +# messages = openreview_client.get_messages(subject = '[ICML 2023] An author ac confidential comment has been received on your.*') +# assert messages and len(messages) == 10 +# recipients = [msg['content']['to'] for msg in messages] +# assert 'peter@mail.com' in recipients + +# messages = openreview_client.get_messages(to='peter@mail.com', subject = '[ICML 2023] An author ac confidential comment has been received on your.*') +# assert messages[0]['content']['text'].startswith('We have received an author ac confidential comment on your submission to ICML 2023.') + +# def test_rebuttal_stage(self, client, openreview_client, helpers): + +# pc_client=openreview.Client(username='pc@icml.cc', password=helpers.strong_password) +# request_form=pc_client.get_notes(invitation='openreview.net/Support/-/Request_Form')[0] + +# now = datetime.datetime.utcnow() + +# # create rebuttal stage in request form +# client.post_invitation(openreview.Invitation( +# id = f'openreview.net/Support/-/Request{request_form.number}/Rebuttal_Stage', +# super = 'openreview.net/Support/-/Rebuttal_Stage', +# invitees = ['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# cdate = openreview.tools.datetime_millis(now), +# reply = { +# 'forum': request_form.id, +# 'referent': request_form.id, +# 'readers': { +# 'description': 'The users who will be allowed to read the above content.', +# 'values' : ['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'] +# } +# }, +# signatures = ['~Super_User1'] +# )) + +# # release only reviews for non position papers +# venue = openreview.helpers.get_conference(client, request_form.id, setup=False) +# venue.review_stage = openreview.stages.ReviewStage( +# due_date = now - datetime.timedelta(days=3), +# release_to_authors=True, +# release_to_reviewers=openreview.stages.ReviewStage.Readers.REVIEWERS_SUBMITTED, +# additional_fields={ +# "summary": { +# "order": 1, +# "description": "Briefly summarize the paper and its contributions. This is not the place to critique the paper; the authors should generally agree with a well-written summary.", +# "value": { +# "param": { +# "maxLength": 200000, +# "type": "string", +# "input": "textarea", +# "markdown": True +# } +# } +# }, +# "strengths_and_weaknesses": { +# "order": 2, +# "description": "Please provide a thorough assessment of the strengths and weaknesses of the paper, touching on each of the following dimensions: originality, quality, clarity, and significance. We encourage people to be broad in their definitions of originality and significance. For example, originality may arise from creative combinations of existing ideas, application to a new domain, or removing restrictive assumptions from prior theoretical results. You can incorporate Markdown and Latex into your review. See https://openreview.net/faq.", +# "value": { +# "param": { +# "maxLength": 200000, +# "type": "string", +# "input": "textarea", +# "markdown": True +# } +# } +# }, +# "questions": { +# "order": 3, +# "description": "Please list up and carefully describe any questions and suggestions for the authors. Think of the things where a response from the author can change your opinion, clarify a confusion or address a limitation. This can be very important for a productive rebuttal and discussion phase with the authors.", +# "value": { +# "param": { +# "maxLength": 200000, +# "type": "string", +# "input": "textarea", +# "markdown": True +# } +# } +# }, +# "limitations": { +# "order": 4, +# "description": "Have the authors adequately addressed the limitations and potential negative societal impact of their work? If not, please include constructive suggestions for improvement. Authors should be rewarded rather than punished for being up front about the limitations of their work and any potential negative societal impact.", +# "value": { +# "param": { +# "maxLength": 200000, +# "type": "string", +# "input": "textarea", +# "markdown": True +# } +# } +# }, +# "ethics_flag": { +# "order": 5, +# "description": "If there are ethical issues with this paper, please flag the paper for an ethics review. For guidance on when this is appropriate, please review the ethics guidelines (https://icml.cc/Conferences/2023/PublicationEthics).", +# "value": { +# "param": { +# "type": "string", +# "enum": [ +# "Yes", +# "No" +# ], +# "input": "radio" +# } +# } +# }, +# "ethics_review_area": { +# "order": 6, +# "description": "If you flagged this paper for ethics review, what area of expertise would it be most useful for the ethics reviewer to have? Please click all that apply.", +# "value": { +# "param": { +# "type": "string[]", +# "enum": [ +# "Discrimination / Bias / Fairness Concerns", +# "Inadequate Data and Algorithm Evaluation", +# "Inappropriate Potential Applications & Impact (e.g., human rights concerns)", +# "Privacy and Security (e.g., consent)", +# "Legal Compliance (e.g., GDPR, copyright, terms of use)", +# "Research Integrity Issues (e.g., plagiarism)", +# "Responsible Research Practice (e.g., IRB, documentation, research ethics)", +# "I don't know" +# ], +# "input": "checkbox", +# "optional": True, +# } +# } +# }, +# "soundness": { +# "order": 7, +# "description": "Please assign the paper a numerical rating on the following scale to indicate the soundness of the technical claims, experimental and research methodology and on whether the central claims of the paper are adequately supported with evidence.", +# "value": { +# "param": { +# "type": "string", +# "enum": [ +# "4 excellent", +# "3 good", +# "2 fair", +# "1 poor" +# ], +# "input": "radio" +# } +# } +# }, +# "presentation": { +# "order": 8, +# "description": "Please assign the paper a numerical rating on the following scale to indicate the quality of the presentation. This should take into account the writing style and clarity, as well as contextualization relative to prior work.", +# "value": { +# "param": { +# "type": "string", +# "enum": [ +# "4 excellent", +# "3 good", +# "2 fair", +# "1 poor" +# ], +# "input": "radio" +# } +# } +# }, +# "contribution": { +# "order": 9, +# "description": "Please assign the paper a numerical rating on the following scale to indicate the quality of the overall contribution this paper makes to the research area being studied. Are the questions being asked important? Does the paper bring a significant originality of ideas and/or execution? Are the results valuable to share with the broader ICML community?", +# "value": { +# "param": { +# "type": "string", +# "enum": [ +# "4 excellent", +# "3 good", +# "2 fair", +# "1 poor" +# ], +# "input": "radio" +# } +# } +# }, +# "rating": { +# "order": 10, +# "description": "Please provide an \"overall score\" for this submission.", +# "value": { +# "param": { +# "type": 'integer', +# "enum": [ +# { 'value': 10, 'description': "10: Award quality: Technically flawless paper with groundbreaking impact, with exceptionally strong evaluation, reproducibility, and resources, and no unaddressed ethical considerations." }, +# { 'value': 9, 'description': "9: Very Strong Accept: Technically flawless paper with groundbreaking impact on at least one area of AI/ML and excellent impact on multiple areas of AI/ML, with flawless evaluation, resources, and reproducibility, and no unaddressed ethical considerations." }, +# { 'value': 8, 'description': "8: Strong Accept: Technically strong paper, with novel ideas, excellent impact on at least one area, or high-to-excellent impact on multiple areas, with excellent evaluation, resources, and reproducibility, and no unaddressed ethical considerations." }, +# { 'value': 7, 'description': "7: Accept: Technically solid paper, with high impact on at least one sub-area, or moderate-to-high impact on more than one areas, with good-to-excellent evaluation, resources, reproducibility, and no unaddressed ethical considerations." }, +# { 'value': 6, 'description': "6: Weak Accept: Technically solid, moderate-to-high impact paper, with no major concerns with respect to evaluation, resources, reproducibility, ethical considerations." }, +# { 'value': 5, 'description': "5: Borderline accept: Technically solid paper where reasons to accept outweigh reasons to reject, e.g., limited evaluation. Please use sparingly." }, +# { 'value': 4, 'description': "4: Borderline reject: Technically solid paper where reasons to reject, e.g., limited evaluation, outweigh reasons to accept, e.g., good evaluation. Please use sparingly." }, +# { 'value': 3, 'description': "3: Reject: For instance, a paper with technical flaws, weak evaluation, inadequate reproducibility and incompletely addressed ethical considerations." }, +# { 'value': 2, 'description': "2: Strong Reject: For instance, a paper with major technical flaws, and/or poor evaluation, limited impact, poor reproducibility and mostly unaddressed ethical considerations." }, +# { 'value': 1, 'description': "1: Very Strong Reject: For instance, a paper with trivial results or unaddressed ethical considerations" } +# ], +# "input": "radio" + +# } +# } +# }, +# "confidence": { +# "order": 11, +# "description": "Please provide a \"confidence score\" for your assessment of this submission to indicate how confident you are in your evaluation.", +# "value": { +# "param": { +# "type": 'integer', +# "enum": [ +# { 'value': 5, 'description': "5: You are absolutely certain about your assessment. You are very familiar with the related work and checked the math/other details carefully." }, +# { 'value': 4, 'description': "4: You are confident in your assessment, but not absolutely certain. It is unlikely, but not impossible, that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work." }, +# { 'value': 3, 'description': "3: You are fairly confident in your assessment. It is possible that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked." }, +# { 'value': 2, 'description': "2: You are willing to defend your assessment, but it is quite likely that you did not understand the central parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked." }, +# { 'value': 1, 'description': "1: Your assessment is an educated guess. The submission is not in your area or the submission was difficult to understand. Math/other details were not carefully checked." } +# ], +# "input": "radio" +# } +# } +# }, +# "code_of_conduct": { +# "description": "While performing my duties as a reviewer (including writing reviews and participating in discussions), I have and will continue to abide by the ICML code of conduct (https://icml.cc/public/CodeOfConduct).", +# "order": 12, +# "value": { +# "param": { +# "type": "string", +# "enum": ["Yes"], +# "input": "checkbox" +# } +# } +# } +# }, +# remove_fields=['title', 'review'], +# source_submissions_query={ +# 'position_paper_track': 'No' +# } +# ) + +# venue.create_review_stage() + +# helpers.await_queue_edit(openreview_client, 'ICML.cc/2024/Conference/-/Official_Review-0-1', count=4) + +# pc_client_v2=openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) + +# # check reviews of a flagged paper is visible to ethics reviewers and authors +# reviews = pc_client_v2.get_notes(invitation='ICML.cc/2024/Conference/Submission1/-/Official_Review') +# assert len(reviews) == 2 +# assert reviews[0].readers == [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Reviewers/Submitted', +# 'ICML.cc/2024/Conference/Submission1/Authors', +# 'ICML.cc/2024/Conference/Submission1/Ethics_Reviewers', +# reviews[0].signatures[0] +# ] + +# # assert position papers' reviews are still hidden +# reviews = pc_client_v2.get_notes(invitation='ICML.cc/2024/Conference/Submission2/-/Official_Review') +# assert len(reviews) == 1 +# assert reviews[0].readers == [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission2/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Submission2/Area_Chairs', +# reviews[0].signatures[0] +# ] + +# # release position paper reviews +# venue.review_stage = openreview.stages.ReviewStage( +# due_date=now - datetime.timedelta(days=3), +# release_to_authors=True, +# release_to_reviewers=openreview.stages.ReviewStage.Readers.REVIEWERS_SUBMITTED, +# name='Position_Paper_Review', +# remove_fields=['title'], +# source_submissions_query={ +# 'position_paper_track': 'Yes' +# } +# ) + +# venue.create_review_stage() + +# helpers.await_queue_edit(openreview_client, 'ICML.cc/2024/Conference/-/Position_Paper_Review-0-1', count=3) + +# # check reviews of a non-flagged paper is not visible to ethics reviewers but it visible to authors +# reviews = pc_client_v2.get_notes(invitation='ICML.cc/2024/Conference/Submission2/-/Official_Review') +# assert len(reviews) == 1 +# assert reviews[0].readers == [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission2/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Submission2/Area_Chairs', +# 'ICML.cc/2024/Conference/Submission2/Reviewers/Submitted', +# 'ICML.cc/2024/Conference/Submission2/Authors', +# reviews[0].signatures[0] +# ] +# edits = openreview_client.get_note_edits(note_id=reviews[0].id) +# for edit in edits: +# assert edit.readers == edit.note.readers +# assert '${2/note/readers}' not in edit.readers + +# now = datetime.datetime.utcnow() +# start_date = now - datetime.timedelta(days=2) +# end_date = now + datetime.timedelta(days=3) +# comment_stage_note = pc_client.post_note(openreview.Note( +# content={ +# 'commentary_start_date': start_date.strftime('%Y/%m/%d'), +# 'commentary_end_date': end_date.strftime('%Y/%m/%d'), +# 'participants': ['Program Chairs', 'Assigned Senior Area Chairs', 'Assigned Area Chairs', 'Assigned Reviewers', 'Authors'], +# 'additional_readers': ['Program Chairs', 'Assigned Senior Area Chairs', 'Assigned Area Chairs', 'Assigned Reviewers', 'Assigned Submitted Reviewers', 'Authors'], +# 'email_program_chairs_about_official_comments': 'No, do not email PCs for each official comment made in the venue' + +# }, +# forum=request_form.forum, +# invitation=f'openreview.net/Support/-/Request{request_form.number}/Comment_Stage', +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# replyto=request_form.forum, +# referent=request_form.forum, +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) + +# helpers.await_queue() + +# test_client = openreview.api.OpenReviewClient(username='test@mail.com', password=helpers.strong_password) + +# reviews = pc_client_v2.get_notes(invitation='ICML.cc/2024/Conference/Submission1/-/Official_Review') +# comment_edit = test_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission1/-/Official_Comment', +# signatures=['ICML.cc/2024/Conference/Submission1/Authors'], +# note=openreview.api.Note( +# replyto = reviews[0].id, +# readers = [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Reviewers/Submitted', +# ], +# content={ +# 'comment': { 'value': 'Thanks for your review!!!' }, +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=comment_edit['id']) + +# invitation = client.get_invitation(f'openreview.net/Support/-/Request{request_form.number}/Rebuttal_Stage') +# invitation.cdate = openreview.tools.datetime_millis(datetime.datetime.utcnow()) +# client.post_invitation(invitation) + +# # post a rebuttal stage note +# now = datetime.datetime.utcnow() +# start_date = now - datetime.timedelta(days=2) +# due_date = now + datetime.timedelta(days=3) +# pc_client.post_note(openreview.Note( +# content={ +# 'rebuttal_start_date': start_date.strftime('%Y/%m/%d'), +# 'rebuttal_deadline': due_date.strftime('%Y/%m/%d'), +# 'number_of_rebuttals': 'Multiple author rebuttals per paper', +# 'email_program_chairs_about_rebuttals': 'No, do not email program chairs about received rebuttals' +# }, +# forum=request_form.forum, +# invitation=f'openreview.net/Support/-/Request{request_form.number}/Rebuttal_Stage', +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# replyto=request_form.forum, +# referent=request_form.forum, +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) + +# helpers.await_queue() + +# submissions = openreview_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', sort='number:asc') + +# assert len(openreview_client.get_invitations(invitation='ICML.cc/2024/Conference/-/Rebuttal')) == 100 +# invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Submission1/-/Rebuttal') +# assert not invitation.maxReplies +# assert invitation.edit['note']['replyto'] == { +# 'param': { +# 'withForum': f'{submissions[0].id}' +# } +# } + +# test_client = openreview.api.OpenReviewClient(username='test@mail.com', password=helpers.strong_password) +# review = reviews[0] + +# rebuttal_edit = test_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission1/-/Rebuttal', +# signatures=['ICML.cc/2024/Conference/Submission1/Authors'], +# note=openreview.api.Note( +# replyto = review.forum, +# content={ +# 'rebuttal': { 'value': 'This is a rebuttal.' } +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=rebuttal_edit['id']) + +# second_rebuttal_edit = test_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission1/-/Rebuttal', +# signatures=['ICML.cc/2024/Conference/Submission1/Authors'], +# note=openreview.api.Note( +# replyto = reviews[0].id, +# content={ +# 'rebuttal': { 'value': 'This is a rebuttal replying to a review.' }, +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=second_rebuttal_edit['id']) + +# rebuttal_id = second_rebuttal_edit['note']['id'] + +# messages = openreview_client.get_messages(subject = '[ICML 2023] Your author rebuttal was posted on Submission Number: 1, Submission Title: "Paper title 1 Version 2"') +# assert len(messages) == 2 +# assert 'test@mail.com' in messages[0]['content']['to'] +# assert messages[0]['content']['replyTo'] == 'pc@icml.cc' +# messages = openreview_client.get_messages(subject = '[ICML 2023] An author rebuttal was posted on Submission Number: 1, Submission Title: "Paper title 1 Version 2"') +# assert len(messages) == 8 +# assert f'https://openreview.net/forum?id={review.forum}¬eId={rebuttal_id}' in messages[4]['content']['text'] +# recipients = [m['content']['to'] for m in messages] +# assert 'peter@mail.com' in recipients +# assert 'andrew@amazon.com' in recipients +# assert 'sac1@gmail.com' in recipients +# assert 'melisa@yahoo.com' in recipients + +# #update rebuttal +# rebuttal_update = test_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission1/-/Rebuttal', +# signatures=['ICML.cc/2024/Conference/Submission1/Authors'], +# note=openreview.api.Note( +# id = rebuttal_id, +# replyto = reviews[0].id, +# content={ +# 'rebuttal': { 'value': 'This is a rebuttal replying to a review UPDATED.' } +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=rebuttal_update['id']) + +# #check no new emails were sent +# messages = openreview_client.get_messages(subject = '[ICML 2023] Your author rebuttal was posted on Submission Number: 1, Submission Title: "Paper title 1 Version 2"') +# assert len(messages) == 2 +# assert 'test@mail.com' in messages[0]['content']['to'] +# messages = openreview_client.get_messages(subject = '[ICML 2023] An author rebuttal was posted on Submission Number: 1, Submission Title: "Paper title 1 Version 2"') +# assert len(messages) == 8 + +# rebuttals = pc_client_v2.get_notes(invitation='ICML.cc/2024/Conference/Submission1/-/Rebuttal') +# assert len(rebuttals) == 2 +# assert rebuttals[0].readers == [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Authors', +# ] + +# # flag a paper after reviews are released and assert readers are correct +# note = openreview_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', number=[2])[0] +# note_edit = pc_client_v2.post_note_edit( +# invitation='ICML.cc/2024/Conference/-/Ethics_Review_Flag', +# note=openreview.api.Note( +# id=note.id, +# content = { +# 'flagged_for_ethics_review': { 'value': True }, +# 'ethics_comments': { 'value': 'These are ethics comments visible to ethics chairs and ethics reviewers' } +# } +# ), +# signatures=['ICML.cc/2024/Conference'] +# ) + +# helpers.await_queue() +# helpers.await_queue_edit(openreview_client, edit_id=note_edit['id']) + +# reviews = openreview_client.get_notes(invitation='ICML.cc/2024/Conference/Submission2/-/Official_Review') +# for review in reviews: +# assert review.readers == [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission2/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Submission2/Area_Chairs', +# 'ICML.cc/2024/Conference/Submission2/Reviewers/Submitted', +# 'ICML.cc/2024/Conference/Submission2/Authors', +# 'ICML.cc/2024/Conference/Submission2/Ethics_Reviewers', +# review.signatures[0] +# ] + +# def test_release_rebuttals(self, openreview_client, helpers): + +# pc_client=openreview.Client(username='pc@icml.cc', password=helpers.strong_password) +# pc_client_v2=openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) +# request_form=pc_client.get_notes(invitation='openreview.net/Support/-/Request_Form')[0] + +# # post a rebuttal stage note +# now = datetime.datetime.utcnow() +# start_date = now - datetime.timedelta(days=2) +# due_date = now + datetime.timedelta(days=3) +# pc_client.post_note(openreview.Note( +# content={ +# 'rebuttal_start_date': start_date.strftime('%Y/%m/%d'), +# 'rebuttal_deadline': due_date.strftime('%Y/%m/%d'), +# 'rebuttal_readers': ['Assigned Senior Area Chairs', 'Assigned Area Chairs', 'Assigned Reviewers who already submitted their review'], +# 'number_of_rebuttals': 'Multiple author rebuttals per paper', +# 'email_program_chairs_about_rebuttals': 'No, do not email program chairs about received rebuttals' +# }, +# forum=request_form.forum, +# invitation=f'openreview.net/Support/-/Request{request_form.number}/Rebuttal_Stage', +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# replyto=request_form.forum, +# referent=request_form.forum, +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) + +# helpers.await_queue() + +# rebuttals = pc_client_v2.get_notes(invitation='ICML.cc/2024/Conference/Submission1/-/Rebuttal') +# assert len(rebuttals) == 2 +# assert rebuttals[0].readers == [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Reviewers/Submitted', +# 'ICML.cc/2024/Conference/Submission1/Authors', +# ] +# assert rebuttals[1].readers == [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Reviewers/Submitted', +# 'ICML.cc/2024/Conference/Submission1/Authors', +# ] + +# def test_meta_review_stage(self, client, openreview_client, helpers): +# pc_client=openreview.Client(username='pc@icml.cc', password=helpers.strong_password) +# pc_client_v2=openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) +# request_form=pc_client.get_notes(invitation='openreview.net/Support/-/Request_Form')[0] + +# now = datetime.datetime.utcnow() +# start_date = now - datetime.timedelta(days=2) +# due_date = now + datetime.timedelta(days=3) +# exp_date = due_date + datetime.timedelta(days=2) + +# venue = openreview.helpers.get_conference(client, request_form.id, setup=False) +# venue.meta_review_stage = openreview.stages.MetaReviewStage( +# start_date=start_date, +# due_date=due_date, +# exp_date=exp_date, +# additional_fields={ +# 'recommendation': { +# 'description': 'Please select a recommendation for the paper', +# 'value': { +# 'param': { +# 'type': 'string', +# 'enum': ['Accept', 'Reject'], +# 'input': 'select' +# } +# }, +# 'order': 2 +# }, +# 'suggestions': { +# 'description': 'Please provide suggestions on how to improve the paper', +# 'value': { +# 'param': { +# 'type': 'string', +# 'maxLength': 5000, +# 'input': 'textarea', +# 'optional': True, +# 'deletable': True +# } +# } +# } +# }, +# remove_fields=['confidence'], +# source_submissions_query={ +# 'position_paper_track': 'No' +# } +# ) + +# venue.create_meta_review_stage() +# helpers.await_queue_edit(openreview_client, 'ICML.cc/2024/Conference/-/Meta_Review-0-1', count=1) + +# invitations = openreview_client.get_invitations(invitation='ICML.cc/2024/Conference/-/Meta_Review') +# assert len(invitations) == 50 +# assert invitations[0].edit['note']['id']['param']['withInvitation'] == invitations[0].id + +# invitations = openreview_client.get_invitations(invitation='ICML.cc/2024/Conference/-/Meta_Review_SAC_Revision') +# assert len(invitations) == 50 + +# sac_revision_invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Submission1/-/Meta_Review_SAC_Revision') +# invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Submission1/-/Meta_Review') +# assert sac_revision_invitation.edit['note']['id']['param']['withInvitation'] == invitation.id +# assert 'suggestions' in invitation.edit['note']['content'] + +# # duedate + 2 days +# exp_date = invitation.duedate + (2*24*60*60*1000) +# assert invitation.expdate == exp_date + +# assert openreview_client.get_invitation('ICML.cc/2024/Conference/Submission1/-/Meta_Review') +# assert not openreview.tools.get_invitation(openreview_client, 'ICML.cc/2024/Conference/Submission2/-/Meta_Review') +# assert openreview_client.get_invitation('ICML.cc/2024/Conference/Submission3/-/Meta_Review') +# assert not openreview.tools.get_invitation(openreview_client, 'ICML.cc/2024/Conference/Submission4/-/Meta_Review') +# assert openreview_client.get_invitation('ICML.cc/2024/Conference/Submission5/-/Meta_Review') + +# ## Create position paper meta reviews +# venue = openreview.helpers.get_conference(client, request_form.id, setup=False) +# venue.meta_review_stage = openreview.stages.MetaReviewStage( +# start_date=start_date, +# due_date=due_date, +# exp_date=exp_date, +# remove_fields=['confidence'], +# name='Position_Paper_Meta_Review', +# source_submissions_query={ +# 'position_paper_track': 'Yes' +# } +# ) + +# venue.create_meta_review_stage() +# helpers.await_queue_edit(openreview_client, 'ICML.cc/2024/Conference/-/Position_Paper_Meta_Review-0-1', count=1) + +# invitations = openreview_client.get_invitations(invitation='ICML.cc/2024/Conference/-/Position_Paper_Meta_Review') +# assert len(invitations) == 50 +# assert invitations[0].edit['note']['id']['param']['withInvitation'] == invitations[0].id + +# invitations = openreview_client.get_invitations(invitation='ICML.cc/2024/Conference/-/Position_Paper_Meta_Review_SAC_Revision') +# assert len(invitations) == 50 + +# sac_revision_invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Submission2/-/Meta_Review_SAC_Revision') +# invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Submission2/-/Meta_Review') +# assert sac_revision_invitation.edit['note']['id']['param']['withInvitation'] == invitation.id +# assert 'metareview' in invitation.edit['note']['content'] +# assert 'suggestions' not in invitation.edit['note']['content'] + +# ac_client = openreview.api.OpenReviewClient(username='ac2@icml.cc', password=helpers.strong_password) +# submissions = ac_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', sort='number:asc') + +# anon_groups = ac_client.get_groups(prefix='ICML.cc/2024/Conference/Submission1/Area_Chair_', signatory='~AC_ICMLTwo1') +# anon_group_id = anon_groups[0].id + +# meta_review_edit = ac_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission1/-/Meta_Review', +# signatures=[anon_group_id], +# note=openreview.api.Note( +# content={ +# 'metareview': { 'value': 'This is a good paper' }, +# 'recommendation': { 'value': 'Accept'} +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=meta_review_edit['id']) + +# #try to delete AC assignment of paper with a submitted metareview +# assignment = pc_client_v2.get_edges(invitation='ICML.cc/2024/Conference/Area_Chairs/-/Assignment', head=submissions[0].id, tail='~AC_ICMLTwo1')[0] +# assignment.ddate = openreview.tools.datetime_millis(datetime.datetime.utcnow()) +# assignment.cdate = None + +# with pytest.raises(openreview.OpenReviewException, match=r'Can not remove assignment, the user ~AC_ICMLTwo1 already posted a Meta Review.'): +# pc_client_v2.post_edge(assignment) + +# ## Post meta review to position paper +# ac_client = openreview.api.OpenReviewClient(username='ac1@icml.cc', password=helpers.strong_password) +# submissions = ac_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', sort='number:asc') + +# anon_groups = ac_client.get_groups(prefix='ICML.cc/2024/Conference/Submission4/Area_Chair_', signatory='~AC_ICMLOne1') +# anon_group_id = anon_groups[0].id + +# meta_review_edit = ac_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission4/-/Meta_Review', +# signatures=[anon_group_id], +# note=openreview.api.Note( +# content={ +# 'metareview': { 'value': 'This is a good paper' }, +# 'recommendation': { 'value': 'Accept (Oral)'} +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=meta_review_edit['id']) + +# ## Extend deadline using a meta invitation and propagate the change to all the children +# new_due_date = openreview.tools.datetime_millis(now + datetime.timedelta(days=10)) +# new_exp_date = openreview.tools.datetime_millis(now + datetime.timedelta(days=15)) +# pc_client_v2.post_invitation_edit( +# invitations='ICML.cc/2024/Conference/-/Edit', +# readers=['ICML.cc/2024/Conference'], +# writers=['ICML.cc/2024/Conference'], +# signatures=['ICML.cc/2024/Conference'], +# invitation=openreview.api.Invitation( +# id='ICML.cc/2024/Conference/-/Position_Paper_Meta_Review', +# edit={ +# 'invitation': { +# 'duedate': new_due_date, +# 'expdate': new_exp_date +# } +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, 'ICML.cc/2024/Conference/-/Position_Paper_Meta_Review-0-1', count=2) +# invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Submission4/-/Meta_Review') +# assert invitation.expdate == new_exp_date + +# def test_meta_review_agreement(self, client, openreview_client, helpers, selenium, request_page): + +# pc_client=openreview.Client(username='pc@icml.cc', password=helpers.strong_password) +# request_form=pc_client.get_notes(invitation='openreview.net/Support/-/Request_Form')[0] +# venue = openreview.get_conference(client, request_form.id, support_user='openreview.net/Support') + +# now = datetime.datetime.utcnow() +# due_date = now + datetime.timedelta(days=3) +# venue.custom_stage = openreview.stages.CustomStage(name='Meta_Review_Agreement', +# reply_to=openreview.stages.CustomStage.ReplyTo.METAREVIEWS, +# source=openreview.stages.CustomStage.Source.ALL_SUBMISSIONS, +# due_date=due_date, +# exp_date=due_date + datetime.timedelta(days=1), +# invitees=[openreview.stages.CustomStage.Participants.SENIOR_AREA_CHAIRS_ASSIGNED], +# readers=[openreview.stages.CustomStage.Participants.SENIOR_AREA_CHAIRS_ASSIGNED], +# content={ +# 'meta_review_agreement': { +# 'order': 1, +# 'description': "If you do not agree with the meta-reviewer’s recommendation, please reach out to the meta-reviewer directly, discuss this submission and arrive at a consensus. If the meta-reviewer and you cannot arrive at a consensus for this submission, please mark \"no\" and describe the disagreement.", +# 'value': { +# 'param': { +# 'type': 'string', +# 'enum': [ +# 'yes', +# 'no' +# ], +# 'input': 'radio' +# } +# } +# }, +# "explanation": { +# "order": 2, +# "description": "If you failed to arrive at consensus with the meta-reviewer, please describe your disagreement here for the program chairs.", +# "value": { +# "param": { +# "maxLength": 5000, +# "type": "string", +# "input": "textarea", +# "optional": True +# } +# } +# } +# }, +# notify_readers=False, +# email_sacs=False) + +# venue.create_custom_stage() + +# assert len(openreview_client.get_invitations(invitation='ICML.cc/2024/Conference/-/Meta_Review_Agreement')) == 2 + +# sac_client = openreview.api.OpenReviewClient(username = 'sac2@icml.cc', password=helpers.strong_password) +# submissions = sac_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', sort='number:asc') + +# ac_client = openreview.api.OpenReviewClient(username='ac2@icml.cc', password=helpers.strong_password) +# anon_groups = ac_client.get_groups(prefix='ICML.cc/2024/Conference/Submission1/Area_Chair_', signatory='~AC_ICMLTwo1') +# anon_group_id = anon_groups[0].id + +# invitation_id = 'ICML.cc/2024/Conference/Submission1/Meta_Review1/-/Meta_Review_Agreement' + +# agreement_edit = sac_client.post_note_edit( +# invitation=invitation_id, +# signatures=['ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs'], +# note=openreview.api.Note( +# content={ +# 'meta_review_agreement': { 'value': 'yes' } +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=agreement_edit['id']) + +# pc_client_v2=openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) +# metareviews = pc_client_v2.get_notes(invitation='ICML.cc/2024/Conference/Submission1/-/Meta_Review') +# agreements = pc_client_v2.get_notes(invitation=invitation_id) +# assert agreements[0].replyto == metareviews[0].id +# assert agreements[0].readers == [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs' +# ] + +# ac_client = openreview.api.OpenReviewClient(username='ac1@icml.cc', password=helpers.strong_password) +# submissions = ac_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', sort='number:asc') + +# anon_groups = ac_client.get_groups(prefix='ICML.cc/2024/Conference/Submission2/Area_Chair_', signatory='~AC_ICMLOne1') +# anon_group_id = anon_groups[0].id + +# # post another metareview and check agreement invitation is created +# meta_review_edit = ac_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission2/-/Meta_Review', +# signatures=[anon_group_id], +# note=openreview.api.Note( +# content={ +# 'metareview': { 'value': 'This is a very bad paper' }, +# 'recommendation': { 'value': 'Reject'} +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=meta_review_edit['id']) + +# assert len(openreview_client.get_invitations(invitation='ICML.cc/2024/Conference/-/Meta_Review_Agreement')) == 3 + +# invitation_id = 'ICML.cc/2024/Conference/Submission2/Meta_Review1/-/Meta_Review_Agreement' +# sac_client = openreview.api.OpenReviewClient(username = 'sac1@gmail.com', password=helpers.strong_password) + +# agreement_edit = sac_client.post_note_edit( +# invitation=invitation_id, +# signatures=['ICML.cc/2024/Conference/Submission2/Senior_Area_Chairs'], +# note=openreview.api.Note( +# content={ +# 'meta_review_agreement': { 'value': 'no' }, +# 'explanation': { 'value': 'I think the paper should be accepted.' } +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=agreement_edit['id']) + +# pc_client_v2=openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) +# metareviews = pc_client_v2.get_notes(invitation='ICML.cc/2024/Conference/Submission2/-/Meta_Review') +# agreements = pc_client_v2.get_notes(invitation=invitation_id) +# assert agreements[0].replyto == metareviews[0].id +# assert agreements[0].readers == [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission2/Senior_Area_Chairs' +# ] + +# submissions = sac_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', sort='number:asc') +# note = submissions[1] + +# # check SACs can't see Metareview Revision button +# request_page(selenium, 'http://localhost:3030/forum?id=' + note.id, sac_client.token, by=By.CLASS_NAME, wait_for_element='invitations-container') +# invitations_container = selenium.find_element(By.CLASS_NAME, 'invitations-container') +# invitation_buttons = invitations_container.find_element(By.CLASS_NAME, 'invitation-buttons') +# buttons = invitation_buttons.find_elements(By.TAG_NAME, 'button') +# assert len(buttons) == 1 +# assert buttons[0].text == 'Official Comment' + +# ## SAC can edit the meta review +# meta_review_edit = sac_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission2/-/Meta_Review_SAC_Revision', +# signatures=['ICML.cc/2024/Conference/Submission2/Senior_Area_Chairs'], +# note=openreview.api.Note( +# id=metareviews[0].id, +# content={ +# 'metareview': { 'value': 'I reverted the AC decision' }, +# 'recommendation': { 'value': 'Accept (Oral)'} +# } +# ) +# ) + +# def test_decision_stage(self, client, openreview_client, helpers): + +# pc_client=openreview.Client(username='pc@icml.cc', password=helpers.strong_password) +# request_form=pc_client.get_notes(invitation='openreview.net/Support/-/Request_Form')[0] + +# # Post a decision stage note +# now = datetime.datetime.utcnow() +# start_date = now - datetime.timedelta(days=2) +# due_date = now + datetime.timedelta(days=3) + +# decision_stage_note = pc_client.post_note(openreview.Note( +# content={ +# 'decision_start_date': start_date.strftime('%Y/%m/%d'), +# 'decision_deadline': due_date.strftime('%Y/%m/%d'), +# 'decision_options': 'Accept, Revision Needed, Reject', +# 'make_decisions_public': 'No, decisions should NOT be revealed publicly when they are posted', +# 'release_decisions_to_authors': 'Yes, decisions should be revealed when they are posted to the paper\'s authors', +# 'release_decisions_to_reviewers': 'No, decisions should not be immediately revealed to the paper\'s reviewers', +# 'release_decisions_to_area_chairs': 'Yes, decisions should be immediately revealed to the paper\'s area chairs', +# 'notify_authors': 'Yes, send an email notification to the authors', +# 'additional_decision_form_options': { +# 'suggestions': { +# 'description': 'Please provide suggestions on how to improve the paper', +# 'value': { +# 'param': { +# 'type': 'string', +# 'maxLength': 5000, +# 'input': 'textarea', +# 'optional': True, +# 'deletable': True +# } +# } +# } +# } +# }, +# forum=request_form.forum, +# invitation=f'openreview.net/Support/-/Request{request_form.number}/Decision_Stage', +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# replyto=request_form.forum, +# referent=request_form.forum, +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) +# assert decision_stage_note +# helpers.await_queue() + +# assert openreview_client.get_invitation('ICML.cc/2024/Conference/Submission1/-/Decision') +# assert openreview_client.get_invitation('ICML.cc/2024/Conference/Submission2/-/Decision') +# assert openreview_client.get_invitation('ICML.cc/2024/Conference/Submission3/-/Decision') +# assert openreview_client.get_invitation('ICML.cc/2024/Conference/Submission4/-/Decision') +# assert openreview_client.get_invitation('ICML.cc/2024/Conference/Submission5/-/Decision') + +# venue = openreview.get_conference(client, request_form.id, support_user='openreview.net/Support') +# submissions = venue.get_submissions(sort='number:asc') +# assert len(submissions) == 100 +# decisions = ['Accept', 'Revision Needed', 'Reject'] +# comment = { +# 'Accept': 'Congratulations on your acceptance.', +# 'Revision Needed': 'A revision is needed from the authors.', +# 'Reject': 'We regret to inform you...' +# } + +# with open(os.path.join(os.path.dirname(__file__), 'data/ICML_decisions.csv'), 'w') as file_handle: +# writer = csv.writer(file_handle) +# writer.writerow([submissions[0].number, 'Accept', comment["Accept"]]) +# writer.writerow([submissions[1].number, 'Reject', comment["Reject"]]) +# writer.writerow([submissions[2].number, 'Revision Needed', comment["Revision Needed"]]) +# for submission in submissions[3:]: +# decision = random.choice(decisions) +# writer.writerow([submission.number, decision, comment[decision]]) + +# decision_stage_invitation = f'openreview.net/Support/-/Request{request_form.number}/Decision_Stage' +# url = pc_client.put_attachment(os.path.join(os.path.dirname(__file__), 'data/ICML_decisions.csv'), +# decision_stage_invitation, 'decisions_file') + +# #post decisions from request form +# decision_stage_note = pc_client.post_note(openreview.Note( +# content={ +# 'decision_start_date': start_date.strftime('%Y/%m/%d'), +# 'decision_deadline': due_date.strftime('%Y/%m/%d'), +# 'decision_options': 'Accept, Revision Needed, Reject', +# 'make_decisions_public': 'No, decisions should NOT be revealed publicly when they are posted', +# 'release_decisions_to_authors': 'No, decisions should NOT be revealed when they are posted to the paper\'s authors', +# 'release_decisions_to_reviewers': 'No, decisions should not be immediately revealed to the paper\'s reviewers', +# 'release_decisions_to_area_chairs': 'Yes, decisions should be immediately revealed to the paper\'s area chairs', +# 'notify_authors': 'No, I will send the emails to the authors', +# 'additional_decision_form_options': { +# 'suggestions': { +# 'description': 'Please provide suggestions on how to improve the paper', +# 'value': { +# 'param': { +# 'type': 'string', +# 'maxLength': 5000, +# 'input': 'textarea', +# 'optional': True, +# 'deletable': True +# } +# } +# } +# }, +# 'decisions_file': url +# }, +# forum=request_form.forum, +# invitation=decision_stage_invitation, +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# replyto=request_form.forum, +# referent=request_form.forum, +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) +# assert decision_stage_note +# helpers.await_queue() + +# decision = openreview_client.get_notes(invitation='ICML.cc/2024/Conference/Submission1/-/Decision')[0] +# assert 'Accept' == decision.content['decision']['value'] +# assert 'Congratulations on your acceptance.' in decision.content['comment']['value'] +# assert decision.readers == [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Area_Chairs' +# ] +# assert decision.nonreaders == [ +# 'ICML.cc/2024/Conference/Submission1/Authors' +# ] + +# decision = openreview_client.get_notes(invitation='ICML.cc/2024/Conference/Submission3/-/Decision')[0] +# assert 'Revision Needed' == decision.content['decision']['value'] + +# # manually change a decision +# pc_client_v2=openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) +# decision_note = pc_client_v2.post_note_edit(invitation='ICML.cc/2024/Conference/Submission3/-/Decision', +# signatures=['ICML.cc/2024/Conference/Program_Chairs'], +# note=openreview.api.Note( +# id=decision.id, +# content={ +# 'decision': {'value': 'Accept'}, +# 'comment': {'value': 'This is a comment.'} +# } +# )) +# helpers.await_queue_edit(openreview_client, edit_id=decision_note['id']) + +# request_form=pc_client.get_notes(invitation='openreview.net/Support/-/Request_Form')[0] + +# #release decisions to authors and reviewers +# decision_stage_note = pc_client.post_note(openreview.Note( +# content={ +# 'decision_start_date': start_date.strftime('%Y/%m/%d'), +# 'decision_deadline': due_date.strftime('%Y/%m/%d'), +# 'decision_options': 'Accept, Revision Needed, Reject', +# 'make_decisions_public': 'No, decisions should NOT be revealed publicly when they are posted', +# 'release_decisions_to_authors': 'Yes, decisions should be revealed when they are posted to the paper\'s authors', +# 'release_decisions_to_reviewers': 'Yes, decisions should be immediately revealed to the paper\'s reviewers', +# 'release_decisions_to_area_chairs': 'Yes, decisions should be immediately revealed to the paper\'s area chairs', +# 'notify_authors': 'No, I will send the emails to the authors', +# 'additional_decision_form_options': { +# 'suggestions': { +# 'description': 'Please provide suggestions on how to improve the paper', +# 'value': { +# 'param': { +# 'type': 'string', +# 'maxLength': 5000, +# 'input': 'textarea', +# 'optional': True, +# 'deletable': True +# } +# } +# } +# }, +# 'decisions_file': request_form.content['decisions_file'] +# }, +# forum=request_form.forum, +# invitation=decision_stage_invitation, +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# replyto=request_form.forum, +# referent=request_form.forum, +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) +# assert decision_stage_note +# helpers.await_queue() + +# decision = openreview_client.get_notes(invitation='ICML.cc/2024/Conference/Submission1/-/Decision')[0] +# assert decision.readers == [ +# 'ICML.cc/2024/Conference/Program_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Area_Chairs', +# 'ICML.cc/2024/Conference/Submission1/Reviewers', +# 'ICML.cc/2024/Conference/Submission1/Authors' +# ] +# assert not decision.nonreaders + +# # assert decisions were not overwritten +# decision = openreview_client.get_notes(invitation='ICML.cc/2024/Conference/Submission3/-/Decision')[0] +# assert 'Accept' == decision.content['decision']['value'] + +# def test_post_decision_stage(self, client, openreview_client, helpers, selenium, request_page): + +# pc_client=openreview.Client(username='pc@icml.cc', password=helpers.strong_password) +# request_form=pc_client.get_notes(invitation='openreview.net/Support/-/Request_Form')[0] + +# submissions = openreview_client.get_notes(content= { 'venueid': 'ICML.cc/2024/Conference/Submission'}, sort='number:asc') +# assert submissions and len(submissions) == 100 + +# # Assert that submissions are still blind +# assert submissions[0].content['authors']['readers'] == ["ICML.cc/2024/Conference","ICML.cc/2024/Conference/Submission1/Authors"] +# assert submissions[0].content['authorids']['readers'] == ["ICML.cc/2024/Conference","ICML.cc/2024/Conference/Submission1/Authors"] +# assert submissions[1].content['authors']['readers'] == ["ICML.cc/2024/Conference","ICML.cc/2024/Conference/Submission2/Authors"] +# assert submissions[1].content['authorids']['readers'] == ["ICML.cc/2024/Conference","ICML.cc/2024/Conference/Submission2/Authors"] +# # Assert that submissions are private +# assert submissions[0].readers == [ +# "ICML.cc/2024/Conference", +# "ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs", +# "ICML.cc/2024/Conference/Submission1/Area_Chairs", +# "ICML.cc/2024/Conference/Submission1/Reviewers", +# "ICML.cc/2024/Conference/Submission1/Authors", +# 'ICML.cc/2024/Conference/Submission1/Ethics_Reviewers' +# ] +# assert submissions[1].readers == [ +# "ICML.cc/2024/Conference", +# "ICML.cc/2024/Conference/Submission2/Senior_Area_Chairs", +# "ICML.cc/2024/Conference/Submission2/Area_Chairs", +# "ICML.cc/2024/Conference/Submission2/Reviewers", +# "ICML.cc/2024/Conference/Submission2/Authors", +# 'ICML.cc/2024/Conference/Submission2/Ethics_Reviewers' +# ] +# assert not submissions[0].odate +# assert not submissions[1].odate + +# invitation = client.get_invitation(f'openreview.net/Support/-/Request{request_form.number}/Post_Decision_Stage') +# invitation.cdate = openreview.tools.datetime_millis(datetime.datetime.utcnow()) +# client.post_invitation(invitation) + +# invitation = pc_client.get_invitation(f'openreview.net/Support/-/Request{request_form.number}/Post_Decision_Stage') + +# assert 'Accept' in invitation.reply['content']['home_page_tab_names']['default'] +# assert invitation.reply['content']['home_page_tab_names']['default']['Accept'] == 'Accept' +# assert 'Revision Needed' in invitation.reply['content']['home_page_tab_names']['default'] +# assert invitation.reply['content']['home_page_tab_names']['default']['Revision Needed'] == 'Revision Needed' +# assert 'Reject' in invitation.reply['content']['home_page_tab_names']['default'] +# assert invitation.reply['content']['home_page_tab_names']['default']['Reject'] == 'Reject' + +# #Post a post decision note, unhide financial_aid and hide pdf +# now = datetime.datetime.utcnow() +# start_date = now - datetime.timedelta(days=2) +# due_date = now + datetime.timedelta(days=3) +# short_name = 'ICML 2023' +# post_decision_stage_note = pc_client.post_note(openreview.Note( +# content={ +# 'reveal_authors': 'Reveal author identities of only accepted submissions to the public', +# 'submission_readers': 'Make accepted submissions public and hide rejected submissions', +# 'hide_fields': ['supplementary_material', 'pdf'], +# 'home_page_tab_names': { +# 'Accept': 'Accept', +# 'Revision Needed': 'Revision Needed', +# 'Reject': 'Submitted' +# }, +# 'send_decision_notifications': 'Yes, send an email notification to the authors', +# 'accept_email_content': f'''Dear {{{{fullname}}}}, + +# Thank you for submitting your paper, {{{{submission_title}}}}, to {short_name}. We are delighted to inform you that your submission has been accepted. Congratulations! +# You can find the final reviews for your paper on the submission page in OpenReview at: {{{{forum_url}}}} + +# Best, +# {short_name} Program Chairs +# ''', +# 'reject_email_content': f'''Dear {{{{fullname}}}}, + +# Thank you for submitting your paper, {{{{submission_title}}}}, to {short_name}. We regret to inform you that your submission was not accepted. +# You can find the final reviews for your paper on the submission page in OpenReview at: {{{{forum_url}}}} + +# Best, +# {short_name} Program Chairs +# ''', +# 'revision_needed_email_content': f'''Dear {{{{fullname}}}}, + +# Thank you for submitting your paper, {{{{submission_title}}}}, to {short_name}. +# You can find the final reviews for your paper on the submission page in OpenReview at: {{{{forum_url}}}} + +# Best, +# {short_name} Program Chairs +# ''' +# }, +# forum=request_form.forum, +# invitation=invitation.id, +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# replyto=request_form.forum, +# referent=request_form.forum, +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) +# assert post_decision_stage_note +# helpers.await_queue() + +# process_logs = client.get_process_logs(id = post_decision_stage_note.id) +# assert len(process_logs) == 1 +# assert process_logs[0]['status'] == 'ok' + +# venue = openreview.get_conference(client, request_form.id, support_user='openreview.net/Support') +# accepted_submissions = venue.get_submissions(accepted=True, sort='number:asc') +# rejected_submissions = venue.get_submissions(venueid='ICML.cc/2024/Conference/Rejected_Submission', sort='number:asc') +# assert (len(accepted_submissions)+len(rejected_submissions)) == 100 + +# messages = openreview_client.get_messages(subject='[ICML 2023] Decision notification for your submission 1: Paper title 1 Version 2') +# assert len(messages) == 5 +# assert messages[0]['content']['replyTo'] == 'pc@icml.cc' +# recipients = [msg['content']['to'] for msg in messages] +# assert 'sac1@gmail.com' in recipients +# assert 'test@mail.com' in recipients +# assert 'peter@mail.com' in recipients +# assert 'melisa@yahoo.com' in recipients +# assert 'andrew@amazon.com' in recipients +# assert 'We are delighted to inform you that your submission has been accepted.' in messages[0]['content']['text'] + +# replies = pc_client.get_notes(forum=request_form.id, invitation=f'openreview.net/Support/-/Request{request_form.number}/Comment', sort='tmdate:desc') +# assert replies[0].content['title'] == 'Post Decision Stage Process Completed' +# assert replies[1].content['title'] == 'Decision Notification Status' +# assert 'Decision notifications have been sent to the authors. You can check the status of the emails by clicking on this link: https://openreview.net/messages?parentGroup=ICML.cc/2024/Conference/Authors' in replies[1].content['comment'] + +# for submission in accepted_submissions: +# assert submission.readers == ['everyone'] +# assert 'readers' not in submission.content['authors'] +# assert 'readers' not in submission.content['authorids'] +# assert 'readers' in submission.content['pdf'] +# assert 'readers' not in submission.content['financial_aid'] +# assert submission.pdate +# assert submission.odate +# assert submission.content['venue']['value'] == 'ICML 2023' +# assert submission.content['venueid']['value'] == 'ICML.cc/2024/Conference' + +# year = datetime.datetime.now().year +# valid_bibtex = '''@inproceedings{ +# user'''+str(year)+'''paper, +# title={Paper title 1 Version 2}, +# author={SomeFirstName User and Peter SomeLastName and Andrew Mc and SAC ICMLOne and Melisa ICML}, +# booktitle={Thirty-ninth International Conference on Machine Learning}, +# year={'''+str(year)+'''}, +# url={https://openreview.net/forum?id=''' + +# valid_bibtex = valid_bibtex + accepted_submissions[0].forum + '''} +# }''' + +# assert '_bibtex' in accepted_submissions[0].content and accepted_submissions[0].content['_bibtex']['value'] == valid_bibtex + +# for submission in rejected_submissions: +# assert submission.readers == [ +# "ICML.cc/2024/Conference", +# f"ICML.cc/2024/Conference/Submission{submission.number}/Senior_Area_Chairs", +# f"ICML.cc/2024/Conference/Submission{submission.number}/Area_Chairs", +# f"ICML.cc/2024/Conference/Submission{submission.number}/Reviewers", +# f"ICML.cc/2024/Conference/Submission{submission.number}/Authors" +# ] +# assert submission.content['authors']['readers'] == ["ICML.cc/2024/Conference",f"ICML.cc/2024/Conference/Submission{submission.number}/Authors"] +# assert submission.content['authorids']['readers'] == ["ICML.cc/2024/Conference",f"ICML.cc/2024/Conference/Submission{submission.number}/Authors"] +# assert not submission.pdate +# assert not submission.odate +# assert submission.content['venue']['value'] == 'Submitted to ICML 2023' +# assert submission.content['venueid']['value'] == 'ICML.cc/2024/Conference/Rejected_Submission' +# assert 'readers' in submission.content['pdf'] +# assert 'readers' not in submission.content['financial_aid'] + +# valid_bibtex = '''@misc{ +# anonymous'''+str(year)+'''paper, +# title={Paper title 2}, +# author={Anonymous}, +# year={'''+str(year)+'''}, +# url={https://openreview.net/forum?id=''' + +# valid_bibtex = valid_bibtex + rejected_submissions[0].forum + '''} +# }''' + +# assert '_bibtex' in rejected_submissions[0].content and rejected_submissions[0].content['_bibtex']['value'] == valid_bibtex + +# #make sure all decision process functions have finished +# for number in range(1, 101): +# helpers.await_queue_edit(openreview_client, invitation=f'ICML.cc/2024/Conference/Submission{number}/-/Decision') + +# authors_accepted_group = openreview_client.get_group('ICML.cc/2024/Conference/Authors/Accepted') +# num_accepted_papers = len(authors_accepted_group.members) + +# # add publication chair +# pc_client=openreview.Client(username='pc@icml.cc', password=helpers.strong_password) +# request_form=pc_client.get_notes(invitation='openreview.net/Support/-/Request_Form')[0] + +# pc_client.post_note(openreview.Note( +# content={ +# 'title': 'Thirty-ninth International Conference on Machine Learning', +# 'Official Venue Name': 'Thirty-ninth International Conference on Machine Learning', +# 'Abbreviated Venue Name': 'ICML 2023', +# 'Official Website URL': 'https://icml.cc', +# 'program_chair_emails': ['pc@icml.cc', 'pc3@icml.cc'], +# 'contact_email': 'pc@icml.cc', +# 'publication_chairs': 'Yes, our venue has Publication Chairs', +# 'publication_chairs_emails': ['publicationchair@icml.com'], +# 'Venue Start Date': '2023/07/01', +# 'Submission Deadline': request_form.content['Submission Deadline'], +# 'Location': 'Virtual', +# 'submission_reviewer_assignment': 'Automatic', +# 'How did you hear about us?': 'ML conferences', +# 'Expected Submissions': '100', +# 'Additional Submission Options': request_form.content['Additional Submission Options'], +# }, +# forum=request_form.forum, +# invitation='openreview.net/Support/-/Request{}/Revision'.format(request_form.number), +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# referent=request_form.forum, +# replyto=request_form.forum, +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) + +# helpers.await_queue() + +# pub_chair_group = openreview_client.get_group('ICML.cc/2024/Conference/Publication_Chairs') +# assert pub_chair_group and 'publicationchair@icml.com' in pub_chair_group.members + +# # check members have not changed +# authors_accepted_group = openreview_client.get_group('ICML.cc/2024/Conference/Authors/Accepted') +# assert len(authors_accepted_group.members) == num_accepted_papers + +# submissions = openreview_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', sort='number:asc') +# submission = submissions[0] + +# # assert PCs can't use Submission invitation after post decision is run +# pc_client_v2=openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) +# request_page(selenium, 'http://localhost:3030/forum?id={}'.format(submission.id), pc_client_v2.token, by=By.CLASS_NAME, wait_for_element='forum-note') +# note_div = selenium.find_element(By.CLASS_NAME, 'forum-note') +# assert note_div +# button_row = note_div.find_element(By.CLASS_NAME, 'invitation-buttons') +# assert button_row +# buttons = button_row.find_elements(By.CLASS_NAME, 'btn-xs') +# assert buttons[0].text == 'Edit ' +# buttons[0].click() +# time.sleep(0.5) +# dropdown = button_row.find_element(By.CLASS_NAME, 'dropdown-menu') +# dropdown_values = dropdown.find_elements(By.TAG_NAME, "a") +# values = [value.text for value in dropdown_values] +# assert ['Post Submission', 'PC Revision', 'Ethics Review Flag'] == values + +# #Post another post decision note +# now = datetime.datetime.utcnow() +# short_name = 'ICML 2023' +# post_decision_stage_note = pc_client.post_note(openreview.Note( +# content={ +# 'reveal_authors': 'Reveal author identities of only accepted submissions to the public', +# 'submission_readers': 'Make accepted submissions public and hide rejected submissions', +# 'hide_fields': ['supplementary_material', 'pdf'], +# 'home_page_tab_names': { +# 'Accept': 'Accept', +# 'Revision Needed': 'Revision Needed', +# 'Reject': 'Submitted' +# }, +# 'send_decision_notifications': 'Yes, send an email notification to the authors', +# 'accept_email_content': f'''Dear {{{{fullname}}}}, + +# Thank you for submitting your paper, {{{{submission_title}}}}, to {short_name}. We are delighted to inform you that your submission has been accepted. Congratulations! +# You can find the final reviews for your paper on the submission page in OpenReview at: {{{{forum_url}}}} + +# Best, +# {short_name} Program Chairs +# ''', +# 'reject_email_content': f'''Dear {{{{fullname}}}}, + +# Thank you for submitting your paper, {{{{submission_title}}}}, to {short_name}. We regret to inform you that your submission was not accepted. +# You can find the final reviews for your paper on the submission page in OpenReview at: {{{{forum_url}}}} + +# Best, +# {short_name} Program Chairs +# ''', +# 'revision_needed_email_content': f'''Dear {{{{fullname}}}}, + +# Thank you for submitting your paper, {{{{submission_title}}}}, to {short_name}. +# You can find the final reviews for your paper on the submission page in OpenReview at: {{{{forum_url}}}} + +# Best, +# {short_name} Program Chairs +# ''' +# }, +# forum=request_form.forum, +# invitation=invitation.id, +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# replyto=request_form.forum, +# referent=request_form.forum, +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) +# assert post_decision_stage_note +# helpers.await_queue() + +# process_logs = client.get_process_logs(id = post_decision_stage_note.id) +# assert len(process_logs) == 1 +# assert process_logs[0]['status'] == 'ok' + +# # check emails were not resent and decision emails status comment was not re-posted +# messages = openreview_client.get_messages(subject='[ICML 2023] Decision notification for your submission 1: Paper title 1 Version 2') +# assert len(messages) == 5 + +# replies = pc_client.get_notes(forum=request_form.id, invitation=f'openreview.net/Support/-/Request{request_form.number}/Comment') +# assert len(replies) == 20 + +# def test_forum_chat(self, openreview_client, helpers): + +# submission_invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/-/Submission') +# assert len(submission_invitation.reply_forum_views) + +# submission = openreview_client.get_notes(invitation='ICML.cc/2024/Conference/-/Submission', number=1)[0] + +# reviewer_client = openreview.api.OpenReviewClient(username='reviewer1@icml.cc', password=helpers.strong_password) + +# anon_groups = reviewer_client.get_groups(prefix='ICML.cc/2024/Conference/Submission1/Reviewer_', signatory='~Reviewer_ICMLOne1') +# anon_group_id = anon_groups[0].id + +# invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Submission1/-/Chat') +# assert invitation.date_processes[0].get('dates') == [] + +# note_edit = reviewer_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission1/-/Chat', +# signatures=[anon_group_id], +# note=openreview.api.Note( +# replyto=submission.id, +# content={ +# 'message': { 'value': 'Hi reviewers, I would like to discuss this paper with you.' } +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=note_edit['id']) + +# invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/Submission1/-/Chat') +# assert invitation.date_processes[0].get('dates') is None +# assert invitation.date_processes[0].get('cron') == '0 */4 * * *' + +# assert len(openreview_client.get_messages(to='reviewer1@icml.cc', subject='[ICML 2023] New conversation in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='reviewer2@icml.cc', subject='[ICML 2023] New conversation in committee members chat for submission 1: Paper title 1 Version 2')) == 1 +# assert len(openreview_client.get_messages(to='melisa@icml.cc', subject='[ICML 2023] New conversation in committee members chat for submission 1: Paper title 1 Version 2')) == 1 +# assert len(openreview_client.get_messages(to='reviewer3@icml.cc', subject='[ICML 2023] New conversation in committee members chat for submission 1: Paper title 1 Version 2')) == 1 +# assert len(openreview_client.get_messages(to='reviewer4@yahoo.com', subject='[ICML 2023] New conversation in committee members chat for submission 1: Paper title 1 Version 2')) == 1 +# assert len(openreview_client.get_messages(to='rachel_bis@icml.cc', subject='[ICML 2023] New conversation in committee members chat for submission 1: Paper title 1 Version 2')) == 1 +# assert len(openreview_client.get_messages(to='ac2@icml.cc', subject='[ICML 2023] New conversation in committee members chat for submission 1: Paper title 1 Version 2')) == 1 +# assert len(openreview_client.get_messages(to='sac2@icml.cc', subject='[ICML 2023] New conversation in committee members chat for submission 1: Paper title 1 Version 2')) == 1 +# assert len(openreview_client.get_messages(to='pc@icml.cc', subject='[ICML 2023] New conversation in committee members chat for submission 1: Paper title 1 Version 2')) == 0 + +# pc_client=openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) + +# note_edit = pc_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission1/-/Chat', +# signatures=['ICML.cc/2024/Conference/Program_Chairs'], +# note=openreview.api.Note( +# replyto=note_edit['note']['id'], +# content={ +# 'message': { 'value': 'Please start the conversation.' } +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=note_edit['id']) + +# assert len(openreview_client.get_messages(to='reviewer1@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='reviewer2@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='melisa@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='reviewer3@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='reviewer4@yahoo.com', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='rachel_bis@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='ac2@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='sac2@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='pc@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 + +# sac_client=openreview.api.OpenReviewClient(username='sac2@icml.cc', password=helpers.strong_password) + +# note_edit = sac_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission1/-/Chat', +# signatures=['ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs'], +# note=openreview.api.Note( +# replyto=note_edit['note']['id'], +# content={ +# 'message': { 'value': 'Chat comment number 3' } +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=note_edit['id']) + +# assert len(openreview_client.get_messages(to='reviewer1@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='reviewer2@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='melisa@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='reviewer3@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='reviewer4@yahoo.com', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='rachel_bis@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='ac2@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='sac2@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='pc@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 + +# note_edit = sac_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission1/-/Chat', +# signatures=['ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs'], +# note=openreview.api.Note( +# replyto=note_edit['note']['id'], +# content={ +# 'message': { 'value': 'Chat comment number 4' } +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=note_edit['id']) + +# assert len(openreview_client.get_messages(to='reviewer1@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='reviewer2@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='melisa@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='reviewer3@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='reviewer4@yahoo.com', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='rachel_bis@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='ac2@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='sac2@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='pc@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 + +# note_edit = sac_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission1/-/Chat', +# signatures=['ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs'], +# note=openreview.api.Note( +# replyto=note_edit['note']['id'], +# content={ +# 'message': { 'value': 'Chat comment number 5' } +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=note_edit['id']) + +# assert len(openreview_client.get_messages(to='reviewer1@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='reviewer2@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='melisa@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='reviewer3@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='reviewer4@yahoo.com', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='rachel_bis@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='ac2@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='sac2@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='pc@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 + +# note_edit = sac_client.post_note_edit( +# invitation='ICML.cc/2024/Conference/Submission1/-/Chat', +# signatures=['ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs'], +# note=openreview.api.Note( +# replyto=note_edit['note']['id'], +# content={ +# 'message': { 'value': 'Chat comment number 6' } +# } +# ) +# ) + +# helpers.await_queue_edit(openreview_client, edit_id=note_edit['id']) + +# assert len(openreview_client.get_messages(to='reviewer1@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 1 +# assert len(openreview_client.get_messages(to='reviewer2@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 1 +# assert len(openreview_client.get_messages(to='melisa@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 1 +# assert len(openreview_client.get_messages(to='reviewer3@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 1 +# assert len(openreview_client.get_messages(to='reviewer4@yahoo.com', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 1 +# assert len(openreview_client.get_messages(to='rachel_bis@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 1 +# assert len(openreview_client.get_messages(to='ac2@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 1 +# assert len(openreview_client.get_messages(to='sac2@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 +# assert len(openreview_client.get_messages(to='pc@icml.cc', subject='[ICML 2023] New messages in committee members chat for submission 1: Paper title 1 Version 2')) == 0 + +# ## Add tag emoji +# tag = sac_client.post_tag(openreview.api.Tag( +# invitation='ICML.cc/2024/Conference/Submission1/-/Chat_Reaction', +# signatures=['ICML.cc/2024/Conference/Submission1/Senior_Area_Chairs'], +# tag='😄', +# replyto=note_edit['note']['id'] +# )) + +# tags = openreview_client.get_tags(invitation='ICML.cc/2024/Conference/Submission1/-/Chat_Reaction', mintmdate=tag.tmdate - 5000) +# assert len(tags) == 1 + +# ## Disable chat +# pc_client=openreview.Client(username='pc@icml.cc', password=helpers.strong_password) +# request_form=pc_client.get_notes(invitation='openreview.net/Support/-/Request_Form')[0] + +# # Post an official comment stage note +# now = datetime.datetime.utcnow() +# start_date = now - datetime.timedelta(days=2) +# end_date = now + datetime.timedelta(days=3) +# comment_stage_note = pc_client.post_note(openreview.Note( +# content={ +# 'commentary_start_date': start_date.strftime('%Y/%m/%d'), +# 'commentary_end_date': end_date.strftime('%Y/%m/%d'), +# 'participants': ['Program Chairs', 'Assigned Senior Area Chairs', 'Assigned Area Chairs', 'Assigned Reviewers'], +# 'additional_readers': ['Program Chairs', 'Assigned Senior Area Chairs', 'Assigned Area Chairs', 'Assigned Reviewers', 'Assigned Submitted Reviewers'], +# 'email_program_chairs_about_official_comments': 'No, do not email PCs for each official comment made in the venue', +# 'enable_chat_between_committee_members': 'No, do not enable chat between committee members' +# }, +# forum=request_form.forum, +# invitation=f'openreview.net/Support/-/Request{request_form.number}/Comment_Stage', +# readers=['ICML.cc/2024/Conference/Program_Chairs', 'openreview.net/Support'], +# replyto=request_form.forum, +# referent=request_form.forum, +# signatures=['~Program_ICMLChair1'], +# writers=[] +# )) + +# helpers.await_queue() +# helpers.await_queue_edit(openreview_client, 'ICML.cc/2024/Conference/-/Official_Comment-0-1', count=1) +# helpers.await_queue_edit(openreview_client, 'ICML.cc/2024/Conference/-/Chat-0-1', count=2) +# helpers.await_queue_edit(openreview_client, 'ICML.cc/2024/Conference/-/Chat_Reaction-0-1', count=2) + +# chat_invitations = openreview_client.get_invitations(invitation='ICML.cc/2024/Conference/-/Chat') +# assert len(chat_invitations) == 0 + +# chat_reaction_invitations = openreview_client.get_invitations(invitation='ICML.cc/2024/Conference/-/Chat_Reaction') +# assert len(chat_reaction_invitations) == 0 + +# submission_invitation = openreview_client.get_invitation('ICML.cc/2024/Conference/-/Submission') +# assert submission_invitation.reply_forum_views is None \ No newline at end of file diff --git a/tests/test_journal.py b/tests/test_journal.py index 5cfeef88f..b6d53de2f 100644 --- a/tests/test_journal.py +++ b/tests/test_journal.py @@ -11,19 +11,12 @@ from openreview.api import Note from openreview.journal import Journal from openreview.journal import JournalRequest -from openreview import ProfileManagement from selenium.webdriver.common.by import By class TestJournal(): @pytest.fixture(scope="class") - def profile_management(self, openreview_client): - profile_management = ProfileManagement(openreview_client, 'openreview.net') - profile_management.setup() - return profile_management - - @pytest.fixture(scope="class") - def journal(self, openreview_client, helpers, profile_management): + def journal(self, openreview_client, helpers): venue_id = 'TMLR' fabian_client=OpenReviewClient(username='fabian@mail.com', password=helpers.strong_password) diff --git a/tests/test_profile_management.py b/tests/test_profile_management.py index 208d6503b..722fedecb 100644 --- a/tests/test_profile_management.py +++ b/tests/test_profile_management.py @@ -5,7 +5,6 @@ import time import re from selenium.webdriver.common.by import By -from openreview import ProfileManagement from openreview.api import OpenReviewClient from openreview.api import Note from openreview.journal import Journal @@ -14,14 +13,7 @@ class TestProfileManagement(): - - @pytest.fixture(scope="class") - def profile_management(self, openreview_client): - profile_management = ProfileManagement(openreview_client, 'openreview.net') - profile_management.setup() - return profile_management - - def test_import_dblp_notes(self, client, openreview_client, profile_management, test_client, helpers): + def test_import_dblp_notes(self, client, openreview_client, test_client, helpers): test_client_v2 = openreview.api.OpenReviewClient(username='test@mail.com', password=helpers.strong_password) @@ -346,7 +338,7 @@ def test_import_dblp_notes(self, client, openreview_client, profile_management, ] - def test_remove_alternate_name(self, openreview_client, profile_management, test_client, helpers): + def test_remove_alternate_name(self, openreview_client, test_client, helpers): john_client = helpers.create_user('john@profile.org', 'John', 'Last', alternates=[], institution='google.com') @@ -863,7 +855,7 @@ def test_request_remove_name_and_decline(self, openreview_client, helpers): The OpenReview Team. ''' - def test_remove_name_from_merged_profile(self, openreview_client, profile_management, helpers): + def test_remove_name_from_merged_profile(self, openreview_client, helpers): ella_client = helpers.create_user('ella@profile.org', 'Ella', 'Last', alternates=[], institution='google.com') @@ -1033,7 +1025,7 @@ def test_remove_name_from_merged_profile(self, openreview_client, profile_manage The OpenReview Team. ''' - def test_remove_duplicated_name(self, openreview_client, profile_management, helpers): + def test_remove_duplicated_name(self, openreview_client, helpers): javier_client = helpers.create_user('javier@profile.org', 'Javier', 'Last', alternates=[], institution='google.com') @@ -1181,7 +1173,7 @@ def test_remove_duplicated_name(self, openreview_client, profile_management, hel The OpenReview Team. ''' - def test_rename_publications_from_api2(self, profile_management, test_client, helpers, openreview_client): + def test_rename_publications_from_api2(self, test_client, helpers, openreview_client): journal=Journal(openreview_client, 'CABJ', '1234', contact_info='cabj@mail.org', full_name='Transactions on Machine Learning Research', short_name='CABJ', submission_name='Submission') journal.setup(support_role='test@mail.com', editors=[]) @@ -1508,7 +1500,7 @@ def test_rename_publications_from_api2(self, profile_management, test_client, he assert openreview_client.get_edges(invitation='CABJ/Reviewers/-/Assignment_Availability', tail='~Paul_Last1')[0].label == 'Unavailable' - def test_remove_name_and_update_relations(self, openreview_client, profile_management, helpers): + def test_remove_name_and_update_relations(self, openreview_client, helpers): juan_client = helpers.create_user('juan@profile.org', 'Juan', 'Last', alternates=[], institution='google.com') @@ -1618,7 +1610,7 @@ def test_remove_name_and_update_relations(self, openreview_client, profile_manag assert profile.content['relations'][1]['name'] == 'Juan Alternate Last' - def test_remove_name_and_accept_automatically(self, openreview_client, profile_management, helpers): + def test_remove_name_and_accept_automatically(self, openreview_client, helpers): nara_client = helpers.create_user('nara@profile.org', 'Nara', 'Last', alternates=[], institution='google.com') @@ -1656,7 +1648,7 @@ def test_remove_name_and_accept_automatically(self, openreview_client, profile_m note = nara_client.get_note(request_note['note']['id']) assert note.content['status']['value'] == 'Accepted' - def test_remove_name_and_do_not_accept_automatically(self, openreview_client, profile_management, helpers): + def test_remove_name_and_do_not_accept_automatically(self, openreview_client, helpers): mara_client = helpers.create_user('mara@profile.org', 'Mara', 'Last', alternates=[], institution='google.com') @@ -1711,7 +1703,7 @@ def test_remove_name_and_do_not_accept_automatically(self, openreview_client, pr assert note.content['status']['value'] == 'Pending' - def test_merge_profiles(self, openreview_client, profile_management, helpers): + def test_merge_profiles(self, openreview_client, helpers): rachel_client = helpers.create_user('rachel@profile.org', 'Rachel', 'Last', alternates=[], institution='google.com') profile = rachel_client.get_profile() @@ -1782,7 +1774,7 @@ def test_merge_profiles(self, openreview_client, profile_management, helpers): The OpenReview Team. ''' - def test_merge_profiles_as_guest(self, openreview_client, profile_management, helpers): + def test_merge_profiles_as_guest(self, openreview_client, helpers): helpers.create_user('marina@profile.org', 'Marina', 'Last', alternates=[], institution='google.com') helpers.create_user('marina@gmail.com', 'Marina', 'Last', alternates=[], institution='google.com') @@ -1845,7 +1837,7 @@ def test_merge_profiles_as_guest(self, openreview_client, profile_management, he ''' - def test_merge_profiles_ignore_request(self, openreview_client, profile_management, helpers): + def test_merge_profiles_ignore_request(self, openreview_client, helpers): melisa_client = helpers.create_user('melisa@profile.org', 'Melisa', 'Last', alternates=[], institution='google.com') profile = melisa_client.get_profile() @@ -1906,7 +1898,7 @@ def test_merge_profiles_ignore_request(self, openreview_client, profile_manageme messages = openreview_client.get_messages(to='melisa@profile.org', subject='Profile merge request has been accepted') assert len(messages) == 0 - def test_remove_email_address(self, profile_management, openreview_client, helpers): + def test_remove_email_address(self, openreview_client, helpers): harold_client = helpers.create_user('harold@profile.org', 'Harold', 'Last', alternates=[], institution='google.com') profile = harold_client.get_profile() @@ -2103,7 +2095,7 @@ def test_remove_email_address(self, profile_management, openreview_client, helpe assert openreview_client.get_group('harold@profile.org').members == ['~Harold_Last1'] - def test_update_relation_after_signup(self, openreview_client, profile_management, helpers): + def test_update_relation_after_signup(self, helpers): carlos_client = helpers.create_user('carlos@profile.org', 'Carlos', 'Last', alternates=[], institution='google.com') @@ -2155,7 +2147,7 @@ def test_update_relation_after_signup(self, openreview_client, profile_managemen assert profile.content['relations'][0]['username'] == '~Zoey_User1' assert 'email' not in profile.content['relations'][0] - def test_anonymous_preprint_server(self, profile_management, openreview_client, helpers): + def test_anonymous_preprint_server(self, openreview_client, helpers): clara_client = helpers.create_user('clara@profile.org', 'Clara', 'Last', alternates=[], institution='google.com') @@ -2236,7 +2228,7 @@ def test_anonymous_preprint_server(self, profile_management, openreview_client, assert len(messages) == 2 - def test_confirm_alternate_email(self, profile_management, openreview_client, helpers, request_page, selenium): + def test_confirm_alternate_email(self, openreview_client, helpers, request_page, selenium): xukun_client = helpers.create_user('xukun@profile.org', 'Xukun', 'First', alternates=[], institution='google.com') @@ -2288,7 +2280,7 @@ def test_confirm_alternate_email(self, profile_management, openreview_client, he content = selenium.find_element(By.ID, 'content') assert 'Click Confirm Email button below to confirm adding xukun@yahoo.com' in content.text - def test_merge_profiles_automatically(self, profile_management, openreview_client, helpers, request_page, selenium): + def test_merge_profiles_automatically(self, openreview_client, helpers, request_page, selenium): akshat_client_1 = helpers.create_user('akshat_1@profile.org', 'Akshat', 'First', alternates=[], institution='google.com') akshat_client_2 = helpers.create_user('akshat_2@profile.org', 'Akshat', 'Last', alternates=[], institution='google.com') @@ -2348,7 +2340,7 @@ def test_merge_profiles_automatically(self, profile_management, openreview_clien content = selenium.find_element(By.ID, 'content') assert 'Activation token is not valid' in content.text - def test_confirm_email_for_inactive_profile(self, profile_management, openreview_client, helpers, request_page, selenium): + def test_confirm_email_for_inactive_profile(self, openreview_client, helpers, request_page, selenium): guest = openreview.api.OpenReviewClient() res = guest.register_user(email = 'confirm_alternate@mail.com', fullname= 'Lionel Messi', password = helpers.strong_password) diff --git a/tests/test_venue_configuration.py b/tests/test_venue_configuration.py new file mode 100644 index 000000000..d110241d7 --- /dev/null +++ b/tests/test_venue_configuration.py @@ -0,0 +1,792 @@ +import pytest +import datetime +import openreview +from openreview.api import Note +from openreview.api import OpenReviewClient +from openreview.venue.configuration import VenueConfiguration + +class TestVenueConfiguration(): + + def test_venue_configuration_setup(self, openreview_client, helpers): + super_id = 'openreview.net' + support_group_id = super_id + '/Support' + + helpers.create_user('sherry@iclr.cc', 'ProgramChair', 'ICLR') + pc_client_v2=openreview.api.OpenReviewClient(username='sherry@iclr.cc', password=helpers.strong_password) + + assert openreview_client.get_invitation('openreview.net/-/Edit') + assert openreview_client.get_invitation('openreview.net/-/Venue_Configuration_Request') + assert openreview_client.get_invitation('openreview.net/-/Comment') + assert openreview_client.get_invitation('openreview.net/-/Deploy') + + now = datetime.datetime.utcnow() + start_date = now + datetime.timedelta(minutes=30) + due_date = now + datetime.timedelta(days=1) + + # post a conference request form + conference_request = openreview_client.post_note_edit(invitation='openreview.net/-/Venue_Configuration_Request', + signatures=['~Super_User1'], + note=openreview.api.Note( + content={ + 'official_venue_name': { 'value': 'The Thirteenth International Conference on Learning Representations' }, + 'abbreviated_venue_name': { 'value': 'ICLR 2025' }, + 'venue_website_url': { 'value': 'https://iclr.cc/Conferences/2025' }, + 'location': { 'value': 'Vienna, Austria' }, + 'venue_start_date': { 'value': now.strftime('%Y/%m/%d') }, + 'program_chair_emails': { 'value': ['sherry@iclr.cc'] }, + 'contact_email': { 'value': 'iclr2024.programchairs@gmail.com' }, + 'publication_chairs': { 'value': 'No, our venue does not have Publication Chairs' }, + 'area_chairs_and_senior_area_chairs': { 'value': 'Yes, our venue has Area Chairs and Senior Area Chairs' }, + 'ethics_chairs_and_reviewers': { 'value': 'No, our venue does not have Ethics Chairs and Reviewers' }, + 'secondary_area_chairs': { 'value': 'No, our venue does not have Secondary Area Chairs' }, + 'submission_start_date': { 'value': start_date.strftime('%Y/%m/%d %H:%M') }, + 'submission_deadline': { 'value': due_date.strftime('%Y/%m/%d %H:%M') }, + 'author_and_reviewer_anonymity': { 'value': 'Double-blind' }, + #'force_profiles_only': { 'value': 'No, allow submissions with email addresses' }, + #'submission_readers': { 'value': 'All program committee (all reviewers, all area chairs, all senior area chairs if applicable)' }, + 'submission_license': { 'value': ['CC BY-NC 4.0'] }, + #'email_pcs_for_new_submissions': { 'value': 'No, do not email PCs.' } + } + )) + + assert conference_request + helpers.await_queue_edit(openreview_client, invitation='openreview.net/-/Venue_Configuration_Request') + + request = openreview_client.get_note(conference_request['note']['id']) + assert openreview_client.get_invitation(f'openreview.net/Venue_Configuration_Request{request.number}/-/Comment') + assert openreview_client.get_invitation(f'openreview.net/Venue_Configuration_Request{request.number}/-/Deploy') + + openreview_client.post_note_edit(invitation=f'openreview.net/Venue_Configuration_Request{request.number}/-/Deploy', + signatures=[support_group_id], + note=openreview.api.Note( + id=request.id, + content={ + 'venue_id': { 'value': 'ICLR.cc/2025/Conference' } + } + )) + + helpers.await_queue_edit(openreview_client, invitation='openreview.net/Venue_Configuration_Request1/-/Deploy') + + assert openreview.tools.get_group(openreview_client, 'ICLR.cc/2025/Conference') + assert openreview.tools.get_group(openreview_client, 'ICLR.cc/2025') + submission_inv = openreview.tools.get_invitation(openreview_client, 'ICLR.cc/2025/Conference/-/Submission') + assert submission_inv and submission_inv.cdate == openreview.tools.datetime_millis(start_date.replace(second=0, microsecond=0)) + assert submission_inv.duedate == openreview.tools.datetime_millis(due_date.replace(second=0, microsecond=0)) + assert submission_inv.expdate == submission_inv.duedate + (30*60*1000) + submission_deadline_inv = openreview.tools.get_invitation(openreview_client, 'ICLR.cc/2025/Conference/-/Submission/Deadlines') + assert submission_deadline_inv and submission_inv.id in submission_deadline_inv.edit['invitation']['id'] + post_submission_inv = openreview.tools.get_invitation(openreview_client, 'ICLR.cc/2025/Conference/-/Post_Submission') + assert post_submission_inv and post_submission_inv.cdate == submission_inv.expdate + + assert openreview.tools.get_group(openreview_client, 'ICLR.cc/2025/Conference/Reviewers') + assert openreview_client.get_invitation('ICLR.cc/2025/Conference/Reviewers/-/Expertise_Selection') + assert openreview_client.get_invitation('ICLR.cc/2025/Conference/Reviewers/-/Matching_Setup') + assert openreview.tools.get_group(openreview_client, 'ICLR.cc/2025/Conference/Area_Chairs') + assert openreview_client.get_invitation('ICLR.cc/2025/Conference/Area_Chairs/-/Expertise_Selection') + assert openreview_client.get_invitation('ICLR.cc/2025/Conference/Area_Chairs/-/Matching_Setup') + assert openreview.tools.get_group(openreview_client, 'ICLR.cc/2025/Conference/Senior_Area_Chairs') + assert openreview_client.get_invitation('ICLR.cc/2025/Conference/Senior_Area_Chairs/-/Expertise_Selection') + assert openreview_client.get_invitation('ICLR.cc/2025/Conference/Senior_Area_Chairs/-/Matching_Setup') + + now = datetime.datetime.utcnow() + new_cdate = openreview.tools.datetime_millis(now - datetime.timedelta(days=3)) + new_duedate = openreview.tools.datetime_millis(now + datetime.timedelta(days=3)) + + # extend Submission duedate with Submission/Deadline invitation + pc_client_v2.post_invitation_edit( + invitations=submission_deadline_inv.id, + content={ + 'activation_date': { 'value': new_cdate }, + 'deadline': { 'value': new_duedate }, + #'expiration_date': { 'value': new_duedate } + } + ) + helpers.await_queue_edit(openreview_client, invitation='ICLR.cc/2025/Conference/-/Submission/Deadlines') + helpers.await_queue_edit(openreview_client, edit_id='ICLR.cc/2025/Conference/-/Post_Submission-0-1', count=2) + + # assert submission deadline and expdate get updated + submission_inv = openreview.tools.get_invitation(openreview_client, 'ICLR.cc/2025/Conference/-/Submission') + assert submission_inv and submission_inv.cdate == new_cdate + assert submission_inv.duedate == new_duedate + assert submission_inv.expdate == new_duedate + 1800000 + post_submission_inv = openreview.tools.get_invitation(openreview_client, 'ICLR.cc/2025/Conference/-/Post_Submission') + assert post_submission_inv and post_submission_inv.cdate == submission_inv.expdate + + matching_setup_inv = openreview.tools.get_invitation(openreview_client, 'ICLR.cc/2025/Conference/Reviewers/-/Matching_Setup') + assert matching_setup_inv and matching_setup_inv.cdate == submission_inv.expdate + + matching_setup_inv = openreview.tools.get_invitation(openreview_client, 'ICLR.cc/2025/Conference/Area_Chairs/-/Matching_Setup') + assert matching_setup_inv and matching_setup_inv.cdate == submission_inv.expdate + + matching_setup_inv = openreview.tools.get_invitation(openreview_client, 'ICLR.cc/2025/Conference/Senior_Area_Chairs/-/Matching_Setup') + assert matching_setup_inv and matching_setup_inv.cdate == submission_inv.expdate + + content_inv = openreview.tools.get_invitation(openreview_client, 'ICLR.cc/2025/Conference/-/Submission/Form_Fields') + assert content_inv + assert 'subject_area' not in submission_inv.edit['note']['content'] + assert 'keywords' in submission_inv.edit['note']['content'] + assert submission_inv.edit['note']['license'] == 'CC BY 4.0' + + ## edit Submission content with Submission/Form_Fields invitation + pc_client_v2.post_invitation_edit( + invitations=content_inv.id, + content = { + 'note_content': { + 'value': { + 'subject_area': { + 'order': 10, + "description": "Select one subject area.", + 'value': { + 'param': { + 'type': 'string', + 'enum': [ + "3D from multi-view and sensors", + "3D from single images", + "Adversarial attack and defense", + "Autonomous driving", + "Biometrics", + "Computational imaging", + "Computer vision for social good", + "Computer vision theory", + "Datasets and evaluation" + ], + "input": "select" + } + } + }, + 'keywords': { + 'delete': True + } + } + }, + 'note_license': { + 'value': [ + {'value': 'CC BY-NC-ND 4.0', 'optional': True, 'description': 'CC BY-NC-ND 4.0'}, + {'value': 'CC BY-NC-SA 4.0', 'optional': True, 'description': 'CC BY-NC-SA 4.0'} + ] + } + } + ) + + submission_inv = openreview.tools.get_invitation(openreview_client, 'ICLR.cc/2025/Conference/-/Submission') + assert submission_inv and 'subject_area' in submission_inv.edit['note']['content'] + assert 'keywords' not in submission_inv.edit['note']['content'] + content_keys = submission_inv.edit['note']['content'].keys() + assert all(field in content_keys for field in ['title', 'authors', 'authorids', 'TLDR', 'abstract', 'pdf']) + assert submission_inv.edit['note']['license']['param']['enum'] == [ + { + "value": "CC BY-NC-ND 4.0", + "optional": True, + "description": "CC BY-NC-ND 4.0" + }, + { + "value": "CC BY-NC-SA 4.0", + "optional": True, + "description": "CC BY-NC-SA 4.0" + } + ] + + notifications_inv = openreview.tools.get_invitation(openreview_client, 'ICLR.cc/2025/Conference/-/Submission/Notifications') + assert notifications_inv + assert 'email_authors' in submission_inv.content and submission_inv.content['email_authors']['value'] + assert 'email_pcs' in submission_inv.content and not submission_inv.content['email_pcs']['value'] + + ## edit Submission invitation content with Submission/Notifications invitation + pc_client_v2.post_invitation_edit( + invitations=notifications_inv.id, + content = { + 'email_authors': { 'value': False }, + 'email_pcs': { 'value': True } + } + ) + + submission_inv = openreview.tools.get_invitation(openreview_client, 'ICLR.cc/2025/Conference/-/Submission') + assert 'email_authors' in submission_inv.content and not submission_inv.content['email_authors']['value'] + assert 'email_pcs' in submission_inv.content and submission_inv.content['email_pcs']['value'] + + def test_post_submissions(self, openreview_client, test_client, helpers): + + test_client = openreview.api.OpenReviewClient(token=test_client.token) + + domains = ['umass.edu', 'amazon.com', 'fb.com', 'cs.umass.edu', 'google.com', 'mit.edu', 'deepmind.com', 'co.ux', 'apple.com', 'nvidia.com'] + for i in range(1,11): + note = openreview.api.Note( + license = 'CC BY-NC-SA 4.0', + content = { + 'title': { 'value': 'Paper title ' + str(i) }, + 'abstract': { 'value': 'This is an abstract ' + str(i) }, + 'authorids': { 'value': ['~SomeFirstName_User1', 'peter@mail.com', 'andrew@' + domains[i % 10]] }, + 'authors': { 'value': ['SomeFirstName User', 'Peter SomeLastName', 'Andrew Mc'] }, + 'subject_area': { 'value': '3D from multi-view and sensors' }, + 'pdf': {'value': '/pdf/' + 'p' * 40 +'.pdf' }, + } + ) + + test_client.post_note_edit(invitation='ICLR.cc/2025/Conference/-/Submission', + signatures=['~SomeFirstName_User1'], + note=note) + + helpers.await_queue_edit(openreview_client, invitation='ICLR.cc/2025/Conference/-/Submission', count=10) + + submissions = openreview_client.get_notes(invitation='ICLR.cc/2025/Conference/-/Submission') + assert len(submissions) == 10 + assert submissions[0].readers == ['ICLR.cc/2025/Conference', '~SomeFirstName_User1', 'peter@mail.com', 'andrew@umass.edu'] + + ## Setup submissions readers after the deadline + pc_client = openreview.api.OpenReviewClient(username='sherry@iclr.cc', password=helpers.strong_password) + + submission_readers_inv = openreview.tools.get_invitation(openreview_client, 'ICLR.cc/2025/Conference/-/Post_Submission/Submission_Readers') + assert submission_readers_inv + + pc_client.post_invitation_edit( + invitations=submission_readers_inv.id, + content = { + 'readers': { 'value': ['ICLR.cc/2025/Conference', 'ICLR.cc/2025/Conference/Submission${{2/id}/number}/Authors'] } + } + ) + + submission_field_readers_inv = openreview.tools.get_invitation(openreview_client, 'ICLR.cc/2025/Conference/-/Post_Submission/Restrict_Field_Visibility') + assert submission_field_readers_inv + + pc_client.post_invitation_edit( + invitations=submission_field_readers_inv.id, + content = { + 'author_readers': { 'value': ['ICLR.cc/2025/Conference', 'ICLR.cc/2025/Conference/Submission${{4/id}/number}/Authors'] }, + 'pdf_readers': { 'value': ['ICLR.cc/2025/Conference', 'ICLR.cc/2025/Conference/Submission${{4/id}/number}/Authors', 'ICLR.cc/2025/Conference/Submission${{4/id}/number}/Reviewers'] }, + } + ) + + now = datetime.datetime.utcnow() + new_cdate = openreview.tools.datetime_millis(now - datetime.timedelta(days=1)) + new_duedate = openreview.tools.datetime_millis(now - datetime.timedelta(minutes=28)) + + pc_client.post_invitation_edit( + invitations='ICLR.cc/2025/Conference/-/Submission/Deadlines', + content={ + 'activation_date': { 'value': new_cdate }, + 'deadline': { 'value': new_duedate }, + #'expiration_date': { 'value': new_duedate } + } + ) + helpers.await_queue_edit(openreview_client, invitation='ICLR.cc/2025/Conference/-/Submission/Deadlines') + helpers.await_queue_edit(openreview_client, edit_id='ICLR.cc/2025/Conference/-/Post_Submission-0-0', count=1) + + submissions = openreview_client.get_notes(invitation='ICLR.cc/2025/Conference/-/Submission', sort='number:asc') + assert len(submissions) == 10 + assert submissions[0].readers == ['ICLR.cc/2025/Conference', 'ICLR.cc/2025/Conference/Submission1/Authors'] + assert submissions[0].content['authors']['readers'] == ['ICLR.cc/2025/Conference', 'ICLR.cc/2025/Conference/Submission1/Authors'] + assert submissions[0].content['authorids']['readers'] == ['ICLR.cc/2025/Conference', 'ICLR.cc/2025/Conference/Submission1/Authors'] + assert submissions[0].content['pdf']['readers'] == ['ICLR.cc/2025/Conference', 'ICLR.cc/2025/Conference/Submission1/Authors', 'ICLR.cc/2025/Conference/Submission1/Reviewers'] + + def test_matching_setup(self, openreview_client, test_client, helpers): + + pc_client = openreview.api.OpenReviewClient(username='sherry@iclr.cc', password=helpers.strong_password) + + pc_client.add_members_to_group('ICLR.cc/2025/Conference/Reviewers', ['reviewer1@iclr.cc', 'reviewer2@iclr.cc', 'reviewer3@iclr.cc']) + + edit = pc_client.post_group_edit( + invitation='ICLR.cc/2025/Conference/Reviewers/-/Matching_Setup', + group=openreview.api.Group( + content={ + 'assignment_mode': { 'value': 'Automatic' }, + 'conflict_policy': { 'value': 'NeurIPS' } + }, + ) + ) + helpers.await_queue_edit(openreview_client, edit_id=edit['id']) + + assert openreview.tools.get_invitation(openreview_client, 'ICLR.cc/2025/Conference/Reviewers/-/Custom_Max_Papers') + assert openreview.tools.get_invitation(openreview_client, 'ICLR.cc/2025/Conference/Reviewers/-/Conflict') + assert openreview.tools.get_invitation(openreview_client, 'ICLR.cc/2025/Conference/Reviewers/-/Aggregate_Score') + assert not openreview.tools.get_invitation(openreview_client, 'ICLR.cc/2025/Conference/Reviewers/-/Affinity_Score') + assert openreview.tools.get_invitation(openreview_client, 'ICLR.cc/2025/Conference/Reviewers/-/Assignment_Configuration') + + + def test_review_stage(self, openreview_client, test_client, helpers): + + pc_client = openreview.api.OpenReviewClient(username='sherry@iclr.cc', password=helpers.strong_password) + assert pc_client.get_invitation('ICLR.cc/2025/Conference/-/Official_Review') + assert pc_client.get_invitation('ICLR.cc/2025/Conference/-/Official_Review/Deadlines') + assert pc_client.get_invitation('ICLR.cc/2025/Conference/-/Official_Review/Form_Fields') + assert pc_client.get_invitation('ICLR.cc/2025/Conference/-/Official_Review/Readers') + + # edit review stage fields + pc_client.post_invitation_edit( + invitations='ICLR.cc/2025/Conference/-/Official_Review/Form_Fields', + content = { + 'note_content': { + 'value': { + "summary": { + "order": 1, + "description": "Briefly summarize the paper and its contributions. This is not the place to critique the paper; the authors should generally agree with a well-written summary.", + "value": { + "param": { + "maxLength": 200000, + "type": "string", + "input": "textarea", + "markdown": True + } + } + }, + "review_rating": { + "order": 10, + "value": { + "param": { + "type": "string", + "enum": [ + "1: strong reject", + "3: reject, not good enough", + "5: marginally below the acceptance threshold", + "6: marginally above the acceptance threshold", + "8: accept, good paper", + "10: strong accept, should be highlighted at the conference" + ], + "input": "radio" + } + }, + "description": "Please provide an \"overall score\" for this submission." + }, + 'review_confidence': { + 'order': 4, + 'value': { + 'param': { + 'type': 'integer', + 'enum': [ + { 'value': 5, 'description': '5: The reviewer is absolutely certain that the evaluation is correct and very familiar with the relevant literature' }, + { 'value': 4, 'description': '4: The reviewer is confident but not absolutely certain that the evaluation is correct' }, + { 'value': 3, 'description': '3: The reviewer is fairly confident that the evaluation is correct' }, + { 'value': 2, 'description': '2: The reviewer is willing to defend the evaluation, but it is quite likely that the reviewer did not understand central parts of the paper' }, + { 'value': 1, 'description': '1: The reviewer\'s evaluation is an educated guess' } + ], + 'input': 'radio' + } + } + }, + 'title': { + 'delete': True + }, + 'review': { + 'delete': True + } + } + }, + 'rating_field_name': { + 'value': 'review_rating' + }, + 'confidence_field_name': { + 'value': 'review_confidence' + } + } + ) + + helpers.await_queue_edit(openreview_client, invitation=f'ICLR.cc/2025/Conference/-/Official_Review/Form_Fields') + + review_inv = openreview.tools.get_invitation(openreview_client, 'ICLR.cc/2025/Conference/-/Official_Review') + assert 'title' not in review_inv.edit['invitation']['edit']['note']['content'] + assert 'review' not in review_inv.edit['invitation']['edit']['note']['content'] + assert 'summary' in review_inv.edit['invitation']['edit']['note']['content'] + assert 'review_rating' in review_inv.edit['invitation']['edit']['note']['content'] and review_inv.edit['invitation']['edit']['note']['content']['review_rating']['value']['param']['enum'][0] == "1: strong reject" + assert 'review_confidence' in review_inv.edit['invitation']['edit']['note']['content'] + + group = openreview_client.get_group('ICLR.cc/2025/Conference') + assert 'review_rating' in group.content and group.content['review_rating']['value'] == 'review_rating' + assert 'review_confidence' in group.content and group.content['review_confidence']['value'] == 'review_confidence' + + ## edit Official Review readers + pc_client.post_invitation_edit( + invitations='ICLR.cc/2025/Conference/-/Official_Review/Readers', + content = { + 'reply_readers': { + 'value': [ + 'ICLR.cc/2025/Conference/Program_Chairs', + 'ICLR.cc/2025/Conference/Submission${5/content/noteNumber/value}/Senior_Area_Chairs', + 'ICLR.cc/2025/Conference/Submission${5/content/noteNumber/value}/Area_Chairs', + 'ICLR.cc/2025/Conference/Submission${5/content/noteNumber/value}/Reviewers' + ] + } + } + ) + + review_inv = openreview.tools.get_invitation(openreview_client, 'ICLR.cc/2025/Conference/-/Official_Review') + assert review_inv.edit['invitation']['edit']['note']['readers'] == [ + 'ICLR.cc/2025/Conference/Program_Chairs', + 'ICLR.cc/2025/Conference/Submission${5/content/noteNumber/value}/Senior_Area_Chairs', + 'ICLR.cc/2025/Conference/Submission${5/content/noteNumber/value}/Area_Chairs', + 'ICLR.cc/2025/Conference/Submission${5/content/noteNumber/value}/Reviewers' + ] + + # create child invitations + now = datetime.datetime.utcnow() + new_cdate = openreview.tools.datetime_millis(now - datetime.timedelta(minutes=30)) + new_duedate = openreview.tools.datetime_millis(now + datetime.timedelta(days=3)) + + pc_client.post_invitation_edit( + invitations='ICLR.cc/2025/Conference/-/Official_Review/Deadlines', + content={ + 'activation_date': { 'value': new_cdate }, + 'deadline': { 'value': new_duedate }, + 'expiration_date': { 'value': new_duedate } + } + ) + helpers.await_queue_edit(openreview_client, edit_id='ICLR.cc/2025/Conference/-/Official_Review-0-1', count=2) + + invitations = openreview_client.get_invitations(invitation='ICLR.cc/2025/Conference/-/Official_Review') + assert len(invitations) == 10 + + invitation = openreview_client.get_invitation('ICLR.cc/2025/Conference/Submission1/-/Official_Review') + assert invitation and invitation.edit['readers'] == [ + 'ICLR.cc/2025/Conference/Program_Chairs', + 'ICLR.cc/2025/Conference/Submission1/Senior_Area_Chairs', + 'ICLR.cc/2025/Conference/Submission1/Area_Chairs', + 'ICLR.cc/2025/Conference/Submission1/Reviewers' + ] + + def test_comment_stage(self, openreview_client, test_client, helpers): + + pc_client = openreview.api.OpenReviewClient(username='sherry@iclr.cc', password=helpers.strong_password) + + now = datetime.datetime.now() + cdate = openreview.tools.datetime_millis(now - datetime.timedelta(minutes=1)) + expdate = openreview.tools.datetime_millis(now + datetime.timedelta(minutes=28)) + + edit = pc_client.post_invitation_edit( + invitations='openreview.net/Support/-/Comment_Template', + signatures=['~ProgramChair_ICLR1'], + content={ + 'venue_id': { 'value': 'ICLR.cc/2025/Conference' }, + 'name': { 'value': 'Confidential_Comment' }, + 'activation_date': { 'value': cdate }, + 'expiration_date': { 'value': expdate }, + 'participants': { 'value': ['Program Chairs', 'Assigned Senior Area Chairs', 'Assigned Area Chairs'] }, + 'email_program_chairs_about_official_comments': { 'value': 'No, do not email PCs.' }, + 'email_senior_area_chairs_about_official_comments' : { 'value': 'Yes, email SACs for every new comment.' }, + } + ) + helpers.await_queue_edit(openreview_client, edit_id=edit['id'], count=1) + helpers.await_queue_edit(openreview_client, edit_id='ICLR.cc/2025/Conference/-/Confidential_Comment-0-1', count=1) + + super_invitation = pc_client.get_invitation('ICLR.cc/2025/Conference/-/Confidential_Comment') + assert super_invitation + assert pc_client.get_invitation('ICLR.cc/2025/Conference/-/Confidential_Comment/Deadlines') + assert pc_client.get_invitation('ICLR.cc/2025/Conference/-/Confidential_Comment/Form_Fields') + assert pc_client.get_invitation('ICLR.cc/2025/Conference/-/Confidential_Comment/Participants_and_Readers') + + invitation = pc_client.get_invitation('ICLR.cc/2025/Conference/Submission1/-/Confidential_Comment') + assert invitation + + assert not super_invitation.content['email_pcs']['value'] + assert super_invitation.content['email_sacs']['value'] + + assert invitation.invitees == ['ICLR.cc/2025/Conference', 'openreview.net/Support', 'ICLR.cc/2025/Conference/Submission1/Senior_Area_Chairs', 'ICLR.cc/2025/Conference/Submission1/Area_Chairs'] + + assert invitation.edit['note']['readers']['param']['items'] == [ + { + "value": "ICLR.cc/2025/Conference/Program_Chairs", + "optional": False + # "description": "Program Chairs." + }, + { + "value": "ICLR.cc/2025/Conference/Submission1/Senior_Area_Chairs", + "optional": False + # "description": "Assigned Senior Area Chairs" + }, + { + "value": "ICLR.cc/2025/Conference/Submission1/Area_Chairs", + "optional": True + # "description": "Assigned Area Chairs" + } + ] + + ## edit Confidential_Comment participants and readers + pc_client.post_invitation_edit( + invitations='ICLR.cc/2025/Conference/-/Confidential_Comment/Participants_and_Readers', + content = { + 'participants': { + 'value': [ + 'ICLR.cc/2025/Conference/Program_Chairs', + 'ICLR.cc/2025/Conference/Submission${3/content/noteNumber/value}/Senior_Area_Chairs', + 'ICLR.cc/2025/Conference/Submission${3/content/noteNumber/value}/Area_Chairs', + 'ICLR.cc/2025/Conference/Submission${3/content/noteNumber/value}/Authors' + ] + }, + 'reply_readers': { + 'value': [ + {'value': 'ICLR.cc/2025/Conference/Program_Chairs', 'optional': False, 'description': 'Program Chairs.'}, + {'value': 'ICLR.cc/2025/Conference/Submission${8/content/noteNumber/value}/Senior_Area_Chairs', 'optional': False, 'description': 'Assigned Senior Area Chairs'}, + {'value': 'ICLR.cc/2025/Conference/Submission${8/content/noteNumber/value}/Area_Chairs', 'optional': True, 'description': 'Assigned Area Chairs'}, + {'value': 'ICLR.cc/2025/Conference/Submission${8/content/noteNumber/value}/Reviewers/Submitted', 'optional': True, 'description': 'Assigned Reviewers who already submitted their review'} + ] + } + } + ) + helpers.await_queue_edit(openreview_client, edit_id='ICLR.cc/2025/Conference/-/Confidential_Comment-0-1', count=2) + + invitation = openreview_client.get_invitation('ICLR.cc/2025/Conference/Submission1/-/Confidential_Comment') + + assert invitation.invitees == ['ICLR.cc/2025/Conference/Program_Chairs', 'ICLR.cc/2025/Conference/Submission1/Senior_Area_Chairs', 'ICLR.cc/2025/Conference/Submission1/Area_Chairs', 'ICLR.cc/2025/Conference/Submission1/Authors'] + assert invitation.edit['note']['readers']['param']['items'] == [ + { + "value": "ICLR.cc/2025/Conference/Program_Chairs", + "optional": False, + "description": "Program Chairs." + }, + { + "value": "ICLR.cc/2025/Conference/Submission1/Senior_Area_Chairs", + "optional": False, + "description": "Assigned Senior Area Chairs" + }, + { + "value": "ICLR.cc/2025/Conference/Submission1/Area_Chairs", + "optional": True, + "description": "Assigned Area Chairs" + }, + { + "value": "ICLR.cc/2025/Conference/Submission1/Reviewers/Submitted", + "optional": True, + "description": "Assigned Reviewers who already submitted their review" + } + ] + + # edit notification settings + pc_client.post_invitation_edit( + invitations='ICLR.cc/2025/Conference/-/Confidential_Comment/Notifications', + content = { + 'email_pcs': { + 'value': True + }, + 'email_sacs': { + 'value': True + } + } + ) + helpers.await_queue_edit(openreview_client, edit_id='ICLR.cc/2025/Conference/-/Confidential_Comment-0-1', count=3) + + invitation = pc_client.get_invitation('ICLR.cc/2025/Conference/-/Confidential_Comment') + assert invitation + + assert invitation.content['email_pcs']['value'] + assert invitation.content['email_sacs']['value'] + + submissions = openreview_client.get_notes(invitation='ICLR.cc/2025/Conference/-/Submission', sort='number:asc') + pc_client.post_note_edit(invitation=f'ICLR.cc/2025/Conference/Submission1/-/Confidential_Comment', + signatures=['ICLR.cc/2025/Conference/Program_Chairs'], + note=openreview.api.Note( + replyto=submissions[0].id, + readers=['ICLR.cc/2025/Conference/Program_Chairs', 'ICLR.cc/2025/Conference/Submission1/Senior_Area_Chairs'], + content={ + 'comment': { 'value': 'this is a comment between PCs and SACs' } + } + )) + + def test_metareview_stage(self, openreview_client, test_client, helpers): + + pc_client = openreview.api.OpenReviewClient(username='sherry@iclr.cc', password=helpers.strong_password) + + now = datetime.datetime.utcnow() + cdate = openreview.tools.datetime_millis(now - datetime.timedelta(minutes=1)) + duedate = openreview.tools.datetime_millis(now + datetime.timedelta(days=10)) + expdate = openreview.tools.datetime_millis(now + datetime.timedelta(days=10)) + + edit = pc_client.post_invitation_edit( + invitations='openreview.net/Support/-/Meta_Review_Template', + signatures=['~ProgramChair_ICLR1'], + content={ + 'venue_id': { 'value': 'ICLR.cc/2025/Conference' }, + 'name': { 'value': 'MetaReview' }, + 'activation_date': { 'value': cdate }, + 'due_date': { 'value': duedate }, + 'expiration_date': { 'value': expdate }, + 'readers': { 'value': ['Program_Chairs', 'Senior_Area_Chairs', 'Area_Chairs', 'Reviewers/Submitted']}, + 'content': { + 'value': { + 'title': { + 'order': 1, + 'description': 'Title', + 'value': { + 'param': { + 'type': 'string', + 'const': 'Meta Review' + } + } + }, + 'metareview': { + 'order': 2, + 'description': 'Please provide an evaluation of the quality, clarity, originality and significance of this work, including a list of its pros and cons. Your comment or reply (max 5000 characters). Add formatting using Markdown and formulas using LaTeX. For more information see https://openreview.net/faq', + 'value': { + 'param': { + 'type': 'string', + 'maxLength': 5000, + 'markdown': True, + 'input': 'textarea' + } + } + }, + 'recommendation': { + 'order': 3, + 'value': { + 'param': { + 'type': 'string', + 'enum': [ + 'Accept', + 'Reject' + ], + 'input': 'radio' + } + } + }, + 'confidence': { + 'order': 4, + 'value': { + 'param': { + 'type': 'float', + 'enum': [ + { 'value': 3, 'description': '3: The area chair is absolutely certain' }, + { 'value': 2.5, 'description': '2.5: The area chair is confident but not absolutely certain' }, + { 'value': 2, 'description': '2: The area chair is somewhat confident' }, + { 'value': 1.5, 'description': '1.5: The area chair is not sure' }, + { 'value': 1.0, 'description': '1.0: The area chair\'s evaluation is an educated guess' } + ], + 'input': 'radio' + } + } + } + } + }, + 'recommendation_field_name': { 'value': 'recommendation' } + } + ) + helpers.await_queue_edit(openreview_client, edit_id=edit['id'], count=1) + + venue_group = openreview_client.get_group('ICLR.cc/2025/Conference') + assert 'meta_review_name' in venue_group.content and venue_group.content['meta_review_name']['value'] == 'MetaReview' + assert 'meta_review_recommendation' in venue_group.content and venue_group.content['meta_review_recommendation']['value'] == 'recommendation' + + assert pc_client.get_invitation('ICLR.cc/2025/Conference/-/MetaReview') + assert pc_client.get_invitation('ICLR.cc/2025/Conference/-/MetaReview/Deadlines') + assert pc_client.get_invitation('ICLR.cc/2025/Conference/-/MetaReview/Form_Fields') + assert pc_client.get_invitation('ICLR.cc/2025/Conference/-/MetaReview/Readers') + + helpers.await_queue_edit(openreview_client, edit_id='ICLR.cc/2025/Conference/-/MetaReview-0-1', count=1) + + #update recommendation field name + pc_client.post_invitation_edit( + invitations='ICLR.cc/2025/Conference/-/MetaReview/Form_Fields', + content = { + 'note_content': { + 'value': { + 'metareview': { + 'order': 1, + 'description': 'Please provide an evaluation of the quality, clarity, originality and significance of this work, including a list of its pros and cons. Your comment or reply (max 5000 characters). Add formatting using Markdown and formulas using LaTeX. For more information see https://openreview.net/faq', + 'value': { + 'param': { + 'type': 'string', + 'maxLength': 5000, + 'markdown': True, + 'input': 'textarea' + } + } + }, + "recommendation_final": { + 'order': 2, + 'value': { + 'param': { + 'type': 'string', + 'enum': [ + 'Accept (Oral)', + 'Accept (Poster)', + 'Reject' + ], + 'input': 'radio' + } + } + }, + 'confidence': { + 'order': 3, + 'value': { + 'param': { + 'type': 'integer', + 'enum': [ + { 'value': 5, 'description': '5: The area chair is absolutely certain' }, + { 'value': 4, 'description': '4: The area chair is confident but not absolutely certain' }, + { 'value': 3, 'description': '3: The area chair is somewhat confident' }, + { 'value': 2, 'description': '2: The area chair is not sure' }, + { 'value': 1, 'description': '1: The area chair\'s evaluation is an educated guess' } + ], + 'input': 'radio' + } + } + }, + 'title': { + 'delete': True + }, + 'recommendation': { + 'delete': True + } + } + }, + 'recommendation_field_name': { + 'value': 'recommendation_final' + } + } + ) + + helpers.await_queue_edit(openreview_client, edit_id='ICLR.cc/2025/Conference/-/MetaReview-0-1', count=2) + + venue_group = openreview_client.get_group('ICLR.cc/2025/Conference') + assert 'meta_review_recommendation' in venue_group.content and venue_group.content['meta_review_recommendation']['value'] == 'recommendation_final' + + invitations = pc_client.get_invitations(invitation='ICLR.cc/2025/Conference/-/MetaReview') + assert len(invitations) == 10 + + invitation = openreview_client.get_invitation('ICLR.cc/2025/Conference/Submission1/-/MetaReview') + assert invitation.edit['readers'] == [ + "ICLR.cc/2025/Conference/Submission1/Senior_Area_Chairs", + "ICLR.cc/2025/Conference/Submission1/Area_Chairs", + "ICLR.cc/2025/Conference/Submission1/Reviewers/Submitted", + "ICLR.cc/2025/Conference/Program_Chairs" + ] + assert 'title' not in invitation.edit['note']['content'] + assert 'recommendation_final' in invitation.edit['note']['content'] + assert invitation.edit['note']['content']['confidence']['value']['param']['type'] == 'integer' + + def test_decision_stage(self, openreview_client, test_client, helpers): + + pc_client = openreview.api.OpenReviewClient(username='sherry@iclr.cc', password=helpers.strong_password) + + now = datetime.datetime.utcnow() + cdate = openreview.tools.datetime_millis(now - datetime.timedelta(minutes=1)) + duedate = openreview.tools.datetime_millis(now + datetime.timedelta(days=5)) + + edit = pc_client.post_invitation_edit( + invitations='openreview.net/Support/-/Decision_Template', + signatures=['~ProgramChair_ICLR1'], + content={ + 'venue_id': { 'value': 'ICLR.cc/2025/Conference' }, + 'name': { 'value': 'Final_Decision' }, + 'activation_date': { 'value': cdate }, + 'due_date': { 'value': duedate }, + 'decision_options': { 'value': ['Accept', 'Revision Needed', 'Reject'] }, + 'readers': { 'value': ['Program Chairs', 'Assigned Senior Area Chairs', 'Assigned Area Chairs', 'Assigned Reviewers', 'Paper Authors']} + } + ) + helpers.await_queue_edit(openreview_client, edit_id=edit['id'], count=1) + + assert pc_client.get_invitation('ICLR.cc/2025/Conference/-/Final_Decision') + assert pc_client.get_invitation('ICLR.cc/2025/Conference/-/Final_Decision/Deadlines') + assert pc_client.get_invitation('ICLR.cc/2025/Conference/-/Final_Decision/Readers') + + helpers.await_queue_edit(openreview_client, edit_id='ICLR.cc/2025/Conference/-/Final_Decision-0-1', count=1) + + invitations = pc_client.get_invitations(invitation='ICLR.cc/2025/Conference/-/Final_Decision') + assert len(invitations) == 10 + + invitation = openreview_client.get_invitation('ICLR.cc/2025/Conference/Submission1/-/Final_Decision') + assert invitation.edit['readers'] == [ + "ICLR.cc/2025/Conference/Program_Chairs", + "ICLR.cc/2025/Conference/Submission1/Senior_Area_Chairs", + "ICLR.cc/2025/Conference/Submission1/Area_Chairs", + "ICLR.cc/2025/Conference/Submission1/Reviewers", + "ICLR.cc/2025/Conference/Submission1/Authors" + ] + assert 'decision' in invitation.edit['note']['content'] + assert invitation.edit['note']['content']['decision']['value']['param']['enum'] == ['Accept', 'Revision Needed', 'Reject'] \ No newline at end of file diff --git a/tests/test_venue_with_tracks.py b/tests/test_venue_with_tracks.py index 2b83a310d..7770a3c94 100644 --- a/tests/test_venue_with_tracks.py +++ b/tests/test_venue_with_tracks.py @@ -8,19 +8,10 @@ import csv from selenium.webdriver.common.by import By from selenium.common.exceptions import NoSuchElementException -from openreview import ProfileManagement class TestVenueWithTracks(): - - @pytest.fixture(scope="class") - def profile_management(self, openreview_client): - profile_management = ProfileManagement(openreview_client, 'openreview.net') - profile_management.setup() - return profile_management - - - def test_create_conference(self, client, openreview_client, helpers, profile_management): + def test_create_conference(self, client, openreview_client, helpers): now = datetime.datetime.utcnow() due_date = now + datetime.timedelta(days=3)