From 5148d87ed794b4d4f7c2d1901128eaffaf5a9821 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Sat, 24 Jun 2023 14:22:03 +0200 Subject: [PATCH 01/14] Add db --- .vscode/settings.json | 2 ++ server/database/schema.prisma | 27 +++++++++++++++++++- server/database/seed.ts | 47 +++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index a9716ce76..328f07117 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,10 +7,12 @@ "composables", "datetime", "esbuild", + "issn", "jiti", "journaltitle", "Nuxt", "nuxtjs", + "scimago", "transpiled", "tsyringe", "upsert" diff --git a/server/database/schema.prisma b/server/database/schema.prisma index 75b5b50bd..ae07a6e7e 100644 --- a/server/database/schema.prisma +++ b/server/database/schema.prisma @@ -110,11 +110,36 @@ model JournalIssue { model Journal { id String @id @default(cuid()) + isCustom Boolean issues JournalIssue[] name String subtitle String? titleAddon String? - issn String? + issn Int[] + scimagoId Int? + country String? + publisher String? + areas String[] + categories String[] + citationInfo JournalCitationInfoYearly[] + hIndex Int? +} + +model JournalCitationInfoYearly { + journalId String + journal Journal @relation(fields: [journalId], references: [id]) + year Int + + docsThisYear Int + docsPrevious3Years Int + citableDocsPrevious3Years Int + citesOutgoing Int + citesOutgoingPerDoc Float + citesIncomingByRecentlyPublished Int + citesIncomingPerDocByRecentlyPublished Float + sjrIndex Float + + @@id([journalId, year]) } model UserDocumentOtherField { diff --git a/server/database/seed.ts b/server/database/seed.ts index f2e542442..508809bb4 100644 --- a/server/database/seed.ts +++ b/server/database/seed.ts @@ -90,6 +90,7 @@ async function seedInternal(prisma: PrismaClientT): Promise { create: { id: 'ckslj094u000309jvdpng93mk', name: 'Circulation', + isCustom: true, }, }, volume: '119', @@ -172,6 +173,7 @@ async function seedInternal(prisma: PrismaClientT): Promise { create: { id: 'ckslj1heh000709jv5ja9dcyn', name: 'British Journal of Nutrition', + isCustom: true, }, }, volume: '99', @@ -258,6 +260,7 @@ async function seedInternal(prisma: PrismaClientT): Promise { create: { id: 'ckslj2ca3000b09jvdmyj6552', name: 'Nutrition & Metabolism', + isCustom: true, }, }, volume: '3', @@ -326,6 +329,50 @@ async function seedInternal(prisma: PrismaClientT): Promise { create: { id: 'ckslj3f10000f09jvc1xifgi9', name: 'Antioxidants & Redox Signaling', + isCustom: false, + issn: [15230864, 15577716], + scimagoId: 27514, + country: 'United States', + publisher: 'Mary Ann Liebert Inc.', + categories: [ + 'Biochemistry (Q1)', + 'Cell Biology (Q1)', + 'Clinical Biochemistry (Q1)', + 'Medicine (miscellaneous) (Q1)', + 'Molecular Biology (Q1)', + 'Physiology (Q1)', + ], + areas: [ + 'Biochemistry, Genetics and Molecular Biology', + 'Medicine', + ], + citationInfo: { + create: [ + { + year: 2022, + docsThisYear: 217, + docsPrevious3Years: 488, + citableDocsPrevious3Years: 487, + citesOutgoing: 19202, + citesOutgoingPerDoc: 130.63, + citesIncomingByRecentlyPublished: 3692, + citesIncomingPerDocByRecentlyPublished: 7.21, + sjrIndex: 1.706, + }, + { + year: 2021, + docsThisYear: 158, + docsPrevious3Years: 530, + citableDocsPrevious3Years: 530, + citesOutgoing: 24155, + citesOutgoingPerDoc: 152.88, + citesIncomingByRecentlyPublished: 4724, + citesIncomingPerDocByRecentlyPublished: 7.59, + sjrIndex: 1.832, + }, + ], + }, + hIndex: 217, }, }, volume: '15', From 31946b8a625b261f74d0974614d823c94aed2980 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Mon, 26 Jun 2023 02:01:58 +0200 Subject: [PATCH 02/14] add graphql --- .../migrations/20230625222959_/migration.sql | 36 +++ .../documents/JournalArticle/schema.graphql | 125 +--------- server/documents/integration.test.ts | 6 +- server/documents/resolvers.ts | 3 +- .../documents/user.document.service.spec.ts | 9 +- server/documents/user.document.service.ts | 2 +- server/journals/journal.service.spec.ts | 56 +++++ server/journals/journal.service.ts | 52 +++++ server/journals/resolvers.ts | 46 ++++ server/journals/schema.graphql | 220 ++++++++++++++++++ server/resolvers.ts | 2 + server/tsyringe.config.ts | 6 + server/tsyringe.ts | 6 + 13 files changed, 445 insertions(+), 124 deletions(-) create mode 100644 server/database/migrations/20230625222959_/migration.sql create mode 100644 server/journals/journal.service.spec.ts create mode 100644 server/journals/journal.service.ts create mode 100644 server/journals/resolvers.ts create mode 100644 server/journals/schema.graphql diff --git a/server/database/migrations/20230625222959_/migration.sql b/server/database/migrations/20230625222959_/migration.sql new file mode 100644 index 000000000..65e2bb1b3 --- /dev/null +++ b/server/database/migrations/20230625222959_/migration.sql @@ -0,0 +1,36 @@ +/* + Warnings: + + - The `issn` column on the `Journal` table would be dropped and recreated. This will lead to data loss if there is data in the column. + - Added the required column `isCustom` to the `Journal` table without a default value. This is not possible if the table is not empty. + +*/ +-- AlterTable +ALTER TABLE "Journal" ADD COLUMN "areas" TEXT[], +ADD COLUMN "categories" TEXT[], +ADD COLUMN "country" TEXT, +ADD COLUMN "hIndex" INTEGER, +ADD COLUMN "isCustom" BOOLEAN NOT NULL, +ADD COLUMN "publisher" TEXT, +ADD COLUMN "scimagoId" INTEGER, +DROP COLUMN "issn", +ADD COLUMN "issn" INTEGER[]; + +-- CreateTable +CREATE TABLE "JournalCitationInfoYearly" ( + "journalId" TEXT NOT NULL, + "year" INTEGER NOT NULL, + "docsThisYear" INTEGER NOT NULL, + "docsPrevious3Years" INTEGER NOT NULL, + "citableDocsPrevious3Years" INTEGER NOT NULL, + "citesOutgoing" INTEGER NOT NULL, + "citesOutgoingPerDoc" DOUBLE PRECISION NOT NULL, + "citesIncomingByRecentlyPublished" INTEGER NOT NULL, + "citesIncomingPerDocByRecentlyPublished" DOUBLE PRECISION NOT NULL, + "sjrIndex" DOUBLE PRECISION NOT NULL, + + CONSTRAINT "JournalCitationInfoYearly_pkey" PRIMARY KEY ("journalId","year") +); + +-- AddForeignKey +ALTER TABLE "JournalCitationInfoYearly" ADD CONSTRAINT "JournalCitationInfoYearly_journalId_fkey" FOREIGN KEY ("journalId") REFERENCES "Journal"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/server/documents/JournalArticle/schema.graphql b/server/documents/JournalArticle/schema.graphql index 241f85e0b..4796e92b8 100644 --- a/server/documents/JournalArticle/schema.graphql +++ b/server/documents/JournalArticle/schema.graphql @@ -1,121 +1,3 @@ -type Journal { - id: ID! - - """ - The name of the journal. - - Biblatex: journaltitle - """ - name: String! - - """ - The subtitle of a journal. - - Biblatex: journalsubtitle - """ - subtitle: String - - """ - An annex to the name of the journal. - This may be useful in case a journal has been renamed or if the journal name isn't unique. - - Biblatex: journaltitleaddon - """ - titleAddon: String - - """ - The International Standard Serial Number of a journal. - - Biblatex: issn - """ - issn: String -} - -input AddJournalInput { - name: String! - subtitle: String - titleAddon: String - issn: String -} - -""" -An issue of a journal. -""" -type JournalIssue implements Node { - id: ID! - - """ - The journal in which the article has been published. - """ - journal: Journal - - """ - The title of a specific issue of a journal. - - Biblatex: issuetitle - """ - title: String - - """ - The subtitle of a specific issue of a journal. - - Biblatex: issuesubtitle - """ - subtitle: String - - """ - An annex to the title of the specific issue of a journal. - This may be useful when a special issue of a journal has a title that doesn't make it clear that it is a special issue and one wants to emphasize that. - - Biblatex: issuetitleaddon - """ - titleAddon: String - - """ - The number of the issue. - - Normally this field will be an integer or an integer range, but it may also be a short designator that is not entirely numeric such as “S1”, “Suppl. 2”, “3es”. - Usually, the number is displayed close to the volume, e.g. 10.2 (volume: 10, number: 2). - - Biblatex: number - """ - number: String - - """ - This field is intended for journals whose individual issues are identified by a designation such as Lent, Michaelmas, Summer or Spring rather than the month or a number. - Usually the issue name is displayed in front of the year and not the volume. - - Biblatex: issue - """ - name: String - - """ - The name or number of a journal series. - Usually, after the journal has restarted publication with a new numbering. - - Biblatex: series - """ - series: String - - """ - The volume of the journal this issue is part of. - - Biblatex: volume - """ - volume: String -} - -input AddJournalIssueInput { - journal: AddJournalInput! - title: String - subtitle: String - titleAddon: String - number: String - name: String - series: String - volume: String -} - """ An article published in a journal disseminates the results of original research and scholarship. It is usually peer-reviewed and published under a separate title in a journal issue or periodical containing other works of the same form. @@ -199,3 +81,10 @@ input AddJournalArticleInput { annotators: [AddEntityInput!] commentators: [AddEntityInput!] } + +extend type Query { + """ + Retrieve a journal by its ID, ISSN or name + """ + journal(id: ID, issn: Int, name: String): Journal +} diff --git a/server/documents/integration.test.ts b/server/documents/integration.test.ts index bec96d6f8..946f60d4a 100644 --- a/server/documents/integration.test.ts +++ b/server/documents/integration.test.ts @@ -144,7 +144,7 @@ describe('query', () => { "id": "ckslizms5000109jv3yx80ujf", "journal": { "id": "ckslj094u000309jvdpng93mk", - "issn": null, + "issn": [], "name": "Circulation", "subtitle": null, "titleAddon": null, @@ -303,7 +303,7 @@ describe('roundtrip', () => { "electronicId": null, "in": { "journal": { - "issn": null, + "issn": [], "name": "Journal of great things", "subtitle": null, "titleAddon": null, @@ -408,7 +408,7 @@ describe('roundtrip', () => { "electronicId": null, "in": { "journal": { - "issn": null, + "issn": [], "name": "Journal of great things", "subtitle": null, "titleAddon": null, diff --git a/server/documents/resolvers.ts b/server/documents/resolvers.ts index 52f9da2a4..97ba7650d 100644 --- a/server/documents/resolvers.ts +++ b/server/documents/resolvers.ts @@ -128,7 +128,8 @@ function convertDocumentInput( name: document.in.journal.name, subtitle: document.in.journal.subtitle, titleAddon: document.in.journal.titleAddon, - issn: document.in.journal.issn, + issn: document.in.journal.issn ?? [], + isCustom: true, }, }, title: document.in.title, diff --git a/server/documents/user.document.service.spec.ts b/server/documents/user.document.service.spec.ts index 4d1b4aa6d..0e9c786bf 100644 --- a/server/documents/user.document.service.spec.ts +++ b/server/documents/user.document.service.spec.ts @@ -53,9 +53,16 @@ const testDocument: UserDocument = { journal: { id: 'test_journal', name: 'Test Journal', - issn: null, + issn: [], subtitle: null, titleAddon: null, + isCustom: true, + scimagoId: null, + country: null, + publisher: null, + areas: [], + categories: [], + hIndex: null, }, }, pageStart: null, diff --git a/server/documents/user.document.service.ts b/server/documents/user.document.service.ts index 0e4a7f678..b676c30cf 100644 --- a/server/documents/user.document.service.ts +++ b/server/documents/user.document.service.ts @@ -22,7 +22,7 @@ import { inject, injectable } from './../tsyringe' export type UserDocument = PlainUserDocument & { other?: UserDocumentOtherField[] - journalIssue?: + journalIssue: | (JournalIssue & { journal: Journal | null }) diff --git a/server/journals/journal.service.spec.ts b/server/journals/journal.service.spec.ts new file mode 100644 index 000000000..2dd0000d8 --- /dev/null +++ b/server/journals/journal.service.spec.ts @@ -0,0 +1,56 @@ +import type { Journal, PrismaClient } from '@prisma/client' +import { mockDeep, mockReset } from 'vitest-mock-extended' +import { register, resolve } from '../tsyringe' + +const prisma = mockDeep() +register('PrismaClient', { useValue: prisma }) +const journalService = resolve('JournalService') + +const testJournal: Journal = { + id: 'test', + name: 'Test Journal', + issn: [12345678], + subtitle: null, + titleAddon: null, + isCustom: true, + scimagoId: null, + country: null, + publisher: null, + areas: [], + categories: [], + hIndex: null, +} + +describe('userDocumentService', () => { + beforeEach(() => { + mockReset(prisma) + }) + + describe('getJournalById', () => { + it('should return a journal with the given ISSN', async () => { + prisma.journal.findFirst.mockResolvedValue(testJournal) + const journal = await journalService.getJournalByIssn(testJournal.issn[0]) + expect(journal).toBeDefined() + expect(journal).toEqual(testJournal) + expect(prisma.journal.findFirst).toBeCalledWith({ + where: { + issn: { has: testJournal.issn[0] }, + isCustom: false, + }, + }) + }) + + it('should return null if no journal with the given ISSN is found', async () => { + prisma.journal.findFirst.mockResolvedValue(null) + const issn = 0 + const journal = await journalService.getJournalByIssn(issn) + expect(journal).toBeNull() + expect(prisma.journal.findFirst).toBeCalledWith({ + where: { + issn: { has: issn }, + isCustom: false, + }, + }) + }) + }) +}) diff --git a/server/journals/journal.service.ts b/server/journals/journal.service.ts new file mode 100644 index 000000000..bf2a252cb --- /dev/null +++ b/server/journals/journal.service.ts @@ -0,0 +1,52 @@ +import { PrismaClient } from '@prisma/client' +import { inject, injectable } from './../tsyringe' + +@injectable() +export class JournalService { + constructor(@inject('PrismaClient') private prisma: PrismaClient) {} + + async getJournalById(id: string) { + return ( + (await this.prisma.journal.findUnique({ + where: { + id, + }, + })) ?? null + ) + } + + async getJournalByIssn(issn: number) { + return ( + (await this.prisma.journal.findFirst({ + where: { + issn: { + has: issn, + }, + isCustom: false, + }, + })) ?? null + ) + } + + async getJournalByName(name: string) { + return ( + (await this.prisma.journal.findFirst({ + where: { + name: { + equals: name, + mode: 'insensitive', + }, + isCustom: false, + }, + })) ?? null + ) + } + + async getCitationInfoYearly(id: string) { + return await this.prisma.journalCitationInfoYearly.findMany({ + where: { + journalId: id, + }, + }) + } +} diff --git a/server/journals/resolvers.ts b/server/journals/resolvers.ts new file mode 100644 index 000000000..bedbe8728 --- /dev/null +++ b/server/journals/resolvers.ts @@ -0,0 +1,46 @@ +import { Journal, QueryJournalArgs, Resolvers } from '#graphql/resolver' +import { Context } from '../context' +import { inject, injectable, resolve } from './../tsyringe' +import { JournalService } from './journal.service' + +@injectable() +export class JournalResolver { + constructor( + @inject('JournalService') + private journalService: JournalService + ) {} + + async citationInfo(journal: Journal) { + return await this.journalService.getCitationInfoYearly(journal.id) + } +} + +@injectable() +export class Query { + constructor( + @inject('JournalService') + private journalService: JournalService + ) {} + + async journal( + _root: Record, + { id, issn, name }: QueryJournalArgs, + _context: Context + ): Promise { + if (id) { + return await this.journalService.getJournalById(id) + } else if (issn) { + return await this.journalService.getJournalByIssn(issn) + } else if (name) { + return await this.journalService.getJournalByName(name) + } + throw new Error('No id, issn or name given') + } +} + +export function resolvers(): Resolvers { + return { + Query: resolve('JournalQuery'), + Journal: resolve('JournalResolver'), + } +} diff --git a/server/journals/schema.graphql b/server/journals/schema.graphql new file mode 100644 index 000000000..61ce1a958 --- /dev/null +++ b/server/journals/schema.graphql @@ -0,0 +1,220 @@ +type Journal { + id: ID! + + """ + The name of the journal. + + Biblatex: journaltitle + """ + name: String! + + """ + The subtitle of a journal. + + Biblatex: journalsubtitle + """ + subtitle: String + + """ + An annex to the name of the journal. + This may be useful in case a journal has been renamed or if the journal name isn't unique. + + Biblatex: journaltitleaddon + """ + titleAddon: String + + """ + The International Standard Serial Numbers of a journal. + + Biblatex: issn + """ + issn: [Int!] + + """ + Specifies whether the information about this journal is user-defined or imported from an external source (like scimagojr) + + Biblatex: no equivalent + """ + isCustom: Boolean! + + """ + The Scimago Journal Rank (SJR) ID of the journal + + Biblatex: no equivalent + """ + scimagoId: Int + + """ + The country of the journal + + Biblatex: no equivalent + """ + country: String + + """ + The publisher of the journal + + Biblatex: no equivalent (?) + """ + publisher: String + + """ + The research areas covered by the journal + + Biblatex: no equivalent + """ + areas: [String!] + + """ + The categories of the journal + + Biblatex: no equivalent + """ + categories: [String!] + + """ + The yearly citation information of the journal + + Biblatex: no equivalent + """ + citationInfo: [JournalCitationInfoYearly!] + + """ + The h-index of the journal + + Biblatex: no equivalent + """ + hIndex: Int +} + +type JournalCitationInfoYearly { + """ + The year for which the citation information is provided + """ + year: Int! + + """ + The total number of documents published in the selected year + """ + docsThisYear: Int! + + """ + The total number of documents published in the three previous years (selected year documents are excluded) + """ + docsPrevious3Years: Int! + + """ + The number of citable documents published by a journal in the three previous years (selected year documents are excluded) + """ + citableDocsPrevious3Years: Int! + + """ + The total number of references to other documents in the selected year + """ + citesOutgoing: Int! + + """ + Average number of references to other documents per document in the selected year + """ + citesOutgoingPerDoc: Float! + + """ + The total number of citations received in the selected year by documents published in the three previous years + """ + citesIncomingByRecentlyPublished: Int! + + """ + The average number of citations received in the selected year per document that was published in the three previous years + """ + citesIncomingPerDocByRecentlyPublished: Float! + + """ + The SCImago Journal Rank (SJR) indicator, which expresses the average number of weighted citations received in the selected year of the documents published in the selected journal in the three previous years + """ + sjrIndex: Float! +} + +input AddJournalInput { + name: String! + subtitle: String + titleAddon: String + issn: [Int!] +} + +""" +An issue of a journal. +""" +type JournalIssue implements Node { + id: ID! + + """ + The journal in which the article has been published. + """ + journal: Journal + + """ + The title of a specific issue of a journal. + + Biblatex: issuetitle + """ + title: String + + """ + The subtitle of a specific issue of a journal. + + Biblatex: issuesubtitle + """ + subtitle: String + + """ + An annex to the title of the specific issue of a journal. + This may be useful when a special issue of a journal has a title that doesn't make it clear that it is a special issue and one wants to emphasize that. + + Biblatex: issuetitleaddon + """ + titleAddon: String + + """ + The number of the issue. + + Normally this field will be an integer or an integer range, but it may also be a short designator that is not entirely numeric such as “S1”, “Suppl. 2”, “3es”. + Usually, the number is displayed close to the volume, e.g. 10.2 (volume: 10, number: 2). + + Biblatex: number + """ + number: String + + """ + This field is intended for journals whose individual issues are identified by a designation such as Lent, Michaelmas, Summer or Spring rather than the month or a number. + Usually the issue name is displayed in front of the year and not the volume. + + Biblatex: issue + """ + name: String + + """ + The name or number of a journal series. + Usually, after the journal has restarted publication with a new numbering. + + Biblatex: series + """ + series: String + + """ + The volume of the journal this issue is part of. + + Biblatex: volume + """ + volume: String +} + +input AddJournalIssueInput { + journal: AddJournalInput! + title: String + subtitle: String + titleAddon: String + number: String + name: String + series: String + volume: String +} diff --git a/server/resolvers.ts b/server/resolvers.ts index 4e8fbfeea..1d0eead59 100644 --- a/server/resolvers.ts +++ b/server/resolvers.ts @@ -3,6 +3,7 @@ import { mergeResolvers } from '@graphql-tools/merge' import { DateTimeResolver, EmailAddressResolver } from 'graphql-scalars' import { resolvers as documentResolvers } from './documents/resolvers' import { resolvers as groupResolvers } from './groups/resolvers' +import { resolvers as journalResolvers } from './journals/resolvers' import { resolvers as userResolvers } from './user/resolvers' export function loadResolvers(): Resolvers { @@ -10,6 +11,7 @@ export function loadResolvers(): Resolvers { userResolvers(), documentResolvers(), groupResolvers(), + journalResolvers(), { // Custom scalar types DateTime: DateTimeResolver, diff --git a/server/tsyringe.config.ts b/server/tsyringe.config.ts index bd8a0eaa2..c8ed3df50 100644 --- a/server/tsyringe.config.ts +++ b/server/tsyringe.config.ts @@ -5,6 +5,8 @@ import * as DocumentResolvers from './documents/resolvers' import { UserDocumentService } from './documents/user.document.service' import * as GroupResolvers from './groups/resolvers' import { GroupService } from './groups/service' +import { JournalService } from './journals/journal.service' +import * as JournalResolvers from './journals/resolvers' import { instanceCachingFactory, register } from './tsyringe' import { AuthService } from './user/auth.service' import PassportInitializer from './user/passport-initializer' @@ -33,6 +35,7 @@ export function registerClasses(): void { register('UserDocumentService', UserDocumentService) register('AuthService', AuthService) register('GroupService', GroupService) + register('JournalService', JournalService) // Resolvers register('DocumentQuery', DocumentResolvers.Query) register('DocumentMutation', DocumentResolvers.Mutation) @@ -49,6 +52,9 @@ export function registerClasses(): void { register('GroupMutation', GroupResolvers.Mutation) register('GroupResolver', GroupResolvers.GroupResolver) + register('JournalQuery', JournalResolvers.Query) + register('JournalResolver', JournalResolvers.JournalResolver) + register('UserQuery', UserResolvers.Query) register('UserMutation', UserResolvers.Mutation) register('UserResolver', UserResolvers.UserResolver) diff --git a/server/tsyringe.ts b/server/tsyringe.ts index c370b651d..aba30a6b8 100644 --- a/server/tsyringe.ts +++ b/server/tsyringe.ts @@ -16,6 +16,8 @@ import type * as DocumentResolvers from './documents/resolvers' import type { UserDocumentService } from './documents/user.document.service' import type * as GroupResolvers from './groups/resolvers' import type { GroupService } from './groups/service' +import type { JournalService } from './journals/journal.service' +import type * as JournalResolvers from './journals/resolvers' import type { AuthService } from './user/auth.service' import type PassportInitializer from './user/passport-initializer' import type * as UserResolvers from './user/resolvers' @@ -51,6 +53,7 @@ export const InjectionSymbols = { ...injectSymbol('UserDocumentService')(), ...injectSymbol('AuthService')(), ...injectSymbol('GroupService')(), + ...injectSymbol('JournalService')(), // Resolvers ...injectSymbol('DocumentQuery')(), ...injectSymbol('DocumentMutation')(), @@ -70,6 +73,9 @@ export const InjectionSymbols = { ...injectSymbol('GroupMutation')(), ...injectSymbol('GroupResolver')(), + ...injectSymbol('JournalQuery')(), + ...injectSymbol('JournalResolver')(), + ...injectSymbol('UserQuery')(), ...injectSymbol('UserMutation')(), ...injectSymbol('UserResolver')(), From 84d4059e6d17944ebc9e7656056d8ee2c9da6fad Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Mon, 26 Jun 2023 21:28:02 +0200 Subject: [PATCH 03/14] fix tests --- server/database/seed.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server/database/seed.ts b/server/database/seed.ts index 508809bb4..f4c986716 100644 --- a/server/database/seed.ts +++ b/server/database/seed.ts @@ -8,6 +8,7 @@ async function seedInternal(prisma: PrismaClientT): Promise { await prisma.user.deleteMany({}) await prisma.userDocument.deleteMany({}) await prisma.userDocumentOtherField.deleteMany({}) + await prisma.journalCitationInfoYearly.deleteMany({}) await prisma.journal.deleteMany({}) await prisma.journalIssue.deleteMany({}) await prisma.group.deleteMany({}) @@ -100,7 +101,7 @@ async function seedInternal(prisma: PrismaClientT): Promise { pageStart: '1433', pageEnd: '1441', publishedAt: '2009', - revisionHash: 'd1265c25d1d45905fc832b9185273aa8', + revisionHash: 'a574258637e9c610636f8c0941734a8a', lastModified: '2021-01-01T00:00:00.000Z', added: '2000-01-01T00:00:00.000Z', }, @@ -183,7 +184,7 @@ async function seedInternal(prisma: PrismaClientT): Promise { pageStart: '1', pageEnd: '11', publishedAt: '2008', - revisionHash: '9c32fd7b106729e7c68275f4e80c178c', + revisionHash: 'd8d6128eaadb987b3c7da7c3c2ece0e9', lastModified: '2021-05-28T12:00:00.000Z', added: '2000-01-01T00:00:00.000Z', }, @@ -268,7 +269,7 @@ async function seedInternal(prisma: PrismaClientT): Promise { }, }, publishedAt: '2006', - revisionHash: '837cd3388b8dcf732f3d1d9dde4d71a0', + revisionHash: '348b9cc86344780ad3b0cd3b9dcc20e6', lastModified: '2022-01-01T00:00:00.000Z', added: '2000-01-01T00:00:00.000Z', }, @@ -382,7 +383,7 @@ async function seedInternal(prisma: PrismaClientT): Promise { pageStart: '2779', pageEnd: '2811', publishedAt: '2011', - revisionHash: 'a751c468e36521f98fb7fb4aac3042c8', + revisionHash: '7682d66bf288d23ab7c4299c58807023', lastModified: '2020-12-01T00:00:00.000Z', added: '2000-01-01T00:00:00.000Z', }, From cf7fed1b2630b84b945c0be70ba9a8eec7c98490 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Tue, 27 Jun 2023 12:57:39 +0200 Subject: [PATCH 04/14] use string instead of int for issn --- server/database/schema.prisma | 2 +- server/database/seed.ts | 2 +- server/documents/JournalArticle/schema.graphql | 2 +- server/journals/journal.service.spec.ts | 4 ++-- server/journals/journal.service.ts | 4 ++-- server/journals/schema.graphql | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/server/database/schema.prisma b/server/database/schema.prisma index ae07a6e7e..add158982 100644 --- a/server/database/schema.prisma +++ b/server/database/schema.prisma @@ -115,7 +115,7 @@ model Journal { name String subtitle String? titleAddon String? - issn Int[] + issn String[] scimagoId Int? country String? publisher String? diff --git a/server/database/seed.ts b/server/database/seed.ts index f4c986716..b5562a87f 100644 --- a/server/database/seed.ts +++ b/server/database/seed.ts @@ -331,7 +331,7 @@ async function seedInternal(prisma: PrismaClientT): Promise { id: 'ckslj3f10000f09jvc1xifgi9', name: 'Antioxidants & Redox Signaling', isCustom: false, - issn: [15230864, 15577716], + issn: ['15230864', '15577716'], scimagoId: 27514, country: 'United States', publisher: 'Mary Ann Liebert Inc.', diff --git a/server/documents/JournalArticle/schema.graphql b/server/documents/JournalArticle/schema.graphql index 4796e92b8..dcf9a00b7 100644 --- a/server/documents/JournalArticle/schema.graphql +++ b/server/documents/JournalArticle/schema.graphql @@ -86,5 +86,5 @@ extend type Query { """ Retrieve a journal by its ID, ISSN or name """ - journal(id: ID, issn: Int, name: String): Journal + journal(id: ID, issn: String, name: String): Journal } diff --git a/server/journals/journal.service.spec.ts b/server/journals/journal.service.spec.ts index 2dd0000d8..567bd8c69 100644 --- a/server/journals/journal.service.spec.ts +++ b/server/journals/journal.service.spec.ts @@ -9,7 +9,7 @@ const journalService = resolve('JournalService') const testJournal: Journal = { id: 'test', name: 'Test Journal', - issn: [12345678], + issn: ['12345678'], subtitle: null, titleAddon: null, isCustom: true, @@ -42,7 +42,7 @@ describe('userDocumentService', () => { it('should return null if no journal with the given ISSN is found', async () => { prisma.journal.findFirst.mockResolvedValue(null) - const issn = 0 + const issn = '00000000' const journal = await journalService.getJournalByIssn(issn) expect(journal).toBeNull() expect(prisma.journal.findFirst).toBeCalledWith({ diff --git a/server/journals/journal.service.ts b/server/journals/journal.service.ts index bf2a252cb..902072b5a 100644 --- a/server/journals/journal.service.ts +++ b/server/journals/journal.service.ts @@ -15,12 +15,12 @@ export class JournalService { ) } - async getJournalByIssn(issn: number) { + async getJournalByIssn(issn: string) { return ( (await this.prisma.journal.findFirst({ where: { issn: { - has: issn, + has: issn.replaceAll('-', '').toLowerCase(), }, isCustom: false, }, diff --git a/server/journals/schema.graphql b/server/journals/schema.graphql index 61ce1a958..2c01e9f90 100644 --- a/server/journals/schema.graphql +++ b/server/journals/schema.graphql @@ -28,7 +28,7 @@ type Journal { Biblatex: issn """ - issn: [Int!] + issn: [String!] """ Specifies whether the information about this journal is user-defined or imported from an external source (like scimagojr) @@ -138,7 +138,7 @@ input AddJournalInput { name: String! subtitle: String titleAddon: String - issn: [Int!] + issn: [String!] } """ From bb13e55ca68697791e8794ad6edf363c504279ff Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Tue, 27 Jun 2023 20:00:55 +0200 Subject: [PATCH 05/14] fix db issn type --- server/database/migrations/20230627180037_/migration.sql | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 server/database/migrations/20230627180037_/migration.sql diff --git a/server/database/migrations/20230627180037_/migration.sql b/server/database/migrations/20230627180037_/migration.sql new file mode 100644 index 000000000..44309c229 --- /dev/null +++ b/server/database/migrations/20230627180037_/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Journal" ALTER COLUMN "issn" SET DATA TYPE TEXT[]; From 0692abd574d09c8c7e86046a67f63b5a97d10c4a Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Mon, 10 Jul 2023 16:11:23 +0200 Subject: [PATCH 06/14] fix revision hash --- server/database/seed.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/database/seed.ts b/server/database/seed.ts index b5562a87f..8011a9a49 100644 --- a/server/database/seed.ts +++ b/server/database/seed.ts @@ -383,7 +383,7 @@ async function seedInternal(prisma: PrismaClientT): Promise { pageStart: '2779', pageEnd: '2811', publishedAt: '2011', - revisionHash: '7682d66bf288d23ab7c4299c58807023', + revisionHash: 'eaf18ca3259f277a05a8790df6bca28f', lastModified: '2020-12-01T00:00:00.000Z', added: '2000-01-01T00:00:00.000Z', }, From eb2ce77361ed50c4fbb4d1659643050a91099ac6 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 28 Jul 2023 09:40:17 +0200 Subject: [PATCH 07/14] Add script to get journal data and update db Co-authored-by: Nitin Suresh --- .gitignore | 2 + scripts/journaldata.py | 303 ++++++++++++++++++ .../migrations/20230718213438_/migration.sql | 8 + .../migrations/20230718214704_/migration.sql | 2 + server/database/schema.prisma | 2 +- server/database/seed.ts | 2 +- 6 files changed, 317 insertions(+), 2 deletions(-) create mode 100644 scripts/journaldata.py create mode 100644 server/database/migrations/20230718213438_/migration.sql create mode 100644 server/database/migrations/20230718214704_/migration.sql diff --git a/.gitignore b/.gitignore index 47071f0d5..97be2857d 100644 --- a/.gitignore +++ b/.gitignore @@ -151,6 +151,8 @@ apollo/introspection.ts apollo/fragment-masking.ts apollo/validation.internal.ts +scripts/journal-data/ + # Yarn: https://yarnpkg.com/getting-started/qa#which-files-should-be-gitignored .pnp.* .yarn/* diff --git a/scripts/journaldata.py b/scripts/journaldata.py new file mode 100644 index 000000000..86bf402ce --- /dev/null +++ b/scripts/journaldata.py @@ -0,0 +1,303 @@ +""" +This script downloads data for multiple years from the Scimago Journal Rank website +(https://www.scimagojr.com/journalrank.php), parses the CSV files, and builds a consolidated +dataset over all the years, in JSON format. +The downloaded data includes various metrics for academic journals such as SJR, +h-index, doc counts, citation counts, etc. + +Usage: +- Add + ``` + generator pyclient { + provider = "prisma-client-py" + recursive_type_depth = 5 + } + ``` + to the `schema.prisma` file, and run `yarn generate` to generate the Prisma client. + +- Update the `current_year` variable to the latest year of data available. +- Set the environment variable `DATABASE_URL` to the postgres database url (using a .env file is recommended). +- If you want to use the Azure database, add your IP address to the Azure exception list under `jabrefdb | Networking`. +- Run this script with `download` argument to downloads data from the specified start year up to the current year. +- Run this script with `db` (or `json`) argument to dump the consolidated dataset in the database (or `scimagojr_combined_data.json`, respectively). +""" + + +import asyncio +import csv +import json +import os +import sys +import urllib.request +from pathlib import Path + +from prisma import Prisma +from prisma.types import JournalCitationInfoYearlyCreateWithoutRelationsInput + +# current_year should be the latest year of data available at https://www.scimagojr.com/journalrank.php +current_year = 2022 +start_year = 1999 +data_directory = Path('scripts/journal-data') + + +class JournalInfoYearly: + def __init__( + self, + sjr: float, + hIndex: int, + totalDocs: int, + totalDocs3Years: int, + totalRefs: int, + totalCites3Years: int, + citableDocs3Years: int, + citesPerDoc2Years: float, + refPerDoc: float, + ): + self.sjr = sjr + self.hIndex = hIndex + self.totalDocs = totalDocs + self.totalDocs3Years = totalDocs3Years + self.totalRefs = totalRefs + self.totalCites3Years = totalCites3Years + self.citableDocs3Years = citableDocs3Years + self.citesPerDoc2Years = citesPerDoc2Years + self.refPerDoc = refPerDoc + + +class JournalInfo: + def __init__( + self, + source_id: int, + issn: str, + title: str, + type: str, + country: str, + region: str, + publisher: str, + coverage: str, + categories: str, + areas: str, + ): + self.source_id = source_id + self.issn = issn + self.title = title + self.type = type + self.country = country + self.region = region + self.publisher = publisher + self.coverage = coverage + self.categories = categories + self.areas = areas + self.yearly: dict[int, JournalInfoYearly] = {} + + +def journal_url(year: int): + """Get url to download info for the given year""" + return f'https://www.scimagojr.com/journalrank.php?year={year}&out=xls' + + +def parse_float(value: str): + """Parse float from string, replacing comma with dot""" + try: + float_val = float(value.replace(',', '.')) + return float_val + except ValueError: + return 0.0 + + +def parse_int(value: str): + """Parse int from string""" + try: + int_val = int(value) + return int_val + except ValueError: + return 0 + + +def get_data_filepath(year: int): + """Get filename for the given year""" + return data_directory / f'scimagojr-journal-{year}.csv' + + +def download_all_data(): + """Download data for all years""" + + # create data directory if it doesn't exist + if not os.path.exists(data_directory): + os.makedirs(data_directory) + + for year in range(start_year, current_year + 1): + # download file for given year + print(f'Downloading data for {year}') + url = journal_url(year) + filepath = get_data_filepath(year) + urllib.request.urlretrieve(url, filepath) + + +def combine_data(): + """Iterate over files and return the consolidated dataset""" + journals: dict[int, JournalInfo] = {} + for year in range(start_year, current_year + 1): + print(f'Processing {year}') + filepath = get_data_filepath(year) + with open(filepath, mode='r', encoding='utf-8') as csv_file: + csv_reader = csv.DictReader(csv_file, delimiter=';') + for row in csv_reader: + # Columns present in the csv: + # 'Rank', 'Sourceid', 'Title', 'Type', 'Issn', 'SJR', 'SJR Best Quartile', 'H index', + # 'Total Docs. (2020)', 'Total Docs. (3years)', 'Total Refs.', 'Total Cites (3years)', + # 'Citable Docs. (3years)', 'Cites / Doc. (2years)', 'Ref. / Doc.', 'Country', 'Region', + # 'Publisher', 'Coverage', 'Categories', 'Areas' + + sourceId = parse_int(row['Sourceid']) + issn = row['Issn'] + if issn == '-': + issn = '' + hIndex = parse_int(row['H index']) + sjr = parse_float(row['SJR']) + totalDocs = parse_int(row[f'Total Docs. ({year})']) + totalDocs3Years = parse_int(row['Total Docs. (3years)']) + totalRefs = parse_int(row['Total Refs.']) + totalCites3Years = parse_int(row['Total Cites (3years)']) + citableDocs3Years = parse_int(row['Citable Docs. (3years)']) + citesPerDoc2Years = parse_float(row['Cites / Doc. (2years)']) + refPerDoc = parse_float(row['Ref. / Doc.']) + + if sourceId not in journals: + # populate non-varying fields + journals[sourceId] = JournalInfo( + source_id=sourceId, + issn=issn, + title=row['Title'], + type=row['Type'], + country=row['Country'], + region=row['Region'], + publisher=row['Publisher'], + coverage=row['Coverage'], + categories=row['Categories'], + areas=row['Areas'], + ) + # populate yearly varying fields + info = journals[sourceId] + info.yearly[year] = JournalInfoYearly( + sjr=sjr, + hIndex=hIndex, + totalDocs=totalDocs, + totalDocs3Years=totalDocs3Years, + totalRefs=totalRefs, + totalCites3Years=totalCites3Years, + citableDocs3Years=citableDocs3Years, + citesPerDoc2Years=citesPerDoc2Years, + refPerDoc=refPerDoc, + ) + + print(f'Number of journals collected: {len(journals)}') + return journals + + +def dump_to_json(journals: dict[int, JournalInfo]): + # write to json file + print('Writing to json') + with open( + data_directory / 'scimagojr_combined_data.json', 'w', encoding='utf-8' + ) as fp: + json.dump(journals, fp, default=vars) + + +async def dump_into_database(journals: dict[int, JournalInfo]): + """Save data from json file to postgres database""" + db = Prisma() + await db.connect() + + # delete all existing yearly data (because its easier than updating) + await db.journalcitationinfoyearly.delete_many() + + for journal in journals.values(): + citation_info: list[JournalCitationInfoYearlyCreateWithoutRelationsInput] = [ + { + 'year': year, + 'docsThisYear': info.totalDocs, + 'docsPrevious3Years': info.totalDocs3Years, + 'citableDocsPrevious3Years': info.citableDocs3Years, + 'citesOutgoing': info.totalCites3Years, + 'citesOutgoingPerDoc': info.citesPerDoc2Years, + 'citesIncomingByRecentlyPublished': info.totalRefs, + 'citesIncomingPerDocByRecentlyPublished': info.refPerDoc, + 'sjrIndex': info.sjr, + } + for year, info in journal.yearly.items() + ] + + await db.journal.upsert( + where={'scimagoId': journal.source_id}, + data={ + 'create': { + 'scimagoId': journal.source_id, + 'isCustom': False, + 'name': journal.title, + 'issn': journal.issn.split(','), + 'country': journal.country, + 'publisher': journal.publisher, + 'areas': journal.areas.split(','), + 'categories': journal.categories.split(','), + 'hIndex': next( + iter(journal.yearly.values()) + ).hIndex, # they are constant + 'citationInfo': {'create': citation_info}, + }, + 'update': { + 'scimagoId': journal.source_id, + 'isCustom': False, + 'name': journal.title, + 'issn': journal.issn.split(','), + 'country': journal.country, + 'publisher': journal.publisher, + 'areas': journal.areas.split(','), + 'categories': journal.categories.split(','), + 'hIndex': next( + iter(journal.yearly.values()) + ).hIndex, # they are constant + 'citationInfo': {'create': citation_info}, + }, + }, + ) + + await db.disconnect() + + +def find_duplicate_issn(): + """Find journals with duplicate issn""" + journals = combine_data() + issn_count: dict[str, list[str]] = {} + for journal in journals.values(): + for issn in journal.issn.split(','): + if issn == '': + continue + journal_list = issn_count.get(issn, []) + journal_list.append(journal.title) + issn_count[issn] = journal_list + + for issn, titles in issn_count.items(): + if len(titles) > 1: + print(issn, titles) + + +def main(argv: list[str]): + """Main function""" + if len(argv) == 1: + print("No arguments provided") + elif argv[1] == "download": + download_all_data() + elif argv[1] == "json": + dump_to_json(combine_data()) + elif argv[1] == "db": + data = combine_data() + asyncio.run(dump_into_database(data)) + elif argv[1] == "duplicates": + find_duplicate_issn() + else: + print("Invalid argument provided") + + +if __name__ == "__main__": + main(sys.argv) diff --git a/server/database/migrations/20230718213438_/migration.sql b/server/database/migrations/20230718213438_/migration.sql new file mode 100644 index 000000000..ecaa1bbde --- /dev/null +++ b/server/database/migrations/20230718213438_/migration.sql @@ -0,0 +1,8 @@ +/* + Warnings: + + - A unique constraint covering the columns `[scimagoId]` on the table `Journal` will be added. If there are existing duplicate values, this will fail. + +*/ +-- CreateIndex +CREATE UNIQUE INDEX "Journal_scimagoId_key" ON "Journal"("scimagoId"); diff --git a/server/database/migrations/20230718214704_/migration.sql b/server/database/migrations/20230718214704_/migration.sql new file mode 100644 index 000000000..b4925ec92 --- /dev/null +++ b/server/database/migrations/20230718214704_/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Journal" ALTER COLUMN "scimagoId" SET DATA TYPE BIGINT; diff --git a/server/database/schema.prisma b/server/database/schema.prisma index add158982..23a0f5b8e 100644 --- a/server/database/schema.prisma +++ b/server/database/schema.prisma @@ -116,7 +116,7 @@ model Journal { subtitle String? titleAddon String? issn String[] - scimagoId Int? + scimagoId BigInt? @unique country String? publisher String? areas String[] diff --git a/server/database/seed.ts b/server/database/seed.ts index 8011a9a49..fab0cbba4 100644 --- a/server/database/seed.ts +++ b/server/database/seed.ts @@ -372,8 +372,8 @@ async function seedInternal(prisma: PrismaClientT): Promise { sjrIndex: 1.832, }, ], + hIndex: 217, }, - hIndex: 217, }, }, volume: '15', From 9aa8394f20e2c0ee721d43aa680016f7d27801ef Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 28 Jul 2023 09:47:10 +0200 Subject: [PATCH 08/14] fix linter --- server/journals/resolvers.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/journals/resolvers.ts b/server/journals/resolvers.ts index bedbe8728..a823ebb68 100644 --- a/server/journals/resolvers.ts +++ b/server/journals/resolvers.ts @@ -7,7 +7,7 @@ import { JournalService } from './journal.service' export class JournalResolver { constructor( @inject('JournalService') - private journalService: JournalService + private journalService: JournalService, ) {} async citationInfo(journal: Journal) { @@ -19,13 +19,13 @@ export class JournalResolver { export class Query { constructor( @inject('JournalService') - private journalService: JournalService + private journalService: JournalService, ) {} async journal( _root: Record, { id, issn, name }: QueryJournalArgs, - _context: Context + _context: Context, ): Promise { if (id) { return await this.journalService.getJournalById(id) From 24a9e14a23f4f0c40fea57694769a3911d83ed6b Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 28 Jul 2023 10:25:43 +0200 Subject: [PATCH 09/14] fix seed --- server/database/seed.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/database/seed.ts b/server/database/seed.ts index fab0cbba4..8011a9a49 100644 --- a/server/database/seed.ts +++ b/server/database/seed.ts @@ -372,8 +372,8 @@ async function seedInternal(prisma: PrismaClientT): Promise { sjrIndex: 1.832, }, ], - hIndex: 217, }, + hIndex: 217, }, }, volume: '15', From 1e083fbe0a8df5e2f2826b4bdf2e2b785484c1fe Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 28 Jul 2023 17:30:31 +0200 Subject: [PATCH 10/14] fix bigint --- graphql.config.json | 6 ++++-- server/api/index.ts | 7 +++++++ server/documents/schema.graphql | 3 --- server/journals/schema.graphql | 2 +- server/resolvers.ts | 7 ++++++- server/schema.graphql | 5 +++++ server/user/schema.graphql | 2 -- 7 files changed, 23 insertions(+), 9 deletions(-) diff --git a/graphql.config.json b/graphql.config.json index dbec0e25a..aa6e1b7d4 100644 --- a/graphql.config.json +++ b/graphql.config.json @@ -21,7 +21,8 @@ "scalars": { "Date": "Date", "DateTime": "Date", - "EmailAddress": "string" + "EmailAddress": "string", + "BigInt": "BigInt" } } }, @@ -42,7 +43,8 @@ "scalarSchemas": { "Date": "z.date()", "DateTime": "z.date()", - "EmailAddress": "z.string().email()" + "EmailAddress": "z.string().email()", + "BigInt": "z.bigint()" }, "importFrom": "~/apollo/graphql", "validationSchemaExportType": "const" diff --git a/server/api/index.ts b/server/api/index.ts index 50d7a7f03..5750a293a 100644 --- a/server/api/index.ts +++ b/server/api/index.ts @@ -81,6 +81,13 @@ http.IncomingMessage.Readable.prototype.unpipe = function (dest) { return this } +// Fix for BigInt not being supported by JSON.stringify by default +// @ts-expect-error: we add json stringify to BigInt +// eslint-disable-next-line no-extend-native +BigInt.prototype.toJSON = function () { + return this.toString() +} + export default defineLazyEventHandler(async () => { const server = new ApolloServer({ schema: await loadSchemaWithResolvers(), diff --git a/server/documents/schema.graphql b/server/documents/schema.graphql index 37377e17c..36c909a51 100644 --- a/server/documents/schema.graphql +++ b/server/documents/schema.graphql @@ -1,6 +1,3 @@ -scalar DateTime -scalar Date - """ Ways in which to filter a list of documents. """ diff --git a/server/journals/schema.graphql b/server/journals/schema.graphql index 2c01e9f90..9eabe01fa 100644 --- a/server/journals/schema.graphql +++ b/server/journals/schema.graphql @@ -42,7 +42,7 @@ type Journal { Biblatex: no equivalent """ - scimagoId: Int + scimagoId: BigInt """ The country of the journal diff --git a/server/resolvers.ts b/server/resolvers.ts index 1d0eead59..8f7324d5f 100644 --- a/server/resolvers.ts +++ b/server/resolvers.ts @@ -1,6 +1,10 @@ import { Resolvers } from '#graphql/resolver' import { mergeResolvers } from '@graphql-tools/merge' -import { DateTimeResolver, EmailAddressResolver } from 'graphql-scalars' +import { + DateTimeResolver, + EmailAddressResolver, + BigIntResolver, +} from 'graphql-scalars' import { resolvers as documentResolvers } from './documents/resolvers' import { resolvers as groupResolvers } from './groups/resolvers' import { resolvers as journalResolvers } from './journals/resolvers' @@ -16,6 +20,7 @@ export function loadResolvers(): Resolvers { // Custom scalar types DateTime: DateTimeResolver, EmailAddress: EmailAddressResolver, + BigInt: BigIntResolver, }, ]) } diff --git a/server/schema.graphql b/server/schema.graphql index 251216305..d6246a638 100644 --- a/server/schema.graphql +++ b/server/schema.graphql @@ -1,3 +1,8 @@ +scalar EmailAddress +scalar DateTime +scalar Date +scalar BigInt + type Query { # eslint-disable-next-line @graphql-eslint/naming-convention -- this is a workaround anyhow to not have an empty type _empty: String diff --git a/server/user/schema.graphql b/server/user/schema.graphql index e570b4bca..99535fafb 100644 --- a/server/user/schema.graphql +++ b/server/user/schema.graphql @@ -1,5 +1,3 @@ -scalar EmailAddress - extend type Query { """ Get user by id. From cb0e9132e09f80b198281683bdd1ac713210c71b Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 28 Jul 2023 17:48:55 +0200 Subject: [PATCH 11/14] use json-bigint-patch lib --- nuxt.config.ts | 4 ++-- package.json | 1 + server/api/index.ts | 8 +------- test/global.setup.ts | 1 + yarn.lock | 8 ++++++++ 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/nuxt.config.ts b/nuxt.config.ts index 41fda3650..839642133 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -18,8 +18,8 @@ export default defineNuxtConfig({ }, nitro: { - // Prevent 'reflect-metadata' from being treeshaked (since we don't explicitly use the import it would otherwise be removed) - moduleSideEffects: ['reflect-metadata'], + // Prevent 'reflect-metadata' and 'json-bigint-patch' from being treeshaked (since we don't explicitly use the import it would otherwise be removed) + moduleSideEffects: ['reflect-metadata', 'json-bigint-patch'], prerender: { // Needed for storybook support (otherwise the file is not created during nuxi generate) routes: ['/_storybook/external-iframe'], diff --git a/package.json b/package.json index 0d6eec886..1a87fb371 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "graphql": "^16.7.1", "graphql-passport": "^0.6.5", "graphql-scalars": "^1.22.2", + "json-bigint-patch": "^0.0.8", "lodash": "^4.17.21", "nodemailer": "^6.8.0", "passport": "^0.6.0", diff --git a/server/api/index.ts b/server/api/index.ts index 5750a293a..d69eb7a48 100644 --- a/server/api/index.ts +++ b/server/api/index.ts @@ -5,6 +5,7 @@ import { startServerAndCreateH3Handler } from '@as-integrations/h3' import { defineCorsEventHandler } from '@nozomuikuta/h3-cors' import http from 'http' import 'reflect-metadata' // Needed for tsyringe +import 'json-bigint-patch' // Needed for bigint support in JSON import { buildContext, Context } from '../context' import { loadSchemaWithResolvers } from '../schema' @@ -81,13 +82,6 @@ http.IncomingMessage.Readable.prototype.unpipe = function (dest) { return this } -// Fix for BigInt not being supported by JSON.stringify by default -// @ts-expect-error: we add json stringify to BigInt -// eslint-disable-next-line no-extend-native -BigInt.prototype.toJSON = function () { - return this.toString() -} - export default defineLazyEventHandler(async () => { const server = new ApolloServer({ schema: await loadSchemaWithResolvers(), diff --git a/test/global.setup.ts b/test/global.setup.ts index 61da913cb..ca22ceb1b 100644 --- a/test/global.setup.ts +++ b/test/global.setup.ts @@ -2,6 +2,7 @@ import prisma from '@prisma/client' import 'dotenv/config' import 'reflect-metadata' +import 'json-bigint-patch' import { beforeAll } from 'vitest' import { constructConfig } from '~/config' import { register } from '~/server/tsyringe' diff --git a/yarn.lock b/yarn.lock index 360f9b6db..227d1ceee 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14531,6 +14531,7 @@ __metadata: graphql-codegen-typescript-validation-schema: ^0.11.1 graphql-passport: ^0.6.5 graphql-scalars: ^1.22.2 + json-bigint-patch: ^0.0.8 lodash: ^4.17.21 mount-vue-component: ^0.10.2 naive-ui: ^2.34.4 @@ -14731,6 +14732,13 @@ __metadata: languageName: node linkType: hard +"json-bigint-patch@npm:^0.0.8": + version: 0.0.8 + resolution: "json-bigint-patch@npm:0.0.8" + checksum: 593de25b2b9dc161cd2c97afda3210602dbe5de1849baee616ecfc25d7daac399400fba7f50a73d69849686bbe9860061a2e04b181f11d0878fde76c3b05801a + languageName: node + linkType: hard + "json-buffer@npm:3.0.0": version: 3.0.0 resolution: "json-buffer@npm:3.0.0" From 09098ac0dceb98edb9a45d0341f24b77e859aac1 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Tue, 22 Aug 2023 18:54:54 +0200 Subject: [PATCH 12/14] chore: add progress bar to journal data update script --- scripts/journaldata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/journaldata.py b/scripts/journaldata.py index 86bf402ce..bab961150 100644 --- a/scripts/journaldata.py +++ b/scripts/journaldata.py @@ -33,6 +33,7 @@ from prisma import Prisma from prisma.types import JournalCitationInfoYearlyCreateWithoutRelationsInput +from tqdm import tqdm # current_year should be the latest year of data available at https://www.scimagojr.com/journalrank.php current_year = 2022 @@ -137,8 +138,7 @@ def download_all_data(): def combine_data(): """Iterate over files and return the consolidated dataset""" journals: dict[int, JournalInfo] = {} - for year in range(start_year, current_year + 1): - print(f'Processing {year}') + for year in tqdm(range(start_year, current_year + 1), desc='Processing data'): filepath = get_data_filepath(year) with open(filepath, mode='r', encoding='utf-8') as csv_file: csv_reader = csv.DictReader(csv_file, delimiter=';') @@ -212,7 +212,7 @@ async def dump_into_database(journals: dict[int, JournalInfo]): # delete all existing yearly data (because its easier than updating) await db.journalcitationinfoyearly.delete_many() - for journal in journals.values(): + for journal in tqdm(journals.values(), desc='Saving to database'): citation_info: list[JournalCitationInfoYearlyCreateWithoutRelationsInput] = [ { 'year': year, From e3975e0bc41374179dc22bdc211dc4d1034e1081 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Mon, 4 Sep 2023 16:28:42 +0200 Subject: [PATCH 13/14] Update index.ts --- server/api/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/server/api/index.ts b/server/api/index.ts index 19c1d8acf..31c881969 100644 --- a/server/api/index.ts +++ b/server/api/index.ts @@ -6,7 +6,6 @@ import { defineCorsEventHandler } from '@nozomuikuta/h3-cors' import http from 'http' import 'json-bigint-patch' // Needed for bigint support in JSON import 'reflect-metadata' // Needed for tsyringe -import 'json-bigint-patch' // Needed for bigint support in JSON import { buildContext, Context } from '../context' import { loadSchemaWithResolvers } from '../schema' From 5c66f06961e5a19baed928ee9085022da0404e87 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Mon, 4 Sep 2023 16:28:52 +0200 Subject: [PATCH 14/14] Update global.setup.ts --- test/global.setup.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/test/global.setup.ts b/test/global.setup.ts index ca11ddadc..842a93932 100644 --- a/test/global.setup.ts +++ b/test/global.setup.ts @@ -3,7 +3,6 @@ import prisma from '@prisma/client' import 'dotenv/config' import 'json-bigint-patch' import 'reflect-metadata' -import 'json-bigint-patch' import { beforeAll } from 'vitest' import { constructConfig } from '~/config' import { register } from '~/server/tsyringe'