Skip to content

Commit

Permalink
Merge branch 'develop' into regression/threads-temp
Browse files Browse the repository at this point in the history
  • Loading branch information
@kodiakhq
kodiakhq[bot] authored Aug 24, 2022
2 parents 1c15862 + 18722ac commit b5b1066
Show file tree
Hide file tree
Showing 37 changed files with 780 additions and 489 deletions.
37 changes: 22 additions & 15 deletions apps/meteor/app/api/server/v1/import.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,14 @@ import {
import { API } from '../api';
import { Imports } from '../../../models/server';
import { Importers } from '../../../importer/server';
import { executeUploadImportFile } from '../../../importer/server/methods/uploadImportFile';
import {
executeUploadImportFile,
executeDownloadPublicImportFile,
executeGetImportProgress,
executeGetImportFileData,
executeStartImport,
executeGetLatestImportOperations,
} from '../../../importer/server/methods';

API.v1.addRoute(
'uploadImportFile',
Expand All @@ -36,12 +43,12 @@ API.v1.addRoute(
{
authRequired: true,
validateParams: isDownloadPublicImportFileParamsPOST,
permissionsRequired: ['run-import'],
},
{
post() {
const { fileUrl, importerKey } = this.bodyParams;

Meteor.call('downloadPublicImportFile', fileUrl, importerKey);
executeDownloadPublicImportFile(this.userId, fileUrl, importerKey);

return API.v1.success();
},
Expand All @@ -53,12 +60,13 @@ API.v1.addRoute(
{
authRequired: true,
validateParams: isStartImportParamsPOST,
permissionsRequired: ['run-import'],
},
{
post() {
const { input } = this.bodyParams;

Meteor.call('startImport', input);
executeStartImport({ input });

return API.v1.success();
},
Expand All @@ -70,10 +78,12 @@ API.v1.addRoute(
{
authRequired: true,
validateParams: isGetImportFileDataParamsGET,
permissionsRequired: ['run-import'],
},
{
get() {
const result = Meteor.call('getImportFileData');
async get() {
const result = await executeGetImportFileData();

return API.v1.success(result);
},
},
Expand All @@ -84,10 +94,11 @@ API.v1.addRoute(
{
authRequired: true,
validateParams: isGetImportProgressParamsGET,
permissionsRequired: ['run-import'],
},
{
get() {
const result = Meteor.call('getImportProgress');
const result = executeGetImportProgress();
return API.v1.success(result);
},
},
Expand All @@ -98,10 +109,11 @@ API.v1.addRoute(
{
authRequired: true,
validateParams: isGetLatestImportOperationsParamsGET,
permissionsRequired: ['view-import-operations'],
},
{
get() {
const result = Meteor.call('getLatestImportOperations');
const result = executeGetLatestImportOperations();
return API.v1.success(result);
},
},
Expand All @@ -118,9 +130,7 @@ API.v1.addRoute(
post() {
const importer = Importers.get('pending-files');
if (!importer) {
throw new Meteor.Error('error-importer-not-defined', 'The Pending File Importer was not found.', {
method: 'downloadPendingFiles',
});
throw new Meteor.Error('error-importer-not-defined', 'The Pending File Importer was not found.', 'downloadPendingFiles');
}

importer.instance = new importer.importer(importer); // eslint-disable-line new-cap
Expand All @@ -144,9 +154,7 @@ API.v1.addRoute(
post() {
const importer = Importers.get('pending-avatars');
if (!importer) {
throw new Meteor.Error('error-importer-not-defined', 'The Pending File Importer was not found.', {
method: 'downloadPendingAvatars',
});
throw new Meteor.Error('error-importer-not-defined', 'The Pending File Importer was not found.', 'downloadPendingAvatars');
}

importer.instance = new importer.importer(importer); // eslint-disable-line new-cap
Expand All @@ -170,7 +178,6 @@ API.v1.addRoute(
get() {
const operation = Imports.findLastImport();
return API.v1.success({
success: true,
operation,
});
},
Expand Down
162 changes: 27 additions & 135 deletions apps/meteor/app/importer-slack-users/server/importer.js
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,7 @@
import { Meteor } from 'meteor/meteor';
import { Accounts } from 'meteor/accounts-base';
import { Random } from 'meteor/random';
import { Settings } from '@rocket.chat/models';

import { RawImports, Base, ProgressStep, Selection, SelectionUser } from '../../importer/server';
import { Base, ProgressStep } from '../../importer/server';
import { RocketChatFile } from '../../file';
import { Users } from '../../models/server';

export class SlackUsersImporter extends Base {
constructor(info, importRecord) {
Expand All @@ -14,171 +10,67 @@ export class SlackUsersImporter extends Base {
const { parse } = require('csv-parse/lib/sync');

this.csvParser = parse;
this.userMap = new Map();
this.admins = []; // Array of ids of the users which are admins
}

prepare(dataURI, sentContentType, fileName) {
this.logger.debug('start preparing import operation');
this.converter.clearImportData();
super.prepare(dataURI, sentContentType, fileName, true);

super.updateProgress(ProgressStep.PREPARING_USERS);
const uriResult = RocketChatFile.dataURIParse(dataURI);
const buf = Buffer.from(uriResult.image, 'base64');
const parsed = this.csvParser(buf.toString());

let userCount = 0;
parsed.forEach((user, index) => {
// Ignore the first column
if (index === 0) {
return;
}

const id = Random.id();
const username = user[0];
const email = user[1];
let isBot = false;
let isDeleted = false;
if (!email) {
return;
}

const name = user[7] || user[8] || username;

const newUser = {
emails: [email],
importIds: [email],
username,
name,
type: 'user',
};

switch (user[2]) {
case 'Admin':
this.admins.push(id);
newUser.roles = ['admin'];
break;
case 'Bot':
isBot = true;
newUser.roles = ['bot'];
newUser.type = 'bot';
break;
case 'Deactivated':
isDeleted = true;
newUser.deleted = true;
break;
}

this.userMap.set(id, new SelectionUser(id, username, email, isDeleted, isBot, true));
this.converter.addUser(newUser);
userCount++;
});

const userArray = Array.from(this.userMap.values());

const usersId = this.collection.insert({
import: this.importRecord._id,
importer: this.name,
type: 'users',
users: userArray,
});
this.users = this.collection.findOne(usersId);
super.updateRecord({ 'count.users': this.userMap.size });
super.addCountToTotal(this.userMap.size);

if (this.userMap.size === 0) {
if (userCount === 0) {
this.logger.error('No users found in the import file.');
super.updateProgress(ProgressStep.ERROR);
return super.getProgress();
}

this.collection.insert({
import: this.importRecord._id,
importer: this.name,
type: 'admins',
admins: this.admins,
});

super.updateProgress(ProgressStep.USER_SELECTION);
return new Selection(this.name, userArray, [], 0);
}

startImport(importSelection) {
const admins = this.collection.findOne({ import: this.importRecord._id, type: 'admins' });
if (admins) {
this.admins = admins.admins || [];
} else {
this.admins = [];
}

this.users = RawImports.findOne({ import: this.importRecord._id, type: 'users' });
// Recreate the userMap from the collection data
this.userMap = new Map();
for (const user of this.users.users) {
const obj = new SelectionUser();
for (const propName in user) {
if (user.hasOwnProperty(propName)) {
obj[propName] = user[propName];
}
}
this.userMap.set(user.user_id, obj);
}

this.reloadCount();

super.startImport(importSelection);
const started = Date.now();

for (const user of importSelection.users) {
const u = this.userMap.get(user.user_id);
u.do_import = user.do_import;

this.userMap.set(user.user_id, u);
}
this.collection.update({ _id: this.users._id }, { $set: { users: Array.from(this.userMap.values()) } });

const startedByUserId = Meteor.userId();
Meteor.defer(() => {
super.updateProgress(ProgressStep.IMPORTING_USERS);

try {
for (const u of this.users.users) {
if (!u.do_import) {
continue;
}

Meteor.runAsUser(startedByUserId, () => {
const existantUser = Users.findOneByEmailAddress(u.email) || Users.findOneByUsernameIgnoringCase(u.username);

let userId;
if (existantUser) {
// since we have an existing user, let's try a few things
userId = existantUser._id;
u.rocketId = existantUser._id;
Users.update({ _id: u.rocketId }, { $addToSet: { importIds: u.user_id } });

Users.setEmail(existantUser._id, u.email);
Users.setEmailVerified(existantUser._id, u.email);
} else {
userId = Accounts.createUser({
username: u.username + Random.id(),
password: Date.now() + u.name + u.email.toUpperCase(),
});

if (!userId) {
console.warn('An error happened while creating a user.');
return;
}

Meteor.runAsUser(userId, () => {
Meteor.call('setUsername', u.username, { joinDefaultChannelsSilenced: true });
Users.setName(userId, u.name);
Users.update({ _id: userId }, { $addToSet: { importIds: u.user_id } });
Users.setEmail(userId, u.email);
Users.setEmailVerified(userId, u.email);
u.rocketId = userId;
});
}

if (this.admins.includes(u.user_id)) {
Meteor.call('setAdminStatus', userId, true);
}

super.addCountCompleted(1);
});
}

Settings.incrementValueById('Slack_Users_Importer_Count', this.users.users.length);
super.updateProgress(ProgressStep.FINISHING);
super.updateProgress(ProgressStep.DONE);
} catch (e) {
this.logger.error(e);
super.updateProgress(ProgressStep.ERROR);
}

const timeTook = Date.now() - started;
this.logger.log(`Slack Users Import took ${timeTook} milliseconds.`);
});

return super.getProgress();
super.addCountToTotal(userCount);
Settings.incrementValueById('Slack_Users_Importer_Count', userCount);
return super.updateRecord({ 'count.users': userCount });
}
}
2 changes: 1 addition & 1 deletion apps/meteor/app/importer-slack/server/importer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ export class SlackImporter extends Base {
}

this.converter.addUser(newUser);
Promise.all(Settings.incrementValueById('Slack_Importer_Count'));
Promise.await(Settings.incrementValueById('Slack_Importer_Count'));
}
}

Expand Down
4 changes: 2 additions & 2 deletions apps/meteor/app/importer/server/classes/ImporterBase.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -291,8 +291,8 @@ export class Base {
this.progress.count.completed = 0;
}

this.progress.count.total = this.importRecord.count.total || 0;
this.progress.count.completed = this.importRecord.count.completed || 0;
this.progress.count.total = this.importRecord.count?.total || 0;
this.progress.count.completed = this.importRecord.count?.completed || 0;
}

/**
Expand Down
7 changes: 1 addition & 6 deletions apps/meteor/app/importer/server/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,7 @@ import { SelectionUser } from './classes/ImporterSelectionUser';
import { ProgressStep } from '../lib/ImporterProgressStep';
import { ImporterInfo } from '../lib/ImporterInfo';
import { Importers } from '../lib/Importers';
import './methods/getImportProgress';
import './methods/startImport';
import './methods/uploadImportFile';
import './methods/getImportFileData';
import './methods/downloadPublicImportFile';
import './methods/getLatestImportOperations';
import './methods';
import './startup/setImportsToInvalid';
import './startup/store';

Expand Down
Loading

0 comments on commit b5b1066

Please sign in to comment.