Chore: Run tests using microservices deployment on CI (#24513)

pull/24568/head^2
Diego Sampaio 4 years ago committed by GitHub
parent 3963953316
commit dd276d1adb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1206
      .github/workflows/build_and_test.yml
  2. 151
      .scripts/start.js
  3. 3
      app/models/server/models/_BaseDb.js
  4. 7
      app/models/server/models/_oplogHandle.ts
  5. 3
      app/models/server/raw/index.ts
  6. 2
      app/slackbridge/server/SlackAdapter.js
  7. 3
      ee/server/NetworkBroker.ts
  8. 12
      ee/server/services/package-lock.json
  9. 4
      ee/server/services/package.json
  10. 33
      ee/server/startup/broker.ts
  11. 7
      ee/server/startup/index.ts
  12. 12
      package-lock.json
  13. 4
      package.json
  14. 7
      server/lib/isRunningMs.ts
  15. 6
      server/services/startup.ts
  16. 9
      server/services/team/service.ts
  17. 3
      server/startup/serverRunning.js
  18. 4
      tests/cypress/integration/12-settings.js
  19. 4
      tests/cypress/pageobjects/Page.js
  20. 4
      tests/cypress/pageobjects/settings.js
  21. 4
      tests/data/api-data.js
  22. 4
      tests/mocks/client/jsdom.ts

File diff suppressed because it is too large Load Diff

@ -2,12 +2,10 @@
const path = require('path');
const fs = require('fs');
const extend = require('util')._extend;
const { spawn } = require('child_process');
const net = require('net');
const processes = [];
let exitCode;
const baseDir = path.resolve(__dirname, '..');
const srcDir = path.resolve(baseDir);
@ -30,7 +28,7 @@ const waitPortRelease = (port, count = 0) =>
if (count > 60) {
return reject();
}
console.log('Port', port, 'not release, waiting 1s...');
console.log('Port', port, 'not released, waiting 1s...');
setTimeout(() => {
waitPortRelease(port, ++count)
.then(resolve)
@ -43,22 +41,39 @@ const appOptions = {
env: {
PORT: 3000,
ROOT_URL: 'http://localhost:3000',
// MONGO_URL: 'mongodb://localhost:27017/test',
// MONGO_OPLOG_URL: 'mongodb://localhost:27017/local',
},
};
function startProcess(opts, callback) {
const proc = spawn(opts.command, opts.params, opts.options);
let killingAllProcess = false;
function killAllProcesses(mainExitCode) {
if (killingAllProcess) {
return;
}
killingAllProcess = true;
if (opts.waitForMessage) {
proc.stdout.on('data', function waitForMessage(data) {
if (data.toString().match(opts.waitForMessage)) {
if (callback) {
callback();
}
}
processes.forEach((p) => {
console.log('Killing process', p.pid);
p.kill();
});
waitPortRelease(appOptions.env.PORT)
.then(() => {
console.log(`Port ${appOptions.env.PORT} was released, exiting with code ${mainExitCode}`);
process.exit(mainExitCode);
})
.catch((error) => {
console.error(`Error waiting port ${appOptions.env.PORT} to be released, exiting with code ${mainExitCode}`);
console.error(error);
process.exit(mainExitCode);
});
}
function startProcess(opts) {
const proc = spawn(opts.command, opts.params, opts.options);
processes.push(proc);
if (opts.onData) {
proc.stdout.on('data', opts.onData);
}
if (!opts.silent) {
@ -73,52 +88,74 @@ function startProcess(opts, callback) {
}
proc.on('exit', function (code, signal) {
processes.splice(processes.indexOf(proc), 1);
if (code != null) {
exitCode = code;
console.log(opts.name, `exited with code ${code}`);
} else {
console.log(opts.name, `exited with signal ${signal}`);
}
processes.splice(processes.indexOf(proc), 1);
processes.forEach((p) => {
console.log('Killing process', p.pid);
p.kill();
});
if (processes.length === 0) {
waitPortRelease(appOptions.env.PORT)
.then(() => {
console.log(`Port ${appOptions.env.PORT} was released, exiting with code ${exitCode}`);
process.exit(exitCode);
})
.catch((error) => {
console.error(`Error waiting port ${appOptions.env.PORT} to be released, exiting with code ${exitCode}`);
console.error(error);
process.exit(exitCode);
});
}
killAllProcesses(code);
});
processes.push(proc);
}
function startApp(callback) {
startProcess(
{
function startRocketChat() {
return new Promise((resolve) => {
const waitServerRunning = (message) => {
if (message.toString().match('SERVER RUNNING')) {
return resolve();
}
};
startProcess({
name: 'Meteor App',
command: 'node',
params: ['/tmp/build-test/bundle/main.js'],
// command: 'node',
// params: ['.meteor/local/build/main.js'],
waitForMessage: appOptions.waitForMessage,
onData: waitServerRunning,
options: {
cwd: srcDir,
env: extend(appOptions.env, process.env),
env: {
...appOptions.env,
...process.env,
},
},
},
callback,
);
});
});
}
async function startMicroservices() {
const startService = (name) => {
return new Promise((resolve) => {
const waitStart = (message) => {
if (message.toString().match('NetworkBroker started successfully')) {
return resolve();
}
};
startProcess({
name: `${name} service`,
command: 'node',
params: ['service.js'],
onData: waitStart,
options: {
cwd: path.resolve(srcDir, 'ee', 'server', 'services', 'dist', 'ee', 'server', 'services', name),
env: {
...appOptions.env,
...process.env,
PORT: 4000,
},
},
});
});
};
await Promise.all([
startService('account'),
startService('authorization'),
startService('ddp-streamer'),
startService('presence'),
startService('stream-hub'),
]);
}
function startChimp() {
@ -126,21 +163,23 @@ function startChimp() {
name: 'Chimp',
command: 'npm',
params: ['test'],
// command: 'exit',
// params: ['2'],
options: {
env: Object.assign({}, process.env, {
env: {
...process.env,
NODE_PATH: `${process.env.NODE_PATH + path.delimiter + srcDir + path.delimiter + srcDir}/node_modules`,
}),
},
},
});
}
function chimpNoMirror() {
appOptions.waitForMessage = 'SERVER RUNNING';
startApp(function () {
startChimp();
});
}
(async () => {
const [, , options = ''] = process.argv;
await startRocketChat();
if (options === '--enterprise') {
await startMicroservices();
}
chimpNoMirror();
startChimp();
})();

@ -8,6 +8,7 @@ import { setUpdatedAt } from '../lib/setUpdatedAt';
import { metrics } from '../../../metrics/server/lib/metrics';
import { getOplogHandle } from './_oplogHandle';
import { SystemLogger } from '../../../../server/lib/logger/system';
import { isRunningMs } from '../../../../server/lib/isRunningMs';
const baseName = 'rocketchat_';
@ -32,7 +33,7 @@ export class BaseDbWatch extends EventEmitter {
super();
this.collectionName = collectionName;
if (!process.env.DISABLE_DB_WATCH) {
if (!isRunningMs()) {
this.initDbWatch();
}
}

@ -5,6 +5,7 @@ import { MongoClient, Cursor, Timestamp, Db } from 'mongodb';
import { escapeRegExp } from '@rocket.chat/string-helpers';
import { urlParser } from './_oplogUrlParser';
import { isRunningMs } from '../../../../server/lib/isRunningMs';
class CustomOplogHandle {
dbName: string;
@ -206,7 +207,7 @@ class CustomOplogHandle {
let oplogHandle: CustomOplogHandle;
if (!process.env.DISABLE_DB_WATCH) {
if (!isRunningMs()) {
const disableOplog = !!(global.Package as any)['disable-oplog'];
if (disableOplog) {
@ -219,10 +220,6 @@ if (!process.env.DISABLE_DB_WATCH) {
}
export const getOplogHandle = async (): Promise<OplogHandle | CustomOplogHandle | undefined> => {
if (process.env.DISABLE_DB_WATCH) {
return;
}
if (oplogHandle) {
return oplogHandle;
}

@ -68,6 +68,7 @@ import RoomsModel from '../models/Rooms';
import SettingsModel from '../models/Settings';
import SubscriptionsModel from '../models/Subscriptions';
import UsersModel from '../models/Users';
import { isRunningMs } from '../../../../server/lib/isRunningMs';
const trashCollection = trash.rawCollection();
@ -154,7 +155,7 @@ const map = {
[Rooms.col.collectionName]: RoomsModel,
};
if (!process.env.DISABLE_DB_WATCH) {
if (!isRunningMs()) {
const models = {
Messages,
Users,

@ -994,7 +994,7 @@ export default class SlackAdapter {
imported: 'slackbridge',
});
} else {
saveRoomName(rocketChannel._id, slackMessage.name, rocketUser, false);
Promise.await(saveRoomName(rocketChannel._id, slackMessage.name, rocketUser, false));
}
}

@ -1,7 +1,6 @@
import { ServiceBroker, Context, ServiceSchema } from 'moleculer';
import { asyncLocalStorage } from '../../server/sdk';
import { api } from '../../server/sdk/api';
import { IBroker, IBrokerNode, IServiceMetrics } from '../../server/sdk/types/IBroker';
import { ServiceClass } from '../../server/sdk/types/ServiceClass';
import { EventSignatures } from '../../server/sdk/lib/Events';
@ -34,8 +33,6 @@ export class NetworkBroker implements IBroker {
constructor(broker: ServiceBroker) {
this.broker = broker;
api.setBroker(this);
this.metrics = broker.metrics;
this.started = this.broker.start();

@ -2263,9 +2263,9 @@
}
},
"pino": {
"version": "7.6.4",
"resolved": "https://registry.npmjs.org/pino/-/pino-7.6.4.tgz",
"integrity": "sha512-ktibPg3ttWONxYQ2Efk1zYbIvofD5zdd/ReoujK84ggEp0REflb9TsXavSjt8u1CdT2mMJe9QQ3ZpyOQxUKipA==",
"version": "7.8.0",
"resolved": "https://registry.npmjs.org/pino/-/pino-7.8.0.tgz",
"integrity": "sha512-Ynw2HRVapiyj+ZGfUcpms+SRgDKFIy0ztaFUf3X6IHh+vsysMvn+tpV/Ej3gyutPp4n9tgH6ZkkCAelSvP5zmQ==",
"requires": {
"fast-redact": "^3.0.0",
"on-exit-leak-free": "^0.2.0",
@ -2289,9 +2289,9 @@
}
},
"pino-pretty": {
"version": "7.5.0",
"resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-7.5.0.tgz",
"integrity": "sha512-vhGvWhb8SZcmO/q8G0NyWTJ6UTfjHzIzHd6kC5Ob8UJH3FIP6f0nJ5+KglmThYTJXjGaaOGe06ulNrNDohCneQ==",
"version": "7.5.1",
"resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-7.5.1.tgz",
"integrity": "sha512-xEOUJiokdBGcZ9d0v7OY6SqEp+rrVH2drE3bHOUsK8elw44eh9V83InZqeL1dFwgD1IDnd6crUoec3hIXxfdBQ==",
"dev": true,
"requires": {
"args": "^5.0.1",

@ -41,7 +41,7 @@
"moleculer": "^0.14.19",
"mongodb": "^3.6.10",
"nats": "^2.4.0",
"pino": "^7.6.4",
"pino": "^7.8.0",
"sodium-native": "^3.3.0",
"sodium-plus": "^0.9.0",
"underscore.string": "^3.3.6",
@ -57,7 +57,7 @@
"@types/mongodb": "^3.6.19",
"@types/node": "^14.17.4",
"@types/ws": "^8.2.3",
"pino-pretty": "^7.5.0",
"pino-pretty": "^7.5.1",
"pm2": "^5.2.0",
"ts-node": "^10.5.0",
"typescript": "^4.3.5"

@ -1,5 +1,6 @@
import EJSON from 'ejson';
import { Errors, Serializers, ServiceBroker } from 'moleculer';
import { pino } from 'pino';
import { api } from '../../../server/sdk/api';
import { isMeteorError, MeteorError } from '../../../server/sdk/errors';
@ -10,7 +11,7 @@ const {
CACHE = 'Memory',
// SERIALIZER = 'MsgPack',
SERIALIZER = 'EJSON',
MOLECULER_LOG_LEVEL = 'error',
MOLECULER_LOG_LEVEL = 'warn',
BALANCE_STRATEGY = 'RoundRobin',
BALANCE_PREFER_LOCAL = 'false',
RETRY_FACTOR = '2',
@ -85,18 +86,25 @@ const network = new ServiceBroker({
},
cacher: CACHE,
serializer: SERIALIZER === 'EJSON' ? new EJSONSerializer() : SERIALIZER,
logLevel: MOLECULER_LOG_LEVEL as any,
// logLevel: {
// // "TRACING": "trace",
// // "TRANS*": "warn",
// BROKER: 'debug',
// TRANSIT: 'debug',
// '**': 'info',
// },
logger: {
type: 'Console',
type: 'Pino',
options: {
formatter: 'short',
level: MOLECULER_LOG_LEVEL,
pino: {
options: {
timestamp: pino.stdTimeFunctions.isoTime,
...(process.env.NODE_ENV !== 'production'
? {
transport: {
target: 'pino-pretty',
options: {
colorize: true,
},
},
}
: {}),
},
},
},
},
registry: {
@ -155,6 +163,9 @@ const network = new ServiceBroker({
},
},
errorRegenerator: new CustomRegenerator(),
started(): void {
console.log('NetworkBroker started successfully.');
},
});
api.setBroker(new NetworkBroker(network));

@ -1,10 +1,9 @@
import './engagementDashboard';
import './seatsCap';
import './services';
import { isRunningMs } from '../../../server/lib/isRunningMs';
const { TRANSPORTER = '' } = process.env;
// only starts network broker if transporter properly configured
if (TRANSPORTER.match(/^(?:nats|TCP)/)) {
// only starts network broker if running in micro services mode
if (isRunningMs()) {
require('./broker');
}

12
package-lock.json generated

@ -29089,9 +29089,9 @@
}
},
"pino": {
"version": "7.6.4",
"resolved": "https://registry.npmjs.org/pino/-/pino-7.6.4.tgz",
"integrity": "sha512-ktibPg3ttWONxYQ2Efk1zYbIvofD5zdd/ReoujK84ggEp0REflb9TsXavSjt8u1CdT2mMJe9QQ3ZpyOQxUKipA==",
"version": "7.8.0",
"resolved": "https://registry.npmjs.org/pino/-/pino-7.8.0.tgz",
"integrity": "sha512-Ynw2HRVapiyj+ZGfUcpms+SRgDKFIy0ztaFUf3X6IHh+vsysMvn+tpV/Ej3gyutPp4n9tgH6ZkkCAelSvP5zmQ==",
"requires": {
"fast-redact": "^3.0.0",
"on-exit-leak-free": "^0.2.0",
@ -29138,9 +29138,9 @@
}
},
"pino-pretty": {
"version": "7.5.0",
"resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-7.5.0.tgz",
"integrity": "sha512-vhGvWhb8SZcmO/q8G0NyWTJ6UTfjHzIzHd6kC5Ob8UJH3FIP6f0nJ5+KglmThYTJXjGaaOGe06ulNrNDohCneQ==",
"version": "7.5.1",
"resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-7.5.1.tgz",
"integrity": "sha512-xEOUJiokdBGcZ9d0v7OY6SqEp+rrVH2drE3bHOUsK8elw44eh9V83InZqeL1dFwgD1IDnd6crUoec3hIXxfdBQ==",
"dev": true,
"requires": {
"args": "^5.0.1",

@ -146,7 +146,7 @@
"jsdom-global": "^3.0.2",
"mocha": "^9.1.3",
"mock-require": "^3.0.3",
"pino-pretty": "^7.5.0",
"pino-pretty": "^7.5.1",
"postcss": "^8.3.5",
"postcss-custom-properties": "^11.0.0",
"postcss-easy-import": "^3.0.0",
@ -284,7 +284,7 @@
"path-to-regexp": "^6.2.0",
"pdfjs-dist": "^2.8.335",
"photoswipe": "^4.1.3",
"pino": "^7.6.4",
"pino": "^7.8.0",
"poplib": "^0.1.7",
"postis": "^2.2.0",
"prom-client": "^12.0.0",

@ -0,0 +1,7 @@
/**
* Checks if the server is running in micro services mode
* @returns {boolean}
*/
export function isRunningMs(): boolean {
return !!process.env.TRANSPORTER?.match(/^(?:nats|TCP)/);
}

@ -13,6 +13,7 @@ import { RoomService } from './room/service';
import { SAUMonitorService } from './sauMonitor/service';
import { TeamService } from './team/service';
import { UiKitCoreApp } from './uikit-core-app/service';
import { isRunningMs } from '../lib/isRunningMs';
const { db } = MongoInternals.defaultRemoteCollectionDriver().mongo;
@ -29,9 +30,8 @@ api.registerService(new SAUMonitorService());
api.registerService(new TeamService(db));
api.registerService(new UiKitCoreApp());
// if TRANSPORTER env var it means the process is running in micro services mode
// in that case we don't need to register services that will run separately
if (!process.env.TRANSPORTER?.match(/^(?:nats|TCP)/)) {
// if the process is running in micro services mode we don't need to register services that will run separately
if (!isRunningMs()) {
(async (): Promise<void> => {
const { Authorization } = await import('./authorization/service');

@ -28,7 +28,6 @@ import {
ITeamUpdateData,
} from '../../sdk/types/ITeamService';
import { ServiceClassInternal } from '../../sdk/types/ServiceClass';
import { canAccessRoom } from '../authorization/canAccessRoom';
import { saveRoomName } from '../../../app/channel-settings/server';
import { saveRoomType } from '../../../app/channel-settings/server/functions/saveRoomType';
import { ISubscription } from '../../../definition/ISubscription';
@ -174,7 +173,7 @@ export class TeamService extends ServiceClassInternal implements ITeamService {
const { name, type, updateRoom = true } = updateData;
if (updateRoom && name) {
saveRoomName(team.roomId, name, user);
await saveRoomName(team.roomId, name, user);
}
if (updateRoom && typeof type !== 'undefined') {
@ -360,7 +359,7 @@ export class TeamService extends ServiceClassInternal implements ITeamService {
// validate access for every room first
for await (const room of validRooms) {
const canSeeRoom = await canAccessRoom(room, user);
const canSeeRoom = await Authorization.canAccessRoom(room, user);
if (!canSeeRoom) {
throw new Error('invalid-room');
}
@ -402,7 +401,7 @@ export class TeamService extends ServiceClassInternal implements ITeamService {
const user = await this.Users.findOneById(uid);
if (!canRemoveAnyRoom) {
const canSeeRoom = await canAccessRoom(room, user);
const canSeeRoom = await Authorization.canAccessRoom(room, user);
if (!canSeeRoom) {
throw new Error('invalid-room');
}
@ -465,7 +464,7 @@ export class TeamService extends ServiceClassInternal implements ITeamService {
const user = await this.Users.findOneById(uid);
if (!canUpdateAnyRoom) {
const canSeeRoom = await canAccessRoom(room, user);
const canSeeRoom = await Authorization.canAccessRoom(room, user);
if (!canSeeRoom) {
throw new Error('invalid-room');
}

@ -10,6 +10,7 @@ import { Info, getMongoInfo } from '../../app/utils/server';
import { Users } from '../../app/models/server';
import { sendMessagesToAdmins } from '../lib/sendMessagesToAdmins';
import { showErrorBox, showWarningBox, showSuccessBox } from '../lib/logger/showBox';
import { isRunningMs } from '../lib/isRunningMs';
const exitIfNotBypassed = (ignore, errorCode = 1) => {
if (typeof ignore === 'string' && ['yes', 'true'].includes(ignore.toLowerCase())) {
@ -47,7 +48,7 @@ Meteor.startup(function () {
msg = msg.join('\n');
if (!process.env.DISABLE_DB_WATCH && !oplogEnabled) {
if (!isRunningMs() && !oplogEnabled) {
msg += [
'',
'',

@ -9,7 +9,9 @@ import { checkIfUserIsValid } from '../../data/checks';
import { adminUsername, adminEmail, adminPassword, username, email, password, reason } from '../../data/user.js';
import { wait } from '../../data/api-data';
const request = supertest('http://localhost:3000');
const apiUrl = process.env.TEST_API_URL || 'http://localhost:3000';
const request = supertest(apiUrl);
const prefix = '/api/v1/';
function api(path) {

@ -1,10 +1,12 @@
const testUrl = process.env.TEST_URL || 'http://localhost:3000';
class Page {
get body() {
return browser.element('body');
}
open(path) {
cy.visit(`http://localhost:3000/${path}`);
cy.visit(`${testUrl}/${path}`);
}
}
module.exports = Page;

@ -2,7 +2,9 @@ import supertest from 'supertest';
import { adminUsername, adminPassword } from '../../data/user.js';
const request = supertest('http://localhost:3000');
const testUrl = process.env.TEST_API_URL || 'http://localhost:3000';
const request = supertest(testUrl);
const prefix = '/api/v1/';
const login = {

@ -4,7 +4,9 @@ import { publicChannelName, privateChannelName } from './channel.js';
import { roleNameUsers, roleNameSubscriptions, roleScopeUsers, roleScopeSubscriptions, roleDescription } from './role.js';
import { username, email, adminUsername, adminPassword } from './user.js';
export const request = supertest('http://localhost:3000');
const apiUrl = process.env.TEST_API_URL || 'http://localhost:3000';
export const request = supertest(apiUrl);
const prefix = '/api/v1/';
export function wait(cb, time) {

@ -1,7 +1,9 @@
import globalJsdom from 'jsdom-global';
const testUrl = process.env.TEST_URL || 'http://localhost:3000';
export const enableJsdom = (): void => {
globalJsdom('<!doctype html><html><head><meta charset="utf-8"></head><body></body></html>', {
url: 'http://localhost:3000',
url: testUrl,
});
};

Loading…
Cancel
Save