From 757c473942bc7a5dfbca3655b92eddbd85a31dbb Mon Sep 17 00:00:00 2001 From: Marcos Spessatto Defendi Date: Tue, 7 Apr 2020 02:17:45 -0300 Subject: [PATCH] [IMPROVE] First data load from existing data on engagement dashboard (#17035) --- app/models/server/index.js | 2 + app/models/server/raw/Analytics.js | 4 ++ app/models/server/raw/Messages.js | 66 +++++++++++++++++++ app/models/server/raw/Users.js | 45 +++++++++++++ .../components/Directory/DirectoryTable.js | 2 +- .../components/EngagementDashboardRoute.js | 1 + .../MessagesTab/MessagesSentSection.js | 12 ++-- .../UsersTab/BusiestChatTimesSection.js | 13 ++-- .../components/UsersTab/NewUsersSection.js | 4 +- ee/app/engagement-dashboard/client/routes.js | 51 ++++---------- ee/app/engagement-dashboard/server/index.js | 10 +++ .../server/lib/messages.js | 31 +++++++-- .../engagement-dashboard/server/lib/users.js | 25 +++++-- server/startup/migrations/index.js | 1 + server/startup/migrations/v182.js | 9 +++ 15 files changed, 213 insertions(+), 63 deletions(-) create mode 100644 server/startup/migrations/v182.js diff --git a/app/models/server/index.js b/app/models/server/index.js index f7bbb89c97a..911268506af 100644 --- a/app/models/server/index.js +++ b/app/models/server/index.js @@ -38,6 +38,7 @@ import LivechatAgentActivity from './models/LivechatAgentActivity'; import LivechatInquiry from './models/LivechatInquiry'; import ReadReceipts from './models/ReadReceipts'; import LivechatExternalMessage from './models/LivechatExternalMessages'; +import Analytics from './models/Analytics'; export { AppsLogsModel } from './models/apps-logs-model'; export { AppsPersistenceModel } from './models/apps-persistence-model'; @@ -88,4 +89,5 @@ export { ReadReceipts, LivechatExternalMessage, LivechatInquiry, + Analytics, }; diff --git a/app/models/server/raw/Analytics.js b/app/models/server/raw/Analytics.js index 7544d897fbd..9fc929795f2 100644 --- a/app/models/server/raw/Analytics.js +++ b/app/models/server/raw/Analytics.js @@ -140,6 +140,10 @@ export class AnalyticsRaw extends BaseRaw { } return this.col.aggregate(params).toArray(); } + + findByTypeBeforeDate({ type, date }) { + return this.find({ type, date: { $lte: date } }); + } } export default new AnalyticsRaw(Analytics.model.rawCollection()); diff --git a/app/models/server/raw/Messages.js b/app/models/server/raw/Messages.js index fcda4e0c85c..6326293c414 100644 --- a/app/models/server/raw/Messages.js +++ b/app/models/server/raw/Messages.js @@ -106,4 +106,70 @@ export class MessagesRaw extends BaseRaw { } return this.col.aggregate(params, { allowDiskUse: true }); } + + getTotalOfMessagesSentByDate({ start, end, options = {} }) { + const params = [ + { $match: { t: { $exists: false }, ts: { $gte: start, $lte: end } } }, + { + $lookup: { + from: 'rocketchat_room', + localField: 'rid', + foreignField: '_id', + as: 'room', + }, + }, + { + $unwind: { + path: '$room', + }, + }, + { + $group: { + _id: { + _id: '$room._id', + name: { + $cond: [{ $ifNull: ['$room.fname', false] }, + '$room.fname', + '$room.name'], + }, + t: '$room.t', + usernames: { + $cond: [{ $ifNull: ['$room.usernames', false] }, + '$room.usernames', + []], + }, + date: { + $concat: [ + { $substr: ['$ts', 0, 4] }, + { $substr: ['$ts', 5, 2] }, + { $substr: ['$ts', 8, 2] }, + ], + }, + }, + messages: { $sum: 1 }, + }, + }, + { + $project: { + _id: 0, + date: { $toInt: '$_id.date' }, + room: { + _id: '$_id._id', + name: '$_id.name', + t: '$_id.t', + usernames: '$_id.usernames', + }, + type: 'messages', + messages: 1, + }, + }, + ]; + if (options.sort) { + params.push({ $sort: options.sort }); + } + if (options.count) { + params.push({ $limit: options.count }); + } + return this.col.aggregate(params).toArray(); + } } diff --git a/app/models/server/raw/Users.js b/app/models/server/raw/Users.js index fcb6b9955bc..f42be432eb2 100644 --- a/app/models/server/raw/Users.js +++ b/app/models/server/raw/Users.js @@ -215,6 +215,51 @@ export class UsersRaw extends BaseRaw { return this.col.aggregate(params).toArray(); } + getTotalOfRegisteredUsersByDate({ start, end, options = {} }) { + const params = [ + { + $match: { + createdAt: { $gte: start, $lte: end }, + }, + }, + { + $group: { + _id: { + $concat: [ + { $substr: ['$createdAt', 0, 4] }, + { $substr: ['$createdAt', 5, 2] }, + { $substr: ['$createdAt', 8, 2] }, + ], + }, + users: { $sum: 1 }, + }, + }, + { + $group: { + _id: { + $toInt: '$_id', + }, + users: { $sum: '$users' }, + }, + }, + { + $project: { + _id: 0, + date: '$_id', + users: 1, + type: 'users', + }, + }, + ]; + if (options.sort) { + params.push({ $sort: options.sort }); + } + if (options.count) { + params.push({ $limit: options.count }); + } + return this.col.aggregate(params).toArray(); + } + updateStatusText(_id, statusText) { const update = { $set: { diff --git a/app/ui/client/views/app/components/Directory/DirectoryTable.js b/app/ui/client/views/app/components/Directory/DirectoryTable.js index 44ea34de335..09925c92e40 100644 --- a/app/ui/client/views/app/components/Directory/DirectoryTable.js +++ b/app/ui/client/views/app/components/Directory/DirectoryTable.js @@ -79,7 +79,7 @@ export function DirectoryTable({ return <> - + } onChange={handleChange} value={text} /> {channels && !channels.length diff --git a/ee/app/engagement-dashboard/client/components/EngagementDashboardRoute.js b/ee/app/engagement-dashboard/client/components/EngagementDashboardRoute.js index 4d232810240..1761ddfdd81 100644 --- a/ee/app/engagement-dashboard/client/components/EngagementDashboardRoute.js +++ b/ee/app/engagement-dashboard/client/components/EngagementDashboardRoute.js @@ -22,3 +22,4 @@ export function EngagementDashboardRoute() { onSelectTab={(tab) => goToEngagementDashboard({ tab })} />; } +EngagementDashboardRoute.displayName = 'EngagementDashboardRoute'; diff --git a/ee/app/engagement-dashboard/client/components/MessagesTab/MessagesSentSection.js b/ee/app/engagement-dashboard/client/components/MessagesTab/MessagesSentSection.js index aefd34d7488..e3d96d4271c 100644 --- a/ee/app/engagement-dashboard/client/components/MessagesTab/MessagesSentSection.js +++ b/ee/app/engagement-dashboard/client/components/MessagesTab/MessagesSentSection.js @@ -63,11 +63,13 @@ export function MessagesSentSection() { const values = Array.from({ length: moment(period.end).diff(period.start, 'days') + 1 }, (_, i) => ({ date: moment(period.start).add(i, 'days').toISOString(), - newUsers: 0, + newMessages: 0, })); - for (const { day, users } of data.days) { + for (const { day, messages } of data.days) { const i = moment(day).diff(period.start, 'days'); - values[i].newUsers += users; + if (i >= 0) { + values[i].newMessages += messages; + } } return [ @@ -106,7 +108,7 @@ export function MessagesSentSection() { - {t('Value_users', { value })} + {t('Value_messages', { value })} } /> diff --git a/ee/app/engagement-dashboard/client/components/UsersTab/BusiestChatTimesSection.js b/ee/app/engagement-dashboard/client/components/UsersTab/BusiestChatTimesSection.js index 86091bd7409..445ef0cb60b 100644 --- a/ee/app/engagement-dashboard/client/components/UsersTab/BusiestChatTimesSection.js +++ b/ee/app/engagement-dashboard/client/components/UsersTab/BusiestChatTimesSection.js @@ -22,12 +22,12 @@ function ContentForHours({ displacement, onPreviousDateClick, onNextDateClick }) const divider = 2; const values = Array.from({ length: 24 / divider }, (_, i) => ({ - hour: divider * i, + hour: String(divider * i), users: 0, })); for (const { hour, users } of data.hours) { const i = Math.floor(hour / divider); - values[i] = values[i] || { hour: divider * i, users: 0 }; + values[i] = values[i] || { hour: String(divider * i), users: 0 }; values[i].users += users; } @@ -134,7 +134,7 @@ function ContentForDays({ displacement, onPreviousDateClick, onNextDateClick }) const data = useEndpointData('GET', 'engagement-dashboard/users/chat-busier/weekly-data', params); const values = useMemo(() => (data ? data.month.map(({ users, day, month, year }) => ({ users, - day: moment.utc([year, month - 1, day, 0, 0, 0]).valueOf(), + day: String(moment.utc([year, month - 1, day, 0, 0, 0]).valueOf()), })).sort(({ day: a }, { day: b }) => a - b) : []), [data]); return <> @@ -185,7 +185,7 @@ function ContentForDays({ displacement, onPreviousDateClick, onNextDateClick }) tickPadding: 4, tickRotation: 0, tickValues: 'every 3 days', - format: (timestamp) => moment(timestamp).format('L'), + format: (timestamp) => moment(parseInt(timestamp, 10)).format('L'), }} axisLeft={null} animate={true} @@ -226,12 +226,13 @@ export function BusiestChatTimesSection() { ['days', t('Days')], ], [t]); + const [displacement, setDisplacement] = useState(0); + const handleTimeUnitChange = (timeUnit) => { setTimeUnit(timeUnit); + setDisplacement(0); }; - const [displacement, setDisplacement] = useState(0); - const handlePreviousDateClick = () => setDisplacement((displacement) => displacement + 1); const handleNextDateClick = () => setDisplacement((displacement) => displacement - 1); diff --git a/ee/app/engagement-dashboard/client/components/UsersTab/NewUsersSection.js b/ee/app/engagement-dashboard/client/components/UsersTab/NewUsersSection.js index 3ffb763f91c..8fcd11a91bd 100644 --- a/ee/app/engagement-dashboard/client/components/UsersTab/NewUsersSection.js +++ b/ee/app/engagement-dashboard/client/components/UsersTab/NewUsersSection.js @@ -67,7 +67,9 @@ export function NewUsersSection() { })); for (const { day, users } of data.days) { const i = moment(day).diff(period.start, 'days'); - values[i].newUsers += users; + if (i >= 0) { + values[i].newUsers += users; + } } return [ diff --git a/ee/app/engagement-dashboard/client/routes.js b/ee/app/engagement-dashboard/client/routes.js index d4d966f5fd9..f8735786bba 100644 --- a/ee/app/engagement-dashboard/client/routes.js +++ b/ee/app/engagement-dashboard/client/routes.js @@ -1,55 +1,30 @@ -import { Blaze } from 'meteor/blaze'; import { HTML } from 'meteor/htmljs'; import { FlowRouter } from 'meteor/kadira:flow-router'; import { BlazeLayout } from 'meteor/kadira:blaze-layout'; -import { Template } from 'meteor/templating'; + import { hasAllPermission } from '../../../../app/authorization'; import { AdminBox } from '../../../../app/ui-utils'; import { hasLicense } from '../../license/client'; +import { createTemplateForComponent } from '../../../../client/createTemplateForComponent'; -Template.EngagementDashboardRoute = new Blaze.Template('EngagementDashboardRoute', - () => HTML.DIV.call(null, { style: 'overflow: hidden; flex: 1 1 auto; height: 1%;' })); - -Template.EngagementDashboardRoute.onRendered(async function() { - const [ - { createElement }, - { render, unmountComponentAtNode }, - { MeteorProvider }, - { EngagementDashboardRoute }, - ] = await Promise.all([ - import('react'), - import('react-dom'), - import('../../../../client/providers/MeteorProvider'), - import('./components/EngagementDashboardRoute'), - ]); - - const container = this.firstNode; - - if (!container) { - return; - } - - this.autorun(() => { - const routeName = FlowRouter.getRouteName(); - if (routeName !== 'engagement-dashboard') { - unmountComponentAtNode(container); - } - }); - - render(createElement(MeteorProvider, { children: createElement(EngagementDashboardRoute) }), container); -}); - -let licensed = false; FlowRouter.route('/admin/engagement-dashboard/:tab?', { name: 'engagement-dashboard', - action: () => { + action: async () => { + const licensed = await hasLicense('engagement-dashboard'); if (!licensed) { return; } - BlazeLayout.render('main', { center: 'EngagementDashboardRoute' }); + const { EngagementDashboardRoute } = await import('./components/EngagementDashboardRoute'); + + BlazeLayout.render('main', { center: await createTemplateForComponent(EngagementDashboardRoute, + {}, + // eslint-disable-next-line new-cap + () => HTML.DIV.call(null, { style: 'overflow: hidden; flex: 1 1 auto; height: 1%;' }), + 'engagement-dashboard'), + }); }, }); @@ -58,8 +33,6 @@ hasLicense('engagement-dashboard').then((enabled) => { return; } - licensed = true; - AdminBox.addOption({ href: 'engagement-dashboard', i18nLabel: 'Engagement Dashboard', diff --git a/ee/app/engagement-dashboard/server/index.js b/ee/app/engagement-dashboard/server/index.js index 0083bfabbf5..4be65208b09 100644 --- a/ee/app/engagement-dashboard/server/index.js +++ b/ee/app/engagement-dashboard/server/index.js @@ -1,6 +1,16 @@ +import { Meteor } from 'meteor/meteor'; + import { onLicense } from '../../license/server'; +import { fillFirstDaysOfMessagesIfNeeded } from './lib/messages'; +import { fillFirstDaysOfUsersIfNeeded } from './lib/users'; onLicense('engagement-dashboard', async () => { await import('./listeners'); await import('./api'); + + Meteor.startup(async () => { + const date = new Date(); + fillFirstDaysOfUsersIfNeeded(date); + fillFirstDaysOfMessagesIfNeeded(date); + }); }); diff --git a/ee/app/engagement-dashboard/server/lib/messages.js b/ee/app/engagement-dashboard/server/lib/messages.js index e6f0d621dcc..8bfb9fe95f4 100644 --- a/ee/app/engagement-dashboard/server/lib/messages.js +++ b/ee/app/engagement-dashboard/server/lib/messages.js @@ -1,7 +1,9 @@ import moment from 'moment'; -import Analytics from '../../../../../app/models/server/raw/Analytics'; +import AnalyticsRaw from '../../../../../app/models/server/raw/Analytics'; import { roomTypes } from '../../../../../app/utils'; +import { Messages } from '../../../../../app/models/server/raw'; +import { Analytics } from '../../../../../app/models/server'; import { convertDateToInt, diffBetweenDaysInclusive, convertIntToDate, getTotalOfWeekItems } from './date'; export const handleMessagesSent = (message, room) => { @@ -9,7 +11,7 @@ export const handleMessagesSent = (message, room) => { if (!roomTypesToShow.includes(room.t)) { return; } - Promise.await(Analytics.saveMessageSent({ + Promise.await(AnalyticsRaw.saveMessageSent({ date: convertDateToInt(message.ts), room, })); @@ -21,25 +23,40 @@ export const handleMessagesDeleted = (message, room) => { if (!roomTypesToShow.includes(room.t)) { return; } - Promise.await(Analytics.saveMessageDeleted({ + Promise.await(AnalyticsRaw.saveMessageDeleted({ date: convertDateToInt(message.ts), room, })); return message; }; +export const fillFirstDaysOfMessagesIfNeeded = async (date) => { + const messagesFromAnalytics = await AnalyticsRaw.findByTypeBeforeDate({ + type: 'messages', + date: convertDateToInt(date), + }).toArray(); + if (!messagesFromAnalytics.length) { + const startOfPeriod = moment(convertIntToDate(date)).subtract(90, 'days').toDate(); + const messages = await Messages.getTotalOfMessagesSentByDate({ + start: startOfPeriod, + end: date, + }); + messages.forEach((message) => Analytics.insert(message)); + } +}; + export const findWeeklyMessagesSentData = async ({ start, end }) => { const daysBetweenDates = diffBetweenDaysInclusive(end, start); const endOfLastWeek = moment(start).clone().subtract(1, 'days').toDate(); const startOfLastWeek = moment(endOfLastWeek).clone().subtract(daysBetweenDates, 'days').toDate(); const today = convertDateToInt(end); const yesterday = convertDateToInt(moment(end).clone().subtract(1, 'days').toDate()); - const currentPeriodMessages = await Analytics.getMessagesSentTotalByDate({ + const currentPeriodMessages = await AnalyticsRaw.getMessagesSentTotalByDate({ start: convertDateToInt(start), end: convertDateToInt(end), options: { count: daysBetweenDates, sort: { _id: -1 } }, }); - const lastPeriodMessages = await Analytics.getMessagesSentTotalByDate({ + const lastPeriodMessages = await AnalyticsRaw.getMessagesSentTotalByDate({ start: convertDateToInt(startOfLastWeek), end: convertDateToInt(endOfLastWeek), options: { count: daysBetweenDates, sort: { _id: -1 } }, @@ -62,7 +79,7 @@ export const findWeeklyMessagesSentData = async ({ start, end }) => { }; export const findMessagesSentOrigin = async ({ start, end }) => { - const origins = await Analytics.getMessagesOrigin({ + const origins = await AnalyticsRaw.getMessagesOrigin({ start: convertDateToInt(start), end: convertDateToInt(end), }); @@ -76,7 +93,7 @@ export const findMessagesSentOrigin = async ({ start, end }) => { }; export const findTopFivePopularChannelsByMessageSentQuantity = async ({ start, end }) => { - const channels = await Analytics.getMostPopularChannelsByMessagesSentQuantity({ + const channels = await AnalyticsRaw.getMostPopularChannelsByMessagesSentQuantity({ start: convertDateToInt(start), end: convertDateToInt(end), options: { count: 5, sort: { messages: -1 } }, diff --git a/ee/app/engagement-dashboard/server/lib/users.js b/ee/app/engagement-dashboard/server/lib/users.js index 5ea0a8b5ee5..9e7403862d7 100644 --- a/ee/app/engagement-dashboard/server/lib/users.js +++ b/ee/app/engagement-dashboard/server/lib/users.js @@ -1,29 +1,46 @@ import moment from 'moment'; -import Analytics from '../../../../../app/models/server/raw/Analytics'; +import AnalyticsRaw from '../../../../../app/models/server/raw/Analytics'; import Sessions from '../../../../../app/models/server/raw/Sessions'; +import { Users } from '../../../../../app/models/server/raw'; +import { Analytics } from '../../../../../app/models/server'; import { convertDateToInt, diffBetweenDaysInclusive, getTotalOfWeekItems, convertIntToDate } from './date'; export const handleUserCreated = (user) => { - Promise.await(Analytics.saveUserData({ + Promise.await(AnalyticsRaw.saveUserData({ date: convertDateToInt(user.ts), user, })); return user; }; +export const fillFirstDaysOfUsersIfNeeded = async (date) => { + const usersFromAnalytics = await AnalyticsRaw.findByTypeBeforeDate({ + type: 'users', + date: convertDateToInt(date), + }).toArray(); + if (!usersFromAnalytics.length) { + const startOfPeriod = moment(date).subtract(90, 'days').toDate(); + const users = await Users.getTotalOfRegisteredUsersByDate({ + start: startOfPeriod, + end: date, + }); + users.forEach((user) => Analytics.insert(user)); + } +}; + export const findWeeklyUsersRegisteredData = async ({ start, end }) => { const daysBetweenDates = diffBetweenDaysInclusive(end, start); const endOfLastWeek = moment(start).clone().subtract(1, 'days').toDate(); const startOfLastWeek = moment(endOfLastWeek).clone().subtract(daysBetweenDates, 'days').toDate(); const today = convertDateToInt(end); const yesterday = convertDateToInt(moment(end).clone().subtract(1, 'days').toDate()); - const currentPeriodUsers = await Analytics.getTotalOfRegisteredUsersByDate({ + const currentPeriodUsers = await AnalyticsRaw.getTotalOfRegisteredUsersByDate({ start: convertDateToInt(start), end: convertDateToInt(end), options: { count: daysBetweenDates, sort: { _id: -1 } }, }); - const lastPeriodUsers = await Analytics.getTotalOfRegisteredUsersByDate({ + const lastPeriodUsers = await AnalyticsRaw.getTotalOfRegisteredUsersByDate({ start: convertDateToInt(startOfLastWeek), end: convertDateToInt(endOfLastWeek), options: { count: daysBetweenDates, sort: { _id: -1 } }, diff --git a/server/startup/migrations/index.js b/server/startup/migrations/index.js index 6eca0c2df49..60ee4afe997 100644 --- a/server/startup/migrations/index.js +++ b/server/startup/migrations/index.js @@ -179,4 +179,5 @@ import './v178'; import './v179'; import './v180'; import './v181'; +import './v182'; import './xrun'; diff --git a/server/startup/migrations/v182.js b/server/startup/migrations/v182.js new file mode 100644 index 00000000000..06f19f56801 --- /dev/null +++ b/server/startup/migrations/v182.js @@ -0,0 +1,9 @@ +import { Migrations } from '../../../app/migrations'; +import { Analytics } from '../../../app/models/server'; + +Migrations.add({ + version: 182, + up() { + Analytics.remove({}); + }, +});