Skip to content

Commit 6b22a82

Browse files
authored
Mfancher/directorymonitoring job creation (#746)
* Routes and tests for directory monitoring created * jobs and adding/removing jobs on updates * bug fixes and updates
1 parent 6718e92 commit 6b22a82

File tree

6 files changed

+365
-7
lines changed

6 files changed

+365
-7
lines changed

Diff for: server/job-scheduler.js

+15
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ const {
3434
createLandingZoneFileMonitoringBreeJob,
3535
createLogicalFileMonitoringBreeJob,
3636
createSuperFileMonitoringBreeJob,
37+
createDirectoryMonitoringBreeJob,
38+
scheduleDirectoryMonitoringOnServerStart,
3739
scheduleSuperFileMonitoringOnServerStart,
3840
scheduleFileMonitoringBreeJob,
3941
scheduleFileMonitoringOnServerStart,
@@ -122,6 +124,7 @@ class JobScheduler {
122124
await this.scheduleTeamsNotificationProcessing();
123125
await this.scheduleOrbitMonitoringOnServerStart();
124126
await this.createOrbitMegaphoneJob();
127+
await this.scheduleDirectoryMonitoringOnServerStart();
125128
})();
126129
}
127130

@@ -288,6 +291,18 @@ class JobScheduler {
288291
});
289292
}
290293

294+
createDirectoryMonitoringBreeJob({ directoryMonitoring_id, name, cron }) {
295+
return createDirectoryMonitoringBreeJob.call(this, {
296+
directoryMonitoring_id,
297+
name,
298+
cron,
299+
});
300+
}
301+
302+
scheduleDirectoryMonitoringOnServerStart() {
303+
return scheduleDirectoryMonitoringOnServerStart.call(this);
304+
}
305+
291306
scheduleSuperFileMonitoringOnServerStart() {
292307
return scheduleSuperFileMonitoringOnServerStart.call(this);
293308
}

Diff for: server/jobSchedularMethods/hpccFiles.js

+45
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,12 @@ const SUBMIT_LANDINGZONE_FILEMONITORING_FILE_NAME =
88
const SUBMIT_LOGICAL_FILEMONITORING_FILE_NAME =
99
"submitLogicalFileMonitoring.js";
1010
const SUBMIT_SUPER_FILEMONITORING_FILE_NAME = "submitSuperFileMonitoring.js";
11+
const SUBMIT_DIRECTORY_MONITORING_FILE_NAME = "submitDirectoryMonitoring.js";
1112
const FILE_MONITORING = "fileMonitoringPoller.js";
1213

1314
const filemonitoring_superfile = models.filemonitoring_superfiles;
1415
const FileMonitoring = models.fileMonitoring;
16+
const directoryMonitoring = models.directoryMonitoring;
1517

1618
function createLandingZoneFileMonitoringBreeJob({
1719
filemonitoring_id,
@@ -34,6 +36,25 @@ function createLandingZoneFileMonitoringBreeJob({
3436
this.bree.add(job);
3537
}
3638

39+
function createDirectoryMonitoringBreeJob({ directoryMonitoring_id, cron }) {
40+
const uniqueJobName = `Directory Monitoring - ${directoryMonitoring_id}`;
41+
const job = {
42+
cron,
43+
name: uniqueJobName,
44+
path: path.join(
45+
__dirname,
46+
"..",
47+
"jobs",
48+
SUBMIT_DIRECTORY_MONITORING_FILE_NAME
49+
),
50+
worker: {
51+
workerData: { directoryMonitoring_id },
52+
},
53+
};
54+
this.bree.add(job);
55+
this.bree.start(uniqueJobName);
56+
}
57+
3758
function createLogicalFileMonitoringBreeJob({ filemonitoring_id, name, cron }) {
3859
const job = {
3960
cron,
@@ -164,10 +185,34 @@ async function scheduleFileMonitoring() {
164185
}
165186
}
166187

188+
async function scheduleDirectoryMonitoringOnServerStart() {
189+
logger.info("📂 DIRECTORY MONITORING STARTED ...");
190+
try {
191+
const activeDirectoryMonitoring = await directoryMonitoring.findAll({
192+
where: {
193+
active: true,
194+
approved: true,
195+
// monitoringAssetType: "landingZoneFile",
196+
},
197+
raw: true,
198+
});
199+
for (const monitoring of activeDirectoryMonitoring) {
200+
await this.createDirectoryMonitoringBreeJob({
201+
directoryMonitoring_id: monitoring.id,
202+
cron: monitoring.cron,
203+
});
204+
}
205+
} catch (err) {
206+
logger.error(err);
207+
}
208+
}
209+
167210
module.exports = {
168211
createLandingZoneFileMonitoringBreeJob,
169212
createLogicalFileMonitoringBreeJob,
170213
createSuperFileMonitoringBreeJob,
214+
createDirectoryMonitoringBreeJob,
215+
scheduleDirectoryMonitoringOnServerStart,
171216
scheduleSuperFileMonitoringOnServerStart,
172217
scheduleFileMonitoringBreeJob,
173218
scheduleFileMonitoringOnServerStart,

Diff for: server/jobs/submitDirectoryMonitoring.js

+256
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,256 @@
1+
const logger = require("../config/logger");
2+
const models = require("../models");
3+
const directoryMonitoring = models.directoryMonitoring;
4+
const teamsWebhooks = models.teams_hook;
5+
const notification = models.notification_queue;
6+
const hpccUtil = require("../utils/hpcc-util");
7+
const { v4: uuidv4 } = require("uuid");
8+
const wildCardStringMatch = require("../utils/wildCardStringMatch");
9+
const { parentPort, workerData } = require("worker_threads");
10+
const moment = require("moment");
11+
12+
(async () => {
13+
try {
14+
//grab all directory monitoring that are active
15+
16+
const directoryMonitoringDetails = await directoryMonitoring.findOne({
17+
where: {
18+
id: workerData.directoryMonitoring_id,
19+
active: true,
20+
},
21+
});
22+
23+
if (!directoryMonitoringDetails) return;
24+
25+
let {
26+
id: directorymonitoring_id,
27+
cluster_id,
28+
directory,
29+
metaData,
30+
metaData: {
31+
machine: Netaddr,
32+
pattern: fileNameWildCard,
33+
landingZone,
34+
lastMonitored,
35+
currentlyMonitoring,
36+
monitoringCondition: {
37+
notifyCondition,
38+
threshold,
39+
fileDetected,
40+
maximumFileCount,
41+
minimumFilecount,
42+
},
43+
notifications,
44+
},
45+
} = directoryMonitoringDetails;
46+
47+
let currentTimeStamp = moment.utc().valueOf();
48+
49+
const Path = `/var/lib/HPCCSystems/${landingZone}/${directory}/`;
50+
51+
const result = await hpccUtil.getDirectories({
52+
clusterId: cluster_id,
53+
Netaddr,
54+
Path,
55+
DirectoryOnly: false,
56+
});
57+
let files = result.filter((item) => !item.isDir);
58+
59+
const newFilesToMonitor = [];
60+
61+
// Notification Details
62+
let emailNotificationDetails;
63+
let teamsNotificationDetails;
64+
for (let notification of notifications) {
65+
if (notification.channel === "eMail") {
66+
emailNotificationDetails = notification;
67+
}
68+
if (notification.channel === "msTeams") {
69+
teamsNotificationDetails = notification;
70+
}
71+
}
72+
73+
let newNotificationDetails = [];
74+
75+
//check if number of files in the directory is within the range
76+
if (files.length < minimumFilecount) {
77+
newNotificationDetails.push({
78+
value: "file_count_below_minimum",
79+
title: `File count below minimum in ${directory.join("/")}`,
80+
text: `Number of files in ${directory.join(
81+
"/"
82+
)} is below the minimum file count of ${minimumFilecount}`,
83+
details: {
84+
"Landing zone": landingZone,
85+
Directory: directory.join("/"),
86+
"File count": files.length,
87+
},
88+
});
89+
}
90+
91+
if (files.length > maximumFileCount) {
92+
newNotificationDetails.push({
93+
value: "file_count_above_maximum",
94+
title: `File count above maximum in ${directory.join("/")}`,
95+
text: `Number of files in ${directory.join(
96+
"/"
97+
)} is above the maximum file count of ${maximumFileCount}`,
98+
details: {
99+
"Landing zone": landingZone,
100+
Directory: directory.join("/"),
101+
"File count": files.length,
102+
},
103+
});
104+
}
105+
106+
//iterate through files, check for notification parameters and update each file if necessary
107+
for (let i = 0; i < files.length; i++) {
108+
let { name: fileName, modifiedtime } = files[i];
109+
110+
let fileModifiedTime = moment(modifiedtime);
111+
112+
fileModifiedTime = fileModifiedTime.utc().valueOf();
113+
114+
//check if file is new
115+
if (
116+
lastMonitored < fileModifiedTime &&
117+
wildCardStringMatch(fileNameWildCard, fileName) &&
118+
//file name not in the currently monitoring array
119+
!currentlyMonitoring.find(
120+
(item) =>
121+
item.name === fileName && item.modifiedTime === fileModifiedTime
122+
)
123+
) {
124+
//Check if user wants to be notified when new file arrives
125+
let notificationDetail;
126+
if (fileDetected) {
127+
notificationDetail = {
128+
name: fileName,
129+
value: "file_detected",
130+
title: `New file uploaded to ${directory}`,
131+
text: "Details about recently added file - ",
132+
details: {
133+
"File Name": fileName,
134+
"Landing zone": landingZone,
135+
Directory: directory,
136+
"File detected at": new Date(fileModifiedTime).toString(),
137+
},
138+
};
139+
}
140+
141+
if (notificationDetail) {
142+
newNotificationDetails.push(notificationDetail);
143+
}
144+
145+
// Start monitoring new file if threshold parameter is set
146+
if (notifyCondition.includes("fileNotMoving")) {
147+
newFilesToMonitor.push({
148+
name: fileName,
149+
modifiedTime: fileModifiedTime,
150+
threshold: currentTimeStamp + threshold * 60 * 1000,
151+
notified: [],
152+
});
153+
}
154+
}
155+
}
156+
157+
if (currentlyMonitoring.length > 0) {
158+
//remove files that have been moved out by checking against files array
159+
currentlyMonitoring = currentlyMonitoring.filter((current) => {
160+
const { name, modifiedTime } = current;
161+
return !files.find(
162+
(file) => file.name === name && file.modifiedtime === modifiedTime
163+
);
164+
});
165+
}
166+
167+
// check for threshold
168+
currentlyMonitoring.forEach((current) => {
169+
const { notified } = current;
170+
171+
const pastExpectedMoveTime = current.threshold < currentTimeStamp;
172+
173+
if (pastExpectedMoveTime && !notified.length) {
174+
newNotificationDetails.push({
175+
name: current.name,
176+
value: "file_not_moving",
177+
title: `${current.name} stuck at ${directory}`,
178+
text: `${current.name} has been stuck at ${directory} longer than ${threshold} minutes`,
179+
details: {
180+
"File Name": current.name,
181+
"Landing zone": landingZone,
182+
Directory: directory,
183+
"File received at": new Date(current.modifiedTime).toString(),
184+
"Expected move time": new Date(current.threshold).toString(),
185+
},
186+
});
187+
}
188+
});
189+
190+
console.log("newNotificationDetails");
191+
console.log(newNotificationDetails);
192+
193+
//send notifications if necessary
194+
for (let notificationDetail of newNotificationDetails) {
195+
//send email notification
196+
if (emailNotificationDetails) {
197+
//TODO: create notification queue with proper template in next isue
198+
logger.verbose("Email notification sent: " + newNotificationDetails);
199+
200+
//once notification is sent, update currently monitoring notified field
201+
currentlyMonitoring = currentlyMonitoring.map((item) => {
202+
if (notificationDetail.name === item.name) {
203+
return {
204+
...item,
205+
notified: [
206+
{
207+
notified: true,
208+
method: "email",
209+
dateNotified: currentTimeStamp,
210+
},
211+
],
212+
};
213+
}
214+
return item;
215+
});
216+
}
217+
218+
//send teams notification
219+
if (teamsNotificationDetails) {
220+
//TODO: create notification queue with proper template in next isue
221+
logger.verbose("Teams notification sent: " + newNotificationDetails);
222+
223+
//once notification is sent, update currently monitoring notified field
224+
currentlyMonitoring = currentlyMonitoring.map((item) => {
225+
if (notificationDetail.name === item.name) {
226+
return {
227+
...item,
228+
notified: [
229+
{
230+
notified: true,
231+
method: "msTeams",
232+
dateNotified: currentTimeStamp,
233+
},
234+
],
235+
};
236+
}
237+
return item;
238+
});
239+
}
240+
}
241+
242+
//update directory monitoring
243+
metaData.lastMonitored = currentTimeStamp;
244+
metaData.currentlyMonitoring = [
245+
...currentlyMonitoring,
246+
...newFilesToMonitor,
247+
];
248+
249+
await directoryMonitoring.update(
250+
{ metaData },
251+
{ where: { id: directorymonitoring_id } }
252+
);
253+
} catch (error) {
254+
logger.error("Error while running Directory Monitoring Jobs: " + error);
255+
}
256+
})();

Diff for: server/notificationTemplates/email/directoryMonitoring.ejs

Whitespace-only changes.

Diff for: server/notificationTemplates/teams/directoryMonitoring.js

Whitespace-only changes.

0 commit comments

Comments
 (0)