diff --git a/webapp/package-lock.json b/webapp/package-lock.json index 73f53129a40..c1602a78e1f 100644 --- a/webapp/package-lock.json +++ b/webapp/package-lock.json @@ -25,6 +25,7 @@ "@ngrx/effects": "^17.1.1", "@ngrx/store": "^17.1.1", "@ngx-translate/core": "^16.0.3", + "@types/levenshtein": "1.0.4", "@webcomponents/webcomponentsjs": "^2.8.0", "bikram-sambat-bootstrap": "^1.6.0", "bootstrap": "^3.4.1", @@ -34,6 +35,7 @@ "eurodigit": "^3.1.3", "font-awesome": "^4.7.0", "jquery": "3.5.1", + "levenshtein": "1.0.5", "lodash-es": "^4.17.21", "moment-locales-webpack-plugin": "^1.2.0", "ngrx-store-logger": "^0.2.4", @@ -4808,6 +4810,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/levenshtein": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@types/levenshtein/-/levenshtein-1.0.4.tgz", + "integrity": "sha512-QiNzDEGuAHoNVa7xjTPGQRecXScckE8bAEpuHipG8lEFPZh4eIBK0dw0K5mu9XdiTiVD8AxwYY8lOxYaP1rZUA==", + "license": "MIT" + }, "node_modules/@types/mime": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", @@ -8687,6 +8695,15 @@ "node": ">=0.10.0" } }, + "node_modules/levenshtein": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/levenshtein/-/levenshtein-1.0.5.tgz", + "integrity": "sha512-UQf1nnmxjl7O0+snDXj2YF2r74Gkya8ZpnegrUBYN9tikh2dtxV/ey8e07BO5wwo0i76yjOvbDhFHdcPEiH9aA==", + "engines": [ + "node >=0.2.0" + ], + "license": "Public Domain" + }, "node_modules/license-webpack-plugin": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/license-webpack-plugin/-/license-webpack-plugin-4.0.2.tgz", @@ -16839,6 +16856,11 @@ "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", "dev": true }, + "@types/levenshtein": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@types/levenshtein/-/levenshtein-1.0.4.tgz", + "integrity": "sha512-QiNzDEGuAHoNVa7xjTPGQRecXScckE8bAEpuHipG8lEFPZh4eIBK0dw0K5mu9XdiTiVD8AxwYY8lOxYaP1rZUA==" + }, "@types/mime": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", @@ -19648,6 +19670,11 @@ "integrity": "sha512-MYUxjSQSBUQmowc0l5nPieOYwMzGPUaTzB6inNW/bdPEG9zOL3eAAD1Qw5ZxSPk7we5dMojHwNODYMV1hq4EVg==", "dev": true }, + "levenshtein": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/levenshtein/-/levenshtein-1.0.5.tgz", + "integrity": "sha512-UQf1nnmxjl7O0+snDXj2YF2r74Gkya8ZpnegrUBYN9tikh2dtxV/ey8e07BO5wwo0i76yjOvbDhFHdcPEiH9aA==" + }, "license-webpack-plugin": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/license-webpack-plugin/-/license-webpack-plugin-4.0.2.tgz", diff --git a/webapp/package.json b/webapp/package.json index ad807a7194d..00660774e8c 100644 --- a/webapp/package.json +++ b/webapp/package.json @@ -61,7 +61,9 @@ "select2": "4.0.3", "signature_pad": "2.3.x", "tslib": "^2.5.3", - "zone.js": "^0.14.4" + "zone.js": "^0.14.4", + "levenshtein": "1.0.5", + "@types/levenshtein": "1.0.4" }, "overrides": { "minimist": ">=1.2.6" diff --git a/webapp/src/css/enketo/medic.less b/webapp/src/css/enketo/medic.less index f4801f4a7df..0785b782389 100644 --- a/webapp/src/css/enketo/medic.less +++ b/webapp/src/css/enketo/medic.less @@ -443,6 +443,58 @@ .pages.or .or-repeat-info[role="page"] { display: block; } + + #duplicate_info { + width: 100%; + min-height: 20px; + padding-left: 20px; + padding-right: 20px; + background-color: #ffe7e8; + + .results_header { + font-size: large; + color: #e33030; + } + + .acknowledge_label { + -webkit-user-select: none; -ms-user-select: none; user-select: none; + } + + .acknowledge_checkbox { + margin-right: 5px; + } + + .divider { + background-color: #e33030; + height: 1px; + margin-top: 5px; + margin-bottom: 5px; + } + + .card { + border: 1px solid #ddd; + padding: 1rem; + margin-bottom: 1rem; + border-radius: 5px; + } + + .nested-section { + margin-left: 1.5rem; + } + + .toggle-button { + background: none; + border: none; + color: #007bff; + cursor: pointer; + font-weight: bold; + padding-left: 0px; + } + + .toggle-button:hover { + text-decoration: underline; + } + } } @media (max-width: @media-mobile) { diff --git a/webapp/src/ts/components/components.module.ts b/webapp/src/ts/components/components.module.ts index 0c930b97e08..c26a5c85a79 100644 --- a/webapp/src/ts/components/components.module.ts +++ b/webapp/src/ts/components/components.module.ts @@ -47,6 +47,7 @@ import { PanelHeaderComponent } from '@mm-components/panel-header/panel-header.c import { SidebarMenuComponent } from '@mm-components/sidebar-menu/sidebar-menu.component'; import { ToolBarComponent } from '@mm-components/tool-bar/tool-bar.component'; import { TrainingCardsFormComponent } from '@mm-components/training-cards-form/training-cards-form.component'; +import {DuplicateInfoComponent} from '@mm-components/duplicate-info/duplicate-info.component'; @NgModule({ declarations: [ @@ -78,6 +79,7 @@ import { TrainingCardsFormComponent } from '@mm-components/training-cards-form/t SidebarMenuComponent, TrainingCardsFormComponent, ToolBarComponent, + DuplicateInfoComponent, ], imports: [ CommonModule, @@ -122,6 +124,7 @@ import { TrainingCardsFormComponent } from '@mm-components/training-cards-form/t SidebarMenuComponent, TrainingCardsFormComponent, ToolBarComponent, + DuplicateInfoComponent, ] }) export class ComponentsModule { } diff --git a/webapp/src/ts/components/duplicate-info/duplicate-info.component.html b/webapp/src/ts/components/duplicate-info/duplicate-info.component.html new file mode 100644 index 00000000000..5e061f5f16a --- /dev/null +++ b/webapp/src/ts/components/duplicate-info/duplicate-info.component.html @@ -0,0 +1,53 @@ +
+

{{ duplicates.length }} {{'potential duplicate item(s) found:' | translate }}

+
+
+ +
+
+
+
+ {{ 'Item number:' | translate }} {{ i + 1 }} +
+

+ {{ 'Name:' | translate }} {{ duplicate.name }} +
+ {{ 'Created on:' | translate }} {{ duplicate.reported_date | date: 'EEE MMM dd yyyy HH:mm:ss' }} +

+ +
+ +
+
+ +
+
+ +
+
+ +
+ +
+
+ + {{ key.key }}: {{ key.value }} + +
+ +
diff --git a/webapp/src/ts/components/duplicate-info/duplicate-info.component.ts b/webapp/src/ts/components/duplicate-info/duplicate-info.component.ts new file mode 100644 index 00000000000..844088f9226 --- /dev/null +++ b/webapp/src/ts/components/duplicate-info/duplicate-info.component.ts @@ -0,0 +1,40 @@ +import { Component, EventEmitter, Input, Output } from '@angular/core'; + +@Component({ + selector: 'mm-duplicate-info', + templateUrl: './duplicate-info.component.html', +}) +export class DuplicateInfoComponent { + @Input() acknowledged: boolean = false; + @Output() acknowledgedChange = new EventEmitter(); + @Output() navigateToDuplicate = new EventEmitter(); + @Input() duplicates: { _id: string; name: string; reported_date: string | Date; [key: string]: string | Date }[] = []; + + toggleAcknowledged() { + this.acknowledged = !this.acknowledged; + this.acknowledgedChange.emit(this.acknowledged); + } + + _navigateToDuplicate(_id: string){ + this.navigateToDuplicate.emit(_id); + } + + // Handles collapse / expand of duplicate doc details + expandedSections = new Map(); + + toggleSection(path: string): void { + this.expandedSections.set(path, !this.expandedSections.get(path)); + } + + isExpanded(path: string): boolean { + return this.expandedSections.get(path) || false; + } + + isObject(value: any): boolean { + return value && typeof value === 'object' && !Array.isArray(value); + } + + getPath(parentPath: string, key: string): string { + return parentPath ? `${parentPath}.${key}` : key; + } +} diff --git a/webapp/src/ts/components/enketo/enketo.component.html b/webapp/src/ts/components/enketo/enketo.component.html index 1a4f81d24b0..773c8d0b6b5 100644 --- a/webapp/src/ts/components/enketo/enketo.component.html +++ b/webapp/src/ts/components/enketo/enketo.component.html @@ -1,5 +1,6 @@
+
diff --git a/webapp/src/ts/modules/contacts/contacts-edit.component.html b/webapp/src/ts/modules/contacts/contacts-edit.component.html index 35faf46137b..1a99d554663 100644 --- a/webapp/src/ts/modules/contacts/contacts-edit.component.html +++ b/webapp/src/ts/modules/contacts/contacts-edit.component.html @@ -9,7 +9,11 @@
- + +
+ +
+
diff --git a/webapp/src/ts/modules/contacts/contacts-edit.component.ts b/webapp/src/ts/modules/contacts/contacts-edit.component.ts index e0407b7caf4..8e3cb82ee24 100644 --- a/webapp/src/ts/modules/contacts/contacts-edit.component.ts +++ b/webapp/src/ts/modules/contacts/contacts-edit.component.ts @@ -5,7 +5,7 @@ import { isEqual as _isEqual } from 'lodash-es'; import { ActivatedRoute, Router } from '@angular/router'; import { LineageModelGeneratorService } from '@mm-services/lineage-model-generator.service'; -import { FormService } from '@mm-services/form.service'; +import { FormService, DuplicatesFoundError, Duplicate } from '@mm-services/form.service'; import { EnketoFormContext } from '@mm-services/enketo.service'; import { ContactTypesService } from '@mm-services/contact-types.service'; import { DbService } from '@mm-services/db.service'; @@ -55,6 +55,18 @@ export class ContactsEditComponent implements OnInit, OnDestroy, AfterViewInit { private trackSave; private trackMetadata = { action: '', form: '' }; + private duplicateCheck; + acknowledged = false; + onAcknowledgeChange(value: boolean) { + this.acknowledged = value; + } + + onNavigateToDuplicate(_id: string){ + this.router.navigate(['/contacts', _id, 'edit']); + } + + duplicates: Duplicate[] = []; + ngOnInit() { this.trackRender = this.performanceService.track(); this.subscribeToStore(); @@ -153,6 +165,10 @@ export class ContactsEditComponent implements OnInit, OnDestroy, AfterViewInit { this.contentError = false; this.errorTranslationKey = false; + // Reset when when navigated to duplicate + this.duplicates = []; + this.acknowledged = false; + try { const contact = await this.getContact(); const contactTypeId = this.contactTypesService.getTypeId(contact) || this.routeSnapshot.params?.type; @@ -272,6 +288,7 @@ export class ContactsEditComponent implements OnInit, OnDestroy, AfterViewInit { private async renderForm(formId: string, titleKey: string) { const formDoc = await this.dbService.get().get(formId); this.xmlVersion = formDoc.xmlVersion; + this.duplicateCheck = formDoc.context?.duplicate_check; this.globalActions.setEnketoEditedStatus(false); @@ -326,7 +343,9 @@ export class ContactsEditComponent implements OnInit, OnDestroy, AfterViewInit { $('form.or').trigger('beforesave'); return this.formService - .saveContact(form, docId, this.enketoContact.type, this.xmlVersion) + .saveContact({ + form, docId, type: this.enketoContact.type, xmlVersion: this.xmlVersion + }, this.duplicateCheck, this.acknowledged) .then((result) => { console.debug('saved contact', result); @@ -345,6 +364,11 @@ export class ContactsEditComponent implements OnInit, OnDestroy, AfterViewInit { this.router.navigate(['/contacts', result.docId]); }) .catch((err) => { + if (err instanceof DuplicatesFoundError){ + this.duplicates = err.duplicates; + err = Error(err.message); + } + console.error('Error submitting form data', err); this.globalActions.setEnketoSavingStatus(false); diff --git a/webapp/src/ts/services/form.service.ts b/webapp/src/ts/services/form.service.ts index 2329c820407..65225f3f3d0 100644 --- a/webapp/src/ts/services/form.service.ts +++ b/webapp/src/ts/services/form.service.ts @@ -26,6 +26,9 @@ import { reduce as _reduce } from 'lodash-es'; import { ContactTypesService } from '@mm-services/contact-types.service'; import { TargetAggregatesService } from '@mm-services/target-aggregates.service'; import { ContactViewModelGeneratorService } from '@mm-services/contact-view-model-generator.service'; +import { ParseProvider } from '@mm-providers/parse.provider'; +import { XmlFormsContextUtilsService } from '@mm-services/xml-forms-context-utils.service'; +import { extractExpression, requestSiblings, getDuplicates, Doc, DuplicateCheck } from './utils/deduplicate'; /** * Service for interacting with forms. This is the primary entry-point for CHT code to render forms and save the @@ -58,6 +61,8 @@ export class FormService { private enketoService: EnketoService, private targetAggregatesService: TargetAggregatesService, private contactViewModelGeneratorService: ContactViewModelGeneratorService, + private readonly parseProvider: ParseProvider, + private readonly xmlFormsDuplicateUtilsService: XmlFormsContextUtilsService, ) { this.inited = this.init(); this.globalActions = new GlobalActions(store); @@ -326,7 +331,39 @@ export class FormService { }, null); } - async saveContact(form, docId, type, xmlVersion) { + async checkForDuplicates(doc, duplicateCheck, acknowledged) { + const parentId = doc ? doc.parent?._id : undefined; + const contactType = doc ? doc.contact_type ?? doc.type : undefined; + const siblings = await requestSiblings(this.dbService, parentId, contactType); + const expression = extractExpression(duplicateCheck); + const isCanonical = doc.is_canonical ? doc.is_canonical === 'true' : false; + acknowledged = acknowledged ?? false; + + if (!isCanonical && expression && !acknowledged){ + const duplicates = getDuplicates( + doc, + siblings, + { + expression, + parseProvider: this.parseProvider, + xmlFormsContextUtilsService: this.xmlFormsDuplicateUtilsService + } + ); + return duplicates; + } + } + + async saveContact( + contactInfo: { + form: any; + docId: string| undefined; + type: string | undefined; + xmlVersion: string | undefined; + }, + duplicateCheck: DuplicateCheck, + acknowledged: boolean + ) { + const { form, docId, type, xmlVersion } = contactInfo; const typeFields = this.contactTypesService.isHardcodedType(type) ? { type } : { type: 'contact', contact_type: type }; @@ -335,6 +372,16 @@ export class FormService { const preparedDocs = await this.applyTransitions(docs); const primaryDoc = preparedDocs.preparedDocs.find(doc => doc.type === type); + + const duplicates = await this.checkForDuplicates( + primaryDoc || preparedDocs.preparedDocs[0], + duplicateCheck, + acknowledged + ); + if (duplicates && duplicates.length > 0){ + throw new DuplicatesFoundError('Duplicates found', duplicates); + } + this.servicesActions.setLastChangedDoc(primaryDoc || preparedDocs.preparedDocs[0]); const bulkDocsResult = await this.dbService.get().bulkDocs(preparedDocs.preparedDocs); const failureMessage = this.generateFailureMessage(bulkDocsResult); @@ -350,4 +397,14 @@ export class FormService { this.enketoService.unload(form); } } - +export class DuplicatesFoundError extends Error { + duplicates: Duplicate[]; + + constructor(message: string, duplicates: Duplicate[]) { + super(message); + this.message = message; + this.duplicates = duplicates; + this.name = 'DuplicatesFoundError'; + } +} +export type Duplicate = Doc; diff --git a/webapp/src/ts/services/utils/deduplicate.ts b/webapp/src/ts/services/utils/deduplicate.ts new file mode 100644 index 00000000000..2aa7a3f6233 --- /dev/null +++ b/webapp/src/ts/services/utils/deduplicate.ts @@ -0,0 +1,94 @@ +import * as Levenshtein from 'levenshtein'; +import { DbService } from '@mm-services/db.service'; +import { ParseProvider } from '@mm-providers/parse.provider'; +import { XmlFormsContextUtilsService } from '@mm-services/xml-forms-context-utils.service'; + + +export type Doc = { _id: string; name: string; reported_date: number;[key: string]: any }; + +const DEFAULT_CONTACT_DUPLICATE_EXPRESSION = 'levenshteinEq(3, current.name, existing.name)'; + +// Normalize the distance by dividing by the length of the longer string. +// This can make the metric more adaptable across different string lengths +const normalizedLevenshteinEq = function (str1: string, str2: string) { + const distance = levenshteinEq(str1, str2); + const maxLen = Math.max(str1.length, str2.length); + return (maxLen === 0) ? 0 : (distance / maxLen); +}; + +// The Levenshtein distance is a measure of the number of edits (insertions, deletions, and substitutions) +// required to change one string into another. +const levenshteinEq = function (str1: string, str2: string): number { + return new Levenshtein(str1, str2).distance; +}; + + +const requestSiblings = async function (dbService: DbService, parentId: string, contactType: string) { + const siblings: Doc[] = []; + const results = parentId && contactType && await dbService.get().query('medic-client/contacts_by_parent', { + startkey: [parentId, contactType], + endkey: [parentId, contactType, {}], + include_docs: true + }); + + if (results) { + // Desc order - reverse order by switching props + siblings.push(...results.rows.map((row: { doc: Doc }) => row.doc) + .sort((a: Doc, b: Doc) => (b.reported_date || 0) - (a.reported_date || 0))); + } + return siblings; +}; + +export type DuplicateCheck = { expression?: string; disabled?: boolean } | undefined; +const extractExpression = function (duplicateCheck: DuplicateCheck) { + // eslint-disable-next-line eqeqeq + if (duplicateCheck != null) { + if (Object.prototype.hasOwnProperty.call(duplicateCheck, 'expression')) { + return duplicateCheck.expression as string; + } else if (Object.prototype.hasOwnProperty.call(duplicateCheck, 'disabled') && duplicateCheck.disabled) { + return null; // No duplicate check should be performed + } + } + + return DEFAULT_CONTACT_DUPLICATE_EXPRESSION; +}; + +const getDuplicates = function ( + doc: Doc, + siblings: Array, + config: { + expression: string; + parseProvider: ParseProvider; + xmlFormsContextUtilsService: XmlFormsContextUtilsService; + } +) { + const { expression, parseProvider, xmlFormsContextUtilsService } = config; + // eslint-disable-next-line eqeqeq + const _siblings: Doc[] = siblings.filter((s) => !((doc._id != null && s._id === doc._id))); + // Remove the currently edited doc from the sibling list + + const duplicates: Array = []; + for (const sibling of _siblings) { + const parsed = parseProvider.parse(expression); + const test = parsed(xmlFormsContextUtilsService, { + current: doc, + existing: sibling, + }); + if (test) { + duplicates.push(sibling); + } + } + + return duplicates; +}; + +export { + normalizedLevenshteinEq, + levenshteinEq, + + requestSiblings, + extractExpression, + getDuplicates, + + DEFAULT_CONTACT_DUPLICATE_EXPRESSION +}; diff --git a/webapp/src/ts/services/xml-forms-context-utils.service.ts b/webapp/src/ts/services/xml-forms-context-utils.service.ts index a3d86cfc2aa..74e8f887946 100644 --- a/webapp/src/ts/services/xml-forms-context-utils.service.ts +++ b/webapp/src/ts/services/xml-forms-context-utils.service.ts @@ -1,5 +1,6 @@ import * as moment from 'moment'; import { Injectable } from '@angular/core'; +import { normalizedLevenshteinEq, levenshteinEq } from './utils/deduplicate'; /** * Util functions available to a form doc's `.context` function for checking if @@ -31,4 +32,12 @@ export class XmlFormsContextUtilsService { return this.getDateDiff(contact, 'years'); } + levenshteinEq(threshold: number, current: string, existing: string){ + return current && existing ? levenshteinEq(current, existing) < threshold : false; + } + + normalizedLevenshteinEq(threshold: number, current: string, existing: string){ + return current && existing ? normalizedLevenshteinEq(current, existing) < threshold : false; + } + } diff --git a/webapp/tests/karma/ts/modules/contacts/contacts-edit.component.spec.ts b/webapp/tests/karma/ts/modules/contacts/contacts-edit.component.spec.ts index 61c63b0d16c..cf43470bc9e 100644 --- a/webapp/tests/karma/ts/modules/contacts/contacts-edit.component.spec.ts +++ b/webapp/tests/karma/ts/modules/contacts/contacts-edit.component.spec.ts @@ -709,7 +709,9 @@ describe('ContactsEdit component', () => { expect(setEnketoSavingStatus.args).to.deep.equal([[true], [false]]); expect(setEnketoError.callCount).to.equal(1); expect(formService.saveContact.callCount).to.equal(1); - expect(formService.saveContact.args[0]).to.deep.equal([ form, null, 'clinic', undefined ]); + expect(formService.saveContact.args[0]).to.deep.equal([ + { form, docId: null, type: 'clinic', xmlVersion: undefined }, undefined, false + ]); expect(router.navigate.callCount).to.equal(1); expect(router.navigate.args[0]).to.deep.equal([['/contacts', 'new_clinic_id']]); }); @@ -744,7 +746,9 @@ describe('ContactsEdit component', () => { expect(setEnketoSavingStatus.args).to.deep.equal([[true], [false]]); expect(setEnketoError.callCount).to.equal(1); expect(formService.saveContact.callCount).to.equal(1); - expect(formService.saveContact.args[0]).to.deep.equal([ form, 'the_person', 'person', undefined ]); + expect(formService.saveContact.args[0]).to.deep.equal( + [ {form, docId: 'the_person', type: 'person', xmlVersion: undefined}, undefined, false ] + ); expect(router.navigate.callCount).to.equal(1); expect(router.navigate.args[0]).to.deep.equal([['/contacts', 'the_person']]); expect(performanceService.track.calledThrice).to.be.true; @@ -792,7 +796,9 @@ describe('ContactsEdit component', () => { expect(setEnketoSavingStatus.args).to.deep.equal([[true], [false]]); expect(setEnketoError.callCount).to.equal(1); expect(formService.saveContact.callCount).to.equal(1); - expect(formService.saveContact.args[0]).to.deep.equal([ form, 'the_patient', 'patient', undefined ]); + expect(formService.saveContact.args[0]).to.deep.equal( + [ { form, docId: 'the_patient', type: 'patient', xmlVersion: undefined }, undefined, false ] + ); expect(router.navigate.callCount).to.equal(1); expect(router.navigate.args[0]).to.deep.equal([['/contacts', 'the_patient']]); expect(performanceService.track.calledThrice).to.be.true; diff --git a/webapp/tests/karma/ts/services/form.service.spec.ts b/webapp/tests/karma/ts/services/form.service.spec.ts index 5b8599081a5..eec8501a382 100644 --- a/webapp/tests/karma/ts/services/form.service.spec.ts +++ b/webapp/tests/karma/ts/services/form.service.spec.ts @@ -37,6 +37,9 @@ import * as FileManager from '../../../../src/js/enketo/file-manager.js'; import { TargetAggregatesService } from '@mm-services/target-aggregates.service'; import { ContactViewModelGeneratorService } from '@mm-services/contact-view-model-generator.service'; +import { ParseProvider } from '@mm-providers/parse.provider'; +import { XmlFormsContextUtilsService } from '@mm-services/xml-forms-context-utils.service'; + describe('Form service', () => { // return a mock form ready for putting in #dbContent const mockEnketoDoc = formInternalId => { @@ -63,6 +66,7 @@ describe('Form service', () => { let dbGetAttachment; let dbGet; let dbBulkDocs; + let dbQuery; let ContactSummary; let Form2Sms; let UserContact; @@ -94,12 +98,15 @@ describe('Form service', () => { let extractLineageService; let targetAggregatesService; let contactViewModelGeneratorService; + let parserProvider; + let xmlFormsContextUtilsService; beforeEach(() => { enketoInit = sinon.stub(); dbGetAttachment = sinon.stub(); dbGet = sinon.stub(); dbBulkDocs = sinon.stub(); + dbQuery = sinon.stub(); ContactSummary = sinon.stub(); Form2Sms = sinon.stub(); UserContact = sinon.stub(); @@ -163,13 +170,16 @@ describe('Form service', () => { targetAggregatesService = { getTargetDocs: sinon.stub() }; contactViewModelGeneratorService = { loadReports: sinon.stub() }; + parserProvider = sinon.stub(); + xmlFormsContextUtilsService = sinon.stub(); + TestBed.configureTestingModule({ providers: [ provideMockStore(), { provide: DbService, useValue: { - get: () => ({ getAttachment: dbGetAttachment, get: dbGet, bulkDocs: dbBulkDocs }) + get: () => ({ getAttachment: dbGetAttachment, get: dbGet, bulkDocs: dbBulkDocs, query: dbQuery }) } }, { provide: ContactSummaryService, useValue: { get: ContactSummary } }, @@ -193,6 +203,8 @@ describe('Form service', () => { { provide: ExtractLineageService, useValue: extractLineageService }, { provide: TargetAggregatesService, useValue: targetAggregatesService }, { provide: ContactViewModelGeneratorService, useValue: contactViewModelGeneratorService }, + { provide: ParseProvider, useValue: parserProvider}, + { provide: XmlFormsContextUtilsService, useValue: xmlFormsContextUtilsService } ], }); @@ -1299,19 +1311,24 @@ describe('Form service', () => { let extractLineageService; let enketoTranslationService; + let parse; + beforeEach(() => { extractLineageService = { extract: sinon.stub() }; enketoTranslationService = { contactRecordToJs: sinon.stub(), }; + parse = sinon.stub(); + parserProvider = { parse }; + TestBed.configureTestingModule({ providers: [ provideMockStore(), { provide: DbService, useValue: { - get: () => ({ getAttachment: dbGetAttachment, get: dbGet, bulkDocs: dbBulkDocs }) + get: () => ({ getAttachment: dbGetAttachment, get: dbGet, bulkDocs: dbBulkDocs, query: dbQuery }) } }, { provide: ContactSummaryService, useValue: { get: ContactSummary } }, @@ -1334,6 +1351,8 @@ describe('Form service', () => { { provide: TranslateService, useValue: translateService }, { provide: TrainingCardsService, useValue: trainingCardsService }, { provide: FeedbackService, useValue: feedbackService }, + { provide: ParseProvider, useValue: parserProvider}, + { provide: XmlFormsContextUtilsService, useValue: xmlFormsContextUtilsService }, ], }); @@ -1357,7 +1376,7 @@ describe('Form service', () => { extractLineageService.extract.returns({ _id: 'abc', parent: { _id: 'def' } }); return service - .saveContact(form, docId, type) + .saveContact({form, docId, type}) .then(() => { assert.equal(dbGet.callCount, 1); assert.equal(dbGet.args[0][0], 'abc'); @@ -1391,7 +1410,7 @@ describe('Form service', () => { extractLineageService.extract.returns({ _id: 'abc', parent: { _id: 'def' } }); return service - .saveContact(form, docId, type) + .saveContact({form, docId, type}) .then(() => { assert.equal(dbGet.callCount, 1); assert.equal(dbGet.args[0][0], 'abc'); @@ -1435,7 +1454,7 @@ describe('Form service', () => { dbBulkDocs.resolves([]); return service - .saveContact(form, docId, type) + .saveContact({form, docId, type}) .then(() => { assert.isTrue(dbBulkDocs.calledOnce); @@ -1494,7 +1513,7 @@ describe('Form service', () => { clock = sinon.useFakeTimers(5000); return service - .saveContact(form, docId, type) + .saveContact({form, docId, type}) .then(() => { assert.equal(dbGet.callCount, 2); assert.deepEqual(dbGet.args[0], ['main1']); @@ -1543,7 +1562,7 @@ describe('Form service', () => { clock = sinon.useFakeTimers(1000); return service - .saveContact(form, docId, type) + .saveContact({form, docId, type}) .then(() => { assert.equal(dbGet.callCount, 1); assert.equal(dbGet.args[0][0], 'abc'); @@ -1580,5 +1599,191 @@ describe('Form service', () => { assert.deepEqual(setLastChangedDoc.args[0], [savedDocs[0]]); }); }); + + it('should throw an error with duplicates found', async function () { + const form = { getDataStr: () => '' }; + const docId = null; + const type = 'some-contact-type'; + + dbGet.resolves({ }); + enketoTranslationService.contactRecordToJs.returns({ + doc: { _id: 'main1', name: 'Main', type: 'main', parent: { _id: 'parent1' } } + }); + extractLineageService.extract.returns({ _id: 'parent1'}); + transitionsService.applyTransitions.callsFake((docs) => { + docs[0].transitioned = true; + return Promise.resolve(docs); + }); + dbQuery.resolves({ + offset: 0, + rows: [ + { id: 'sib1', + doc: { + _id: 'sib1', + name: 'Sibling1', + parent: { _id: 'parent1' }, + type, + reported_date: 1736845534000 + } + }, + { + id: 'sib2', + doc: { + _id: 'sib2', + name: 'Sibling2', + parent: { _id: 'parent1' }, + type, + reported_date: 1736845534000 + } + }, + ], + total_rows: 2 + }); + dbBulkDocs.resolves([]); + clock = sinon.useFakeTimers(1000); + // eslint-disable-next-line @typescript-eslint/no-unused-vars, no-undef + parse.callsFake(() => (XmlFormsContextUtilsService, ctx) => true); + try { + await service.saveContact({form, docId, type, xmlVersion: undefined}, undefined, false); + // Fail the test if no error is thrown + throw new Error('Expected saveContact to throw an error, but it did not.'); + } catch (e) { + expect(e.message).to.include('Duplicates found'); + expect(e.duplicates).to.have.lengthOf(2); + expect(e.duplicates).to.deep.equal([ + { + _id: 'sib1', + name: 'Sibling1', + parent: { _id: 'parent1' }, + type, + reported_date: 1736845534000 + }, + { + _id: 'sib2', + name: 'Sibling2', + parent: { _id: 'parent1' }, + type, + reported_date: 1736845534000 + } + ]); + } + }); + + it('should pass duplicate check when duplicates are acknowledged', async function () { + const form = { getDataStr: () => '' }; + const docId = null; + const type = 'some-contact-type'; + + dbGet.resolves({ }); + enketoTranslationService.contactRecordToJs.returns({ + doc: { _id: 'main1', name: 'Main', type: 'main', parent: { _id: 'parent1' } } + }); + extractLineageService.extract.returns({ _id: 'parent1'}); + transitionsService.applyTransitions.callsFake((docs) => { + docs[0].transitioned = true; + return Promise.resolve(docs); + }); + dbQuery.resolves({ + offset: 0, + rows: [ + { id: 'sib1', + doc: { + _id: 'sib1', + name: 'Sibling1', + parent: { _id: 'parent1' }, + type, + reported_date: 1736845534000 + } + }, + { + id: 'sib2', + doc: { + _id: 'sib2', + name: 'Sibling2', + parent: { _id: 'parent1' }, + type, + reported_date: 1736845534000 + } + }, + ], + total_rows: 2 + }); + dbBulkDocs.resolves([]); + clock = sinon.useFakeTimers(1000); + + await service.saveContact({form, docId, type, xmlVersion: undefined}, undefined, true); + assert.equal(transitionsService.applyTransitions.callCount, 1); + assert.deepEqual(transitionsService.applyTransitions.args[0], [[ + { + _id: 'main1', + name: 'Main', + type: 'contact', + contact_type: type, + parent: { _id: 'parent1' }, + reported_date: 1000, + contact: undefined, + transitioned: true + } + ]]); + }); + + it('should pass duplicate check when record is marked as canonical', async function () { + const form = { getDataStr: () => '' }; + const docId = null; + const type = 'some-contact-type'; + + dbGet.resolves({ }); + enketoTranslationService.contactRecordToJs.returns({ + doc: { _id: 'main1', name: 'Main', type: 'main', parent: { _id: 'parent1' }, is_canonical: 'true' } + }); + extractLineageService.extract.returns({ _id: 'parent1'}); + transitionsService.applyTransitions.callsFake((docs) => { + docs[0].transitioned = true; + return Promise.resolve(docs); + }); + dbQuery.resolves({ + offset: 0, + rows: [ + { id: 'sib1', + doc: { + _id: 'sib1', + name: 'Sibling1', + parent: { _id: 'parent1' }, + type, + reported_date: 1736845534000 + } + }, + { + id: 'sib2', + doc: { + _id: 'sib2', + name: 'Sibling2', + parent: { _id: 'parent1' }, + type, + reported_date: 1736845534000 + } + }, + ], + total_rows: 2 + }); + dbBulkDocs.resolves([]); + clock = sinon.useFakeTimers(1000); + + await service.saveContact({form, docId, type, xmlVersion: undefined}, undefined, false); + assert.equal(transitionsService.applyTransitions.callCount, 1); + assert.deepEqual(transitionsService.applyTransitions.args[0], [[ + { + _id: 'main1', + name: 'Main', + type: 'contact', + contact_type: type, + parent: { _id: 'parent1' }, + reported_date: 1000, + contact: undefined, + transitioned: true, + is_canonical: 'true' + } + ]]); + }); }); }); diff --git a/webapp/tests/karma/ts/services/utils/deduplicate.spec.ts b/webapp/tests/karma/ts/services/utils/deduplicate.spec.ts new file mode 100644 index 00000000000..be5a6e89d3a --- /dev/null +++ b/webapp/tests/karma/ts/services/utils/deduplicate.spec.ts @@ -0,0 +1,156 @@ +import { TestBed } from '@angular/core/testing'; +import sinon from 'sinon'; +import { expect } from 'chai'; + +import { DbService } from '@mm-services/db.service'; +import { ParseProvider } from '@mm-providers/parse.provider'; +import { XmlFormsContextUtilsService } from '@mm-services/xml-forms-context-utils.service'; +import { + normalizedLevenshteinEq, + levenshteinEq, + requestSiblings, + extractExpression, + DEFAULT_CONTACT_DUPLICATE_EXPRESSION, + getDuplicates, +} from '../../../../../src/ts/services/utils/deduplicate'; + +describe('Deduplicate', () => { + let dbService; + let query; + + beforeEach(() => { + query = sinon.stub(); + dbService = { + get: () => ({ query }) + }; + + TestBed.configureTestingModule({ + providers: [ + { provide: DbService, useValue: dbService } + ] + }); + }); + + afterEach(() => { + sinon.restore(); + }); + + describe('normalizedLevenshteinEq', () => { + it('should return return a score of 3', () => { + // Score/distance / maxLength + // 3 (3 characters need to be added to make str1 = str2) / 5 (Test123 is the larger string) + // ~ 0.42857142857142855 + expect(normalizedLevenshteinEq('Test123', 'Test')).lessThanOrEqual(0.42857142857142855); + }); + }); + + describe('levenshteinEq', () => { + it('should return return a score of 3', () => { + expect(levenshteinEq('Test123', 'Test')).to.equal(3); + }); + }); + + describe('requestSiblings', () => { + it('should return results filtered by parent and contact type', async function () { + query.resolves({ + offset: 0, + rows: [ + { id: 'sib1', doc: { _id: 'sib1', name: 'Sibling1', parent: { _id: 'parent1' }, contact_type: 'some_type' } }, + { id: 'sib2', doc: { _id: 'sib2', name: 'Sibling2', parent: { _id: 'parent1' }, contact_type: 'some_type' }}, + ], + total_rows: 6 + }); + const siblings = await requestSiblings(dbService, 'parent1', 'some_type'); + expect(siblings.length).to.equal(2); + expect(siblings).to.deep.equal([ + { _id: 'sib1', name: 'Sibling1', parent: { _id: 'parent1' }, contact_type: 'some_type' }, + { _id: 'sib2', name: 'Sibling2', parent: { _id: 'parent1' }, contact_type: 'some_type' }, + ]); + }); + }); + + describe('extractExpression', () => { + it('should return a default expression when none is provided', () => { + expect(extractExpression(undefined)).to.equal(DEFAULT_CONTACT_DUPLICATE_EXPRESSION); + }); + }); + + describe('getDuplicates', () => { + let pipesService; + let parseProvider; + beforeEach(() => { + pipesService = { + getPipeNameVsIsPureMap: sinon.stub().returns(new Map([['date', { pure: true }]])), + meta: sinon.stub(), + getInstance: sinon.stub(), + }; + parseProvider = new ParseProvider(pipesService); + }); + + it('should return duplicates based on default matching', () => { + const doc = { + _id: 'new', + name: 'Test', + parent: { _id: 'parent1' }, + contact_type: 'some_type', + reported_date: 1736845534000 + }; + const siblings = [ + { + _id: 'sib1', + name: 'Test1', + parent: { _id: 'parent1' }, + contact_type: 'some_type', + reported_date: 1736845534000 + }, + { + _id: 'sib2', + name: 'Test2', + parent: { _id: 'parent1' }, + contact_type: 'some_type', + reported_date: 1736845534000 + }, + { + _id: 'sib3', + name: 'Test the things', + parent: { _id: 'parent1' }, + contact_type: 'some_type', + reported_date: 1736845534000 + }, + { + _id: 'sib4', + name: 'Testimony', + parent: { _id: 'parent1' }, + contact_type: 'some_type', + reported_date: 1736845534000 + }, + ]; + const results = getDuplicates( + doc, + siblings, + { + expression: DEFAULT_CONTACT_DUPLICATE_EXPRESSION, + parseProvider, + xmlFormsContextUtilsService: new XmlFormsContextUtilsService() + } + ); + expect(results.length).equal(2); + expect(results).to.deep.equal([ + { + _id: 'sib1', + name: 'Test1', + parent: { _id: 'parent1' }, + contact_type: 'some_type', + reported_date: 1736845534000 + }, + { + _id: 'sib2', + name: 'Test2', + parent: { _id: 'parent1' }, + contact_type: 'some_type', + reported_date: 1736845534000 + }, + ]); + }); + }); +});