From 17831fc2ec0ac8362d4d29d67833bc1e07eecc65 Mon Sep 17 00:00:00 2001 From: Bryon Lewis <61746913+BryonLewis@users.noreply.github.com> Date: Fri, 5 Apr 2024 09:46:47 -0400 Subject: [PATCH] update numerical filter (#120) * update numerical filter * reloading filters properly, added counts to filter results * count of filter display, initialization of filter URL parameters * fix url parameters * linting * version nudge --- client/package.json | 2 +- .../web-girder/api/divemetadata.service.ts | 2 + .../web-girder/views/DIVEMetadataFilter.vue | 62 ++++++++++++++++--- .../views/DIVEMetadataFilterItem.vue | 51 +++++++++------ .../web-girder/views/DIVEMetadataSearch.vue | 14 ++++- server/dive_server/views_metadata.py | 16 +++-- server/dive_utils/metadata/models.py | 14 +++-- 7 files changed, 123 insertions(+), 38 deletions(-) diff --git a/client/package.json b/client/package.json index 6420d0e..153ea71 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "dive-dsa", - "version": "1.10.2", + "version": "1.10.3", "author": { "name": "Kitware, Inc.", "email": "Bryon.Lewis@kitware.com" diff --git a/client/platform/web-girder/api/divemetadata.service.ts b/client/platform/web-girder/api/divemetadata.service.ts index 8baa211..45f92ac 100644 --- a/client/platform/web-girder/api/divemetadata.service.ts +++ b/client/platform/web-girder/api/divemetadata.service.ts @@ -38,6 +38,8 @@ export interface DIVEMetadataFilterValueResults { export interface DIVEMetadataResults { pageResults: MetadataResultItem[]; totalPages: number; + filtered: number; + count: number; } export interface MetadataResultItem { diff --git a/client/platform/web-girder/views/DIVEMetadataFilter.vue b/client/platform/web-girder/views/DIVEMetadataFilter.vue index 83895a7..101b1c1 100644 --- a/client/platform/web-girder/views/DIVEMetadataFilter.vue +++ b/client/platform/web-girder/views/DIVEMetadataFilter.vue @@ -5,6 +5,7 @@ import { import { computed, defineComponent, onBeforeMount, PropType, Ref, ref, watch, } from 'vue'; +import { intersection } from 'lodash'; import DIVEMetadataFilterItemVue from './DIVEMetadataFilterItem.vue'; import DIVEMetadataCloneVue from './DIVEMetadataClone.vue'; @@ -20,6 +21,14 @@ export default defineComponent({ type: Number, default: 0, }, + count: { + type: Number, + default: 0, + }, + filtered: { + type: Number, + default: 0, + }, id: { type: String, default: '', @@ -35,7 +44,7 @@ export default defineComponent({ }, }, setup(props, { emit }) { - const search = ref(props.rootFilter.search || ''); + const search: Ref = ref(props.rootFilter.search || ''); const filters: Ref = ref({}); const splitFilters = computed(() => { const advanced: DIVEMetadataFilterValueResults['metadataKeys'] = {}; @@ -50,6 +59,7 @@ export default defineComponent({ return { advanced, displayed }; }); const filtersOn = ref(false); + const defaultEnabledKeys: Ref = ref([]); // If items should default to on because they are in the URL parameters const currentFilter: Ref = ref({}); const pageList = computed(() => { const list = []; @@ -65,12 +75,20 @@ export default defineComponent({ }; onBeforeMount(async () => { await getFilters(); + if (props.rootFilter.metadataFilters) { + const metadataKeys = Object.keys(props.rootFilter.metadataFilters); + const advancedKeys = Object.keys(splitFilters.value.advanced); + defaultEnabledKeys.value = metadataKeys; + if (intersection(metadataKeys, advancedKeys)) { + filtersOn.value = true; + } + } loadCurrentFilter(); }); const loadCurrentFilter = () => { if (props.rootFilter.metadataFilters && Object.keys(props.rootFilter.metadataFilters).length) { - currentFilter.value = props.rootFilter.metadataFilters; + currentFilter.value.metadataFilters = props.rootFilter.metadataFilters; } if (props.rootFilter.search) { search.value = props.rootFilter.search; @@ -86,6 +104,15 @@ export default defineComponent({ emit('updateFilters', currentFilter.value); }); + const clearFilter = (key: string) => { + if (!currentFilter.value.metadataFilters) { + currentFilter.value.metadataFilters = {}; + } + if (currentFilter.value.metadataFilters[key]) { + delete currentFilter.value.metadataFilters[key]; + } + emit('updateFilters', currentFilter.value); + }; const updateFilter = (key: string, { value, category } : {value: string | string[] | number | boolean | number[], category: MetadataFilterItem['category']}) => { if (!currentFilter.value.metadataFilters) { currentFilter.value.metadataFilters = {}; @@ -105,7 +132,6 @@ export default defineComponent({ value, }; } - emit('updateFilters', currentFilter.value); }; @@ -114,8 +140,13 @@ export default defineComponent({ }; const getDefaultValue = (key: string) => { - if (props.rootFilter?.metadataFilters && props.rootFilter.metadataFilters[key]) { - return props.rootFilter.metadataFilters[key].value; + if (props.rootFilter?.metadataFilters) { + if (props.rootFilter.metadataFilters[key] && props.rootFilter.metadataFilters[key].category === 'numerical') { + return props.rootFilter.metadataFilters[key].range; + } + if (props.rootFilter.metadataFilters[key]) { + return props.rootFilter.metadataFilters[key].value; + } } return undefined; }; @@ -128,8 +159,10 @@ export default defineComponent({ filtersOn, search, currentFilter, + defaultEnabledKeys, changePage, updateFilter, + clearFilter, getDefaultValue, }; }, @@ -175,7 +208,14 @@ export default defineComponent({ >
- +
@@ -186,12 +226,20 @@ export default defineComponent({ mt-3" >
- +
+ Filtered:{{ filtered }} / {{ count }} import { MetadataFilterKeysItem } from 'platform/web-girder/api/divemetadata.service'; import { - defineComponent, ref, PropType, watch, onMounted, Ref, + defineComponent, ref, PropType, watch, Ref, } from 'vue'; export default defineComponent({ @@ -19,13 +19,18 @@ export default defineComponent({ type: [String, Number, Array, Boolean] as PropType, default: undefined, }, + defaultEnabled: { + type: Boolean, + default: false, + }, }, setup(props, { emit }) { const set = ref(props.filterItem.set); const value: Ref = ref(props.defaultValue); const rangeFilterEnabled = ref(false); const categoryLimit = ref(20); - watch(value, () => { + const enabled = ref(props.defaultEnabled); // numerical enabled filter + watch([value, enabled], () => { const update = { value: value.value, category: props.filterItem.category, @@ -33,20 +38,25 @@ export default defineComponent({ if (props.filterItem.category === 'categorical' && props.filterItem.count > categoryLimit.value) { update.category = 'search'; } - emit('update-value', update); - }); - - onMounted(() => { - if (props.filterItem.category === 'numerical' && props.filterItem.range) { - value.value = [props.filterItem.range.min, props.filterItem.range.max]; + if (props.filterItem.category === 'numerical' && !enabled.value) { + emit('clear-filter'); + return; // skip emitting the value unless the checkbox is enabled } + emit('update-value', update); }); - + if (enabled.value) { + const update = { + value: value.value, + category: props.filterItem.category, + }; + emit('update-value', update); + } return { set, value, rangeFilterEnabled, categoryLimit, + enabled, }; }, }); @@ -72,15 +82,20 @@ export default defineComponent({
- + + + +
diff --git a/client/platform/web-girder/views/DIVEMetadataSearch.vue b/client/platform/web-girder/views/DIVEMetadataSearch.vue index 14ca78b..276b6d2 100644 --- a/client/platform/web-girder/views/DIVEMetadataSearch.vue +++ b/client/platform/web-girder/views/DIVEMetadataSearch.vue @@ -33,6 +33,8 @@ export default defineComponent({ const displayConfig: Ref = ref({ display: [], hide: [] }); const totalPages = ref(0); const currentPage = ref(0); + const count = ref(0); + const filtered = ref(0); const filters: Ref = ref(props.filter || {}); const locationStore = { _id: props.id, @@ -44,6 +46,8 @@ export default defineComponent({ const processFilteredMetadataResults = (data: DIVEMetadataResults) => { folderList.value = data.pageResults; totalPages.value = data.totalPages; + filtered.value = data.filtered; + count.value = data.count; }; const getData = async () => { const { data } = await filterDiveMetadata(props.id, { ...filters.value }); @@ -58,7 +62,11 @@ export default defineComponent({ }; const updateURLParams = () => { - router.replace({ path: props.id, params: { id: props.id }, query: { filter: JSON.stringify(filters.value) } }); + if ((filters.value.metadataFilters && Object.keys(filters.value.metadataFilters).length) || filters.value.search) { + router.replace({ path: props.id, params: { id: props.id }, query: { filter: JSON.stringify(filters.value) } }); + } else { + window.location.href = window.location.href.replace(/filter=.*/, ''); + } }; onMounted(() => { @@ -99,6 +107,8 @@ export default defineComponent({ const openClone = ref(false); return { totalPages, + count, + filtered, currentPage, changePage, locationStore, @@ -122,6 +132,8 @@ export default defineComponent({ :current-page="currentPage" :root-filter="filters" :total-pages="totalPages" + :count="count" + :filtered="filtered" :display-config="displayConfig" @update:currentPage="changePage($event)" @updateFilters="updateFilter($event)" diff --git a/server/dive_server/views_metadata.py b/server/dive_server/views_metadata.py index c0eb490..13b3b99 100644 --- a/server/dive_server/views_metadata.py +++ b/server/dive_server/views_metadata.py @@ -69,7 +69,7 @@ def load_metadata_json(search_folder, type='ndjson'): Folder().childItems( search_folder, filters={"lowerName": {"$regex": regex}}, - sort=[("created", pymongo.ASCENDING)], + sort=[("updated", pymongo.ASCENDING)], ) ) if len(json_items) > 0: @@ -195,8 +195,6 @@ def process_metadata(self, folder, sibling_path, fileType, matcher, path_key, di ] } results = list(Folder().findWithPermissions(query=query, user=user)) - print(query) - print(f"RESULTS LENGTH: {len(results)}") if len(results) > 0: matched = False key_path = item.get(path_key, False) @@ -228,9 +226,11 @@ def process_metadata(self, folder, sibling_path, fileType, matcher, path_key, di else: errorLog.append(f"Could not find any results for Video file {item[matcher]}") for key in item.keys(): - if key not in metadataKeys.keys(): + if key not in metadataKeys.keys() and item[key] is not None: datatype = python_to_javascript_type(type(item[key])) metadataKeys[key] = {"type": datatype, "set": set(), "count": 1} + if item[key] is None: + continue # we skip null values for processing if metadataKeys[key]['type'] == 'string': metadataKeys[key]['set'].add(item[key]) metadataKeys[key]['count'] += 1 @@ -262,13 +262,13 @@ def process_metadata(self, folder, sibling_path, fileType, matcher, path_key, di del metadataKeys[key]['set'] else: del metadataKeys[key]['set'] - DIVE_MetadataKeys().createMetadataKeys(datasetFolder, folder, user, metadataKeys) + DIVE_MetadataKeys().createMetadataKeys(folder, user, metadataKeys) # add metadata to root folder for folder['meta'][DIVEMetadataMarker] = True folder['meta'][DIVEMetadataFilter] = displayKeys Folder().save(folder) - return {"results": f"added {added} folders", "errors": errorLog} + return {"results": f"added {added} folders", "errors": errorLog, "metadataKeys": metadataKeys} @access.user @autoDescribeRoute( @@ -313,6 +313,8 @@ def filter_folder(self, folder, filters, limit, offset, sort): user = self.getCurrentUser() query = self.get_filter_query(folder, user, filters) + total_query = self.get_filter_query(folder, user, {}) + total_items = DIVE_Metadata().find(total_query).count() metadata_items = DIVE_Metadata().find( query, offset=offset, limit=limit, sort=sort, user=self.getCurrentUser() ) @@ -321,6 +323,8 @@ def filter_folder(self, folder, filters, limit, offset, sort): structured_results = { 'totalPages': pages, 'pageResults': list(metadata_items), + 'count': total_items, + 'filtered': metadata_items.count() } return structured_results diff --git a/server/dive_utils/metadata/models.py b/server/dive_utils/metadata/models.py index 4c03480..dc10d0a 100644 --- a/server/dive_utils/metadata/models.py +++ b/server/dive_utils/metadata/models.py @@ -54,7 +54,9 @@ def createMetadata(self, folder, root, owner, metadata, created_date=None): metadata=metadata, created=created, ) - existing = self.save(existing) + else: + existing['metadata'] = metadata + existing = self.save(existing) return existing def validate(self, doc): @@ -78,7 +80,7 @@ def _cleanupDeletedEntity(self, event): # remove data if the folderId matches entityDoc = event.info folderId = entityDoc['_id'] - dive_dataset = self.findOne({'DIVEDataset': str(folderId)}) + dive_dataset = self.findOne({'root': str(folderId)}) if dive_dataset is not None: self.remove(dive_dataset) @@ -96,8 +98,8 @@ def initialize(self): ] ) - def createMetadataKeys(self, folder, root, owner, metadataKeys, created_date=None): - existing = self.findOne({'DIVEDataset': str(folder['_id'])}) + def createMetadataKeys(self, root, owner, metadataKeys, created_date=None): + existing = self.findOne({'root': str(root['_id'])}) if not existing: if created_date is None: created = datetime.datetime.utcnow() @@ -109,7 +111,9 @@ def createMetadataKeys(self, folder, root, owner, metadataKeys, created_date=Non metadataKeys=metadataKeys, created=created, ) - existing = self.save(existing) + else: + existing['metadataKeys'] = metadataKeys + self.save(existing) return existing def validate(self, doc):