update numerical filter (#120)

* update numerical filter * reloading filters properly, added counts to filter results * count of filter display, initialization of filter URL parameters * fix url parameters * linting * version nudge
DigitalSlideArchive · Apr 5, 2024 · 17831fc · 17831fc
1 parent e978219
commit 17831fc
Show file tree

Hide file tree

Showing 7 changed files with 123 additions and 38 deletions.
diff --git a/client/package.json b/client/package.json
@@ -1,6 +1,6 @@
 {
   "name": "dive-dsa",
-  "version": "1.10.2",
+  "version": "1.10.3",
   "author": {
     "name": "Kitware, Inc.",
     "email": "Bryon.Lewis@kitware.com"

diff --git a/client/platform/web-girder/api/divemetadata.service.ts b/client/platform/web-girder/api/divemetadata.service.ts
@@ -38,6 +38,8 @@ export interface DIVEMetadataFilterValueResults {
 export interface DIVEMetadataResults {
     pageResults: MetadataResultItem[];
     totalPages: number;
+    filtered: number;
+    count: number;
 }
 
 export interface MetadataResultItem {

diff --git a/client/platform/web-girder/views/DIVEMetadataFilter.vue b/client/platform/web-girder/views/DIVEMetadataFilter.vue
@@ -5,6 +5,7 @@ import {
 import {
   computed, defineComponent, onBeforeMount, PropType, Ref, ref, watch,
 } from 'vue';
+import { intersection } from 'lodash';
 import DIVEMetadataFilterItemVue from './DIVEMetadataFilterItem.vue';
 import DIVEMetadataCloneVue from './DIVEMetadataClone.vue';
 
@@ -20,6 +21,14 @@ export default defineComponent({
       type: Number,
       default: 0,
     },
+    count: {
+      type: Number,
+      default: 0,
+    },
+    filtered: {
+      type: Number,
+      default: 0,
+    },
     id: {
       type: String,
       default: '',
@@ -35,7 +44,7 @@ export default defineComponent({
     },
   },
   setup(props, { emit }) {
-    const search = ref(props.rootFilter.search || '');
+    const search: Ref<string> = ref(props.rootFilter.search || '');
     const filters: Ref<DIVEMetadataFilterValueResults['metadataKeys']> = ref({});
     const splitFilters = computed(() => {
       const advanced: DIVEMetadataFilterValueResults['metadataKeys'] = {};
@@ -50,6 +59,7 @@ export default defineComponent({
       return { advanced, displayed };
     });
     const filtersOn = ref(false);
+    const defaultEnabledKeys: Ref<string[]> = ref([]); // If items should default to on because they are in the URL parameters
     const currentFilter: Ref<DIVEMetadataFilter> = ref({});
     const pageList = computed(() => {
       const list = [];
@@ -65,12 +75,20 @@ export default defineComponent({
     };
     onBeforeMount(async () => {
       await getFilters();
+      if (props.rootFilter.metadataFilters) {
+        const metadataKeys = Object.keys(props.rootFilter.metadataFilters);
+        const advancedKeys = Object.keys(splitFilters.value.advanced);
+        defaultEnabledKeys.value = metadataKeys;
+        if (intersection(metadataKeys, advancedKeys)) {
+          filtersOn.value = true;
+        }
+      }
       loadCurrentFilter();
     });
 
     const loadCurrentFilter = () => {
       if (props.rootFilter.metadataFilters && Object.keys(props.rootFilter.metadataFilters).length) {
-        currentFilter.value = props.rootFilter.metadataFilters;
+        currentFilter.value.metadataFilters = props.rootFilter.metadataFilters;
       }
       if (props.rootFilter.search) {
         search.value = props.rootFilter.search;
@@ -86,6 +104,15 @@ export default defineComponent({
       emit('updateFilters', currentFilter.value);
     });
 
+    const clearFilter = (key: string) => {
+      if (!currentFilter.value.metadataFilters) {
+        currentFilter.value.metadataFilters = {};
+      }
+      if (currentFilter.value.metadataFilters[key]) {
+        delete currentFilter.value.metadataFilters[key];
+      }
+      emit('updateFilters', currentFilter.value);
+    };
     const updateFilter = (key: string, { value, category } : {value: string | string[] | number | boolean | number[], category: MetadataFilterItem['category']}) => {
       if (!currentFilter.value.metadataFilters) {
         currentFilter.value.metadataFilters = {};
@@ -105,7 +132,6 @@ export default defineComponent({
           value,
         };
       }
-
       emit('updateFilters', currentFilter.value);
     };
 
@@ -114,8 +140,13 @@ export default defineComponent({
     };
 
     const getDefaultValue = (key: string) => {
-      if (props.rootFilter?.metadataFilters && props.rootFilter.metadataFilters[key]) {
-        return props.rootFilter.metadataFilters[key].value;
+      if (props.rootFilter?.metadataFilters) {
+        if (props.rootFilter.metadataFilters[key] && props.rootFilter.metadataFilters[key].category === 'numerical') {
+          return props.rootFilter.metadataFilters[key].range;
+        }
+        if (props.rootFilter.metadataFilters[key]) {
+          return props.rootFilter.metadataFilters[key].value;
+        }
       }
       return undefined;
     };
@@ -128,8 +159,10 @@ export default defineComponent({
       filtersOn,
       search,
       currentFilter,
+      defaultEnabledKeys,
       changePage,
       updateFilter,
+      clearFilter,
       getDefaultValue,
     };
   },
@@ -175,7 +208,14 @@ export default defineComponent({
       >
         <v-spacer />
         <div v-for="(filterItem, key) in splitFilters.displayed" :key="`filterItem_${key}`">
-          <DIVEMetadataFilterItemVue :label="key" :default-value="getDefaultValue(key)" :filter-item="filterItem" @update-value="updateFilter(key, $event)" />
+          <DIVEMetadataFilterItemVue
+            :label="key"
+            :default-value="getDefaultValue(key)"
+            :filter-item="filterItem"
+            :default-enabled="defaultEnabledKeys.includes(key)"
+            @update-value="updateFilter(key, $event)"
+            @clear-filter="clearFilter(key)"
+          />
         </div>
         <v-spacer />
       </v-row>
@@ -186,12 +226,20 @@ export default defineComponent({
           mt-3"
       >
         <div v-for="(filterItem, key) in splitFilters.advanced" :key="`filterItem_${key}`">
-          <DIVEMetadataFilterItemVue :label="key" :default-value="getDefaultValue(key)" :filter-item="filterItem" @update-value="updateFilter(key, $event, filterItem.category)" />
+          <DIVEMetadataFilterItemVue
+            :label="key"
+            :default-value="getDefaultValue(key)"
+            :filter-item="filterItem"
+            :default-enabled="defaultEnabledKeys.includes(key)"
+            @update-value="updateFilter(key, $event)"
+            @clear-filter="clearFilter(key)"
+          />
         </div>
       </v-row>
       <v-row class="mt-3">
         <slot name="leftOptions" />
         <v-spacer />
+        <v-chip><span class="pr-1">Filtered:</span>{{ filtered }} / {{ count }}</v-chip>
         <v-select
           class="mx-2 pa-0 fit"
           style="max-width: 50px"

diff --git a/client/platform/web-girder/views/DIVEMetadataFilterItem.vue b/client/platform/web-girder/views/DIVEMetadataFilterItem.vue
@@ -1,7 +1,7 @@
 <script lang="ts">
 import { MetadataFilterKeysItem } from 'platform/web-girder/api/divemetadata.service';
 import {
-  defineComponent, ref, PropType, watch, onMounted, Ref,
+  defineComponent, ref, PropType, watch, Ref,
 } from 'vue';
 
 export default defineComponent({
@@ -19,34 +19,44 @@ export default defineComponent({
       type: [String, Number, Array, Boolean] as PropType<boolean | string | number | string[] | number[]>,
       default: undefined,
     },
+    defaultEnabled: {
+      type: Boolean,
+      default: false,
+    },
   },
   setup(props, { emit }) {
     const set = ref(props.filterItem.set);
     const value: Ref<undefined | boolean | number | string | string[] | number[]> = ref(props.defaultValue);
     const rangeFilterEnabled = ref(false);
     const categoryLimit = ref(20);
-    watch(value, () => {
+    const enabled = ref(props.defaultEnabled); // numerical enabled filter
+    watch([value, enabled], () => {
       const update = {
         value: value.value,
         category: props.filterItem.category,
       };
       if (props.filterItem.category === 'categorical' && props.filterItem.count > categoryLimit.value) {
         update.category = 'search';
       }
-      emit('update-value', update);
-    });
-
-    onMounted(() => {
-      if (props.filterItem.category === 'numerical' && props.filterItem.range) {
-        value.value = [props.filterItem.range.min, props.filterItem.range.max];
+      if (props.filterItem.category === 'numerical' && !enabled.value) {
+        emit('clear-filter');
+        return; // skip emitting the value unless the checkbox is enabled
       }
+      emit('update-value', update);
     });
-
+    if (enabled.value) {
+      const update = {
+        value: value.value,
+        category: props.filterItem.category,
+      };
+      emit('update-value', update);
+    }
     return {
       set,
       value,
       rangeFilterEnabled,
       categoryLimit,
+      enabled,
     };
   },
 });
@@ -72,15 +82,20 @@ export default defineComponent({
       <v-checkbox v-model="value" :label="label" />
     </div>
     <div v-else-if="filterItem.category === 'numerical' && filterItem.range">
-      <v-range-slider
-        v-model="value"
-        :min="filterItem.range.min"
-        :max="filterItem.range.max"
-        :label="label"
-        style="min-width: 250px;"
-        thumb-label="always"
-        class="pt-7"
-      />
+      <v-row dense align="center">
+        <v-checkbox v-model="enabled" hide-details="" />
+        <v-range-slider
+          v-model="value"
+          :min="filterItem.range.min"
+          :max="filterItem.range.max"
+          :disabled="!enabled"
+          :label="label"
+          style="min-width: 250px;"
+          thumb-label="always"
+          class="pt-7"
+          hide-details=""
+        />
+      </v-row>
     </div>
   </div>
 </template>

diff --git a/client/platform/web-girder/views/DIVEMetadataSearch.vue b/client/platform/web-girder/views/DIVEMetadataSearch.vue
@@ -33,6 +33,8 @@ export default defineComponent({
     const displayConfig: Ref<FilterDisplayConfig> = ref({ display: [], hide: [] });
     const totalPages = ref(0);
     const currentPage = ref(0);
+    const count = ref(0);
+    const filtered = ref(0);
     const filters: Ref<DIVEMetadataFilter> = ref(props.filter || {});
     const locationStore = {
       _id: props.id,
@@ -44,6 +46,8 @@ export default defineComponent({
     const processFilteredMetadataResults = (data: DIVEMetadataResults) => {
       folderList.value = data.pageResults;
       totalPages.value = data.totalPages;
+      filtered.value = data.filtered;
+      count.value = data.count;
     };
     const getData = async () => {
       const { data } = await filterDiveMetadata(props.id, { ...filters.value });
@@ -58,7 +62,11 @@ export default defineComponent({
     };
 
     const updateURLParams = () => {
-      router.replace({ path: props.id, params: { id: props.id }, query: { filter: JSON.stringify(filters.value) } });
+      if ((filters.value.metadataFilters && Object.keys(filters.value.metadataFilters).length) || filters.value.search) {
+        router.replace({ path: props.id, params: { id: props.id }, query: { filter: JSON.stringify(filters.value) } });
+      } else {
+        window.location.href = window.location.href.replace(/filter=.*/, '');
+      }
     };
 
     onMounted(() => {
@@ -99,6 +107,8 @@ export default defineComponent({
     const openClone = ref(false);
     return {
       totalPages,
+      count,
+      filtered,
       currentPage,
       changePage,
       locationStore,
@@ -122,6 +132,8 @@ export default defineComponent({
       :current-page="currentPage"
       :root-filter="filters"
       :total-pages="totalPages"
+      :count="count"
+      :filtered="filtered"
       :display-config="displayConfig"
       @update:currentPage="changePage($event)"
       @updateFilters="updateFilter($event)"

diff --git a/server/dive_server/views_metadata.py b/server/dive_server/views_metadata.py
@@ -69,7 +69,7 @@ def load_metadata_json(search_folder, type='ndjson'):
         Folder().childItems(
             search_folder,
             filters={"lowerName": {"$regex": regex}},
-            sort=[("created", pymongo.ASCENDING)],
+            sort=[("updated", pymongo.ASCENDING)],
         )
     )
     if len(json_items) > 0:
@@ -195,8 +195,6 @@ def process_metadata(self, folder, sibling_path, fileType, matcher, path_key, di
                     ]
                 }
                 results = list(Folder().findWithPermissions(query=query, user=user))
-                print(query)
-                print(f"RESULTS LENGTH: {len(results)}")
                 if len(results) > 0:
                     matched = False
                     key_path = item.get(path_key, False)
@@ -228,9 +226,11 @@ def process_metadata(self, folder, sibling_path, fileType, matcher, path_key, di
                 else:
                     errorLog.append(f"Could not find any results for Video file {item[matcher]}")
                 for key in item.keys():
-                    if key not in metadataKeys.keys():
+                    if key not in metadataKeys.keys() and item[key] is not None:
                         datatype = python_to_javascript_type(type(item[key]))
                         metadataKeys[key] = {"type": datatype, "set": set(), "count": 1}
+                    if item[key] is None:
+                        continue  # we skip null values for processing
                     if metadataKeys[key]['type'] == 'string':
                         metadataKeys[key]['set'].add(item[key])
                         metadataKeys[key]['count'] += 1
@@ -262,13 +262,13 @@ def process_metadata(self, folder, sibling_path, fileType, matcher, path_key, di
                     del metadataKeys[key]['set']
                 else:
                     del metadataKeys[key]['set']
-            DIVE_MetadataKeys().createMetadataKeys(datasetFolder, folder, user, metadataKeys)
+            DIVE_MetadataKeys().createMetadataKeys(folder, user, metadataKeys)
             # add metadata to root folder for
             folder['meta'][DIVEMetadataMarker] = True
             folder['meta'][DIVEMetadataFilter] = displayKeys
             Folder().save(folder)
 
-        return {"results": f"added {added} folders", "errors": errorLog}
+        return {"results": f"added {added} folders", "errors": errorLog, "metadataKeys": metadataKeys}
 
     @access.user
     @autoDescribeRoute(
@@ -313,6 +313,8 @@ def filter_folder(self, folder, filters, limit, offset, sort):
 
         user = self.getCurrentUser()
         query = self.get_filter_query(folder, user, filters)
+        total_query = self.get_filter_query(folder, user, {})
+        total_items = DIVE_Metadata().find(total_query).count()
         metadata_items = DIVE_Metadata().find(
             query, offset=offset, limit=limit, sort=sort, user=self.getCurrentUser()
         )
@@ -321,6 +323,8 @@ def filter_folder(self, folder, filters, limit, offset, sort):
             structured_results = {
                 'totalPages': pages,
                 'pageResults': list(metadata_items),
+                'count': total_items,
+                'filtered': metadata_items.count()
             }
             return structured_results