various fixes and updates

client/platform/web-girder/api/divemetadata.service.ts

@@ -55,7 +55,7 @@ function getMetadataFilterValues(folderId: string, keys?: string[]) {
   });
 }
 
-function filterDiveMetadata(folderId: string, filters: DIVEMetadataFilter, offset = 0, limit = 50, sort = 'filename', sortdir = -1) {
+function filterDiveMetadata(folderId: string, filters: DIVEMetadataFilter, offset = 0, limit = 50, sort = 'filename', sortdir = 1) {
   return girderRest.get<DIVEMetadataResults>(`dive_metadata/${folderId}/filter`, {
     params: {
       filters, offset, limit, sort, sortdir,

client/platform/web-girder/views/DIVEMetadataClone.vue

@@ -82,7 +82,6 @@ export default defineComponent({
       >
         <GirderFileManager
           new-folder-enabled
-          root-location-disabled
           no-access-control
           :location="location"
           @update:location="setLocation"

client/platform/web-girder/views/DIVEMetadataFilter.vue

@@ -6,6 +6,7 @@ import {
   computed, defineComponent, onBeforeMount, PropType, Ref, ref, watch,
 } from 'vue';
 import { intersection } from 'lodash';
+import { usePrompt } from 'dive-common/vue-utilities/prompt-service';
 import DIVEMetadataFilterItemVue from './DIVEMetadataFilterItem.vue';
 import DIVEMetadataCloneVue from './DIVEMetadataClone.vue';
 
@@ -44,6 +45,18 @@ export default defineComponent({
     },
   },
   setup(props, { emit }) {
+    const { prompt } = usePrompt();
+
+    const checkConfig = async () => {
+      if (typeof (props.displayConfig) === 'string') {
+        await prompt({
+          title: 'Filter Error',
+          text: 'The default filter for this folder is not a JSON Object it appears to be a string',
+        });
+      }
+    };
+    watch(() => props.displayConfig, () => checkConfig());
+
     const search: Ref<string> = ref(props.rootFilter.search || '');
     const filters: Ref<DIVEMetadataFilterValueResults['metadataKeys']> = ref({});
     const splitFilters = computed(() => {
@@ -61,6 +74,15 @@ export default defineComponent({
     const filtersOn = ref(false);
     const defaultEnabledKeys: Ref<string[]> = ref([]); // If items should default to on because they are in the URL parameters
     const currentFilter: Ref<DIVEMetadataFilter> = ref({});
+    const sortParams = computed(() => {
+      if (splitFilters.value) {
+        return ['filename', ...Object.keys(splitFilters.value.displayed), ...Object.keys(splitFilters.value.advanced)];
+      }
+      return ['filename'];
+    });
+    const sortValue = ref('filename');
+    const sortDir = ref(1);
+
     const pageList = computed(() => {
       const list = [];
       for (let i = 0; i < props.totalPages; i += 1) {
@@ -86,7 +108,7 @@ export default defineComponent({
       loadCurrentFilter();
     });
 
-    const loadCurrentFilter = () => {
+    const loadCurrentFilter = async () => {
       if (props.rootFilter.metadataFilters && Object.keys(props.rootFilter.metadataFilters).length) {
         currentFilter.value.metadataFilters = props.rootFilter.metadataFilters;
       }
@@ -99,9 +121,21 @@ export default defineComponent({
       loadCurrentFilter();
     });
 
+    watch(filtersOn, (newVal, oldVal) => {
+      if (!newVal && oldVal) {
+        // We remove all of the old filters then
+        Object.keys(splitFilters.value.advanced).forEach((key) => {
+          if (currentFilter.value && currentFilter.value.metadataFilters && currentFilter.value.metadataFilters[key]) {
+            delete currentFilter.value.metadataFilters[key];
+          }
+        });
+        emit('updateFilters', { filter: currentFilter.value, sortVal: sortValue.value, sortDir: sortDir.value });
+      }
+    });
+
     watch(search, () => {
       currentFilter.value.search = search.value;
-      emit('updateFilters', currentFilter.value);
+      emit('updateFilters', { filter: currentFilter.value, sortVal: sortValue.value, sortDir: sortDir.value });
     });
 
     const clearFilter = (key: string) => {
@@ -111,7 +145,7 @@ export default defineComponent({
       if (currentFilter.value.metadataFilters[key]) {
         delete currentFilter.value.metadataFilters[key];
       }
-      emit('updateFilters', currentFilter.value);
+      emit('updateFilters', { filter: currentFilter.value, sortVal: sortValue.value, sortDir: sortDir.value });
     };
     const updateFilter = (key: string, { value, category } : {value: string | string[] | number | boolean | number[], category: MetadataFilterItem['category']}) => {
       if (!currentFilter.value.metadataFilters) {
@@ -132,20 +166,24 @@ export default defineComponent({
           value,
         };
       }
-      emit('updateFilters', currentFilter.value);
+      emit('updateFilters', { filter: currentFilter.value, sortVal: sortValue.value, sortDir: sortDir.value });
     };
 
     const changePage = async (page: number) => {
       emit('update:currentPage', page - 1);
     };
 
+    watch([sortValue, sortDir], () => {
+      emit('updateFilters', { filter: currentFilter.value, sortVal: sortValue.value, sortDir: sortDir.value });
+    });
+
     const getDefaultValue = (key: string) => {
-      if (props.rootFilter?.metadataFilters) {
-        if (props.rootFilter.metadataFilters[key] && props.rootFilter.metadataFilters[key].category === 'numerical') {
-          return props.rootFilter.metadataFilters[key].range;
+      if (currentFilter.value.metadataFilters) {
+        if (currentFilter.value.metadataFilters[key] && currentFilter.value.metadataFilters[key].category === 'numerical') {
+          return currentFilter.value.metadataFilters[key].range;
         }
-        if (props.rootFilter.metadataFilters[key]) {
-          return props.rootFilter.metadataFilters[key].value;
+        if (currentFilter.value.metadataFilters[key]) {
+          return currentFilter.value.metadataFilters[key].value;
         }
       }
       return undefined;
@@ -164,6 +202,9 @@ export default defineComponent({
       updateFilter,
       clearFilter,
       getDefaultValue,
+      sortValue,
+      sortParams,
+      sortDir,
     };
   },
 });
@@ -172,7 +213,7 @@ export default defineComponent({
 <template>
   <v-card class="pb-2">
     <v-container>
-      <v-row>
+      <v-row class="pt-2">
         <v-btn
           class="ma-1 pa-0"
           :depressed="filtersOn"
@@ -190,6 +231,24 @@ export default defineComponent({
           </v-icon>
           Advanced Filters
         </v-btn>
+        <v-spacer />
+        <v-chip><span class="pr-1">Filtered:</span>{{ filtered }} / {{ count }}</v-chip>
+        <v-select
+          v-model="sortValue"
+          class="mx-2 pa-0 fit"
+          style="max-width: 150px"
+          x-small
+          :items="sortParams"
+          dense
+          label="Sort"
+          hide-details
+        />
+        <v-icon v-if="sortDir === 1" @click="sortDir = -1">
+          mdi-sort-descending
+        </v-icon>
+        <v-icon v-else-if="sortDir === -1" @click="sortDir = 1">
+          mdi-sort-ascending
+        </v-icon>
       </v-row>
       <v-row
         no-wrap
@@ -239,7 +298,6 @@ export default defineComponent({
       <v-row class="mt-3">
         <slot name="leftOptions" />
         <v-spacer />
-        <v-chip><span class="pr-1">Filtered:</span>{{ filtered }} / {{ count }}</v-chip>
         <v-select
           class="mx-2 pa-0 fit"
           style="max-width: 50px"

client/platform/web-girder/views/DIVEMetadataFilterItem.vue

@@ -31,6 +31,12 @@ export default defineComponent({
     const categoryLimit = ref(20);
     const enabled = ref(props.defaultEnabled); // numerical enabled filter
     watch([value, enabled], () => {
+      if (enabled.value) {
+        if (value.value === undefined && props.filterItem.category === 'numerical' && props.filterItem.range) {
+          value.value = [props.filterItem.range.min, props.filterItem.range.max];
+        }
+      }
+
       const update = {
         value: value.value,
         category: props.filterItem.category,
@@ -45,6 +51,9 @@ export default defineComponent({
       emit('update-value', update);
     });
     if (enabled.value) {
+      if (value.value === undefined && props.filterItem.category === 'numerical' && props.filterItem.range) {
+        value.value = [props.filterItem.range.min, props.filterItem.range.max];
+      }
       const update = {
         value: value.value,
         category: props.filterItem.category,

client/platform/web-girder/views/DIVEMetadataSearch.vue

@@ -63,7 +63,7 @@ export default defineComponent({
 
     const updateURLParams = () => {
       if ((filters.value.metadataFilters && Object.keys(filters.value.metadataFilters).length) || filters.value.search) {
-        router.replace({ path: props.id, params: { id: props.id }, query: { filter: JSON.stringify(filters.value) } });
+        window.location.href = window.location.href.replace(/metadata\/.*/, `metadata/${props.id}?filter=${(JSON.stringify(filters.value))}`);
       } else {
         window.location.href = window.location.href.replace(/filter=.*/, '');
       }
@@ -74,21 +74,33 @@ export default defineComponent({
       getData();
     });
 
-    const updateFilter = async (filter?: DIVEMetadataFilter) => {
+    const storedSortVal = ref('filename');
+    const storedSortDir = ref(1);
+
+    const updateFilter = async ({ filter, sortVal, sortDir } : { filter?:DIVEMetadataFilter, sortVal?: string, sortDir?: number}) => {
       if (filter) {
         filters.value = filter;
         currentPage.value = 0;
         currentFilter.value = filter;
       }
-      const sort = 'filename';
+      if (sortVal) {
+        storedSortVal.value = sortVal;
+      }
+      if (sortDir) {
+        storedSortDir.value = sortDir;
+      }
       updateURLParams();
-      const { data } = await filterDiveMetadata(props.id, { ...filters.value }, currentPage.value * 50, 50, sort);
+      let updatedSortVal = sortVal;
+      if (updatedSortVal !== 'filename') {
+        updatedSortVal = `metadata.${updatedSortVal}`;
+      }
+      const { data } = await filterDiveMetadata(props.id, { ...filters.value }, currentPage.value * 50, 50, updatedSortVal, sortDir);
       processFilteredMetadataResults(data);
     };
 
     const changePage = async (page: number) => {
       currentPage.value = page;
-      await updateFilter();
+      await updateFilter({ filter: currentFilter.value, sortVal: storedSortVal.value, sortDir: storedSortDir.value });
     };
 
     const advanced = computed(() => {

server/dive_server/views_metadata.py

@@ -154,16 +154,23 @@ def __init__(self, resourceName):
             required=True,
         )
         .jsonParam(
-            "displayKeys",
+            "displayConfig",
             "List of Main Display Keys for the metadata and keys to hide from the filter",
             required=True,
             default={
                 "display": ['Batch', 'SampleDate', 'SubjectId', 'StudyId', 'ExperimentTag'],
                 "hide": ["ETag", "ETagDuplicated", "Size"],
             },
         )
+        .param(
+            "categoricalLimit",
+            "Above this number make a field a search field instead of a dropdown",
+            paramType="formData",
+            dataType="integer",
+            default=50,
+        )
     )
-    def process_metadata(self, folder, sibling_path, fileType, matcher, path_key, displayKeys):
+    def process_metadata(self, folder, sibling_path, fileType, matcher, path_key, displayConfig, categoricalLimit):
         # Process the current folder for the specified fileType using the matcher to generate DIVE_Metadata
         # make sure the folder is set to a DIVE Metadata folder using DIVE_METADATA = True
         user = self.getCurrentUser()
@@ -198,29 +205,46 @@ def process_metadata(self, folder, sibling_path, fileType, matcher, path_key, di
                 if len(results) > 0:
                     matched = False
                     key_path = item.get(path_key, False)
-                    modified_key_path = remove_before_folder(key_path, root_name)
+                    base_modified_key_path = remove_before_folder(key_path, root_name)
+                    childFolders = list(
+                        Folder().childFolders(folder, 'folder', user=user)
+                    )
+                    modified_key_paths = [{"root": root_name, "modified_path": base_modified_key_path}]
+                    print(f" Lenth of child folders: {len(childFolders)}")
+                    print(childFolders)
+                    for childFolder in childFolders:
+                        print(f"Child Item: {childFolder['name']} path: {key_path}")
+                        modified_key_paths.append({"root": childFolder["name"], "modified_path": remove_before_folder(key_path, childFolder['name'])})
+                    resource_path = ""
+                    print(modified_key_paths)
                     for datasetFolder in results:
                         resource_path = path_util.getResourcePath(
                             'folder', datasetFolder, user=user
                         )
                         # lets modify the path so it contains only the root folder down
-                        resource_path = remove_before_folder(resource_path, root_name)
-                        resource_path = resource_path.replace(
-                            f'/Video {item[matcher]}', f'/{item[matcher]}'
-                        )
-                        # now we check to see if the path matches the DIVE dataset item found.
-                        if modified_key_path:
-                            if modified_key_path == resource_path:
-                                item['pathMatches'] = True
-                                DIVE_Metadata().createMetadata(datasetFolder, folder, user, item)
-                                added += 1
-                                matched = True
-                            else:
-                                item['pathMatches'] = False
+                        for rootObj in modified_key_paths:
+                            root = rootObj['root']
+                            modified_path = rootObj['modified_path']
+                            resource_path = remove_before_folder(resource_path, root)
+                            resource_path = resource_path.replace(
+                                f'/Video {item[matcher]}', f'/{item[matcher]}'
+                            )
+                            # now we check to see if the path matches the DIVE dataset item found.
+                            if modified_path:
+                                if modified_path == resource_path:
+                                    item['pathMatches'] = True
+                                    DIVE_Metadata().createMetadata(datasetFolder, folder, user, item)
+                                    added += 1
+                                    matched = True
+                                    break
+                                else:
+                                    item['pathMatches'] = False
+                        if matched:
+                            break
 
                     if not matched:
                         errorLog.append(
-                            f"using matcher: {matcher} and key_path: {key_path} Could not find any matching key file path for Video file {item[matcher]} with path: {modified_key_path}"
+                            f"using matcher: {matcher} and key_path: {key_path} Could not find any matching key file path for Video file {item[matcher]} with path: {resource_path}"
                         )
 
                 else:
@@ -249,8 +273,9 @@ def process_metadata(self, folder, sibling_path, fileType, matcher, path_key, di
             # now we need to determine what is categorical vs what is a search field
             for key in metadataKeys.keys():
                 item = metadataKeys[key]
+                metadataKeys[key]["unique"] = len(item["set"])
                 if item["type"] in ['string', 'array'] and (
-                    item["count"] < 50 or item["count"] < len(item["set"])
+                    item["count"] < categoricalLimit or (item["count"] <= len(item["set"]) and len(item["set"]) < categoricalLimit)
                 ):
                     metadataKeys[key]["category"] = "categorical"
                     metadataKeys[key]['set'] = list(metadataKeys[key]['set'])
@@ -265,14 +290,14 @@ def process_metadata(self, folder, sibling_path, fileType, matcher, path_key, di
             DIVE_MetadataKeys().createMetadataKeys(folder, user, metadataKeys)
             # add metadata to root folder for
             folder['meta'][DIVEMetadataMarker] = True
-            folder['meta'][DIVEMetadataFilter] = displayKeys
+            folder['meta'][DIVEMetadataFilter] = displayConfig
             Folder().save(folder)
 
         return {"results": f"added {added} folders", "errors": errorLog, "metadataKeys": metadataKeys}
 
     @access.user
     @autoDescribeRoute(
-        Description("Get a list of filter keys for a specific folder").modelParam(
+        Description("Get a list of filter keys for a specific folder.  This is more used for debugging values in the metadata").modelParam(
             "id",
             description="Base folder ID",
             model=Folder,
@@ -305,7 +330,7 @@ def get_metadata_keys(
             "JSON Settings for the filtering",
             required=False,
         )
-        .pagingParams(defaultSort='created')
+        .pagingParams(defaultSort='filename')
     )
     def filter_folder(self, folder, filters, limit, offset, sort):
         if folder['meta'].get(DIVEMetadataMarker, False) is False:
@@ -390,12 +415,7 @@ def get_filter_query(self, folder, user, filters):
             query = {'$and': [query]}
             if 'search' in filters.keys():
                 query["$and"].append(
-                    {
-                        '$or': [
-                            {'filename': {'$regex': re.escape(filters['search'])}},
-                            {'Probes': {'$regex': re.escape(filters['search'])}},
-                        ]
-                    }
+                    {'filename': {'$regex': re.escape(filters['search'])}},
                 )
             # Now we need to go through the other filters and create querys for them
             # each filter in metadataFilters will have a type associated with it