Merge pull request #415 from KNMI/trace-drop-layermetadata-issue

Fix bug where directory reader could not figure out the file type (opendir ent->d_type == DT_UNKNOWN)

Dockerfile

@@ -6,7 +6,7 @@ USER root
 LABEL maintainer="adaguc@knmi.nl"
 
 # Version should be same as in Definitions.h
-LABEL version="2.28.2"
+LABEL version="2.28.3"
 
 # Try to update image packages
 RUN apt-get -q -y update \

NEWS.md

@@ -1,3 +1,6 @@
+**Version 2.28.3 2024-10-24**
+- Fix bug where directory reader could not figure out the file type (opendir ent->d_type == DT_UNKNOWN)
+
 **Version 2.28.2 2024-10-23**
 - Fix bug which caused sheduled metadata updates to only work intermittently.
 

adagucserverEC/CDBAdapterPostgreSQL.cpp

@@ -1190,9 +1190,9 @@ bool CDBAdapterPostgreSQL::tryAdvisoryLock(size_t key) {
   auto result = store->getRecord(0)->get("result");
   bool succesfullylocked = result != nullptr && result->equals("t");
   if (succesfullylocked) {
-    CDBDebug("pg_try_advisory_lock succesfullylocked");
+    CDBDebug("pg_try_advisory_lock succesfully locked");
   } else {
-    CDBDebug("pg_try_advisory_lock NOT succesfullylocked");
+    CDBDebug("pg_try_advisory_lock NOT succesfully locked");
   }
   delete store;
   return succesfullylocked;
@@ -1214,9 +1214,9 @@ bool CDBAdapterPostgreSQL::advisoryUnLock(size_t key) {
   auto result = store->getRecord(0)->get("result");
   bool succesfullyunlocked = result != nullptr && result->equals("t");
   if (succesfullyunlocked) {
-    CDBDebug("pg_advisory_unlock succesfullyunlocked");
+    CDBDebug("pg_advisory_unlock succesfully unlocked");
   } else {
-    CDBWarning("pg_advisory_unlock NOT succesfullyunlocked");
+    CDBWarning("pg_advisory_unlock NOT succesfully unlocked");
   }
   delete store;
   return succesfullyunlocked;

adagucserverEC/CRequest.cpp

@@ -103,7 +103,7 @@ int CRequest::setConfigFile(const char *pszConfigFile) {
     // Include additional config files given as argument
     if (configFileList.size() > 1) {
       for (size_t j = 1; j < configFileList.size() - 1; j++) {
-        // CDBDebug("Include '%s'",configFileList[j].c_str());
+        // CDBDebug("Include '%s'", configFileList[j].c_str());
         status = srvParam->parseConfigFile(configFileList[j]);
         if (status != 0) {
           CDBError("There is an error with include '%s'", configFileList[j].c_str());
@@ -239,8 +239,6 @@ int CRequest::setConfigFile(const char *pszConfigFile) {
         CDBError("Configuration error at layer %d: <FilePath> not defined", j);
         return 1;
       }
-
-      // bool layerConfigCacheAvailable = false;
       try {
         /* Create the list of layers from a directory list */
         const char *baseDir = srvParam->cfg->Layer[j]->FilePath[0]->value.c_str();

adagucserverEC/Definitions.h

@@ -28,7 +28,7 @@
 #ifndef Definitions_H
 #define Definitions_H
 
-#define ADAGUCSERVER_VERSION "2.28.2" // Please also update in the Dockerfile to the same version
+#define ADAGUCSERVER_VERSION "2.28.3" // Please also update in the Dockerfile to the same version
 
 // CConfigReaderLayerType
 #define CConfigReaderLayerTypeUnknown 0

adagucserverEC/utils/ConfigurationUtils.cpp

@@ -2,12 +2,18 @@
 
 std::vector<std::string> getEnabledDatasetsConfigurations(CServerParams *srvParam) {
   std::vector<std::string> datasetList;
+  if (srvParam->cfg->Dataset.size() == 0) {
+    CDBWarning("No dataset paths are configured");
+  }
   for (auto dataset : srvParam->cfg->Dataset) {
     if (dataset->attr.enabled.equals("true") && dataset->attr.location.empty() == false) {
       if (srvParam->verbose) {
-        CDBDebug("Dataset locations %s", dataset->attr.location.c_str());
+        CDBDebug("Checking dataset location %s", dataset->attr.location.c_str());
       }
       auto files = CDirReader::listDir(dataset->attr.location.c_str(), false, "^.*\\.xml$");
+      if (files.size() == 0) {
+        CDBWarning("No datasets found in directory [%s]", dataset->attr.location.c_str());
+      }
       datasetList.insert(datasetList.end(), files.begin(), files.end());
     }
   }
@@ -23,14 +29,15 @@ bool checkIfPathIsFile(CT::string filePath) {
 void serverLogFunctionNothing(const char *) {}
 
 /* Set config file from environment variable ADAGUC_CONFIG */
-int setCRequestConfigFromEnvironment(CRequest *request, const char *additionalDataset) {
+int setCRequestConfigFromEnvironment(CRequest *request, CT::string additionalDataset) {
   char *configfile = getenv("ADAGUC_CONFIG");
   if (configfile != NULL) {
     CT::string configWithAdditionalDataset = configfile;
-    if (additionalDataset != nullptr && strlen(additionalDataset) > 0) {
+    if (additionalDataset.empty() == false) {
       configWithAdditionalDataset.concat(",");
       configWithAdditionalDataset.concat(additionalDataset);
     }
+
     int status = request->setConfigFile(configWithAdditionalDataset.c_str());
 
     /* Check logging level */

adagucserverEC/utils/ConfigurationUtils.h

@@ -10,6 +10,6 @@ std::vector<std::string> getEnabledDatasetsConfigurations(CServerParams *srvPara
 
 bool checkIfPathIsFile(CT::string filePath);
 
-int setCRequestConfigFromEnvironment(CRequest *request, const char *additionalDataset = nullptr);
+int setCRequestConfigFromEnvironment(CRequest *request, CT::string additionalDataset = "");
 
 #endif

adagucserverEC/utils/UpdateLayerMetadata.cpp

@@ -15,6 +15,11 @@ int updateLayerMetadata(CRequest &request) {
   auto datasetList = getEnabledDatasetsConfigurations(srvParam);
   // TODO: Remove dimension tables which don't have a matching configuration
 
+  if (datasetList.size() == 0) {
+    CDBWarning("No datasets found in dataset configuration directories. Not going to do update or cleanup.");
+    return 1;
+  }
+
   std::set<DatasetAndLayerPair> dataSetConfigsWithLayers;
 
   for (auto &dataset : datasetList) {
@@ -25,6 +30,9 @@ int updateLayerMetadata(CRequest &request) {
 
     if (dataset.length() > 0) {
       CDBDebug("\n\n *********************************** Updating metadatatable for dataset [%s] **************************************************", dataset.c_str());
+    } else {
+      CDBWarning("Datasetname is empty");
+      continue;
     }
     CRequest requestPerDataset;
 
@@ -33,14 +41,19 @@ int updateLayerMetadata(CRequest &request) {
       CDBError("Unable to read configuration file");
       continue;
     }
+    if (requestPerDataset.getServerParams()->cfg->Layer.size() == 0) {
+      CDBWarning("Found no layers in dataset %s", dataset.c_str());
+      continue;
+    } else {
+      CDBDebug("Found %d layer(s) in dataset %s", requestPerDataset.getServerParams()->cfg->Layer.size(), dataset.c_str());
+    }
     for (auto layer : requestPerDataset.getServerParams()->cfg->Layer) {
       CT::string layerName;
       makeUniqueLayerName(&layerName, layer);
       if (isalpha(layerName.charAt(0)) == 0) {
         CT::string tmp = layerName;
         layerName.print("ID_%s", tmp.c_str());
       }
-
       dataSetConfigsWithLayers.insert(std::make_pair(datasetBaseName, layerName.c_str()));
     }
     CT::string layerPathToScan;
@@ -52,6 +65,8 @@ int updateLayerMetadata(CRequest &request) {
     }
   }
 
+  CDBDebug("Found %d layers in total over all datasets in this instance", dataSetConfigsWithLayers.size());
+
   // Check for datasets and or layers which are not configured anymore.
   json dataset;
   json layer;

hclasses/CDirReader.cpp

@@ -53,18 +53,30 @@ std::vector<std::string> CDirReader::listDir(const char *directory, bool recursi
   if ((dir = opendir(directory)) != NULL) {
     /* print all the files and directories within directory */
     while ((ent = readdir(dir)) != NULL) {
-      if (ent->d_type == DT_REG || ent->d_type == DT_DIR) {
-        if (ent->d_name[0] != '.') { // Omit dirs starting with a .
-          CT::string fullName = directory;
-          fullName.concat("/");
-          fullName.concat(ent->d_name);
+      CT::string fullName = directory;
+      fullName.concat("/");
+      fullName.concat(ent->d_name);
+      // Deal with filesystems that don't provide d_type
+      auto d_type = ent->d_type;
+      if (d_type == DT_UNKNOWN) {
+        struct stat path_stat {};
+        int ret = stat(fullName.c_str(), &path_stat);
+        if (ret == 0 && S_ISREG(path_stat.st_mode)) {
+          d_type = DT_REG;
+        }
+        if (ret == 0 && S_ISDIR(path_stat.st_mode)) {
+          d_type = DT_DIR;
+        }
+      }
 
-          if (((filesAndOrDirs & CDIRREADER_INCLUDE_FILES) && ent->d_type == DT_REG) || ((filesAndOrDirs & CDIRREADER_INCLUDE_DIRECTORIES) && ent->d_type == DT_DIR)) {
+      if (d_type == DT_REG || d_type == DT_DIR) {
+        if (ent->d_name[0] != '.') { // Omit files and directories starting with a .
+          if (((filesAndOrDirs & CDIRREADER_INCLUDE_FILES) && d_type == DT_REG) || ((filesAndOrDirs & CDIRREADER_INCLUDE_DIRECTORIES) && d_type == DT_DIR)) {
             if (ext_filter == nullptr || std::regex_match(ent->d_name, self_regex)) {
               result.push_back(makeCleanPath(fullName.c_str()).c_str());
             }
           }
-          if (recursive && ent->d_type == DT_DIR) {
+          if (recursive && d_type == DT_DIR) {
             auto dirFiles = listDir(fullName.c_str(), recursive, ext_filter, filesAndOrDirs, exceptionOnError);
             // Move elements from dirFiles to result.
             // dirFiles is left in undefined but safe-to-destruct state.
@@ -77,6 +89,7 @@ std::vector<std::string> CDirReader::listDir(const char *directory, bool recursi
     closedir(dir);
   } else {
     if (exceptionOnError) {
+      CDBWarning("Unable to open dir [%s]", directory);
       /* could not open directory */
       throw "Error";
     }