diff --git a/.clang-format-ignore b/.clang-format-ignore new file mode 100644 index 00000000..e9bf8bf4 --- /dev/null +++ b/.clang-format-ignore @@ -0,0 +1,2 @@ +./build/* +./src/pmgd/* diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 00000000..b67148d1 --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,5 @@ +# Contributing to VDMS +Thank you for even being interested in contributing to VDMS. +As an open-source project in a rapidly developing field, we are extremely open to contributions, whether they involve new features, improved infrastructure, better documentation, or bug fixes. + +To learn how to contribute to VDMS, please follow the [Developer Guide](https://github.com/IntelLabs/vdms/wiki/Developer-Guide). diff --git a/.github/coverage/python.develop.coverage_report.txt b/.github/coverage/python.develop.coverage_report.txt index 9f0fd59c..f0e234b2 100644 --- a/.github/coverage/python.develop.coverage_report.txt +++ b/.github/coverage/python.develop.coverage_report.txt @@ -1,6 +1,6 @@ Name Stmts Miss Cover Missing -------------------------------------------------------------------- /vdms/client/python/vdms/__init__.py 2 0 100% -/vdms/client/python/vdms/vdms.py 99 2 98% 152, 167 +/vdms/client/python/vdms/vdms.py 95 2 98% 148, 163 -------------------------------------------------------------------- -TOTAL 101 2 98% +TOTAL 97 2 98% diff --git a/.github/coverage/python.develop.coverage_value.txt b/.github/coverage/python.develop.coverage_value.txt index e2b6d0f9..d6834c32 100644 --- a/.github/coverage/python.develop.coverage_value.txt +++ b/.github/coverage/python.develop.coverage_value.txt @@ -1 +1 @@ -98.02 +97.94 diff --git a/.github/scripts/auto-formatter.sh b/.github/scripts/auto-formatter.sh deleted file mode 100755 index a609b0bf..00000000 --- a/.github/scripts/auto-formatter.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -e - -check_package(){ - PACKAGE_TYPE=$1 - PACKAGE_NAME=$2 - - if [ $PACKAGE_TYPE == "apt" ]; then - if hash $PACKAGE_NAME 2>/dev/null; then - echo "$PACKAGE_NAME exists!" - else - echo "Installing $PACKAGE_NAME" - sudo apt-get install $PACKAGE_NAME - fi - elif [ $PACKAGE_TYPE == "python" ]; then - if python3 -c "import $PACKAGE_NAME" 2>/dev/null; then - echo "$PACKAGE_NAME exists!" - else - echo "Installing $PACKAGE_NAME" - python3 -m pip install --upgrade --no-cache-dir "${PACKAGE_NAME}" - fi - else - echo "UNKNOWN Package type ($PACKAGE_TYPE). Choose apt or python" - exit 1; - fi -} - -REPO_DIR=$(dirname "$(dirname "$(dirname "$(readlink -f "$0")")")") -echo "SCAN DIR: ${REPO_DIR}" - -# Convert files from Windows-style line endings (CRLF) to Linux-style line endings (LF) -check_package apt dos2unix -find ${REPO_DIR} -type f -exec dos2unix -v -k -s -o {} ';' - -# Run Clang-Format on C++ Code (Google C++ Style) -check_package apt clang-format -find "${REPO_DIR}" -type f -not -path "${REPO_DIR}/src/pmgd/*" \ - -not -path "${REPO_DIR}/build/*" \ - -regex '.*\.\(cc\|cpp\|h\|hpp\)' | xargs clang-format -i - -# Run Linter on Python Code -check_package python 'black>=23.1.0' -black ${REPO_DIR}/ --exclude="client/python/vdms/queryMessage_pb2.py" diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 0582b32e..d676a9c8 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -19,8 +19,18 @@ concurrency: cancel-in-progress: true jobs: + PreCommit: + # Apply formatting and linting + permissions: + contents: write + issues: write + pull-requests: write + uses: ./.github/workflows/_precommit.yml + secrets: inherit + Get-Envs: name: Get Environment vars + needs: [PreCommit] runs-on: ubuntu-latest outputs: CHECKIN_DOCKERFILE: .github/scripts/docker-compose.yml @@ -60,9 +70,9 @@ jobs: Update: permissions: contents: write + pull-requests: write # name: Lint & Update Reports needs: [Testing, Results] - if: ${{ always() }} uses: ./.github/workflows/_CI_update.yml with: coverage_value_updated: ${{ needs.Testing.outputs.coverage_value_updated }} diff --git a/.github/workflows/_CI_coverage.yml b/.github/workflows/_CI_coverage.yml index 765aca59..38b9554d 100644 --- a/.github/workflows/_CI_coverage.yml +++ b/.github/workflows/_CI_coverage.yml @@ -73,7 +73,7 @@ jobs: target_cpp_value: ${{ steps.target_values.outputs.cpp }} target_py_value: ${{ steps.target_values.outputs.py }} steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.1.7 with: ref: ${{ env.TARGET_BRANCH_REF }} fetch-depth: 0 @@ -105,7 +105,7 @@ jobs: coverage_value_updated: ${{ steps.report_coverage.outputs.cov_changed }} coverage_test_status: ${{ steps.report_coverage.outputs.coverage_test_status }} steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.1.7 with: submodules: true @@ -273,7 +273,7 @@ jobs: echo "coverage_test_status=${test_status}" >> $GITHUB_OUTPUT - name: Upload New coverage results - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4 + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 with: name: coverage_artifact path: .github/coverage/*.new.*.txt diff --git a/.github/workflows/_CI_update.yml b/.github/workflows/_CI_update.yml index c1e47665..c653e3db 100644 --- a/.github/workflows/_CI_update.yml +++ b/.github/workflows/_CI_update.yml @@ -16,34 +16,30 @@ jobs: commit_job: permissions: contents: write - name: Commit Code Updates + pull-requests: write + name: Commit Code Updates env: - COMMIT_MSG: "Automated updates: Format and/or coverage" + COMMIT_MSG: "Automated coverage update" DOCKER_ARTIFACT_DIR: "Docker_artifacts" runs-on: ubuntu-latest steps: # Checkout code doesn't persist across jobs - name: Checkout Source Branch - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} - token: ${{ secrets.FACELESS_TOKEN || github.token }} - run: mkdir -p ${{ env.DOCKER_ARTIFACT_DIR }} - name: Retrieve Current Coverage Files if: ${{ inputs.coverage_value_updated }} == 'true' && ${{ inputs.coverage_test_status }} == 'passed' - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: name: coverage_artifact path: .github/coverage/ - - name: Format C++ Code (clang-format), Python (black code), and apply dos2unix - run: | - ./.github/scripts/auto-formatter.sh - - name: Update coverage reports with latest coverage # Change latest coverage as develop (future target) if: ${{ inputs.coverage_value_updated }} == 'true' && ${{ inputs.coverage_test_status }} == 'passed' @@ -67,9 +63,8 @@ jobs: echo "changes=$changes" >> $GITHUB_OUTPUT if [ "$changes" != "Empty" ]; then - git config user.name ${{ secrets.FACELESS_NAME }} - git config user.email ${{ secrets.FACELESS_NAME }}@intel.com - git remote set-url origin https://x-access-token:${{ secrets.FACELESS_TOKEN }}@github.com/${{ github.event.pull_request.head.repo.full_name }} + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' git add .github/coverage/* git commit -am "${{ env.COMMIT_MSG }}" git push @@ -81,6 +76,6 @@ jobs: FILE_CHANGES: ${{ steps.update_commit.outputs.changes }} run: | if [ "${{ env.FILE_CHANGES }}" != "Empty" ]; then - echo "Please provide sys-vdms write access to fork (if applicable)." + echo "Error issuing commit. Coverage reports were NOT updated" exit 1 fi diff --git a/.github/workflows/_precommit.yml b/.github/workflows/_precommit.yml new file mode 100644 index 00000000..ac1ca80a --- /dev/null +++ b/.github/workflows/_precommit.yml @@ -0,0 +1,32 @@ +name: Pre-commit + +on: + workflow_call: + + +permissions: {} + +jobs: + Pre-Commit: + runs-on: ubuntu-latest + permissions: + contents: write + issues: write + pull-requests: write + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.1.7 + with: + submodules: false + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + with: + python-version: 3.x + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 + - uses: pre-commit-ci/lite-action@5d6cc0eb514c891a40562a58a8e71576c5c7fb43 # v1.1.0 + if: always() + - if: failure() + run: | + echo 'Pre-commit failed. CI will restart or please fix requested issues.' + pre-commit clean + exit 1 + - run: pre-commit clean + diff --git a/.gitmodules b/.gitmodules index 0c55c59d..3cd09509 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,4 +2,3 @@ path = src/pmgd url = https://github.com/omp87/pmgd.git branch = develop - \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..edcbe919 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,32 @@ +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.9.6 + hooks: + # Run the linter. + - id: ruff + args: [ + "--fix", + "--ignore=E402", + "--extend-exclude=client/python/vdms/queryMessage_pb2.py" + ] + # Run the formatter. + - id: ruff-format + args: [ + "--exclude=client/python/vdms/queryMessage_pb2.py" + ] + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: mixed-line-ending + args: ["--fix=auto"] + - id: trailing-whitespace + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v19.1.7 + hooks: + - id: clang-format + types: [c, c++] + args: [ + "--style=Google", + "-i" + ] diff --git a/README.md b/README.md index 2170247a..c43d1b6f 100644 --- a/README.md +++ b/README.md @@ -57,3 +57,11 @@ Industrial and Applications @ VLDB 2021 | [Paper](http://vldb.org/pvldb/vol14/p3 2nd USENIX Workshop @ HotEdge 2019 | [Paper](https://www.usenix.org/system/files/hotedge19-paper-altarawneh.pdf), [Presentation](https://www.usenix.org/sites/default/files/conference/protected-files/hotedge19_slides_altarawneh.pdf) | VDMS in Edge-to-cloud architecture for video streaming application Learning Systems @ NIPS 2018 | [Paper](https://export.arxiv.org/abs/1810.11832), [Cite](https://dblp.uni-trier.de/rec/bibtex/journals/corr/abs-1810-11832) | Systems for Machine Learning [Workshop](http://learningsys.org/nips18/cfp.html) @ NIPS HotStorage @ ATC 2017 | [Paper](https://www.usenix.org/conference/hotstorage17/program/presentation/gupta-cledat), [Presentation](https://www.usenix.org/conference/hotstorage17/program/presentation/gupta-cledat), [Cite](https://www.usenix.org/biblio/export/bibtex/203374)| Positioning Paper at USENIX ATC 2017 Workshop + + +## Contributing + +Thank you for your interest in contributing to VDMS. +As an open-source project in a rapidly developing field, we are extremely open to contributions, whether they involve new features, improved infrastructure, better documentation, or bug fixes. + +To learn how to contribute to VDMS, please follow the [Developer Guide](https://github.com/IntelLabs/vdms/wiki/Developer-Guide). diff --git a/client/cpp/BoundingBoxQueryParser.h b/client/cpp/BoundingBoxQueryParser.h index 7ba8d075..ceee97ae 100644 --- a/client/cpp/BoundingBoxQueryParser.h +++ b/client/cpp/BoundingBoxQueryParser.h @@ -1,20 +1,19 @@ #include "CSVParserUtil.h" namespace VDMS { class BoundingBoxQueryParser : public CSVParserUtil { -private: + private: vector rectangleKeys{"x", "y", "w", "h"}; void parseRectangle(string row, string queryType, Json::Value &aquery); -public: + public: VDMS::Response ParseAddBoundingBox(vector row, vector cols); // VDMS::Response ParseUpdateBoundingBox(vector row, vector& // cols); }; -}; // namespace VDMS +}; // namespace VDMS -VDMS::Response -VDMS::BoundingBoxQueryParser::ParseAddBoundingBox(vector row, - vector columnNames) { +VDMS::Response VDMS::BoundingBoxQueryParser::ParseAddBoundingBox( + vector row, vector columnNames) { if (row.empty() || row[0].empty()) { throw "please provide rectangle details"; } @@ -49,8 +48,7 @@ VDMS::BoundingBoxQueryParser::ParseAddBoundingBox(vector row, } } - if (cons) - allquery.append(aqueryf); + if (cons) allquery.append(aqueryf); allquery.append(aquery); // std::cout< #include #include @@ -7,20 +5,23 @@ #include #include +#include "CSVParserUtil.h" +#include "rapidcsv.h" + namespace VDMS { class CSVParser { -public: + public: CSVParser(std::string filename, size_t num_threads, std::string server, int port) - : m_filename(filename), m_num_threads(num_threads), vdms_server(server), + : m_filename(filename), + m_num_threads(num_threads), + vdms_server(server), vdms_port(port) {} ~CSVParser() {} - std::vector - parse_csv_lines(const std::string &filename, int start_line, int end_line, - std::vector &local_results, - const size_t thread_id) { - + std::vector parse_csv_lines( + const std::string &filename, int start_line, int end_line, + std::vector &local_results, const size_t thread_id) { rapidcsv::Document csv(filename); std::vector columnNames = csv.GetColumnNames(); VDMS::CSVParserUtil csv_util(vdms_server, vdms_port, columnNames, @@ -58,7 +59,7 @@ class CSVParser { std::mutex mutex; std::vector threads; std::vector> all_local_results(m_num_threads); - std::vector all_results; // Local vector for each thread + std::vector all_results; // Local vector for each thread all_results.reserve(num_lines); for (size_t i = 0; i < m_num_threads; i++) { @@ -76,7 +77,6 @@ class CSVParser { } size_t allResultsSizeBefore = all_results.size(); for (const auto &local_results : all_local_results) { - // Extend the size of all_results to accommodate local_results all_results.resize(all_results.size() + local_results.size()); @@ -89,10 +89,10 @@ class CSVParser { return all_results; } -private: + private: std::string m_filename; size_t m_num_threads; std::string vdms_server; int vdms_port; }; -}; // namespace VDMS \ No newline at end of file +}; // namespace VDMS \ No newline at end of file diff --git a/client/cpp/CSVParserUtil.h b/client/cpp/CSVParserUtil.h index 1aad7534..d0865401 100644 --- a/client/cpp/CSVParserUtil.h +++ b/client/cpp/CSVParserUtil.h @@ -1,21 +1,22 @@ #pragma once -#include "VDMSClient.h" -#include "rapidcsv.h" #include +#include + #include #include #include -#include #include #include #include +#include "VDMSClient.h" +#include "rapidcsv.h" + using namespace std; using namespace std::chrono; namespace VDMS { class CSVParserUtil { - enum QueryType { EntityClass, ConnectionClass, @@ -57,7 +58,7 @@ class CSVParserUtil { std::map commands; std::map command_list; -public: + public: CSVParserUtil(); CSVParserUtil(const std::string &, int port, const std::vector, int id); @@ -93,11 +94,11 @@ class CSVParserUtil { VDMS::Response send_to_vdms(const Json::Value &json_query, const std::vector blobs = {}); -public: + public: std::string vdms_server; int vdms_port; std::vector _columnNames; int id; std::unique_ptr vdms_client; }; -}; // namespace VDMS +}; // namespace VDMS diff --git a/client/cpp/ConnectionQueryParser.h b/client/cpp/ConnectionQueryParser.h index d57a3d5e..ac55713d 100644 --- a/client/cpp/ConnectionQueryParser.h +++ b/client/cpp/ConnectionQueryParser.h @@ -1,16 +1,15 @@ #include "CSVParserUtil.h" namespace VDMS { class ConnectionQueryParser : public CSVParserUtil { -public: + public: VDMS::Response ParseAddConnection(vector row, vector &cols); // VDMS::Response ParseUpdateConnection(vector row, vector // & cols); }; -}; // namespace VDMS +}; // namespace VDMS -VDMS::Response -VDMS::ConnectionQueryParser::ParseAddConnection(vector row, - vector &columnNames) { +VDMS::Response VDMS::ConnectionQueryParser::ParseAddConnection( + vector row, vector &columnNames) { Json::Value aquery; Json::Value allquery; Json::Value find_query1, find_query2, find_query; diff --git a/client/cpp/DescriptorQueryParser.h b/client/cpp/DescriptorQueryParser.h index 1e7dc3b0..521784a8 100644 --- a/client/cpp/DescriptorQueryParser.h +++ b/client/cpp/DescriptorQueryParser.h @@ -2,15 +2,14 @@ #include "CSVParserUtil.h" namespace VDMS { class DescriptorQueryParser : public CSVParserUtil { -public: + public: VDMS::Response ParseAddDescriptor(vector row, vector &columnNames, int id); }; -}; // namespace VDMS +}; // namespace VDMS VDMS::Response VDMS::DescriptorQueryParser::ParseAddDescriptor( vector row, vector &columnNames, int id) { - if (row[0] == "") { throw "Set not provided"; } @@ -31,7 +30,6 @@ VDMS::Response VDMS::DescriptorQueryParser::ParseAddDescriptor( aquery["AddDescriptor"]["label"] = row[j]; } if (columnNames[j] == "inputdata") { - parseBlobFile(row[j], &descriptor); if (descriptor == nullptr) { std::cout << "Failed to parse blob file" << std::endl; diff --git a/client/cpp/DescriptorSetQueryParser.h b/client/cpp/DescriptorSetQueryParser.h index dbe2b554..b590a307 100644 --- a/client/cpp/DescriptorSetQueryParser.h +++ b/client/cpp/DescriptorSetQueryParser.h @@ -2,13 +2,13 @@ #include "CSVParserUtil.h" namespace VDMS { class DescriptorSetQueryParser : public CSVParserUtil { -public: + public: VDMS::Response ParseAddDescriptorSet(vector row, vector &columnNames); bool isValidMetric(string &metric); bool isValidEngine(string &engine); }; -}; // namespace VDMS +}; // namespace VDMS VDMS::Response VDMS::DescriptorSetQueryParser::ParseAddDescriptorSet( vector row, vector &columnNames) { if (row[0] == "") { @@ -28,13 +28,11 @@ VDMS::Response VDMS::DescriptorSetQueryParser::ParseAddDescriptorSet( aquery["AddDescriptorSet"]["dimensions"] = stoi(row[j]); } if (columnNames[j] == "distancemetric") { - if (!isValidMetric(row[j])) - throw "Metric value is not valid"; + if (!isValidMetric(row[j])) throw "Metric value is not valid"; aquery["AddDescriptorSet"]["metric"] = row[j]; } if (columnNames[j] == "searchengine") { - if (!isValidEngine(row[j])) - throw "Engine value is not valid"; + if (!isValidEngine(row[j])) throw "Engine value is not valid"; aquery["AddDescriptorSet"]["engine"] = row[j]; } } diff --git a/client/cpp/EntityQueryParser.h b/client/cpp/EntityQueryParser.h index 4f966a73..0fd503ac 100644 --- a/client/cpp/EntityQueryParser.h +++ b/client/cpp/EntityQueryParser.h @@ -1,16 +1,17 @@ #pragma once -#include "CSVParserUtil.h" #include +#include "CSVParserUtil.h" + namespace VDMS { class EntityQueryParser : public CSVParserUtil { -public: + public: VDMS::Response ParseAddEntity(vector row, vector &cols); // VDMS::Response ParseUpdateEntity(vector row, vector & // cols); }; -}; // namespace VDMS +}; // namespace VDMS VDMS::Response VDMS::EntityQueryParser::ParseAddEntity(vector row, vector &cols) { @@ -28,15 +29,11 @@ VDMS::Response VDMS::EntityQueryParser::ParseAddEntity(vector row, aquery[command_name]["_ref"] = 11; for (int j = 1; j < cols.size(); j++) { - if (!row[j].empty()) { - string columnType = cols[j].substr(0, 5); if (columnType == "prop_") { - parseProperty(cols[j], row[j], command_name, aquery); } else if (columnType == "cons_") { - parseConstraints(cols[j], row[j], command_name, aquery); } } diff --git a/client/cpp/ImageQueryParser.h b/client/cpp/ImageQueryParser.h index f56c18ff..aac4bd0d 100644 --- a/client/cpp/ImageQueryParser.h +++ b/client/cpp/ImageQueryParser.h @@ -2,27 +2,25 @@ #include "CSVParserUtil.h" namespace VDMS { class ImageQueryParser : public CSVParserUtil { -private: + private: std::mutex file_access_mutex; -public: + public: // ImageQueryParser(); VDMS::Response ParseAddImage(vector row, vector columnNames); // VDMS::Response ParseUpdateImage(vector row, vector // columnNames); bool ValidImageFormat(string data); }; -}; // namespace VDMS +}; // namespace VDMS -VDMS::Response -VDMS::ImageQueryParser::ParseAddImage(vector row, - vector columnNames) { +VDMS::Response VDMS::ImageQueryParser::ParseAddImage( + vector row, vector columnNames) { Json::Value aquery; Json::Value fullquery; std::vector blobs; // - if (row[0].empty()) - throw "Image path is not specified"; + if (row[0].empty()) throw "Image path is not specified"; if (columnNames.size() == 0) { throw std::invalid_argument("Error: Column names vector is empty."); } @@ -46,8 +44,7 @@ VDMS::ImageQueryParser::ParseAddImage(vector row, for (int j = 1; j < columnNames.size(); j++) { if (!row[j].empty()) { if (columnNames[j] == "format") { - if (!ValidImageFormat(row[j])) - throw "Invalid image format"; + if (!ValidImageFormat(row[j])) throw "Invalid image format"; aquery["AddImage"]["format"] = row[j]; } else if (columnNames[j].find("prop_") != string::npos) { VDMS::CSVParserUtil::parseProperty(columnNames[j], row[j], command_name, diff --git a/client/cpp/VDMSClient.h b/client/cpp/VDMSClient.h index 24b89a24..78322a10 100644 --- a/client/cpp/VDMSClient.h +++ b/client/cpp/VDMSClient.h @@ -29,9 +29,10 @@ #pragma once -#include "comm/Connection.h" #include #include + +#include "comm/Connection.h" // #include "CSVParser.h" namespace VDMS { @@ -51,7 +52,7 @@ class VDMSClient { // disconnect and connect specifically, then we can add explicit calls. comm::ConnClient _conn; -public: + public: VDMSClient(std::string addr = "localhost", int port = VDMS_PORT, const std::string &cert_file = "", const std::string &key_file = "", const std::string &ca_file = ""); @@ -61,4 +62,4 @@ class VDMSClient { const std::vector blobs = {}); // void parse_csv_file(std::string filename, std::string , int); }; -}; // namespace VDMS +}; // namespace VDMS diff --git a/client/cpp/VideoQueryParser.h b/client/cpp/VideoQueryParser.h index 6ac5747a..e234932c 100644 --- a/client/cpp/VideoQueryParser.h +++ b/client/cpp/VideoQueryParser.h @@ -2,20 +2,18 @@ #include "CSVParserUtil.h" namespace VDMS { class VideoQueryParser : public CSVParserUtil { -public: + public: VDMS::Response ParseAddVideo(vector row, vector &columnNames); bool isValidCodec(string &row); bool isValidContainer(string &row); }; -} // namespace VDMS -VDMS::Response -VDMS::VideoQueryParser::ParseAddVideo(vector row, - vector &columnNames) { +} // namespace VDMS +VDMS::Response VDMS::VideoQueryParser::ParseAddVideo( + vector row, vector &columnNames) { Json::Value aquery; Json::Value fullquery; std::vector blobs; - if (row[0] == "") - throw "Video not provided"; + if (row[0] == "") throw "Video not provided"; std::string command_name = "AddVideo"; std::string video_name = row[0]; @@ -39,14 +37,12 @@ VDMS::VideoQueryParser::ParseAddVideo(vector row, } if (columnNames[j] == "compressto") { - if (!isValidCodec(row[j])) - throw "Invalid codec value"; + if (!isValidCodec(row[j])) throw "Invalid codec value"; aquery["AddVideo"]["codec"] = row[j]; } if (columnNames[j] == "format") { - if (!isValidContainer(row[j])) - throw "Invalid container value"; + if (!isValidContainer(row[j])) throw "Invalid container value"; aquery["AddVideo"]["container"] = row[j]; } @@ -61,7 +57,7 @@ VDMS::VideoQueryParser::ParseAddVideo(vector row, aquery["AddVideo"]["index_frames"] = false; } else { aquery["AddVideo"]["index_frames"] = - false; // or set to a default value + false; // or set to a default value } } } diff --git a/client/cpp/rapidcsv.h b/client/cpp/rapidcsv.h index 2f2d5325..6f815f09 100644 --- a/client/cpp/rapidcsv.h +++ b/client/cpp/rapidcsv.h @@ -63,7 +63,8 @@ struct ConverterParams { std::numeric_limits::signaling_NaN(), const long long pDefaultInteger = 0) : mHasDefaultConverter(pHasDefaultConverter), - mDefaultFloat(pDefaultFloat), mDefaultInteger(pDefaultInteger) {} + mDefaultFloat(pDefaultFloat), + mDefaultInteger(pDefaultInteger) {} /** * @brief specifies if conversion of non-numerical strings shall be @@ -102,8 +103,9 @@ class no_converter : public std::exception { * strings. Only intended for rapidcsv internal usage, but exposed externally to * allow specialization for custom datatype conversions. */ -template class Converter { -public: +template +class Converter { + public: /** * @brief Constructor * @param pConverterParams specifies how conversion of non-numerical @@ -196,7 +198,7 @@ template class Converter { } } -private: + private: const ConverterParams &mConverterParams; }; @@ -278,8 +280,11 @@ struct SeparatorParams { const bool pHasCR = sPlatformHasCR, const bool pQuotedLinebreaks = false, const bool pAutoQuote = true) - : mSeparator(pSeparator), mTrim(pTrim), mHasCR(pHasCR), - mQuotedLinebreaks(pQuotedLinebreaks), mAutoQuote(pAutoQuote) {} + : mSeparator(pSeparator), + mTrim(pTrim), + mHasCR(pHasCR), + mQuotedLinebreaks(pQuotedLinebreaks), + mAutoQuote(pAutoQuote) {} /** * @brief specifies the column separator. @@ -325,7 +330,8 @@ struct LineReaderParams { explicit LineReaderParams(const bool pSkipCommentLines = false, const char pCommentPrefix = '#', const bool pSkipEmptyLines = false) - : mSkipCommentLines(pSkipCommentLines), mCommentPrefix(pCommentPrefix), + : mSkipCommentLines(pSkipCommentLines), + mCommentPrefix(pCommentPrefix), mSkipEmptyLines(pSkipEmptyLines) {} /** @@ -348,7 +354,7 @@ struct LineReaderParams { * @brief Class representing a CSV document. */ class Document { -public: + public: /** * @brief Constructor * @param pPath specifies the path of an existing CSV-file @@ -368,8 +374,10 @@ class Document { const SeparatorParams &pSeparatorParams = SeparatorParams(), const ConverterParams &pConverterParams = ConverterParams(), const LineReaderParams &pLineReaderParams = LineReaderParams()) - : mPath(pPath), mLabelParams(pLabelParams), - mSeparatorParams(pSeparatorParams), mConverterParams(pConverterParams), + : mPath(pPath), + mLabelParams(pLabelParams), + mSeparatorParams(pSeparatorParams), + mConverterParams(pConverterParams), mLineReaderParams(pLineReaderParams) { if (!mPath.empty()) { ReadCsv(); @@ -394,7 +402,9 @@ class Document { const SeparatorParams &pSeparatorParams = SeparatorParams(), const ConverterParams &pConverterParams = ConverterParams(), const LineReaderParams &pLineReaderParams = LineReaderParams()) - : mPath(), mLabelParams(pLabelParams), mSeparatorParams(pSeparatorParams), + : mPath(), + mLabelParams(pLabelParams), + mSeparatorParams(pSeparatorParams), mConverterParams(pConverterParams), mLineReaderParams(pLineReaderParams) { ReadCsv(pStream); @@ -731,7 +741,8 @@ class Document { * @param pRowIdx zero-based row index. * @returns vector of row data. */ - template std::vector GetRow(const size_t pRowIdx) const { + template + std::vector GetRow(const size_t pRowIdx) const { const ssize_t rowIdx = pRowIdx + (mLabelParams.mColumnNameIdx + 1); std::vector row; Converter converter(mConverterParams); @@ -1224,7 +1235,7 @@ class Document { return rownames; } -private: + private: void ReadCsv() { std::ifstream stream; stream.exceptions(std::ifstream::failbit | std::ifstream::badbit); @@ -1514,7 +1525,7 @@ class Document { } } -private: + private: std::string mPath; LabelParams mLabelParams; SeparatorParams mSeparatorParams; @@ -1528,4 +1539,4 @@ class Document { bool mIsLE = false; #endif }; -} // namespace rapidcsv \ No newline at end of file +} // namespace rapidcsv \ No newline at end of file diff --git a/client/python/vdms/__init__.py b/client/python/vdms/__init__.py index 7268a1fd..32881741 100644 --- a/client/python/vdms/__init__.py +++ b/client/python/vdms/__init__.py @@ -1,3 +1,3 @@ name = "vdms" -from .vdms import * +from .vdms import * # noqa: F403 diff --git a/client/python/vdms/vdms.py b/client/python/vdms/vdms.py index 7ca21a41..a66478e3 100644 --- a/client/python/vdms/vdms.py +++ b/client/python/vdms/vdms.py @@ -26,15 +26,11 @@ # #! /usr/bin/python -import os import ssl import sys import json -import time -import urllib import socket import struct -from threading import Thread # VDMS Protobuf import (autogenerated) from . import queryMessage_pb2 diff --git a/config-vdms.json b/config-vdms.json index c8318dbf..f6920b62 100755 --- a/config-vdms.json +++ b/config-vdms.json @@ -4,7 +4,7 @@ // "key_file": "key.pem", // "ca_file": "ca.pem", "autoreplicate_interval":-1, // it should be > 0 - "unit":"s", + "unit":"s", "max_simultaneous_clients": 100, // "backup_path":"backups_test", // set this if you want different path to store the back up file "db_root_path": "db", diff --git a/distributed/docker-compose.yml b/distributed/docker-compose.yml index 55843730..2e4b91f3 100644 --- a/distributed/docker-compose.yml +++ b/distributed/docker-compose.yml @@ -15,7 +15,7 @@ services: container_name: broker depends_on: - zookeeper - + ports: - "9092:9092" - "19092:19092" @@ -52,7 +52,7 @@ services: - "8082:8082" environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' create-topics: image: confluentinc/cp-kafka:5.5.0 hostname: create-topics @@ -64,7 +64,7 @@ services: kafka-topics --create --if-not-exists --zookeeper zookeeper:2181 --partitions 2 --replication-factor 1 --topic vdms-query'" environment: KAFKA_BROKER_ID: ignored - KAFKA_ZOOKEEPER_CONNECT: ignored + KAFKA_ZOOKEEPER_CONNECT: ignored data_store_1: build: '../docker/base' image: vdms_dist:1 @@ -74,25 +74,25 @@ services: network_mode: "host" #networks: # - default - - + + data_store_2: image: 'vdms_dist:1' container_name: 'data_store_2' network_mode: host ports: - - '55561:55561' - # # volumes: + - '55561:55561' + # # volumes: # # - type: 'bind' # # - 'vdms_data_2:/vdms/build/db' # environment: # NETWORK_PORT: 55561 # hostname: 'data_store_2' - + # volumes: # - 'vdms_data_1': # - 'vdms_data_2': - + diff --git a/distributed/helpers.h b/distributed/helpers.h index a3345d4e..f30bd3c8 100644 --- a/distributed/helpers.h +++ b/distributed/helpers.h @@ -1,18 +1,18 @@ #ifndef SENDERS_KAFKA_HELPER_H_ #define SENDERS_KAFKA_HELPER_H_ -#include "VDMSClient.h" -#include "queryMessage.pb.h" +#include +#include +#include + #include #include #include #include #include -#include -#include #include -#include - +#include "VDMSClient.h" +#include "queryMessage.pb.h" #include "utils.h" using namespace std::chrono; @@ -93,13 +93,11 @@ Json::Value add_set(std::string &name) { descriptor_set["name"] = name; descriptor_set["dimensions"] = 1000; set_query["AddDescriptorSet"] = descriptor_set; - if (add_set) - tuple.append(set_query); + if (add_set) tuple.append(set_query); return tuple; } Json::Value construct_descriptor(std::string &name) { - Json::Value AddDesc; Json::Value Desc; Json::Value tuple; @@ -151,7 +149,7 @@ Json::Value construct_query() { Json::Value person_json, bounding_box, add_bounding_box, add_FV_entity, add_person_entity, edge, connect, tuple_data; person_json["_ref"] = - 1; // to assure the differences between the used references in the DB + 1; // to assure the differences between the used references in the DB person_json["class"] = "Person"; person_json["properties"]["Id"] = "1234"; person_json["properties"]["imaginary_node"] = 1; @@ -190,8 +188,7 @@ std::string img_query() { image.resize(file.tellg()); file.seekg(0, std::ios::beg); - if (!file.read(&image[0], image.size())) - std::cout << "error" << std::endl; + if (!file.read(&image[0], image.size())) std::cout << "error" << std::endl; std::vector blobs; std::string *bytes_str = new std::string(image); diff --git a/distributed/kafka_receiver.h b/distributed/kafka_receiver.h index be02f544..fafdb52f 100644 --- a/distributed/kafka_receiver.h +++ b/distributed/kafka_receiver.h @@ -3,29 +3,28 @@ #define KAFKA_RECIVER #include -#include +#include +#include #include + +#include #include #include -//#include "utils/hash_utils.h" - -#include -#include #include "utils.h" // LOG::FLAGS_minloglevel = 100; class BaseReceiver { -public: + public: BaseReceiver() {} virtual ~BaseReceiver() {} virtual bool Init() = 0; - virtual std::unique_ptr - Receive(const std::string &aux = "", const std::string &color = WHITE) = 0; + virtual std::unique_ptr Receive( + const std::string &aux = "", const std::string &color = WHITE) = 0; }; class KafkaReceiver : public BaseReceiver { -public: + public: long duration; KafkaReceiver(const std::string &endpoint) @@ -79,12 +78,10 @@ class KafkaReceiver : public BaseReceiver { } std::unique_ptr ret; while (true) { - RdKafka::Message *msg = consumer_->consume(topics_[topic_str].get(), 0, 10000); if (msg->err() == RdKafka::ERR_NO_ERROR) { - // LOG(INFO) << color <<"Kafka reads message at offset " << // msg->offset(); LOG(INFO) << color << "Receiver " @@ -107,7 +104,7 @@ class KafkaReceiver : public BaseReceiver { return ret; } -private: + private: std::unique_ptr conf_; std::unique_ptr tconf_; std::unique_ptr consumer_; diff --git a/distributed/kafka_sender.h b/distributed/kafka_sender.h index 8856c5d3..54a57c1f 100644 --- a/distributed/kafka_sender.h +++ b/distributed/kafka_sender.h @@ -1,16 +1,16 @@ #ifndef SENDERS_KAFKA_SENDER_H_ #define SENDERS_KAFKA_SENDER_H_ -#include -#include - #include #include +#include +#include + #include "utils.h" class BaseSender { -public: + public: BaseSender() {} virtual ~BaseSender() {} @@ -20,7 +20,7 @@ class BaseSender { }; class KafkaSender : public BaseSender { -public: + public: KafkaSender(const std::string &endpoint) : conf_(RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL)), tconf_(RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC)) { @@ -78,7 +78,7 @@ class KafkaSender : public BaseSender { producer_->poll(0); } -private: + private: std::unique_ptr conf_; std::unique_ptr tconf_; std::unique_ptr producer_; diff --git a/docker/README.md b/docker/README.md index 72292d15..cc0a2b0e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -2,7 +2,7 @@ Here are the instructions to build a docker image of the VDMS Server. -The build command is: +The build command is: cd base Dockerfile | docker build -t vdms:latest . @@ -15,12 +15,12 @@ If you are under a proxy, use: To run the docker image as a container, include the --net flag. This flag is needed as the server will be accepting connections on the default VDMS port (55555). - // Run the image interactively + // Run the image interactively docker run -it --net=host vdms:latest // or - // Run the image and deattach it from your bash + // Run the image and deattach it from your bash docker run -d --net=host vdms:latest diff --git a/docker/override_default_config.py b/docker/override_default_config.py index 21a43585..5b31aa0d 100644 --- a/docker/override_default_config.py +++ b/docker/override_default_config.py @@ -91,7 +91,7 @@ def main(args): try: updated_config_value = int(env_value) - except: + except Exception: if any(b in env_value.lower() for b in ["true", "false"]): updated_config_value = bool(env_value) else: diff --git a/include/VDMSConfigHelper.h b/include/VDMSConfigHelper.h index 1b7a760b..e24446c8 100644 --- a/include/VDMSConfigHelper.h +++ b/include/VDMSConfigHelper.h @@ -30,15 +30,15 @@ * */ #pragma once +#include +#include +#include + #include #include -#include #include #include -#include -#include - namespace VDMS { // E N U M S @@ -57,4 +57,4 @@ const std::map aws_log_level_map = { {"debug", Aws::Utils::Logging::LogLevel::Debug}, {"trace", Aws::Utils::Logging::LogLevel::Trace}}; -} // namespace VDMS +} // namespace VDMS diff --git a/include/vcl/CustomVCL.h b/include/vcl/CustomVCL.h index 069ea26b..5d01dadd 100644 --- a/include/vcl/CustomVCL.h +++ b/include/vcl/CustomVCL.h @@ -1,15 +1,13 @@ #pragma once +#include +#include #include #include #include - #include #include #include -#include -#include - #include #include "../ExceptionsCommand.h" diff --git a/include/vcl/DescriptorSet.h b/include/vcl/DescriptorSet.h index c35820a0..29020f02 100644 --- a/include/vcl/DescriptorSet.h +++ b/include/vcl/DescriptorSet.h @@ -34,17 +34,17 @@ #pragma once -#include "DescriptorParams.h" -#include "Exception.h" -#include "RemoteConnection.h" -#include "utils.h" +#include + #include #include #include -#include - +#include "DescriptorParams.h" +#include "Exception.h" +#include "RemoteConnection.h" #include "timers/TimerMap.h" +#include "utils.h" namespace VCL { @@ -61,8 +61,7 @@ enum DistanceMetric { L2, IP }; // enum class Storage { LOCAL = 0, AWS = 1 }; class DescriptorSet { - -public: + public: typedef std::vector DescIdVector; typedef std::vector LabelIdVector; typedef std::vector DistanceVector; @@ -74,7 +73,7 @@ class DescriptorSet { TimerMap timers; -private: + private: DescriptorSetData *_set; DescriptorSetEngine _eng; @@ -84,7 +83,7 @@ class DescriptorSet { void write_set_info(); void read_set_info(const std::string &set_path); -public: + public: /** * Loads an existing collection located at set_path * @@ -376,4 +375,4 @@ class DescriptorSet { */ void set_connection(RemoteConnection *remote); }; -}; // namespace VCL +}; // namespace VCL diff --git a/include/vcl/Exception.h b/include/vcl/Exception.h index eba12e66..475b2827 100644 --- a/include/vcl/Exception.h +++ b/include/vcl/Exception.h @@ -65,16 +65,16 @@ enum ExceptionType { struct Exception { // Which exception - int num; ///< Exception number - const char *name; ///< Exception name + int num; ///< Exception number + const char *name; ///< Exception name // Additional information std::string msg; int errno_val; // Where it was thrown - const char *file; ///< Source file name - int line; ///< Source line number + const char *file; ///< Source file name + int line; ///< Source line number Exception(int exc, const char *exc_name, const char *f, int l) : num(exc), name(exc_name), msg(), errno_val(0), file(f), line(l) {} @@ -88,8 +88,8 @@ struct Exception { : num(exc), name(exc_name), msg(m), errno_val(err), file(f), line(l) {} }; -#define VCLException(name, ...) \ +#define VCLException(name, ...) \ VCL::Exception(VCL::name, #name, ##__VA_ARGS__, __FILE__, __LINE__) -}; // namespace VCL +}; // namespace VCL extern void print_exception(const VCL::Exception &e, FILE *f = stdout); diff --git a/include/vcl/Image.h b/include/vcl/Image.h index 277b0255..5652be38 100644 --- a/include/vcl/Image.h +++ b/include/vcl/Image.h @@ -33,28 +33,26 @@ */ #pragma once -#include +#include +#include +#include #include -#include +#include #include #include #include #include #include +#include +#include #include "Exception.h" #include "RemoteConnection.h" #include "TDBImage.h" -#include "utils.h" -#include -#include -#include -#include - #include "VDMSConfigHelper.h" - #include "timers/TimerMap.h" +#include "utils.h" namespace VCL { @@ -65,7 +63,7 @@ namespace VCL { typedef cv::Rect Rectangle; class Image { -public: + public: // enum class Storage { LOCAL = 0, AWS = 1 }; /* *********************** */ @@ -258,9 +256,8 @@ class Image { * @return A vector containing the encoded image * @see OpenCV documentation for imencode for more details */ - std::vector - get_encoded_image(VCL::Format format, - const std::vector ¶ms = std::vector()); + std::vector get_encoded_image( + VCL::Format format, const std::vector ¶ms = std::vector()); /** * Gets encoded image data in a buffer in a async way @@ -271,9 +268,8 @@ class Image { * @return A vector containing the encoded image * @see OpenCV documentation for imencode for more details */ - std::vector - get_encoded_image_async(VCL::Format format, - const std::vector ¶ms = std::vector()); + std::vector get_encoded_image_async( + VCL::Format format, const std::vector ¶ms = std::vector()); /** * Executes the operations in the operation vector @@ -502,12 +498,13 @@ class Image { * @param buffer The buffer that will contain the image * data */ - template void copy_to_buffer(T *buffer); + template + void copy_to_buffer(T *buffer); TimerMap timers; std::vector op_labels; -private: + private: // Forward declaration of Operation class, to be used of _operations // list class Operation; @@ -608,7 +605,7 @@ class Image { * () operator */ class Operation { - protected: + protected: /** The format of the image for this operation */ Format _format; @@ -618,9 +615,9 @@ class Image { * @param format The format for the operation * @see Image.h for more details on Format */ - Operation(VCL::Format format) : _format(format){}; + Operation(VCL::Format format) : _format(format) {}; - public: + public: /** * Implemented by the specific operation, performs what * the operation is supposed to do @@ -639,11 +636,11 @@ class Image { * Extends Operation, reads image from the file system */ class Read : public Operation { - private: + private: /** The full path to the object to read */ std::string _fullpath; - public: + public: /** * Constructor, sets the format and path for reading * @@ -672,7 +669,7 @@ class Image { * format */ class Write : public Operation { - private: + private: /** The full path of where to write the image */ std::string _fullpath; /** The format the image used to be stored as */ @@ -680,7 +677,7 @@ class Image { /** Whether to store the metadata */ bool _metadata; - public: + public: /** * Constructor, sets the formats and path for writing * @@ -709,11 +706,11 @@ class Image { * Extends Operation, resizes the image to the specified size */ class Resize : public Operation { - private: + private: /** Gives the height and width to resize the image to */ Rectangle _rect; - public: + public: /** * Constructor, sets the size to resize to and the format * @@ -722,7 +719,7 @@ class Image { * @see Image.h for more details on ::Format and Rectangle */ Resize(const Rectangle &rect, VCL::Format format) - : Operation(format), _rect(rect){}; + : Operation(format), _rect(rect) {}; /** * Resizes an image to the given dimensions @@ -741,11 +738,11 @@ class Image { * Extends Operation, crops the image to the specified area */ class Crop : public Operation { - private: + private: /** Gives the dimensions and coordinates of the desired area */ Rectangle _rect; - public: + public: /** * Constructor, sets the area to crop to and the format * @@ -755,7 +752,7 @@ class Image { * @see Image.h for more details on ::Format and Rectangle */ Crop(const Rectangle &rect, VCL::Format format) - : Operation(format), _rect(rect){}; + : Operation(format), _rect(rect) {}; /** * Crops the image to the given area @@ -775,11 +772,11 @@ class Image { * threshold and sets that pixel to 0 */ class Threshold : public Operation { - private: + private: /** Minimum value pixels should be */ int _threshold; - public: + public: /** * Constructor, sets the threshold value and format * @@ -788,7 +785,7 @@ class Image { * @see Image.h for more details on ::Format */ Threshold(const int value, VCL::Format format) - : Operation(format), _threshold(value){}; + : Operation(format), _threshold(value) {}; /** * Performs the thresholding operation @@ -806,11 +803,11 @@ class Image { /** Extends Operation, performs a flip operation that */ class Flip : public Operation { - private: + private: /** Minimum value pixels should be */ int _code; - public: + public: /** * Constructor, sets the flip code value. * @@ -818,7 +815,8 @@ class Image { * @param format The current format of the image data * @see Image.h for more details on ::Format */ - Flip(const int code, VCL::Format format) : Operation(format), _code(code){}; + Flip(const int code, VCL::Format format) + : Operation(format), _code(code) {}; /** * Performs the flip operation @@ -836,12 +834,12 @@ class Image { /** Extends Operation, performs a flip operation that */ class Rotate : public Operation { - private: + private: /** Minimum value pixels should be */ float _angle; bool _keep_size; - public: + public: /** * Constructor, sets the flip code value. * @@ -849,7 +847,7 @@ class Image { * @see Image.h for more details on Format */ Rotate(float angle, bool keep_size, VCL::Format format) - : Operation(format), _angle(angle), _keep_size(keep_size){}; + : Operation(format), _angle(angle), _keep_size(keep_size) {}; /** * Performs the flip operation @@ -867,12 +865,12 @@ class Image { /** Extends Operation, performs a remote operation that */ class SyncRemoteOperation : public Operation { - private: + private: /** Minimum value pixels should be */ std::string _url; Json::Value _options; - public: + public: /** * Constructor, sets the flip code value. * @@ -882,7 +880,7 @@ class Image { */ SyncRemoteOperation(std::string url, Json::Value options, VCL::Format format) - : Operation(format), _url(url), _options(options){}; + : Operation(format), _url(url), _options(options) {}; /** * Performs the remote operation @@ -902,12 +900,12 @@ class Image { /** Extends Operation, performs a remote operation that */ class RemoteOperation : public Operation { - private: + private: /** Minimum value pixels should be */ std::string _url; Json::Value _options; - public: + public: /** * Constructor, sets the flip code value. * @@ -916,7 +914,7 @@ class Image { * @see Image.h for more details on Format */ RemoteOperation(std::string url, Json::Value options, VCL::Format format) - : Operation(format), _url(url), _options(options){}; + : Operation(format), _url(url), _options(options) {}; /** * Performs the remote operation @@ -934,11 +932,11 @@ class Image { /** Extends Operation, performs a user operation that */ class UserOperation : public Operation { - private: + private: /** Minimum value pixels should be */ Json::Value _options; - public: + public: /** * Constructor, sets the flip code value. * @@ -946,7 +944,7 @@ class Image { * @see Image.h for more details on Format */ UserOperation(Json::Value options, VCL::Format format) - : Operation(format), _options(options){}; + : Operation(format), _options(options) {}; /** * Performs the user operation @@ -1001,4 +999,4 @@ class Image { */ void set_format(const std::string &extension); }; -}; // namespace VCL +}; // namespace VCL diff --git a/include/vcl/KeyFrame.h b/include/vcl/KeyFrame.h index ed905ac5..0fd6bb40 100644 --- a/include/vcl/KeyFrame.h +++ b/include/vcl/KeyFrame.h @@ -54,13 +54,13 @@ typedef std::vector KeyFrameList; typedef std::vector EncodedFrameList; class KeyFrameOp { -protected: + protected: struct FormatContext { AVFormatContext *fmt_context; // For now, we only process videos with a SINGLE video stream. - AVStream *video_stream; // Pointer to the video stream in the ctx - unsigned video_stream_idx; // Index to the video stream in the ctx + AVStream *video_stream; // Pointer to the video stream in the ctx + unsigned video_stream_idx; // Index to the video stream in the ctx }; FormatContext _fctx; @@ -72,7 +72,7 @@ class KeyFrameOp { int init_stream(void); std::string error_msg(int errnum, const std::string &opt = ""); -public: + public: KeyFrameOp(std::string filename); virtual ~KeyFrameOp(); }; @@ -81,14 +81,14 @@ class KeyFrameOp { /* KEY_FRAME_PARSER */ /* *********************** */ class KeyFrameParser : public KeyFrameOp { -private: + private: KeyFrameList _frame_list; int fill_frame_list(void) noexcept; -public: - KeyFrameParser(std::string filename) : KeyFrameOp(filename){}; - ~KeyFrameParser() override{}; + public: + KeyFrameParser(std::string filename) : KeyFrameOp(filename) {}; + ~KeyFrameParser() override {}; const KeyFrameList &parse(void); }; @@ -97,7 +97,7 @@ class KeyFrameParser : public KeyFrameOp { /* KEY_FRAME_DECODER */ /* *********************** */ class KeyFrameDecoder : public KeyFrameOp { -private: + private: enum class H264Format { AVCC = 0, AnnexB = 1 }; struct DecoderContext { @@ -108,9 +108,11 @@ class KeyFrameDecoder : public KeyFrameOp { H264Format byte_stream_format; DecoderContext() - : bsf_context(NULL), video_codec_context(NULL), - frame_codec_context(NULL), sws_context(NULL), - byte_stream_format(H264Format::AVCC){}; + : bsf_context(NULL), + video_codec_context(NULL), + frame_codec_context(NULL), + sws_context(NULL), + byte_stream_format(H264Format::AVCC) {}; }; struct FrameInterval { @@ -139,11 +141,11 @@ class KeyFrameDecoder : public KeyFrameOp { int populate_interval_map(const std::vector &frames); int encode_frames(void); -public: + public: KeyFrameDecoder(std::string filename); ~KeyFrameDecoder() override; void set_key_frames(const KeyFrameList &key_frames); EncodedFrameList &decode(const std::vector &frames); }; -} // namespace VCL +} // namespace VCL diff --git a/include/vcl/RemoteConnection.h b/include/vcl/RemoteConnection.h index 6071513a..3a7ddac3 100644 --- a/include/vcl/RemoteConnection.h +++ b/include/vcl/RemoteConnection.h @@ -33,11 +33,6 @@ #pragma once -#include -#include - -#include "Exception.h" - #include #include #include @@ -45,10 +40,15 @@ #include #include +#include +#include + +#include "Exception.h" + namespace VCL { class RemoteConnection { -public: + public: RemoteConnection(); ~RemoteConnection(); @@ -65,7 +65,7 @@ class RemoteConnection { Aws::String _bucket_name; -private: + private: bool _remote_connected = false; Aws::SDKOptions *_aws_sdk_options; @@ -84,4 +84,4 @@ class RemoteConnection { const std::string &errorMessage = ""); // void LogEntry(std::string functionName); }; -} // namespace VCL +} // namespace VCL diff --git a/include/vcl/Video.h b/include/vcl/Video.h index 49a47727..d475a9cd 100644 --- a/include/vcl/Video.h +++ b/include/vcl/Video.h @@ -31,35 +31,32 @@ #pragma once -#include -#include // For shared_ptr -#include +#include +#include +#include +#include // For shared_ptr #include #include #include #include - -#include "KeyFrame.h" -#include "vcl/Image.h" +#include #include "../utils/include/stats/SystemStats.h" #include "Exception.h" +#include "KeyFrame.h" #include "VDMSConfigHelper.h" +#include "timers/TimerMap.h" #include "utils.h" +#include "vcl/Image.h" #include "zip.h" -#include -#include - -#include "timers/TimerMap.h" namespace VCL { -typedef cv::Rect Rectangle; // specify an ROI inside a video +typedef cv::Rect Rectangle; // specify an ROI inside a video class Video { - -public: + public: enum Codec { NOCODEC = 0, MJPG, XVID, H263, H264, AVC1 }; std::string NOERRORSTRING = ""; @@ -84,7 +81,7 @@ class Video { USEROPERATION }; - RemoteConnection *_remote; // Remote connection (if one exists) + RemoteConnection *_remote; // Remote connection (if one exists) TimerMap timers; std::vector op_labels; @@ -438,7 +435,7 @@ class Video { */ int execute_operations(bool isRemote = false); -private: + private: class Operation; // Forward declaration of VideoTest class, that is used for the unit @@ -459,13 +456,13 @@ class Video { // Query Error response std::string _query_error_response = ""; - bool _flag_stored; // Flag to avoid unnecessary read/write + bool _flag_stored; // Flag to avoid unnecessary read/write VideoSize _size; float _fps; - Codec _codec; // (h.264, etc). + Codec _codec; // (h.264, etc). // Pointer to key frame decoder object, allocated when key frames // are set, and used whenever frames are decoded using key-frame @@ -498,8 +495,7 @@ class Video { * () operator */ class Operation { - - public: + public: /** * Implemented by the specific operation, performs what * the operation is supposed to do @@ -531,17 +527,17 @@ class Video { * Extends Operation, resizes the Video to the specified size */ class Resize : public Operation { - private: + private: /** Gives the height and width to resize the Video to */ cv::Size _size; - public: + public: /** * Constructor, sets the size to resize to and the format * * @param size Struct that contains w and h */ - Resize(const cv::Size &size) : _size(size){}; + Resize(const cv::Size &size) : _size(size) {}; /** * Resizes an Video to the given dimensions @@ -562,13 +558,13 @@ class Video { /* *********************** */ class Interval : public Operation { - private: + private: int _start; int _stop; int _step; Video::Unit _u; - public: + public: /** * Constructor, sets the size to resize to and the format * @@ -578,7 +574,7 @@ class Video { * @param step Number of frames to be skipped in between. */ Interval(Video::Unit u, const int start, const int stop, int step) - : _u(u), _start(start), _stop(stop), _step(step){}; + : _u(u), _start(start), _stop(stop), _step(step) {}; /** * Resizes an Video to the given dimensions @@ -600,18 +596,18 @@ class Video { * Extends Operation, crops the Video to the specified area */ class Crop : public Operation { - private: + private: /** Gives the dimensions and coordinates of the desired area */ Rectangle _rect; - public: + public: /** * Constructor, sets the area to crop to and the format * * @param rect Contains dimensions and coordinates of * desired area */ - Crop(const Rectangle &rect) : _rect(rect){}; + Crop(const Rectangle &rect) : _rect(rect) {}; /** * Crops the Video to the given area @@ -636,17 +632,17 @@ class Video { * threshold and sets that pixel to 0 */ class Threshold : public Operation { - private: + private: /** Minimum value pixels should be */ int _threshold; - public: + public: /** * Constructor, sets the threshold value and format * * @param value Minimum value pixels should be */ - Threshold(const int value) : _threshold(value){}; + Threshold(const int value) : _threshold(value) {}; /** * Performs the thresholding operation @@ -666,11 +662,11 @@ class Video { /** Extends Operation, performs a synchronous remote operation */ class SyncRemoteOperation : public Operation { - private: + private: std::string _url; Json::Value _options; - public: + public: /** * * Constructor, sets the remote url and client options @@ -679,7 +675,7 @@ class Video { * @param options client parameters for the operation */ SyncRemoteOperation(std::string url, Json::Value options) - : _url(url), _options(options){}; + : _url(url), _options(options) {}; /** * Performs the remote operation @@ -699,11 +695,11 @@ class Video { /** Extends Operation, performs an asynchronous remote operation */ class RemoteOperation : public Operation { - private: + private: std::string _url; Json::Value _options; - public: + public: /** * * Constructor, sets the remote url and client options @@ -712,7 +708,7 @@ class Video { * @param options client parameters for the operation */ RemoteOperation(std::string url, Json::Value options) - : _url(url), _options(options){}; + : _url(url), _options(options) {}; /** * Performs the remote operation @@ -732,17 +728,17 @@ class Video { /** Extends Operation, performs a udf */ class UserOperation : public Operation { - private: + private: Json::Value _options; - public: + public: /** * * Constructor, sets the client options * * @param options client parameters for the operation */ - UserOperation(Json::Value options) : _options(options){}; + UserOperation(Json::Value options) : _options(options) {}; /** * Performs the remote operation @@ -756,7 +752,7 @@ class Video { OperationType get_type() { return USEROPERATION; }; }; -protected: + protected: /* *********************** */ /* UTILITIES */ /* *********************** */ @@ -830,4 +826,4 @@ class Video { std::string fname); }; -} // namespace VCL \ No newline at end of file +} // namespace VCL \ No newline at end of file diff --git a/include/vcl/utils.h b/include/vcl/utils.h index 33774503..a9de9771 100644 --- a/include/vcl/utils.h +++ b/include/vcl/utils.h @@ -30,6 +30,7 @@ #pragma once #include + #include namespace VCL { @@ -104,4 +105,4 @@ std::string create_unique(const std::string &path, * value of PNG and JPG format */ Format read_image_format(void *buffer, long size); -}; // namespace VCL +}; // namespace VCL diff --git a/remote_function/README.md b/remote_function/README.md index dbee2719..34e14481 100644 --- a/remote_function/README.md +++ b/remote_function/README.md @@ -14,7 +14,7 @@ This submodule is required to execute VDMS operation on a remote server using Fl Any operation can be added to the module by creating a python file of the same name as the operation and adding it to the `functions` folder. Any operaion file should follow the following setup to define a `run` function that the endpoint will use; ``` def run(ipfilename, format, options): - + # ipfilename: Name of the input file to be read from # format: Format of the input file # options: Any inputs that the UDF will require from the client @@ -100,7 +100,7 @@ def run(ipfilename, format, options): global car_cascade_src img = cv2.imread(ipfilename) - + # These lines # represent the # code logic diff --git a/remote_function/functions/caption.py b/remote_function/functions/caption.py index d086b1e1..a3c3fdcc 100644 --- a/remote_function/functions/caption.py +++ b/remote_function/functions/caption.py @@ -1,9 +1,5 @@ import cv2 -import numpy as np -from datetime import datetime -from collections import deque import skvideo.io -import imutils import uuid @@ -14,7 +10,7 @@ def run(ipfilename, format, options): video = skvideo.io.FFmpegWriter(opfilename, {"-pix_fmt": "bgr24"}) print(options) - i = 0 + while True: (grabbed, frame) = vs.read() if not grabbed: diff --git a/remote_function/functions/metadata.py b/remote_function/functions/metadata.py index 944863b3..7fce0764 100644 --- a/remote_function/functions/metadata.py +++ b/remote_function/functions/metadata.py @@ -1,9 +1,4 @@ import cv2 -import numpy as np -from datetime import datetime -from collections import deque -import skvideo.io -import imutils import uuid import json @@ -22,10 +17,8 @@ def facedetectbbox(frame): def run(ipfilename, format, options): - # Extract metadata for video files if options["media_type"] == "video": - vs = cv2.VideoCapture(ipfilename) frameNum = 1 metadata = {} diff --git a/remote_function/udf_server.py b/remote_function/udf_server.py index d5f0dfc3..29424bc2 100644 --- a/remote_function/udf_server.py +++ b/remote_function/udf_server.py @@ -56,7 +56,7 @@ def image_api(): return_string += r_meta.encode("utf-8") os.remove(tmpfile) - if return_string == "" or return_string == None: + if return_string == "" or return_string is None: return "error" return return_string @@ -94,7 +94,7 @@ def remove_tempfile(response): os.remove(response_file) os.remove(video_file) os.remove(metadata_file) - except Exception as e: + except Exception: print("Some files cannot be deleted or are not present") return response @@ -121,7 +121,7 @@ def handle_bad_request(e): if __name__ == "__main__": - if sys.argv[1] == None: + if sys.argv[1] is None: print("Port missing\n Correct Usage: python3 udf_server.py ") else: app.run(host="0.0.0.0", port=int(sys.argv[1])) diff --git a/src/AutoDeleteNode.h b/src/AutoDeleteNode.h index d2ea1605..42d522f0 100644 --- a/src/AutoDeleteNode.h +++ b/src/AutoDeleteNode.h @@ -29,24 +29,23 @@ * */ -#include +#include +#include #include +#include #include #include -#include -#include - #include "iterator.h" #include "node.h" class AutoDeleteNode { -private: + private: Json::UInt64 _expiration_timestamp; - void *_node; // can use void pointer because query only seraches for Nodes and - // not edges -public: + void *_node; // can use void pointer because query only seraches for Nodes + // and not edges + public: AutoDeleteNode(Json::UInt64 new_expiration_timestamp, void *n_node); ~AutoDeleteNode(); Json::UInt64 GetExpirationTimestamp(); diff --git a/src/BackendNeo4j.h b/src/BackendNeo4j.h index d8b321b6..b10dbd27 100644 --- a/src/BackendNeo4j.h +++ b/src/BackendNeo4j.h @@ -4,23 +4,23 @@ #pragma once -#include #include #include -#include #include #include +#include +#include + void print_val(neo4j_value_t val, int val_type); int val_check(neo4j_value_t val); class BackendNeo4j { - tbb::concurrent_bounded_queue conn_pool; neo4j_config_t *config; std::string tgt_db; -public: + public: // constructor BackendNeo4j(unsigned int nr_conns, char *tgt_url, char *user, char *pass, uint_fast32_t flags); diff --git a/src/BlobCommand.h b/src/BlobCommand.h index 5aafaeb3..4d7b3a47 100644 --- a/src/BlobCommand.h +++ b/src/BlobCommand.h @@ -30,21 +30,21 @@ */ #pragma once -#include "vcl/CustomVCL.h" -#include "vcl/Image.h" #include #include #include #include "ExceptionsCommand.h" #include "RSCommand.h" +#include "vcl/CustomVCL.h" +#include "vcl/Image.h" namespace VDMS { // Helper classes for handling various JSON commands. class BlobCommand : public RSCommand { -public: + public: BlobCommand(const std::string &cmd_name); virtual int construct_protobuf(PMGDQuery &tx, const Json::Value &root, @@ -55,10 +55,9 @@ class BlobCommand : public RSCommand { }; class AddBlob : public BlobCommand { - std::string _storage_bin; -public: + public: AddBlob(); int construct_protobuf(PMGDQuery &tx, const Json::Value &root, @@ -69,7 +68,7 @@ class AddBlob : public BlobCommand { }; class UpdateBlob : public BlobCommand { -public: + public: UpdateBlob(); int construct_protobuf(PMGDQuery &tx, const Json::Value &root, @@ -78,7 +77,7 @@ class UpdateBlob : public BlobCommand { }; class FindBlob : public BlobCommand { -public: + public: FindBlob(); int construct_protobuf(PMGDQuery &tx, const Json::Value &root, const std::string &blob, int grp_id, @@ -90,4 +89,4 @@ class FindBlob : public BlobCommand { const std::string &blob); }; -}; // namespace VDMS +}; // namespace VDMS diff --git a/src/BoundingBoxCommand.h b/src/BoundingBoxCommand.h index 9aa8a3ba..bf64b5b1 100644 --- a/src/BoundingBoxCommand.h +++ b/src/BoundingBoxCommand.h @@ -40,7 +40,7 @@ namespace VDMS { class AddBoundingBox : public RSCommand { -public: + public: AddBoundingBox(); int construct_protobuf(PMGDQuery &tx, const Json::Value &root, @@ -49,7 +49,7 @@ class AddBoundingBox : public RSCommand { }; class UpdateBoundingBox : public RSCommand { -public: + public: UpdateBoundingBox(); int construct_protobuf(PMGDQuery &tx, const Json::Value &root, @@ -60,7 +60,7 @@ class UpdateBoundingBox : public RSCommand { class FindBoundingBox : public RSCommand { // bool _use_aws_storage; -public: + public: FindBoundingBox(); int construct_protobuf(PMGDQuery &tx, const Json::Value &root, @@ -73,4 +73,4 @@ class FindBoundingBox : public RSCommand { const std::string &blob); }; -}; // namespace VDMS +}; // namespace VDMS diff --git a/src/CommunicationManager.h b/src/CommunicationManager.h index fd9668e6..76df1b6b 100644 --- a/src/CommunicationManager.h +++ b/src/CommunicationManager.h @@ -45,8 +45,8 @@ namespace VDMS { class CommunicationManager { static const int MAX_CONNECTED_CLIENTS = 500; std::string DEFAULT_QUERY_HANDLER = - "pmgd"; // TODO need to move this someplace central between server and - // comm manager + "pmgd"; // TODO need to move this someplace central between server and + // comm manager std::string _q_handler; // For the thread pool @@ -63,11 +63,11 @@ class CommunicationManager { bool _shutdown; -public: + public: CommunicationManager(); ~CommunicationManager(); void process_queue(); void add_connection(comm::Connection *c); void shutdown(); }; -}; // namespace VDMS +}; // namespace VDMS diff --git a/src/DescriptorsCommand.h b/src/DescriptorsCommand.h index e872cd6e..b3c6a3c7 100644 --- a/src/DescriptorsCommand.h +++ b/src/DescriptorsCommand.h @@ -30,16 +30,16 @@ */ #pragma once +#include +#include + #include #include #include #include -#include -#include - #include "DescriptorsManager.h" -#include "QueryHandlerPMGD.h" // to provide the database connection +#include "QueryHandlerPMGD.h" // to provide the database connection #include "tbb/concurrent_unordered_map.h" namespace VDMS { @@ -48,7 +48,7 @@ typedef std::pair, std::vector> IDDistancePair; // This class encapsulates common behavior of Descriptors-related cmds. class DescriptorsCommand : public RSCommand { -protected: + protected: DescriptorsManager *_dm; VCL::DescriptorSetEngine _eng; bool output_vcl_timing; @@ -69,7 +69,7 @@ class DescriptorsCommand : public RSCommand { bool check_blob_size(const std::string &blob, const int dimensions, const long n_desc); -public: + public: DescriptorsCommand(const std::string &cmd_name); virtual bool need_blob(const Json::Value &cmd) { return false; } @@ -87,7 +87,7 @@ class DescriptorsCommand : public RSCommand { class FindDescriptorSet : public DescriptorsCommand { std::string _storage_sets; -public: + public: FindDescriptorSet(); int construct_protobuf(PMGDQuery &tx, const Json::Value &root, const std::string &blob, int grp_id, @@ -106,12 +106,12 @@ class AddDescriptorSet : public DescriptorsCommand { uint64_t _flinng_num_hash_tables; uint64_t _flinng_hashes_per_table; uint64_t - _flinng_sub_hash_bits; // sub_hash_bits * hashes_per_table must be - // less than 32, otherwise segfault will happen + _flinng_sub_hash_bits; // sub_hash_bits * hashes_per_table must be + // less than 32, otherwise segfault will happen uint64_t _flinng_cut_off; // bool _use_aws_storage; -public: + public: AddDescriptorSet(); int construct_protobuf(PMGDQuery &tx, const Json::Value &root, @@ -140,7 +140,7 @@ class AddDescriptor : public DescriptorsCommand { const std::string &blob, int grp_id, Json::Value &error); -public: + public: AddDescriptor(); int construct_protobuf(PMGDQuery &tx, const Json::Value &root, @@ -156,8 +156,7 @@ class AddDescriptor : public DescriptorsCommand { }; class ClassifyDescriptor : public DescriptorsCommand { - -public: + public: ClassifyDescriptor(); int construct_protobuf(PMGDQuery &tx, const Json::Value &root, @@ -173,15 +172,14 @@ class ClassifyDescriptor : public DescriptorsCommand { }; class FindDescriptor : public DescriptorsCommand { - -private: + private: void convert_properties(Json::Value &entities, Json::Value &list, std::string set_name); void populate_blobs(const std::string &set_path, std::string set_name, const Json::Value &results, Json::Value &entities, protobufs::queryMessage &query_res); -public: + public: FindDescriptor(); int construct_protobuf(PMGDQuery &tx, const Json::Value &root, @@ -195,4 +193,4 @@ class FindDescriptor : public DescriptorsCommand { protobufs::queryMessage &response, const std::string &blob); }; -} // namespace VDMS +} // namespace VDMS diff --git a/src/DescriptorsManager.h b/src/DescriptorsManager.h index 711a7f64..e2bae1fb 100644 --- a/src/DescriptorsManager.h +++ b/src/DescriptorsManager.h @@ -46,7 +46,7 @@ class DescriptorsManager { DescriptorsManager(); -public: + public: static bool init(); static DescriptorsManager *instance(); @@ -59,4 +59,4 @@ class DescriptorsManager { VCL::DescriptorSet *get_descriptors_handler(std::string path); void flush(); }; -}; // namespace VDMS +}; // namespace VDMS diff --git a/src/Exception.h b/src/Exception.h index 3b2a2763..12820e25 100644 --- a/src/Exception.h +++ b/src/Exception.h @@ -41,21 +41,21 @@ enum ExceptionServerType { SignalHandler, NullConnection, - Undefined = 100, // Any undefined error + Undefined = 100, // Any undefined error }; struct ExceptionServer { // Which exception - int num; ///< Exception number - const char *name; ///< Exception name + int num; ///< Exception number + const char *name; ///< Exception name // Additional information std::string msg; int errno_val; // Where it was thrown - const char *file; ///< Source file name - int line; ///< Source line number + const char *file; ///< Source file name + int line; ///< Source line number ExceptionServer(int exc, const char *exc_name, const char *f, int l) : num(exc), name(exc_name), msg(), errno_val(0), file(f), line(l) {} @@ -69,8 +69,8 @@ struct ExceptionServer { : num(exc), name(exc_name), msg(m), errno_val(err), file(f), line(l) {} }; -#define ExceptionServer(name, ...) \ +#define ExceptionServer(name, ...) \ ExceptionServer(VDMS::name, #name, ##__VA_ARGS__, __FILE__, __LINE__) -}; // namespace VDMS +}; // namespace VDMS extern void print_exception(const VDMS::ExceptionServer &e, FILE *f = stdout); diff --git a/src/ExceptionsCommand.h b/src/ExceptionsCommand.h index 3287db57..19a6afd0 100644 --- a/src/ExceptionsCommand.h +++ b/src/ExceptionsCommand.h @@ -46,21 +46,21 @@ enum ExceptionCommandType { LockTimeout, LockError, - Undefined = 100, // Any undefined error + Undefined = 100, // Any undefined error }; struct ExceptionCommand { // Which exception - int num; ///< Exception number - const char *name; ///< Exception name + int num; ///< Exception number + const char *name; ///< Exception name // Additional information std::string msg; int errno_val; // Where it was thrown - const char *file; ///< Source file name - int line; ///< Source line number + const char *file; ///< Source file name + int line; ///< Source line number ExceptionCommand(int exc, const char *exc_name, const char *f, int l) : num(exc), name(exc_name), msg(), errno_val(0), file(f), line(l) {} @@ -74,8 +74,8 @@ struct ExceptionCommand { : num(exc), name(exc_name), msg(m), errno_val(err), file(f), line(l) {} }; -#define ExceptionCommand(name, ...) \ +#define ExceptionCommand(name, ...) \ ExceptionCommand(VDMS::name, #name, ##__VA_ARGS__, __FILE__, __LINE__) -}; // namespace VDMS +}; // namespace VDMS extern void print_exception(const VDMS::ExceptionCommand &e, FILE *f = stdout); diff --git a/src/ImageCommand.h b/src/ImageCommand.h index 982d5d5a..70800af8 100644 --- a/src/ImageCommand.h +++ b/src/ImageCommand.h @@ -30,23 +30,23 @@ */ #pragma once -#include "vcl/CustomVCL.h" -#include "vcl/Image.h" +#include + #include #include #include #include "ExceptionsCommand.h" #include "RSCommand.h" - -#include +#include "vcl/CustomVCL.h" +#include "vcl/Image.h" namespace VDMS { // Helper classes for handling various JSON commands. class ImageCommand : public RSCommand { -public: + public: ImageCommand(const std::string &cmd_name); virtual int construct_protobuf(PMGDQuery &tx, const Json::Value &root, @@ -64,7 +64,7 @@ class ImageCommand : public RSCommand { // corresponding VCL::Image::Format type. VCL::Format get_requested_format(const Json::Value &cmd); -protected: + protected: bool output_vcl_timing; }; @@ -75,7 +75,7 @@ class AddImage : public ImageCommand { std::string _storage_bin; // bool _use_aws_storage; -public: + public: AddImage(); int construct_protobuf(PMGDQuery &tx, const Json::Value &root, @@ -86,7 +86,7 @@ class AddImage : public ImageCommand { }; class UpdateImage : public ImageCommand { -public: + public: UpdateImage(); int construct_protobuf(PMGDQuery &tx, const Json::Value &root, @@ -100,7 +100,7 @@ class UpdateImage : public ImageCommand { class FindImage : public ImageCommand { // bool _use_aws_storage; -public: + public: FindImage(); int construct_protobuf(PMGDQuery &tx, const Json::Value &root, const std::string &blob, int grp_id, @@ -112,4 +112,4 @@ class FindImage : public ImageCommand { const std::string &blob); }; -}; // namespace VDMS +}; // namespace VDMS diff --git a/src/ImageLoop.h b/src/ImageLoop.h index 80b5e8dc..56a39922 100644 --- a/src/ImageLoop.h +++ b/src/ImageLoop.h @@ -29,15 +29,16 @@ * */ -#include "vcl/Image.h" #include #include #include #include #include +#include "vcl/Image.h" + class ImageLoop { -public: + public: ImageLoop() = default; ImageLoop(const ImageLoop &) = delete; ImageLoop(ImageLoop &&) noexcept = delete; @@ -56,7 +57,7 @@ class ImageLoop { bool is_loop_running(); void close_no_operation_loop(std::string imageid); -private: + private: int _nrof_entities = 0; bool destroyed = false; bool _remote_running = false; diff --git a/src/Neo4JCommands.h b/src/Neo4JCommands.h index dc6b45ce..43230a74 100644 --- a/src/Neo4JCommands.h +++ b/src/Neo4JCommands.h @@ -31,15 +31,16 @@ #pragma once +#include + #include "queryMessage.pb.h" #include "vcl/Image.h" #include "vcl/VCL.h" -#include namespace VDMS { class Neo4jCommand { -protected: + protected: const std::string _cmd_name; std::map _valid_params_map; @@ -50,7 +51,7 @@ class Neo4jCommand { virtual Json::Value check_responses(Json::Value &responses); bool _use_aws_storage; -public: + public: enum ErrorCode { Success = 0, Error = -1, @@ -77,7 +78,7 @@ class Neo4jNeoAdd : public Neo4jCommand { std::string _storage_jpg; std::string _storage_bin; -public: + public: Neo4jNeoAdd(); bool need_blob(const Json::Value &cmd); int data_processing(std::string &tx, const Json::Value &root, @@ -89,8 +90,7 @@ class Neo4jNeoAdd : public Neo4jCommand { }; class Neo4jNeoFind : public Neo4jCommand { - -public: + public: Neo4jNeoFind(); bool need_blob(const Json::Value &cmd) { return false; } int data_processing(std::string &tx, const Json::Value &root, @@ -101,4 +101,4 @@ class Neo4jNeoFind : public Neo4jCommand { const std::string &blob); }; -} // namespace VDMS +} // namespace VDMS diff --git a/src/Neo4jQueryHelpers.h b/src/Neo4jQueryHelpers.h index b23167ec..6b4ae87f 100644 --- a/src/Neo4jQueryHelpers.h +++ b/src/Neo4jQueryHelpers.h @@ -50,7 +50,6 @@ void PrintJSONValue(const Json::Value &val) { } void json_printer(const Json::Value &root, unsigned short depth) { - printf("\n---------JSON Dumper-----------\n"); for (Json::Value::const_iterator itr = root.begin(); itr != root.end(); itr++) { diff --git a/src/OpsIOCoordinator.h b/src/OpsIOCoordinator.h index 2b0dc400..9dda9ea4 100644 --- a/src/OpsIOCoordinator.h +++ b/src/OpsIOCoordinator.h @@ -31,9 +31,9 @@ extern VCL::RemoteConnection *global_s3_connection; -std::vector -do_single_img_ops(const Json::Value &orig_query, - std::vector &raw_data, std::string cmd_name); +std::vector do_single_img_ops( + const Json::Value &orig_query, std::vector &raw_data, + std::string cmd_name); std::vector s3_retrieval(std::string obj_name, VCL::RemoteConnection *connection); int s3_upload(std::string obj_name, std::vector upload_data, diff --git a/src/PMGDIterators.h b/src/PMGDIterators.h index d2fd8e96..7b7da016 100644 --- a/src/PMGDIterators.h +++ b/src/PMGDIterators.h @@ -39,9 +39,10 @@ namespace VDMS { -template class PMGDQueryHandler::ReusableIterator { +template +class PMGDQueryHandler::ReusableIterator { // Iterator for the starting nodes. - Ti _ti; // Type Iterator + Ti _ti; // Type Iterator // TODO Is list the best data structure // if we could potentially sort? @@ -55,8 +56,7 @@ template class PMGDQueryHandler::ReusableIterator { bool _next() { if (_it != _traversed.end()) { ++_it; - if (_it != _traversed.end()) - return true; + if (_it != _traversed.end()) return true; } if (bool(_ti)) { _it = _traversed.insert(_traversed.end(), &static_cast(*_ti)); @@ -67,8 +67,7 @@ template class PMGDQueryHandler::ReusableIterator { } T *ref() { - if (!bool(*this)) - throw PMGDException(NullIterator, "Null impl"); + if (!bool(*this)) throw PMGDException(NullIterator, "Null impl"); return *_it; } @@ -87,7 +86,7 @@ template class PMGDQueryHandler::ReusableIterator { } }; -public: + public: // Make sure this is not auto-declared. The move one won't be. ReusableIterator(const ReusableIterator &) = delete; ReusableIterator(Ti ti) : _ti(ti), _it(_traversed.begin()) { _next(); } @@ -152,12 +151,15 @@ class PMGDQueryHandler::MultiNeighborIteratorImpl bool _next(); -public: + public: MultiNeighborIteratorImpl(ReusableNodeIterator *start_ni, SearchExpression search_neighbors, PMGD::Direction dir, PMGD::StringID edge_tag) - : _start_ni(start_ni), _search_neighbors(search_neighbors), - _neighb_i(NULL), _dir(dir), _edge_tag(edge_tag) { + : _start_ni(start_ni), + _search_neighbors(search_neighbors), + _neighb_i(NULL), + _dir(dir), + _edge_tag(edge_tag) { _next(); } @@ -224,19 +226,22 @@ class PMGDQueryHandler::NodeEdgeIteratorImpl } } -public: + public: NodeEdgeIteratorImpl(const SearchExpression &expr, ReusableNodeIterator *src_ni = NULL, ReusableNodeIterator *dest_ni = NULL) - : _expr(expr), _num_predicates(_expr.num_node_predicates()), - _src_ni(src_ni), _dest_ni(dest_ni), _pred_start(0), _check_dest(false) + : _expr(expr), + _num_predicates(_expr.num_node_predicates()), + _src_ni(src_ni), + _dest_ni(dest_ni), + _pred_start(0), + _check_dest(false) { _edge_it.reset(new PMGD::EdgeIterator(return_iterator())); // If the first criteria did not return any edges, // there is no node checking on either side. - if (!bool(*_edge_it)) - return; + if (!bool(*_edge_it)) return; if (_dest_ni != NULL) { for (; bool(*_dest_ni); _dest_ni->next()) _dest_nodes.insert(&(**_dest_ni)); @@ -244,8 +249,7 @@ class PMGDQueryHandler::NodeEdgeIteratorImpl _dest_ni = NULL; _check_dest = true; } - if (!check_predicates()) - next(); + if (!check_predicates()) next(); } operator bool() const { return bool(*_edge_it); } @@ -259,4 +263,4 @@ class PMGDQueryHandler::NodeEdgeIteratorImpl return &static_cast(**_edge_it); } }; -} // namespace VDMS +} // namespace VDMS diff --git a/src/PMGDQuery.h b/src/PMGDQuery.h index 3551a906..1fab5175 100644 --- a/src/PMGDQuery.h +++ b/src/PMGDQuery.h @@ -30,13 +30,13 @@ */ #pragma once -#include - -#include "PMGDQueryHandler.h" // to provide the database connection - #include #include +#include + +#include "PMGDQueryHandler.h" // to provide the database connection + namespace VDMS { /* This class takes care of the transaction and conversion @@ -49,11 +49,11 @@ class PMGDQuery { unsigned _current_group_id; PMGDQueryHandler &_pmgd_qh; unsigned _current_ref; - bool _readonly; // Stays true unless some write cmd sets it to false. - bool _resultdeletion; // Indicates whether the results should be deleted - bool _resultexpiration; // Indicates whether the result should be stored in - // expiration_queue This takes place only during an - // add where the _expiration flag is true + bool _readonly; // Stays true unless some write cmd sets it to false. + bool _resultdeletion; // Indicates whether the results should be deleted + bool _resultexpiration; // Indicates whether the result should be stored in + // expiration_queue This takes place only during an + // add where the _expiration flag is true Json::Value _json_responses; @@ -76,7 +76,7 @@ class PMGDQuery { Json::Value construct_error_response(PMGDCmdResponse *response); -public: + public: PMGDQuery(PMGDQueryHandler &pmgd_qh); ~PMGDQuery(); @@ -120,4 +120,4 @@ class PMGDQuery { void DeleteExpired(); }; -} // namespace VDMS +} // namespace VDMS diff --git a/src/PMGDQueryHandler.h b/src/PMGDQueryHandler.h index fab24bcb..d9288b3e 100644 --- a/src/PMGDQueryHandler.h +++ b/src/PMGDQueryHandler.h @@ -39,7 +39,7 @@ #include "AutoDeleteNode.h" #include "pmgd.h" -#include "pmgdMessages.pb.h" // Protobuff implementation +#include "pmgdMessages.pb.h" // Protobuff implementation #define PMGD_QUERY_RETRY_LIMIT 10 @@ -63,7 +63,8 @@ typedef std::vector PMGDCmds; typedef std::vector PMGDCmdResponses; class PMGDQueryHandler { - template class ReusableIterator; + template + class ReusableIterator; typedef ReusableIterator ReusableNodeIterator; typedef ReusableIterator ReusableEdgeIterator; @@ -74,15 +75,16 @@ class PMGDQueryHandler { static PMGD::Graph *_db; static std::list _expiration_timestamp_queue; static std::vector - _cleanup_filename_list; // files cannot be deleted until after blobs are - // added + _cleanup_filename_list; // files cannot be deleted until after blobs are + // added PMGD::Transaction *_tx; - bool _readonly; // Variable changes per TX based on process_queries parameter. - bool _resultdeletion; // Variable that indicates whether results of query - // should be - bool _autodelete_init; // Varibale that indicates whether we need to add nodes - // from query into deletion_queue + bool + _readonly; // Variable changes per TX based on process_queries parameter. + bool _resultdeletion; // Variable that indicates whether results of query + // should be + bool _autodelete_init; // Varibale that indicates whether we need to add + // nodes from query into deletion_queue // deleted after result is complete // Map an integer ID to a NodeIterator (reset at the end of each transaction). @@ -106,7 +108,8 @@ class PMGDQueryHandler { int add_edge(const PMGD::protobufs::AddEdge &ce, PMGDCmdResponse *response); int update_edge(const PMGD::protobufs::UpdateEdge &ue, PMGDCmdResponse *response); - template void set_property(Element &e, const PMGDProp &p); + template + void set_property(Element &e, const PMGDProp &p); int query_node(const PMGDQueryNode &qn, PMGDCmdResponse *response, bool autodelete_init = false); int query_edge(const PMGDQueryEdge &qe, PMGDCmdResponse *response); @@ -139,7 +142,7 @@ class PMGDQueryHandler { int delete_expired_nodes(); -public: + public: class NodeEdgeIteratorImpl; static void init(); static void destroy(); @@ -168,7 +171,7 @@ class PMGDQueryHandler { void print_node_idx_stats(char *tag_name, char *prop_id); }; -}; // namespace VDMS +}; // namespace VDMS void insert_into_queue(std::list *queue, AutoDeleteNode *new_element); diff --git a/src/QueryHandlerBase.h b/src/QueryHandlerBase.h index 5da77dab..ec45ede8 100644 --- a/src/QueryHandlerBase.h +++ b/src/QueryHandlerBase.h @@ -5,20 +5,21 @@ #ifndef VDMS_QUERYHANDLERBASE_H #define VDMS_QUERYHANDLERBASE_H -#include "QueryMessage.h" // Protobuff implementation #include #include +#include "QueryMessage.h" // Protobuff implementation + // Json parsing files #include + #include #include namespace VDMS { class QueryHandlerBase { - -protected: + protected: // valijson valijson::Validator _validator; static valijson::Schema *_schema; @@ -40,11 +41,11 @@ class QueryHandlerBase { void virtual process_query(protobufs::queryMessage &proto_query, protobufs::queryMessage &response) = 0; -public: + public: QueryHandlerBase(); void virtual process_connection(comm::Connection *c); }; -} // namespace VDMS +} // namespace VDMS -#endif // VDMS_QUERYHANDLERBASE_H +#endif // VDMS_QUERYHANDLERBASE_H diff --git a/src/QueryHandlerExample.h b/src/QueryHandlerExample.h index 477ec0c5..cfbea916 100644 --- a/src/QueryHandlerExample.h +++ b/src/QueryHandlerExample.h @@ -40,6 +40,7 @@ // Json parsing files #include + #include #include @@ -48,11 +49,11 @@ namespace VDMS { typedef ::google::protobuf::RepeatedPtrField BlobArray; class QueryHandlerExample : public QueryHandlerBase { -public: + public: static void init(); QueryHandlerExample(); void process_connection(comm::Connection *c); void process_query(protobufs::queryMessage &proto_query, protobufs::queryMessage &response); }; -} // namespace VDMS +} // namespace VDMS diff --git a/src/QueryHandlerNeo4j.h b/src/QueryHandlerNeo4j.h index 4c1f79e1..09bb5566 100644 --- a/src/QueryHandlerNeo4j.h +++ b/src/QueryHandlerNeo4j.h @@ -38,8 +38,7 @@ namespace VDMS { class QueryHandlerNeo4j : public QueryHandlerBase { - -protected: + protected: static BackendNeo4j *neoconn_pool; static std::unordered_map _rs_cmds; friend class QueryHandlerTester; @@ -47,11 +46,11 @@ class QueryHandlerNeo4j : public QueryHandlerBase { int parse_commands(const protobufs::queryMessage &proto_query, Json::Value &root); -public: + public: static void init(); QueryHandlerNeo4j(); void process_query(protobufs::queryMessage &proto_query, protobufs::queryMessage &response); }; -} // namespace VDMS +} // namespace VDMS diff --git a/src/QueryHandlerPMGD.h b/src/QueryHandlerPMGD.h index d848e4b6..8aec2041 100644 --- a/src/QueryHandlerPMGD.h +++ b/src/QueryHandlerPMGD.h @@ -30,7 +30,7 @@ */ #pragma once -#include "PMGDQueryHandler.h" // to provide the database connection +#include "PMGDQueryHandler.h" // to provide the database connection #include "QueryHandlerBase.h" #include "RSCommand.h" #include "Server.h" @@ -38,8 +38,7 @@ namespace VDMS { class QueryHandlerPMGD : public QueryHandlerBase { - -protected: + protected: friend class QueryHandlerTester; static std::unordered_map _rs_cmds; @@ -51,7 +50,7 @@ class QueryHandlerPMGD : public QueryHandlerBase { int parse_commands(const protobufs::queryMessage &proto_query, Json::Value &root); -public: + public: static void init(); QueryHandlerPMGD(); @@ -68,4 +67,4 @@ class QueryHandlerPMGD : public QueryHandlerBase { bool output_query_level_timing; }; -} // namespace VDMS +} // namespace VDMS diff --git a/src/QueryMessage.h b/src/QueryMessage.h index 78820914..8122c5ab 100644 --- a/src/QueryMessage.h +++ b/src/QueryMessage.h @@ -38,10 +38,10 @@ namespace VDMS { class QueryMessage { comm::Connection *_conn; -public: + public: QueryMessage(comm::Connection *conn); protobufs::queryMessage get_query(); void send_response(protobufs::queryMessage cmd); }; -}; // namespace VDMS +}; // namespace VDMS diff --git a/src/RSCommand.h b/src/RSCommand.h index 827010c0..ff9254bc 100644 --- a/src/RSCommand.h +++ b/src/RSCommand.h @@ -30,7 +30,6 @@ */ #pragma once -#include "tbb/concurrent_unordered_map.h" #include #include #include @@ -38,6 +37,7 @@ #include "PMGDQuery.h" #include "queryMessage.pb.h" +#include "tbb/concurrent_unordered_map.h" // Json parsing files #include @@ -46,7 +46,7 @@ namespace VDMS { // Helper classes for handling various JSON commands. class RSCommand { -protected: + protected: const std::string _cmd_name; std::map _valid_params_map; @@ -58,7 +58,7 @@ class RSCommand { virtual Json::Value check_responses(Json::Value &responses); -public: + public: enum ErrorCode { Success = 0, Error = -1, @@ -84,10 +84,10 @@ class RSCommand { }; class AddEntity : public RSCommand { -private: + private: std::string _storage_blob; -public: + public: AddEntity(); int construct_protobuf(PMGDQuery &query, const Json::Value &root, const std::string &blob, int grp_id, @@ -97,7 +97,7 @@ class AddEntity : public RSCommand { }; class AddConnection : public RSCommand { -public: + public: AddConnection(); int construct_protobuf(PMGDQuery &query, const Json::Value &root, const std::string &blob, int grp_id, @@ -105,7 +105,7 @@ class AddConnection : public RSCommand { }; class UpdateEntity : public RSCommand { -public: + public: UpdateEntity(); int construct_protobuf(PMGDQuery &query, const Json::Value &root, const std::string &blob, int grp_id, @@ -113,7 +113,7 @@ class UpdateEntity : public RSCommand { }; class UpdateConnection : public RSCommand { -public: + public: UpdateConnection(); int construct_protobuf(PMGDQuery &query, const Json::Value &root, const std::string &blob, int grp_id, @@ -121,7 +121,7 @@ class UpdateConnection : public RSCommand { }; class FindEntity : public RSCommand { -public: + public: FindEntity(); int construct_protobuf(PMGDQuery &query, const Json::Value &root, const std::string &blob, int grp_id, @@ -134,7 +134,7 @@ class FindEntity : public RSCommand { }; class DeleteExpired : public RSCommand { -public: + public: DeleteExpired(); int construct_protobuf(PMGDQuery &query, const Json::Value &root, const std::string &blob, int grp_id, @@ -147,10 +147,10 @@ class DeleteExpired : public RSCommand { }; class FindConnection : public RSCommand { -public: + public: FindConnection(); int construct_protobuf(PMGDQuery &query, const Json::Value &root, const std::string &blob, int grp_id, Json::Value &error); }; -}; // namespace VDMS +}; // namespace VDMS diff --git a/src/SearchExpression.h b/src/SearchExpression.h index 151af8cc..0ca47cc4 100644 --- a/src/SearchExpression.h +++ b/src/SearchExpression.h @@ -31,9 +31,10 @@ #pragma once -#include "pmgd.h" #include +#include "pmgd.h" + /// Search expression to query a PMGD Lake database /// /// Search expression is currently limited to be the @@ -67,7 +68,7 @@ class SearchExpression { /// A pointer to the database PMGD::Graph &_db; -public: + public: /// Construction requires a handle to a database SearchExpression(PMGD::Graph &db, PMGD::StringID tag, bool p_or) : _db(db), _tag(tag), _or(p_or) {} @@ -98,4 +99,4 @@ class SearchExpression { PMGD::EdgeIterator eval_edges(); }; -}; // namespace VDMS +}; // namespace VDMS diff --git a/src/Server.h b/src/Server.h index e3dfdfa3..33ebb483 100644 --- a/src/Server.h +++ b/src/Server.h @@ -31,13 +31,12 @@ #pragma once +#include #include #include "CommunicationManager.h" #include "VDMSConfig.h" #include "pmgd.h" -#include -#include namespace VDMS { struct ReplicationConfig { @@ -57,18 +56,24 @@ struct ReplicationConfig { int pmgd_num_allocators; ReplicationConfig() - : backup_path("."), db_path("db"), replication_time("-1"), - autoreplication_unit("s"), backup_flag("false"), - images_path("db/images"), descriptor_path("db/descriptors"), - blobs_path("db/blobs"), server_port(55555), - max_simultaneous_clients(10), autoreplicate_interval(-1), - autodelete_interval(-1), expiration_time(86400), + : backup_path("."), + db_path("db"), + replication_time("-1"), + autoreplication_unit("s"), + backup_flag("false"), + images_path("db/images"), + descriptor_path("db/descriptors"), + blobs_path("db/blobs"), + server_port(55555), + max_simultaneous_clients(10), + autoreplicate_interval(-1), + autodelete_interval(-1), + expiration_time(86400), pmgd_num_allocators(5) { // Additional initialization code if needed } }; class Server { - // Defining constants/defaults within the class itself is a bit weird. // Consider refactoring static const int DEFAULT_PORT = 55555; @@ -99,7 +104,7 @@ class Server { // used to select as well as initialize any state for query handlers void setup_query_handler(); -public: + public: VDMSConfig *cfg; Server(std::string config_file, std::string cert_file, std::string key_file, std::string ca_file); @@ -111,4 +116,4 @@ class Server { ~Server(); }; -}; // namespace VDMS +}; // namespace VDMS diff --git a/src/VDMSConfig.h b/src/VDMSConfig.h index 298e3301..cdb0399e 100644 --- a/src/VDMSConfig.h +++ b/src/VDMSConfig.h @@ -31,6 +31,10 @@ #pragma once +#include +#include +#include + #include #include #include @@ -38,10 +42,6 @@ #include #include -#include -#include -#include - #include "VDMSConfigHelper.h" // Parameters in the JSON config file @@ -93,8 +93,7 @@ const std::string PARAM_AWS_LOG_LEVEL = "aws_log_level"; namespace VDMS { class VDMSConfig { - -public: + public: static bool init(std::string config_file); static bool destroy(); @@ -139,13 +138,13 @@ class VDMSConfig { return aws_log_level; } -protected: + protected: static VDMSConfig *cfg; static std::mutex _mutex; VDMSConfig(std::string config_file); ~VDMSConfig() {} -private: + private: Json::Value json_config; // Dirs @@ -162,9 +161,9 @@ class VDMSConfig { std::string path_tmp; StorageType storage_type; - bool aws_flag; // use aws flag - std::string aws_bucket_name; // aws bucket name - bool use_endpoint; // Use Mocked S3 server or real AWS S3 + bool aws_flag; // use aws flag + std::string aws_bucket_name; // aws bucket name + bool use_endpoint; // Use Mocked S3 server or real AWS S3 std::optional endpoint_override; std::optional proxy_host; @@ -196,4 +195,4 @@ class VDMSConfig { } }; -}; // namespace VDMS +}; // namespace VDMS diff --git a/src/VideoCommand.h b/src/VideoCommand.h index 1f3fccab..8dd846d6 100644 --- a/src/VideoCommand.h +++ b/src/VideoCommand.h @@ -30,20 +30,20 @@ */ #pragma once -#include "vcl/Video.h" #include #include #include #include "ExceptionsCommand.h" #include "RSCommand.h" +#include "vcl/Video.h" namespace VDMS { // Helper classes for handling various JSON commands. class VideoCommand : public RSCommand { -protected: + protected: void enqueue_operations(VCL::Video &video, const Json::Value &op, bool is_addition = false); @@ -52,7 +52,7 @@ class VideoCommand : public RSCommand { virtual Json::Value check_responses(Json::Value &responses); bool output_vcl_timing; -public: + public: VideoCommand(const std::string &cmd_name); virtual int construct_protobuf(PMGDQuery &tx, const Json::Value &root, @@ -67,7 +67,7 @@ class AddVideo : public VideoCommand { std::string _storage_video; // bool _use_aws_storage; -public: + public: AddVideo(); int construct_protobuf(PMGDQuery &tx, const Json::Value &root, @@ -83,7 +83,7 @@ class AddVideo : public VideoCommand { }; class UpdateVideo : public VideoCommand { -public: + public: UpdateVideo(); int construct_protobuf(PMGDQuery &tx, const Json::Value &root, @@ -100,7 +100,7 @@ class FindVideo : public VideoCommand { // bool _use_aws_storage; std::vector shortlisted_videos; -public: + public: FindVideo(); int construct_protobuf(PMGDQuery &tx, const Json::Value &root, @@ -117,7 +117,7 @@ class FindFrames : public VideoCommand { // bool _use_aws_storage; bool get_interval_index(const Json::Value &cmd, Json::ArrayIndex &op_index); -public: + public: FindFrames(); int construct_protobuf(PMGDQuery &tx, const Json::Value &root, @@ -130,4 +130,4 @@ class FindFrames : public VideoCommand { const std::string &blob) override; }; -}; // namespace VDMS +}; // namespace VDMS diff --git a/src/VideoLoop.h b/src/VideoLoop.h index 76d58672..b0dd1ad4 100644 --- a/src/VideoLoop.h +++ b/src/VideoLoop.h @@ -29,21 +29,21 @@ * */ -#include "vcl/Image.h" -#include "vcl/Video.h" #include #include #include -#include -#include - #include #include #include #include +#include +#include + +#include "vcl/Image.h" +#include "vcl/Video.h" class VideoLoop { -public: + public: VideoLoop() = default; VideoLoop(const VideoLoop &) = delete; VideoLoop(VideoLoop &&) noexcept = delete; @@ -86,7 +86,7 @@ class VideoLoop { */ void close_no_operation_loop(std::string videoId); -private: + private: // Number of entities in the VDMS query response int _nrof_entities = 0; diff --git a/src/defines.h b/src/defines.h index afa2d653..6963faf6 100644 --- a/src/defines.h +++ b/src/defines.h @@ -57,7 +57,7 @@ // Descriptor Set #define VDMS_DESC_SET_TAG "VD:DESCSET" -#define VDMS_DESC_SET_EDGE_TAG "VD:DESCSETLINK" // link between set and desc +#define VDMS_DESC_SET_EDGE_TAG "VD:DESCSETLINK" // link between set and desc #define VDMS_DESC_SET_PATH_PROP "VD:descSetPath" #define VDMS_DESC_SET_NAME_PROP "VD:name" #define VDMS_DESC_SET_DIM_PROP "VD:dimensions" diff --git a/src/vcl/DescriptorParams.h b/src/vcl/DescriptorParams.h index c54ba2d7..33385f1d 100644 --- a/src/vcl/DescriptorParams.h +++ b/src/vcl/DescriptorParams.h @@ -34,32 +34,31 @@ #pragma once +#include +#include +#include +#include + #include #include #include #include #include -#include -#include -#include -#include - #include "vcl/DescriptorSet.h" namespace VCL { class DescriptorParams { - -public: + public: /* Params needed for FLINNG */ // constants for now until we derive them from N and dimensions uint64_t num_rows; uint64_t cells_per_row; uint64_t num_hash_tables; uint64_t hashes_per_table; - uint64_t sub_hash_bits; // sub_hash_bits * hashes_per_table must be less than - // 32, otherwise segfault will happen + uint64_t sub_hash_bits; // sub_hash_bits * hashes_per_table must be less than + // 32, otherwise segfault will happen uint64_t cut_off; DescriptorParams(uint64_t numrows = 3, uint64_t cellsperrow = (1 << 12), @@ -74,4 +73,4 @@ class DescriptorParams { this->cut_off = cutoff; } }; -}; // namespace VCL +}; // namespace VCL diff --git a/src/vcl/DescriptorSetData.h b/src/vcl/DescriptorSetData.h index c2b2c423..1bc1dc31 100644 --- a/src/vcl/DescriptorSetData.h +++ b/src/vcl/DescriptorSetData.h @@ -34,24 +34,23 @@ #pragma once +#include +#include +#include +#include + #include #include #include #include #include -#include -#include -#include -#include - #include "vcl/DescriptorSet.h" namespace VCL { class DescriptorSet::DescriptorSetData { - -protected: + protected: std::string _set_path; unsigned _dimensions; uint64_t _n_total; @@ -100,7 +99,7 @@ class DescriptorSet::DescriptorSetData { void write_labels_map(); void read_labels_map(); -public: + public: /** * Loads an existing collection located at collection_path * or created a new collection if it does not exist @@ -275,4 +274,4 @@ class DescriptorSet::DescriptorSetData { void set_labels_map(std::map &labels); }; -}; // namespace VCL \ No newline at end of file +}; // namespace VCL \ No newline at end of file diff --git a/src/vcl/FaissDescriptorSet.h b/src/vcl/FaissDescriptorSet.h index d484969e..0516bb51 100644 --- a/src/vcl/FaissDescriptorSet.h +++ b/src/vcl/FaissDescriptorSet.h @@ -34,24 +34,23 @@ #pragma once +#include +#include +#include +#include + #include #include -#include #include #include #include #include "DescriptorSetData.h" -#include -#include -#include - namespace VCL { class FaissDescriptorSet : public DescriptorSet::DescriptorSetData { - -protected: + protected: std::string _faiss_file; faiss::Index *_index; @@ -64,7 +63,7 @@ class FaissDescriptorSet : public DescriptorSet::DescriptorSetData { void train_core(float *descriptors, unsigned n); -public: + public: FaissDescriptorSet(const std::string &set_path); FaissDescriptorSet(const std::string &set_path, unsigned dim); @@ -94,16 +93,14 @@ class FaissDescriptorSet : public DescriptorSet::DescriptorSetData { }; class FaissFlatDescriptorSet : public FaissDescriptorSet { - -public: + public: FaissFlatDescriptorSet(const std::string &set_path); FaissFlatDescriptorSet(const std::string &set_path, unsigned dim, DistanceMetric metric); }; class FaissIVFFlatDescriptorSet : public FaissDescriptorSet { - -public: + public: FaissIVFFlatDescriptorSet(const std::string &set_path); FaissIVFFlatDescriptorSet(const std::string &set_path, unsigned dim, DistanceMetric metric); @@ -112,8 +109,7 @@ class FaissIVFFlatDescriptorSet : public FaissDescriptorSet { }; class FaissHNSWFlatDescriptorSet : public FaissDescriptorSet { - -public: + public: FaissHNSWFlatDescriptorSet(const std::string &set_path); FaissHNSWFlatDescriptorSet(const std::string &set_path, unsigned dim, DistanceMetric metric); @@ -122,4 +118,4 @@ class FaissHNSWFlatDescriptorSet : public FaissDescriptorSet { float *distances); }; -}; // namespace VCL +}; // namespace VCL diff --git a/src/vcl/FlinngDescriptorSet.h b/src/vcl/FlinngDescriptorSet.h index 288dbe00..7b4bbd40 100644 --- a/src/vcl/FlinngDescriptorSet.h +++ b/src/vcl/FlinngDescriptorSet.h @@ -34,28 +34,28 @@ #pragma once +#include + #include #include -#include #include #include #include #include "DescriptorSetData.h" -//#include "../../../FLINNG/include/lib_flinng.h" //todo update make files for -// flinng lib include directory +// #include "../../../FLINNG/include/lib_flinng.h" //todo update make files for +// flinng lib include directory #include "DescriptorParams.h" #include "lib_flinng.h" namespace VCL { class FlinngDescriptorSet : public DescriptorSet::DescriptorSetData { - -protected: + protected: std::string _flinng_file; bool is_finalized; - flinng::BaseDenseFlinng32 *_index; // FLinng have a base class by this name + flinng::BaseDenseFlinng32 *_index; // FLinng have a base class by this name // depending on metric to be used will point to the right index flinng::FlinngBuilder *_builder; @@ -69,7 +69,7 @@ class FlinngDescriptorSet : public DescriptorSet::DescriptorSetData { void getFlinngParams(VCL::DescriptorParams *par, flinng::FlinngBuilder *builder); -public: + public: FlinngDescriptorSet(const std::string &set_path); FlinngDescriptorSet(const std::string &set_path, unsigned dim, DistanceMetric metric, VCL::DescriptorParams *par = NULL); @@ -104,4 +104,4 @@ class FlinngDescriptorSet : public DescriptorSet::DescriptorSetData { void store(); void store(std::string set_path); }; -}; // namespace VCL +}; // namespace VCL diff --git a/src/vcl/TDBDescriptorSet.h b/src/vcl/TDBDescriptorSet.h index ff31d5e5..00a2821c 100644 --- a/src/vcl/TDBDescriptorSet.h +++ b/src/vcl/TDBDescriptorSet.h @@ -34,13 +34,13 @@ #pragma once +#include + #include #include -#include #include -#include - #include +#include #include "DescriptorSetData.h" #include "TDBObject.h" @@ -53,20 +53,19 @@ typedef std::vector DistanceData; class TDBDescriptorSet : public DescriptorSet::DescriptorSetData, public TDBObject { - -protected: + protected: const unsigned long MAX_DESC = 100000; const unsigned long METADATA_OFFSET = MAX_DESC - 2; // this is caching data - std::vector _label_ids; // we need to move this + std::vector _label_ids; // we need to move this void compute_distances(float *q, DistanceData &d, DescBuffer &data); virtual void read_descriptor_metadata() = 0; virtual void write_descriptor_metadata() = 0; -public: + public: /** * Loads an existing collection located at collection_path * or created a new collection if it does not exist @@ -103,8 +102,7 @@ class TDBDescriptorSet : public DescriptorSet::DescriptorSetData, }; class TDBDenseDescriptorSet : public TDBDescriptorSet { - -private: + private: // This is for caching, accelerates searches fairly well. bool _flag_buffer_updated; std::vector _buffer; @@ -113,13 +111,13 @@ class TDBDenseDescriptorSet : public TDBDescriptorSet { void read_descriptor_metadata(); void write_descriptor_metadata(); -public: + public: TDBDenseDescriptorSet(const std::string &collection_path); TDBDenseDescriptorSet(const std::string &collection_path, unsigned dim, DistanceMetric metric); - ~TDBDenseDescriptorSet(){}; + ~TDBDenseDescriptorSet() {}; long add(float *descriptors, unsigned n_descriptors, long *classes); @@ -130,8 +128,7 @@ class TDBDenseDescriptorSet : public TDBDescriptorSet { }; class TDBSparseDescriptorSet : public TDBDescriptorSet { - -private: + private: void read_descriptor_metadata(); void write_descriptor_metadata(); @@ -142,13 +139,13 @@ class TDBSparseDescriptorSet : public TDBDescriptorSet { void search(float *query, unsigned n_queries, unsigned k, long *descriptors, float *distances, long *labels); -public: + public: TDBSparseDescriptorSet(const std::string &collection_path); TDBSparseDescriptorSet(const std::string &collection_path, unsigned dim, DistanceMetric metric); - ~TDBSparseDescriptorSet(){}; + ~TDBSparseDescriptorSet() {}; long add(float *descriptors, unsigned n_descriptors, long *classes); @@ -161,4 +158,4 @@ class TDBSparseDescriptorSet : public TDBDescriptorSet { void get_labels(long *ids, unsigned n, long *labels); }; -}; // namespace VCL +}; // namespace VCL diff --git a/src/vcl/TDBImage.h b/src/vcl/TDBImage.h index 997b1b69..6c533a37 100644 --- a/src/vcl/TDBImage.h +++ b/src/vcl/TDBImage.h @@ -32,11 +32,12 @@ #pragma once +#include + #include #include "TDBObject.h" #include "vcl/Exception.h" -#include namespace VCL { @@ -47,11 +48,10 @@ namespace VCL { typedef cv::Rect Rectangle; class TDBImage : public TDBObject { - /* *********************** */ /* VARIABLES */ /* *********************** */ -private: + private: // Image dimensions uint64_t _img_height, _img_width, _img_channels; long _img_size; @@ -63,7 +63,7 @@ class TDBImage : public TDBObject { unsigned char *_raw_data; std::vector _full_array; -public: + public: /* *********************** */ /* CONSTRUCTORS */ /* *********************** */ @@ -87,7 +87,8 @@ class TDBImage : public TDBObject { * @param buffer The raw pixel data * @param size The length of the buffer */ - template TDBImage(T *buffer, long size); + template + TDBImage(T *buffer, long size); /** * Creates a TDBImage object from an existing TDBImage @@ -152,7 +153,8 @@ class TDBImage : public TDBObject { * data when the function ends * @param buffer_size The length of buffer (not in bytes) */ - template void get_buffer(T *buffer, long buffer_size); + template + void get_buffer(T *buffer, long buffer_size); /* *********************** */ /* SET FUNCTIONS */ @@ -249,7 +251,7 @@ class TDBImage : public TDBObject { */ bool operator==(const TDBImage &rhs); -private: + private: /* *********************** */ /* GET FUNCTIONS */ /* *********************** */ @@ -389,4 +391,4 @@ class TDBImage : public TDBObject { double linear_interpolation(double x1, double val1, double x2, double val2, double x); }; -}; // namespace VCL +}; // namespace VCL diff --git a/src/vcl/TDBObject.h b/src/vcl/TDBObject.h index 2b76b9fb..ffef954f 100644 --- a/src/vcl/TDBObject.h +++ b/src/vcl/TDBObject.h @@ -32,25 +32,24 @@ #pragma once -#include - #include + +#include #include +#include #include #include "vcl/Exception.h" #include "vcl/RemoteConnection.h" #include "vcl/utils.h" -#include namespace VCL { class TDBObject { - /* *********************** */ /* VARIABLES */ /* *********************** */ -protected: + protected: // Path variables std::string _group; std::string _name; @@ -82,7 +81,7 @@ class TDBObject { tiledb::Config _config; -public: + public: /* *********************** */ /* ENUMS */ /* *********************** */ @@ -313,7 +312,7 @@ class TDBObject { const std::vector &subarray, std::vector &values, std::string &attribute); -protected: + protected: /* *********************** */ /* GET FUNCTIONS */ /* *********************** */ @@ -362,7 +361,7 @@ class TDBObject { */ void set_from_schema(const std::string &object_id); -private: + private: /** * Sets the TileDB type of the attribute values, currently * all are unsigned characters @@ -434,4 +433,4 @@ class TDBObject { */ void find_tile_extents(); }; -}; // namespace VCL +}; // namespace VCL diff --git a/tests/csv_samples/connection.csv b/tests/csv_samples/connection.csv index 59d44594..fcc2c6f1 100644 --- a/tests/csv_samples/connection.csv +++ b/tests/csv_samples/connection.csv @@ -1,5 +1,5 @@ ConnectionClass,Person@id,Person@id,prop_type -BloodRelation,1,2,brother +BloodRelation,1,2,brother BloodRelation,14,16,sister BloodRelation,14,15,mother BloodRelation,14,13,father \ No newline at end of file diff --git a/tests/python/TestBoundingBox.py b/tests/python/TestBoundingBox.py index 4fefaeea..863f071d 100644 --- a/tests/python/TestBoundingBox.py +++ b/tests/python/TestBoundingBox.py @@ -45,7 +45,7 @@ def insertBoundingBox(self, db, props=None): bb["rectangle"] = bb_coords # adds some prop - if not props is None: + if props is not None: bb["properties"] = props query = {} @@ -419,7 +419,6 @@ def test_findBoundingBoxBlob(self): self.assertEqual(len(img_array), self.number_of_inserts) for i in range(0, self.number_of_inserts): - coord = self.number_of_inserts - i - 1 self.assertEqual(response[i]["FindBoundingBox"]["status"], 0) self.assertEqual( response[i]["FindBoundingBox"]["entities"][0]["name"], diff --git a/tests/python/TestConnections.py b/tests/python/TestConnections.py index d7a6b466..4a708e09 100644 --- a/tests/python/TestConnections.py +++ b/tests/python/TestConnections.py @@ -24,7 +24,6 @@ # THE SOFTWARE. # -from threading import Thread import TestCommand diff --git a/tests/python/TestDescriptors.py b/tests/python/TestDescriptors.py index 20f850cd..b70afa04 100644 --- a/tests/python/TestDescriptors.py +++ b/tests/python/TestDescriptors.py @@ -41,7 +41,6 @@ def add_descriptor( constraints: dict = None, results: dict = None, ): - descriptor: dict = {"set": setname} if "Add" in command_str and label is not None: @@ -207,7 +206,6 @@ def test_addSetAndDescriptorsDimMismatch(self): self.disconnect(db) def test_AddSetAndWrongBatchSize(self): - db = self.create_connection() # Create and verify descriptor set @@ -255,7 +253,6 @@ def test_AddSetAndWrongBatchSize(self): self.disconnect(db) def test_AddSetAndInsertBatch(self): - db = self.create_connection() # Create and verify descriptor set @@ -316,7 +313,6 @@ def test_AddSetAndInsertBatch(self): self.disconnect(db) def test_AddBatchAndFindKNN(self): - db = self.create_connection() # Create and verify descriptor set diff --git a/tests/python/TestEntities.py b/tests/python/TestEntities.py index c4257b0a..8114ba46 100644 --- a/tests/python/TestEntities.py +++ b/tests/python/TestEntities.py @@ -42,7 +42,7 @@ def addSingleEntity(self, thID, results, db): try: self.assertEqual(response[0]["AddEntity"]["status"], 0) - except: + except Exception: results[thID] = -1 results[thID] = 0 @@ -70,7 +70,7 @@ def findEntity(self, thID, results, db): response[0]["FindEntity"]["entities"][0]["lastname"], "Ferro" ) self.assertEqual(response[0]["FindEntity"]["entities"][0]["threadid"], thID) - except: + except Exception: results[thID] = -1 results[thID] = 0 diff --git a/tests/python/TestImages.py b/tests/python/TestImages.py index 4667e9b4..ca65dd1f 100644 --- a/tests/python/TestImages.py +++ b/tests/python/TestImages.py @@ -84,7 +84,7 @@ def insertImage(self, db, props=None, collections=None, format="png"): fd.close() # adds some prop - if not props is None: + if props is not None: props["test_case"] = "test_case_prop" op_params_resize = {} diff --git a/tests/python/TestRetail.py b/tests/python/TestRetail.py index c427ee70..0453ccaa 100644 --- a/tests/python/TestRetail.py +++ b/tests/python/TestRetail.py @@ -179,7 +179,7 @@ def single(self, thID, db, results): cmd = list(response[i].items())[0][0] self.assertEqual(response[i][cmd]["status"], 0) - except: + except Exception: results[thID] = -1 results[thID] = 0 diff --git a/tests/python/TestTLS.py b/tests/python/TestTLS.py index f5f59da7..ed0d1eef 100644 --- a/tests/python/TestTLS.py +++ b/tests/python/TestTLS.py @@ -1,7 +1,6 @@ import ssl import unittest import vdms -import json import os diff --git a/tests/python/TestVideos.py b/tests/python/TestVideos.py index 4070c595..96f233ae 100644 --- a/tests/python/TestVideos.py +++ b/tests/python/TestVideos.py @@ -113,7 +113,7 @@ def insertVideo(self, db, props=None): fd.close() # adds some prop - if not props is None: + if props is not None: props["test_case"] = "test_case_prop" query = self.create_video( diff --git a/tests/python/longquery.py b/tests/python/longquery.py index 1e613e92..34114353 100644 --- a/tests/python/longquery.py +++ b/tests/python/longquery.py @@ -24,8 +24,6 @@ # THE SOFTWARE. # -import os - def queryPerson(id): query = [ diff --git a/tests/python/main.py b/tests/python/main.py index cacc1ca7..7492e172 100644 --- a/tests/python/main.py +++ b/tests/python/main.py @@ -24,15 +24,7 @@ # THE SOFTWARE. # -from threading import Thread -import sys -import os -import urllib -import time -import json import unittest -import numpy as np -import vdms if __name__ == "__main__": unittest.main() diff --git a/tests/remote_function_test/functions/caption.py b/tests/remote_function_test/functions/caption.py index d086b1e1..a3c3fdcc 100644 --- a/tests/remote_function_test/functions/caption.py +++ b/tests/remote_function_test/functions/caption.py @@ -1,9 +1,5 @@ import cv2 -import numpy as np -from datetime import datetime -from collections import deque import skvideo.io -import imutils import uuid @@ -14,7 +10,7 @@ def run(ipfilename, format, options): video = skvideo.io.FFmpegWriter(opfilename, {"-pix_fmt": "bgr24"}) print(options) - i = 0 + while True: (grabbed, frame) = vs.read() if not grabbed: diff --git a/tests/remote_function_test/functions/flip.py b/tests/remote_function_test/functions/flip.py index a82cd3b8..ec248584 100644 --- a/tests/remote_function_test/functions/flip.py +++ b/tests/remote_function_test/functions/flip.py @@ -1,4 +1,3 @@ -import time import cv2 diff --git a/tests/remote_function_test/functions/metadata.py b/tests/remote_function_test/functions/metadata.py index b03ac749..4dd270e7 100644 --- a/tests/remote_function_test/functions/metadata.py +++ b/tests/remote_function_test/functions/metadata.py @@ -1,9 +1,4 @@ import cv2 -import numpy as np -from datetime import datetime -from collections import deque -import skvideo.io -import imutils import uuid import json @@ -22,9 +17,7 @@ def facedetectbbox(frame): def run(ipfilename, format, options): - if options["media_type"] == "video": - vs = cv2.VideoCapture(ipfilename) frameNum = 1 metadata = {} diff --git a/tests/remote_function_test/udf_server.py b/tests/remote_function_test/udf_server.py index 939034ae..33a78ece 100644 --- a/tests/remote_function_test/udf_server.py +++ b/tests/remote_function_test/udf_server.py @@ -91,7 +91,7 @@ def remove_tempfile(response): os.remove(response_file) os.remove(video_file) os.remove(metadata_file) - except Exception as e: + except Exception: print("Some files cannot be deleted or are not present") return response @@ -118,7 +118,7 @@ def handle_bad_request(e): if __name__ == "__main__": - if sys.argv[1] == None: + if sys.argv[1] is None: print("Port missing\n Correct Usage: python3 udf_server.py ") else: app.run(host="0.0.0.0", port=int(sys.argv[1])) diff --git a/tests/server/AutoDeleteNodeTest.json b/tests/server/AutoDeleteNodeTest.json index 47755f84..a20369d4 100644 --- a/tests/server/AutoDeleteNodeTest.json +++ b/tests/server/AutoDeleteNodeTest.json @@ -1,15 +1,15 @@ [ { - "FindEntity": + "FindEntity": { - "results": + "results": { - "list": + "list": ["_expiration"] }, - "constraints": + "constraints": { - "_deletion": + "_deletion": ["==", 1], "type_id": ["==", 0] @@ -17,32 +17,32 @@ } }, { - "FindImage": + "FindImage": { - "results": + "results": { - "list": + "list": ["_expiration"] }, - "constraints": + "constraints": { - "_expiration": + "_expiration": [">", 0], "type_id": ["==", 1] } } }, { - "FindVideo": + "FindVideo": { - "results": + "results": { - "list": + "list": ["_expiration"] }, - "constraints": + "constraints": { - "_deletion": + "_deletion": ["==", 1], "type_id": ["==", 2] } diff --git a/tests/server/CustomFunctionNoProcess.json b/tests/server/CustomFunctionNoProcess.json index f9441ed4..83016988 100644 --- a/tests/server/CustomFunctionNoProcess.json +++ b/tests/server/CustomFunctionNoProcess.json @@ -1,14 +1,14 @@ [ { - "AddImage": + "AddImage": { "format": "png", "properties": { "name": "brain.png" - }, - "operations": + }, + "operations": [ { "type": diff --git a/tests/server/QueryHandlerTester.h b/tests/server/QueryHandlerTester.h index bd5c261f..4a839438 100644 --- a/tests/server/QueryHandlerTester.h +++ b/tests/server/QueryHandlerTester.h @@ -35,7 +35,7 @@ namespace VDMS { class QueryHandlerPMGDTester { QueryHandlerPMGD &_qh; -public: + public: QueryHandlerPMGDTester(QueryHandlerPMGD &qh) : _qh(qh) {} void pq(protobufs::queryMessage &proto_query, @@ -47,7 +47,7 @@ class QueryHandlerPMGDTester { class QueryHandlerExampleTester { QueryHandlerExample &_qh; -public: + public: QueryHandlerExampleTester(QueryHandlerExample &qh) : _qh(qh) {} void pq(protobufs::queryMessage &proto_query, @@ -55,4 +55,4 @@ class QueryHandlerExampleTester { _qh.process_query(proto_query, response); } }; -}; // namespace VDMS \ No newline at end of file +}; // namespace VDMS \ No newline at end of file diff --git a/tests/tls_test/prep_certs.py b/tests/tls_test/prep_certs.py index 9cf60d7e..ec23db86 100644 --- a/tests/tls_test/prep_certs.py +++ b/tests/tls_test/prep_certs.py @@ -5,15 +5,11 @@ from cryptography.hazmat.primitives.serialization import ( Encoding, PrivateFormat, - BestAvailableEncryption, NoEncryption, ) from cryptography.hazmat.backends import default_backend import datetime import os -import socket -import ssl -import time def generate_private_key(): @@ -44,8 +40,7 @@ def generate_ca_certificate(subject_name, private_key): .not_valid_before(current_time) .not_valid_after( # Our certificate will be valid for 10 days - current_time - + datetime.timedelta(days=10) + current_time + datetime.timedelta(days=10) ) .add_extension( x509.BasicConstraints(ca=True, path_length=None), @@ -83,8 +78,7 @@ def generate_signed_certificate( .not_valid_before(current_time) .not_valid_after( # Our certificate will be valid for 10 days - current_time - + datetime.timedelta(days=10) + current_time + datetime.timedelta(days=10) ) .add_extension( x509.BasicConstraints(ca=False, path_length=None), @@ -109,7 +103,6 @@ def write_to_disk(directory, name, key, cert): if __name__ == "__main__": - ##################################################################################### # GENERATE TRUSTED CERTS AND KEYS ##################################################################################### diff --git a/tests/tls_test/run_tls_test_client.py b/tests/tls_test/run_tls_test_client.py index 3a8f0cda..e5e9a741 100644 --- a/tests/tls_test/run_tls_test_client.py +++ b/tests/tls_test/run_tls_test_client.py @@ -1,6 +1,5 @@ import socket import ssl -import time def print_and_flush(message): @@ -76,7 +75,6 @@ def handle_connection(self, ssock): if __name__ == "__main__": - tls_client = TLSServer( ca_cert_path="/tmp/trusted_ca_cert.pem", server_cert_path="/tmp/trusted_server_cert.pem", diff --git a/tests/tls_test/run_tls_test_server.py b/tests/tls_test/run_tls_test_server.py index 7ad420f4..85cb386c 100644 --- a/tests/tls_test/run_tls_test_server.py +++ b/tests/tls_test/run_tls_test_server.py @@ -36,7 +36,7 @@ def create_connection(self): print_and_flush("Connection established.") self.handle_connection(ssock) return - except (ConnectionRefusedError, socket.timeout) as e: + except (ConnectionRefusedError, socket.timeout): time.sleep( 0.1 ) # wait a bit before retrying to avoid flooding with attempts @@ -84,7 +84,6 @@ def handle_connection(self, ssock): if __name__ == "__main__": - tls_client = TLSClient( ca_cert_path="/tmp/trusted_ca_cert.pem", client_cert_path="/tmp/trusted_client_cert.pem", diff --git a/tests/udf_test/functions/caption.py b/tests/udf_test/functions/caption.py index 50486e4a..592524bc 100644 --- a/tests/udf_test/functions/caption.py +++ b/tests/udf_test/functions/caption.py @@ -1,9 +1,5 @@ import cv2 -import numpy as np -from datetime import datetime -from collections import deque import skvideo.io -import imutils import time @@ -17,7 +13,7 @@ def run(settings, message, input_params): vs = cv2.VideoCapture(ipfilename) video = skvideo.io.FFmpegWriter(opfilename, {"-pix_fmt": "bgr24"}) - i = 0 + while True: (grabbed, frame) = vs.read() if not grabbed: diff --git a/tests/udf_test/functions/metadata.py b/tests/udf_test/functions/metadata.py index 4f9a3eb8..5eacc023 100644 --- a/tests/udf_test/functions/metadata.py +++ b/tests/udf_test/functions/metadata.py @@ -1,10 +1,4 @@ import cv2 -import numpy as np -from datetime import datetime -from collections import deque -import skvideo.io -import imutils -import time import json face_cascade = cv2.CascadeClassifier( @@ -23,10 +17,8 @@ def facedetectbbox(frame): def run(settings, message, input_params): ipfilename = message - format = message.strip().split(".")[-1] if input_params["media_type"] == "video": - vs = cv2.VideoCapture(ipfilename) frameNum = 1 metadata = {} diff --git a/tests/unit_tests/config-aws-tests.json b/tests/unit_tests/config-aws-tests.json index ffc3c49e..4805ee46 100644 --- a/tests/unit_tests/config-aws-tests.json +++ b/tests/unit_tests/config-aws-tests.json @@ -2,7 +2,7 @@ // This is the run-time config file // Sets database paths and other parameters { - + "port": 55557, "db_root_path": "test_db_1", "storage_type": "aws", //local, aws, etc diff --git a/tests/unit_tests/config-for-vdms-config-v1-tests.json b/tests/unit_tests/config-for-vdms-config-v1-tests.json index bb4d8404..fa30f45d 100644 --- a/tests/unit_tests/config-for-vdms-config-v1-tests.json +++ b/tests/unit_tests/config-for-vdms-config-v1-tests.json @@ -2,7 +2,7 @@ // This is the run-time config file // Sets database paths and other parameters { - + "port": 55557, "db_root_path": "test_db_1", "use_endpoint": true, diff --git a/tests/unit_tests/config-for-vdms-config-v2-tests.json b/tests/unit_tests/config-for-vdms-config-v2-tests.json index 7495ee82..fac38c56 100644 --- a/tests/unit_tests/config-for-vdms-config-v2-tests.json +++ b/tests/unit_tests/config-for-vdms-config-v2-tests.json @@ -2,7 +2,7 @@ // This is the run-time config file // Sets database paths and other parameters { - + "port": 55557, "db_root_path": "test_db_1", "use_endpoint": true, diff --git a/tests/unit_tests/config-for-vdms-config-v3-tests.json b/tests/unit_tests/config-for-vdms-config-v3-tests.json index 686ec7ea..3c817f37 100644 --- a/tests/unit_tests/config-for-vdms-config-v3-tests.json +++ b/tests/unit_tests/config-for-vdms-config-v3-tests.json @@ -2,7 +2,7 @@ // This is the run-time config file // Sets database paths and other parameters { - + "port": 55557, "db_root_path": "test_db_1", "storage_type": "local", diff --git a/tests/unit_tests/config-tests.json b/tests/unit_tests/config-tests.json index a5540d28..3386e927 100644 --- a/tests/unit_tests/config-tests.json +++ b/tests/unit_tests/config-tests.json @@ -2,7 +2,7 @@ // This is the run-time config file // Sets database paths and other parameters { - + "port": 55557, "db_root_path": "test_db_1", diff --git a/tests/unit_tests/connection.json b/tests/unit_tests/connection.json index 95cea2fa..245ed095 100644 --- a/tests/unit_tests/connection.json +++ b/tests/unit_tests/connection.json @@ -1,5 +1,5 @@ [ { - "FindEntity" : + "FindEntity" : { "class" :"Person", "_ref" : 1, @@ -18,7 +18,7 @@ } }, -{ +{ "AddConnection":{ "class" : "Has", "ref1" : 1, diff --git a/tests/unit_tests/helpers.h b/tests/unit_tests/helpers.h index 5dbbcc50..fb273740 100644 --- a/tests/unit_tests/helpers.h +++ b/tests/unit_tests/helpers.h @@ -34,7 +34,6 @@ #include #include #include - #include #include #include diff --git a/tests/unit_tests/meta_data_helper.h b/tests/unit_tests/meta_data_helper.h index a3e6b825..bdaa9402 100644 --- a/tests/unit_tests/meta_data_helper.h +++ b/tests/unit_tests/meta_data_helper.h @@ -1,24 +1,25 @@ #pragma once +#include +#include +#include + #include #include #include #include #include #include -#include -#include #include #include -#include #include #include "VDMSClient.h" +#include "gtest/gtest.h" #include "helpers.h" #include "vcl/VCL.h" -#include "gtest/gtest.h" class Meta_Data { -public: + public: std::shared_ptr _aclient; std::string _server_name = "localhost"; int _port = 55558; diff --git a/tests/unit_tests/queries.json b/tests/unit_tests/queries.json index e161674b..30744133 100644 --- a/tests/unit_tests/queries.json +++ b/tests/unit_tests/queries.json @@ -19,7 +19,7 @@ "City" : "Jodhpur", "Gender" : "F", "age" : 20.0, - + "email" : "mymail.1@gmail.com", "has_dog" : true, "height" : 34.234100341796875, diff --git a/tests/unit_tests/two_entities.json b/tests/unit_tests/two_entities.json index a28aff14..d155cfff 100644 --- a/tests/unit_tests/two_entities.json +++ b/tests/unit_tests/two_entities.json @@ -4,16 +4,16 @@ { "class" : "Person", "_ref":1, - + "properties" : { "Address" : "90 royal lane kingsland", "City" : "Jodhpur", "Gender" : "F", "age" : 20, - "email" : "mymail.1@gmail.com", + "email" : "mymail.1@gmail.com", "name": "Ali1" - + } } }, @@ -22,7 +22,7 @@ { "class" : "Movie", "_ref":2, - + "properties" :{ "name" :"Ali Baba1", "year":2021 diff --git a/user_defined_operations/functions/caption.py b/user_defined_operations/functions/caption.py index 50486e4a..592524bc 100644 --- a/user_defined_operations/functions/caption.py +++ b/user_defined_operations/functions/caption.py @@ -1,9 +1,5 @@ import cv2 -import numpy as np -from datetime import datetime -from collections import deque import skvideo.io -import imutils import time @@ -17,7 +13,7 @@ def run(settings, message, input_params): vs = cv2.VideoCapture(ipfilename) video = skvideo.io.FFmpegWriter(opfilename, {"-pix_fmt": "bgr24"}) - i = 0 + while True: (grabbed, frame) = vs.read() if not grabbed: diff --git a/user_defined_operations/functions/metadata.py b/user_defined_operations/functions/metadata.py index 0b69761a..b0b7d3ef 100644 --- a/user_defined_operations/functions/metadata.py +++ b/user_defined_operations/functions/metadata.py @@ -1,10 +1,4 @@ import cv2 -import numpy as np -from datetime import datetime -from collections import deque -import skvideo.io -import imutils -import time import json face_cascade = cv2.CascadeClassifier( @@ -23,11 +17,9 @@ def facedetectbbox(frame): def run(settings, message, input_params): ipfilename = message - format = message.strip().split(".")[-1] # Extract metadata for video files if input_params["media_type"] == "video": - vs = cv2.VideoCapture(ipfilename) frameNum = 1 metadata = {} diff --git a/utils/include/comm/Connection.h b/utils/include/comm/Connection.h index d9799204..544fb525 100644 --- a/utils/include/comm/Connection.h +++ b/utils/include/comm/Connection.h @@ -29,16 +29,17 @@ #pragma once -#include "ExceptionComm.h" #include #include + #include +#include "ExceptionComm.h" + namespace comm { class Connection { - -public: + public: Connection(); Connection(int socket_fd, SSL *_ssl); ~Connection(); @@ -56,7 +57,7 @@ class Connection { void set_buffer_size_limit(uint32_t buffer_size_limit); -protected: + protected: const unsigned MAX_PORT_NUMBER = 65535; const unsigned MAX_RETRIES = 100; @@ -73,8 +74,7 @@ class Connection { // Implements a TCP/IP server class ConnServer { - -public: + public: ConnServer(int port, const std::string &cert_file, const std::string &key_file, const std::string &ca_file); ~ConnServer(); @@ -82,11 +82,11 @@ class ConnServer { ConnServer(const ConnServer &) = delete; Connection accept(); -private: + private: const unsigned MAX_CONN_QUEUE = 2048; const unsigned MAX_PORT_NUMBER = 65535; - int _port; // Server port + int _port; // Server port std::string _cert_file; std::string _key_file; std::string _ca_file; @@ -96,8 +96,7 @@ class ConnServer { // Implements a TCP/IP client class ConnClient : public Connection { - -public: + public: struct ServerAddress { std::string addr; int port; @@ -111,7 +110,7 @@ class ConnClient : public Connection { ConnClient(const ConnClient &) = delete; ~ConnClient() {} -private: + private: ConnClient(); void connect(); void setupTLS(); @@ -126,4 +125,4 @@ class ConnClient : public Connection { SSL_CTX *_ssl_ctx; }; -}; // namespace comm +}; // namespace comm diff --git a/utils/include/comm/ExceptionComm.h b/utils/include/comm/ExceptionComm.h index ae811250..315a8487 100644 --- a/utils/include/comm/ExceptionComm.h +++ b/utils/include/comm/ExceptionComm.h @@ -36,9 +36,9 @@ namespace comm { enum ExceptionCommType { FATAL_Internal_Error, - WriteFail, // For write/send failure - ReadFail, // For read/recv failure - BindFail, // Fail to bind a port + WriteFail, // For write/send failure + ReadFail, // For read/recv failure + BindFail, // Fail to bind a port SocketFail, ListentFail, @@ -58,21 +58,21 @@ enum ExceptionCommType { SSL_CA_FAIL, SSL_CONNECT_FAIL, - Undefined = 100, // Any undefined error + Undefined = 100, // Any undefined error }; struct ExceptionComm { // Which exception - int num; // Exception number - const char *name; // Exception name + int num; // Exception number + const char *name; // Exception name // Additional information std::string msg; int errno_val; // Where it was thrown - const char *file; // Source file name - int line; // Source line number + const char *file; // Source file name + int line; // Source line number ExceptionComm(int exc, const char *exc_name, const char *f, int l) : num(exc), name(exc_name), msg(), errno_val(0), file(f), line(l) {} @@ -86,8 +86,8 @@ struct ExceptionComm { : num(exc), name(exc_name), msg(m), errno_val(err), file(f), line(l) {} }; -#define ExceptionComm(name, ...) \ +#define ExceptionComm(name, ...) \ ExceptionComm(comm::name, #name, ##__VA_ARGS__, __FILE__, __LINE__) -}; // namespace comm +}; // namespace comm extern void print_exception(const comm::ExceptionComm &e, FILE *f = stdout); diff --git a/utils/include/stats/SystemStats.h b/utils/include/stats/SystemStats.h index 17082fbc..083d1403 100644 --- a/utils/include/stats/SystemStats.h +++ b/utils/include/stats/SystemStats.h @@ -29,13 +29,14 @@ #pragma once -#include #include #include #include -#include #include +#include +#include + #include "sys/sysinfo.h" /******************************* S T R U C T S ********************************/ @@ -143,7 +144,7 @@ struct ClocksInfo { // *************************************************************************** class SystemStats { -public: + public: SystemStats(); virtual ~SystemStats(); @@ -164,7 +165,7 @@ class SystemStats { virtual void log_stats(std::string pName); virtual bool query_sufficient_memory(int size_requested); -protected: + protected: virtual FILE *get_stats_fd(); virtual FILE *get_cpu_info_fd(); virtual std::string get_filename_prefix(); @@ -181,9 +182,8 @@ class SystemStats { virtual bool is_overflow_in_time_detected(clock_t now, struct tms time_sample); - virtual bool - is_overflow_in_totals_detected(const TotalsInfo ¤t_total_info, - const TotalsInfo &last_totals_info); + virtual bool is_overflow_in_totals_detected( + const TotalsInfo ¤t_total_info, const TotalsInfo &last_totals_info); virtual void set_memory_stats(const MemoryStats &memory_stats); virtual void set_cpu_stats(const CPUStats &cpu_stats); @@ -201,6 +201,6 @@ class SystemStats { ClocksInfo m_last_clocks_info; int m_numProcessors; -private: + private: virtual int parse_line(char *line); }; diff --git a/utils/include/timers/TimerMap.h b/utils/include/timers/TimerMap.h index 98d43a14..7819d356 100644 --- a/utils/include/timers/TimerMap.h +++ b/utils/include/timers/TimerMap.h @@ -36,13 +36,12 @@ #include class TimerMap { - -private: + private: std::map *> timemap; std::mutex maplock; -public: + public: ~TimerMap(); /** @@ -108,4 +107,4 @@ class TimerMap { void clear_all_timers(); }; -#endif // CLEAN_VDMS_TIMERMAP_H +#endif // CLEAN_VDMS_TIMERMAP_H diff --git a/utils/src/api_schema/api_schema.json b/utils/src/api_schema/api_schema.json index 31fa384f..9157ead0 100644 --- a/utils/src/api_schema/api_schema.json +++ b/utils/src/api_schema/api_schema.json @@ -343,7 +343,7 @@ "required": ["type", "url"], "additionalProperties": false }, - + "operationRemote" : { "type": "object", "properties": { @@ -536,7 +536,7 @@ "additionalProperties": false }, - + "UpdateVideoTop": { @@ -736,10 +736,10 @@ "storeIndex" : { "type": "boolean" }, "constraints": { "type": "object" }, "link": { "$ref": "#/definitions/blockLink" } - + }, "required": ["set"], - + "additionalProperties": false }, @@ -828,7 +828,7 @@ "from_file_path": { "type": "string"}, "link": { "$ref": "#/definitions/blockLink" }, "properties": { "type": "object" } - + }, "additionalProperties": false }, @@ -872,7 +872,7 @@ }, "additionalProperties": false }, - + "UpdateVideo": { "properties": { "_ref": { "$ref": "#/definitions/refInt" }, diff --git a/utils/src/api_schema/createApiString.py b/utils/src/api_schema/createApiString.py index 409f129f..e123f244 100644 --- a/utils/src/api_schema/createApiString.py +++ b/utils/src/api_schema/createApiString.py @@ -25,7 +25,6 @@ # import sys -import os with open(sys.argv[1], "r") as schema_file: file = schema_file.readlines()