diff --git a/.annotation_safe_list.yml b/.annotation_safe_list.yml index c57eeeb2503d..e91fe39cd613 100644 --- a/.annotation_safe_list.yml +++ b/.annotation_safe_list.yml @@ -142,8 +142,6 @@ workflow.AssessmentWorkflowStep: # Via edx-celeryutils celery_utils.ChordData: ".. no_pii:": "No PII" -celery_utils.FailedTask: - ".. no_pii:": "No PII" # Via completion XBlock completion.BlockCompletion: diff --git a/.github/workflows/js-tests-paver.yml b/.github/workflows/js-tests-paver.yml new file mode 100644 index 000000000000..566063fdfd22 --- /dev/null +++ b/.github/workflows/js-tests-paver.yml @@ -0,0 +1,84 @@ +name: Javascript tests PAVER + +on: + pull_request: + push: + branches: + - master + +jobs: + run_tests: + name: JS + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + node-version: [18, 20] + python-version: + - "3.11" + + steps: + - uses: actions/checkout@v4 + - name: Fetch master to compare coverage + run: git fetch --depth=1 origin master + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + + - name: Setup npm + run: npm i -g npm@10.5.x + + - name: Install Firefox 123.0 + run: | + sudo apt-get purge firefox + wget "https://ftp.mozilla.org/pub/firefox/releases/123.0/linux-x86_64/en-US/firefox-123.0.tar.bz2" + tar -xjf firefox-123.0.tar.bz2 + sudo mv firefox /opt/firefox + sudo ln -s /opt/firefox/firefox /usr/bin/firefox + + - name: Install Required System Packages + run: sudo apt-get update && sudo apt-get install libxmlsec1-dev ubuntu-restricted-extras xvfb + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Get pip cache dir + id: pip-cache-dir + run: | + echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT + + - name: Cache pip dependencies + id: cache-dependencies + uses: actions/cache@v4 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/base.txt') }} + restore-keys: ${{ runner.os }}-pip- + + - name: Install Required Python Dependencies + run: | + make base-requirements + + - uses: c-hive/gha-npm-cache@v1 + - name: Run JS Tests + env: + TEST_SUITE: js-unit + SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh + run: | + npm install -g jest + xvfb-run --auto-servernum ./scripts/all-tests.sh + + - name: Save Job Artifacts + uses: actions/upload-artifact@v4 + with: + name: Build-Artifacts + path: | + reports/**/* + test_root/log/*.png + test_root/log/*.log + **/TEST-*.xml + overwrite: true diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index 4496a4b61c41..463352e1c552 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -64,13 +64,13 @@ jobs: make base-requirements - uses: c-hive/gha-npm-cache@v1 + + - name: Install npm + run: npm ci + - name: Run JS Tests - env: - TEST_SUITE: js-unit - SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh run: | - npm install -g jest - xvfb-run --auto-servernum ./scripts/all-tests.sh + npm run test - name: Save Job Artifacts uses: actions/upload-artifact@v4 diff --git a/.github/workflows/quality-checks-paver.yml b/.github/workflows/quality-checks-paver.yml new file mode 100644 index 000000000000..beb9fea8007f --- /dev/null +++ b/.github/workflows/quality-checks-paver.yml @@ -0,0 +1,82 @@ +name: Quality checks PAVER + +on: + pull_request: + push: + branches: + - master + - open-release/lilac.master + +jobs: + run_tests: + name: Quality Others + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-22.04] + python-version: + - "3.11" + node-version: [20] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Fetch base branch for comparison + run: git fetch --depth=1 origin ${{ github.base_ref }} + + - name: Install Required System Packages + run: sudo apt-get update && sudo apt-get install libxmlsec1-dev + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + + - name: Setup npm + run: npm i -g npm@8.5.x + + - name: Get pip cache dir + id: pip-cache-dir + run: | + echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT + + - name: Cache pip dependencies + id: cache-dependencies + uses: actions/cache@v4 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }} + restore-keys: ${{ runner.os }}-pip- + + - name: Install Required Python Dependencies + env: + PIP_SRC: ${{ runner.temp }} + run: | + make test-requirements + + - name: Run Quality Tests + env: + TEST_SUITE: quality + SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh + PIP_SRC: ${{ runner.temp }} + TARGET_BRANCH: ${{ github.base_ref }} + run: | + ./scripts/all-tests.sh + + - name: Save Job Artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: Build-Artifacts + path: | + **/reports/**/* + test_root/log/**/*.log + *.log + overwrite: true diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index 310f9f83bf3d..2452f54da14b 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -60,16 +60,29 @@ jobs: PIP_SRC: ${{ runner.temp }} run: | make test-requirements - + + - name: Install npm + env: + PIP_SRC: ${{ runner.temp }} + run: npm ci + + - name: Install python packages + env: + PIP_SRC: ${{ runner.temp }} + run: | + pip install -e . + - name: Run Quality Tests env: - TEST_SUITE: quality - SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh PIP_SRC: ${{ runner.temp }} TARGET_BRANCH: ${{ github.base_ref }} run: | - ./scripts/all-tests.sh - + make pycodestyle + npm run lint + make xsslint + make pii_check + make check_keywords + - name: Save Job Artifacts if: always() uses: actions/upload-artifact@v4 diff --git a/.pii_annotations.yml b/.pii_annotations.yml index 328520738f10..b9e2fcb1b063 100644 --- a/.pii_annotations.yml +++ b/.pii_annotations.yml @@ -1,7 +1,7 @@ source_path: ./ report_path: pii_report safelist_path: .annotation_safe_list.yml -coverage_target: 94.5 +coverage_target: 83.3 # See OEP-30 for more information on these values and what they mean: # https://open-edx-proposals.readthedocs.io/en/latest/oep-0030-arch-pii-markup-and-auditing.html#docstring-annotations annotations: diff --git a/.stylelintignore b/.stylelintignore deleted file mode 100644 index cd53bacf3cf9..000000000000 --- a/.stylelintignore +++ /dev/null @@ -1,5 +0,0 @@ -xmodule/css -common/static/sass/bourbon -common/static/xmodule/modules/css -common/test/test-theme -lms/static/sass/vendor diff --git a/Makefile b/Makefile index 15bab5df67a9..62681f6f3711 100644 --- a/Makefile +++ b/Makefile @@ -204,3 +204,37 @@ migrate: migrate-lms migrate-cms # Part of https://github.com/openedx/wg-developer-experience/issues/136 ubuntu-requirements: ## Install ubuntu 22.04 system packages needed for `pip install` to work on ubuntu. sudo apt install libmysqlclient-dev libxmlsec1-dev + +xsslint: ## check xss for quality issuest + python scripts/xsslint/xss_linter.py \ + --rule-totals \ + --config=scripts.xsslint_config \ + --thresholds=scripts/xsslint_thresholds.json + +pycodestyle: ## check python files for quality issues + pycodestyle . + +## Re-enable --lint flag when this issue https://github.com/openedx/edx-platform/issues/35775 is resolved +pii_check: ## check django models for pii annotations + DJANGO_SETTINGS_MODULE=cms.envs.test \ + code_annotations django_find_annotations \ + --config_file .pii_annotations.yml \ + --app_name cms \ + --coverage \ + --lint + + DJANGO_SETTINGS_MODULE=lms.envs.test \ + code_annotations django_find_annotations \ + --config_file .pii_annotations.yml \ + --app_name lms \ + --coverage \ + --lint + +check_keywords: ## check django models for reserve keywords + DJANGO_SETTINGS_MODULE=cms.envs.test \ + python manage.py cms check_reserved_keywords \ + --override_file db_keyword_overrides.yml + + DJANGO_SETTINGS_MODULE=lms.envs.test \ + python manage.py lms check_reserved_keywords \ + --override_file db_keyword_overrides.yml diff --git a/package.json b/package.json index 92f7de9124dc..2f09f8a7df90 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,26 @@ "compile-sass-dev": "scripts/compile_sass.py --env=development", "watch": "{ npm run watch-webpack& npm run watch-sass& } && sleep infinity", "watch-webpack": "npm run webpack-dev -- --watch", - "watch-sass": "scripts/watch_sass.sh" + "watch-sass": "scripts/watch_sass.sh", + "lint": "python scripts/eslint.py", + "test": "npm run test-cms && npm run test-lms && npm run test-xmodule && npm run test-common && npm run test-jest", + "test-kind-vanilla": "npm run test-cms-vanilla && npm run test-xmodule-vanilla && npm run test-common-vanilla", + "test-kind-require": "npm run test-cms-require && npm run test-common-require", + "test-kind-webpack": "npm run test-cms-webpack && npm run test-lms-webpack && npm run test-xmodule-webpack", + "test-cms": "npm run test-cms-vanilla && npm run test-cms-require", + "test-cms-vanilla": "npm run test-suite -- cms/static/karma_cms.conf.js", + "test-cms-require": "npm run test-suite -- cms/static/karma_cms_squire.conf.js", + "test-cms-webpack": "npm run test-suite -- cms/static/karma_cms_webpack.conf.js", + "test-lms": "echo 'WARNING: Webpack JS tests are disabled. No LMS JS tests will be run. See https://github.com/openedx/edx-platform/issues/35956 for details.'", + "test-lms-webpack": "npm run test-suite -- lms/static/karma_lms.conf.js", + "test-xmodule": "npm run test-xmodule-vanilla", + "test-xmodule-vanilla": "npm run test-suite -- xmodule/js/karma_xmodule.conf.js", + "test-xmodule-webpack": "npm run test-suite -- xmodule/js/karma_xmodule_webpack.conf.js", + "test-common": "npm run test-common-vanilla && npm run test-common-require", + "test-common-vanilla": "npm run test-suite -- common/static/karma_common.conf.js", + "test-common-require": "npm run test-suite -- common/static/karma_common_requirejs.conf.js", + "test-suite": "${NODE_WRAPPER:-xvfb-run --auto-servernum} node --max_old_space_size=4096 node_modules/.bin/karma start --single-run=true --capture-timeout=60000 --browsers=FirefoxNoUpdates", + "test-jest": "jest" }, "dependencies": { "@babel/core": "7.26.0", diff --git a/pavelib/paver_tests/test_timer.py b/pavelib/paver_tests/test_timer.py index bc9817668347..5ccbf74abcf9 100644 --- a/pavelib/paver_tests/test_timer.py +++ b/pavelib/paver_tests/test_timer.py @@ -77,9 +77,17 @@ def test_times(self): messages = self.get_log_messages() assert len(messages) == 1 - assert 'duration' in messages[0] and messages[0]['duration'] == 35.6 - assert 'started_at' in messages[0] and messages[0]['started_at'] == start.isoformat(' ') - assert 'ended_at' in messages[0] and messages[0]['ended_at'] == end.isoformat(' ') + # I'm not using assertDictContainsSubset because it is + # removed in python 3.2 (because the arguments were backwards) + # and it wasn't ever replaced by anything *headdesk* + assert 'duration' in messages[0] + assert 35.6 == messages[0]['duration'] + + assert 'started_at' in messages[0] + assert start.isoformat(' ') == messages[0]['started_at'] + + assert 'ended_at' in messages[0] + assert end.isoformat(' ') == messages[0]['ended_at'] @patch.object(timer, 'PAVER_TIMER_LOG', None) def test_no_logs(self): @@ -91,18 +99,28 @@ def test_arguments(self): messages = self.get_log_messages(args=(1, 'foo'), kwargs=dict(bar='baz')) assert len(messages) == 1 - assert 'args' in messages[0] and messages[0]['args'] == [repr(1), repr('foo')] - assert 'kwargs' in messages[0] and messages[0]['kwargs'] == {'bar': repr('baz')} + # I'm not using assertDictContainsSubset because it is + # removed in python 3.2 (because the arguments were backwards) + # and it wasn't ever replaced by anything *headdesk* + assert 'args' in messages[0] + assert [repr(1), repr('foo')] == messages[0]['args'] + assert 'kwargs' in messages[0] + assert {'bar': repr('baz')} == messages[0]['kwargs'] @patch.object(timer, 'PAVER_TIMER_LOG', '/tmp/some-log') def test_task_name(self): messages = self.get_log_messages() assert len(messages) == 1 - assert 'task' in messages[0] and messages[0]['task'] == 'pavelib.paver_tests.test_timer.identity' + # I'm not using assertDictContainsSubset because it is + # removed in python 3.2 (because the arguments were backwards) + # and it wasn't ever replaced by anything *headdesk* + assert 'task' in messages[0] + assert 'pavelib.paver_tests.test_timer.identity' == messages[0]['task'] @patch.object(timer, 'PAVER_TIMER_LOG', '/tmp/some-log') def test_exceptions(self): + @timer.timed def raises(): """ @@ -113,7 +131,11 @@ def raises(): messages = self.get_log_messages(task=raises, raises=Exception) assert len(messages) == 1 - assert 'exception' in messages[0] and messages[0]['exception'] == 'Exception: The Message!' + # I'm not using assertDictContainsSubset because it is + # removed in python 3.2 (because the arguments were backwards) + # and it wasn't ever replaced by anything *headdesk* + assert 'exception' in messages[0] + assert 'Exception: The Message!' == messages[0]['exception'] @patch.object(timer, 'PAVER_TIMER_LOG', '/tmp/some-log-%Y-%m-%d-%H-%M-%S.log') def test_date_formatting(self): diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 8f8d87059aa5..91c294b95079 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -504,7 +504,7 @@ edx-opaque-keys[django]==2.11.0 # ora2 edx-organizations==6.13.0 # via -r requirements/edx/kernel.in -edx-proctoring==5.0.1 +edx-proctoring==4.18.4 # via # -r requirements/edx/kernel.in # edx-proctoring-proctortrack diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index f8b512ba5874..387ee3305eb9 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -795,7 +795,7 @@ edx-organizations==6.13.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -edx-proctoring==5.0.1 +edx-proctoring==4.18.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index c46d1fbd5715..79c0ddcda54a 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -587,7 +587,7 @@ edx-opaque-keys[django]==2.11.0 # ora2 edx-organizations==6.13.0 # via -r requirements/edx/base.txt -edx-proctoring==5.0.1 +edx-proctoring==4.18.4 # via # -r requirements/edx/base.txt # edx-proctoring-proctortrack diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index e978fde4b1db..0df7a7d68a38 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -610,7 +610,7 @@ edx-opaque-keys[django]==2.11.0 # ora2 edx-organizations==6.13.0 # via -r requirements/edx/base.txt -edx-proctoring==5.0.1 +edx-proctoring==4.18.4 # via # -r requirements/edx/base.txt # edx-proctoring-proctortrack diff --git a/scripts/eslint.py b/scripts/eslint.py new file mode 100644 index 000000000000..4ee42d6daff0 --- /dev/null +++ b/scripts/eslint.py @@ -0,0 +1,73 @@ +""" # pylint: disable=django-not-configured +Check code quality using pycodestyle, pylint, and diff_quality. +""" + +import re +import subprocess +import shlex +import sys + + +class BuildFailure(Exception): + pass + + +def fail_quality(message): + """ + Fail the specified quality check. + """ + + raise BuildFailure(message) + + +def run_eslint(): + """ + Runs eslint on static asset directories. + If limit option is passed, fails build if more violations than the limit are found. + """ + violations_limit = 1285 + + command = [ + "node", + "--max_old_space_size=4096", + "node_modules/.bin/eslint", + "--ext", ".js", + "--ext", ".jsx", + "--format=compact", + "lms", + "cms", + "common", + "openedx", + "xmodule", + ] + print("Running command:", shlex.join(command)) + result = subprocess.run( + command, + text=True, + check=False, + capture_output=True + ) + + print(result.stdout) + last_line = result.stdout.strip().splitlines()[-1] if result.stdout.strip().splitlines() else "" + regex = r'^\d+' + try: + num_violations = int(re.search(regex, last_line).group(0)) if last_line else 0 + # Fail if number of violations is greater than the limit + if num_violations > violations_limit: + fail_quality( + "FAILURE: Too many eslint violations ({count}).\nThe limit is {violations_limit}.".format(count=num_violations, violations_limit=violations_limit)) + else: + print(f"successfully run eslint with '{num_violations}' violations") + + # An AttributeError will occur if the regex finds no matches. + except (AttributeError, ValueError): + fail_quality(f"FAILURE: Number of eslint violations could not be found in '{last_line}'") + + +if __name__ == "__main__": + try: + run_eslint() + except BuildFailure as e: + print(e) + sys.exit(1) diff --git a/scripts/xsslint/xss_linter.py b/scripts/xsslint/xss_linter.py index a35038c3de6d..b32c54aa5cd7 100755 --- a/scripts/xsslint/xss_linter.py +++ b/scripts/xsslint/xss_linter.py @@ -4,6 +4,316 @@ """ +import argparse +import importlib +import json +import os +import re +import sys + +from functools import reduce +from io import StringIO +from xsslint.reporting import SummaryResults +from xsslint.rules import RuleSet +from xsslint.utils import is_skip_dir + + +class BuildFailure(Exception): + pass + + +def fail_quality(message): + """ + Fail the specified quality check. + """ + + raise BuildFailure(message) + + +def _load_config_module(module_path): + cwd = os.getcwd() + if cwd not in sys.path: + # Enable config module to be imported relative to wherever the script was run from. + sys.path.append(cwd) + return importlib.import_module(module_path) + + +def _build_ruleset(template_linters): + """ + Combines the RuleSets from the provided template_linters into a single, aggregate RuleSet. + + Arguments: + template_linters: A list of linting objects. + + Returns: + The combined RuleSet. + """ + return reduce( + lambda combined, current: combined + current.ruleset, + template_linters, + RuleSet() + ) + + +def _process_file(full_path, template_linters, options, summary_results, out): + """ + For each linter, lints the provided file. This means finding and printing + violations. + + Arguments: + full_path: The full path of the file to lint. + template_linters: A list of linting objects. + options: A list of the options. + summary_results: A SummaryResults with a summary of the violations. + out: output file + + """ + num_violations = 0 + directory = os.path.dirname(full_path) + file_name = os.path.basename(full_path) + try: + for template_linter in template_linters: + results = template_linter.process_file(directory, file_name) + results.print_results(options, summary_results, out) + except BaseException as e: + raise Exception(f"Failed to process path: {full_path}") from e + + +def _process_os_dir(directory, files, template_linters, options, summary_results, out): + """ + Calls out to lint each file in the passed list of files. + + Arguments: + directory: Directory being linted. + files: All files in the directory to be linted. + template_linters: A list of linting objects. + options: A list of the options. + summary_results: A SummaryResults with a summary of the violations. + out: output file + + """ + for current_file in sorted(files, key=lambda s: s.lower()): + full_path = os.path.join(directory, current_file) + _process_file(full_path, template_linters, options, summary_results, out) + + +def _process_os_dirs(starting_dir, template_linters, options, summary_results, out): + """ + For each linter, lints all the directories in the starting directory. + + Arguments: + starting_dir: The initial directory to begin the walk. + template_linters: A list of linting objects. + options: A list of the options. + summary_results: A SummaryResults with a summary of the violations. + out: output file + + """ + skip_dirs = options.get('skip_dirs', ()) + for root, dirs, files in os.walk(starting_dir): + if is_skip_dir(skip_dirs, root): + del dirs + continue + dirs.sort(key=lambda s: s.lower()) + _process_os_dir(root, files, template_linters, options, summary_results, out) + + +def _get_xsslint_counts(result_contents): + """ + This returns a dict of violations from the xsslint report. + + Arguments: + filename: The name of the xsslint report. + + Returns: + A dict containing the following: + rules: A dict containing the count for each rule as follows: + violation-rule-id: N, where N is the number of violations + total: M, where M is the number of total violations + + """ + + rule_count_regex = re.compile(r"^(?P[a-z-]+):\s+(?P\d+) violations", re.MULTILINE) + total_count_regex = re.compile(r"^(?P\d+) violations total", re.MULTILINE) + violations = {'rules': {}} + for violation_match in rule_count_regex.finditer(result_contents): + try: + violations['rules'][violation_match.group('rule_id')] = int(violation_match.group('count')) + except ValueError: + violations['rules'][violation_match.group('rule_id')] = None + try: + violations['total'] = int(total_count_regex.search(result_contents).group('count')) + # An AttributeError will occur if the regex finds no matches. + # A ValueError will occur if the returned regex cannot be cast as a float. + except (AttributeError, ValueError): + violations['total'] = None + return violations + + +def _check_violations(options, results): + xsslint_script = "xss_linter.py" + try: + thresholds_option = options['thresholds'] + # Read the JSON file + with open(thresholds_option, 'r') as file: + violation_thresholds = json.load(file) + + except ValueError: + violation_thresholds = None + if isinstance(violation_thresholds, dict) is False or \ + any(key not in ("total", "rules") for key in violation_thresholds.keys()): + print('xsslint') + fail_quality("""FAILURE: Thresholds option "{thresholds_option}" was not supplied using proper format.\n""" + """Here is a properly formatted example, '{{"total":100,"rules":{{"javascript-escape":0}}}}' """ + """with property names in double-quotes.""".format(thresholds_option=thresholds_option)) + + try: + metrics_str = "Number of {xsslint_script} violations: {num_violations}\n".format( + xsslint_script=xsslint_script, num_violations=int(results['total']) + ) + if 'rules' in results and any(results['rules']): + metrics_str += "\n" + rule_keys = sorted(results['rules'].keys()) + for rule in rule_keys: + metrics_str += "{rule} violations: {count}\n".format( + rule=rule, + count=int(results['rules'][rule]) + ) + except TypeError: + print('xsslint') + fail_quality("FAILURE: Number of {xsslint_script} violations could not be found".format( + xsslint_script=xsslint_script + )) + + error_message = "" + # Test total violations against threshold. + if 'total' in list(violation_thresholds.keys()): + if violation_thresholds['total'] < results['total']: + error_message = "Too many violations total ({count}).\nThe limit is {violations_limit}.".format( + count=results['total'], violations_limit=violation_thresholds['total'] + ) + + # Test rule violations against thresholds. + if 'rules' in violation_thresholds: + threshold_keys = sorted(violation_thresholds['rules'].keys()) + for threshold_key in threshold_keys: + if threshold_key not in results['rules']: + error_message += ( + "\nNumber of {xsslint_script} violations for {rule} could not be found" + ).format( + xsslint_script=xsslint_script, rule=threshold_key + ) + elif violation_thresholds['rules'][threshold_key] < results['rules'][threshold_key]: + error_message += \ + "\nToo many {rule} violations ({count}).\nThe {rule} limit is {violations_limit}.".format( + rule=threshold_key, count=results['rules'][threshold_key], + violations_limit=violation_thresholds['rules'][threshold_key], + ) + + if error_message: + print('xsslint') + fail_quality("FAILURE: XSSLinter Failed.\n{error_message}\n" + "run the following command to hone in on the problem:\n" + "./scripts/xss-commit-linter.sh -h".format(error_message=error_message)) + else: + print("successfully run xsslint") + + +def _lint(file_or_dir, template_linters, options, summary_results, out): + """ + For each linter, lints the provided file or directory. + + Arguments: + file_or_dir: The file or initial directory to lint. + template_linters: A list of linting objects. + options: A list of the options. + summary_results: A SummaryResults with a summary of the violations. + out: output file + + """ + + if file_or_dir is not None and os.path.isfile(file_or_dir): + _process_file(file_or_dir, template_linters, options, summary_results, out) + else: + directory = "." + if file_or_dir is not None: + if os.path.exists(file_or_dir): + directory = file_or_dir + else: + raise ValueError(f"Path [{file_or_dir}] is not a valid file or directory.") + _process_os_dirs(directory, template_linters, options, summary_results, out) + + summary_results.print_results(options, out) + result_output = _get_xsslint_counts(out.getvalue()) + _check_violations(options, result_output) + + +def main(): + """ + Used to execute the linter. Use --help option for help. + + Prints all violations. + """ + epilog = "For more help using the xss linter, including details on how to\n" + epilog += "understand and fix any violations, read the docs here:\n" + epilog += "\n" + # pylint: disable=line-too-long + epilog += " https://edx.readthedocs.org/projects/edx-developer-guide/en/latest/conventions/preventing_xss.html#xss-linter\n" + + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description='Checks that templates are safe.', + epilog=epilog, + ) + parser.add_argument( + '--list-files', dest='list_files', action='store_true', + help='Only display the filenames that contain violations.' + ) + parser.add_argument( + '--rule-totals', dest='rule_totals', action='store_true', + help='Display the totals for each rule.' + ) + parser.add_argument( + '--summary-format', dest='summary_format', + choices=['eslint', 'json'], default='eslint', + help='Choose the display format for the summary.' + ) + parser.add_argument( + '--verbose', dest='verbose', action='store_true', + help='Print multiple lines where possible for additional context of violations.' + ) + parser.add_argument( + '--config', dest='config', action='store', default='xsslint.default_config', + help='Specifies the config module to use. The config module should be in Python package syntax.' + ) + parser.add_argument( + '--thresholds', dest='thresholds', action='store', + help='Specifies the config module to use. The config module should be in Python package syntax.' + ) + parser.add_argument('path', nargs="?", default=None, help='A file to lint or directory to recursively lint.') + + args = parser.parse_args() + config = _load_config_module(args.config) + options = { + 'list_files': args.list_files, + 'rule_totals': args.rule_totals, + 'summary_format': args.summary_format, + 'verbose': args.verbose, + 'skip_dirs': getattr(config, 'SKIP_DIRS', ()), + 'thresholds': args.thresholds + } + template_linters = getattr(config, 'LINTERS', ()) + if not template_linters: + raise ValueError(f"LINTERS is empty or undefined in the config module ({args.config}).") + + ruleset = _build_ruleset(template_linters) + summary_results = SummaryResults(ruleset) + _lint(args.path, template_linters, options, summary_results, out=StringIO()) + + if __name__ == "__main__": - from xsslint.main import main - main() + try: + main() + except BuildFailure as e: + print(e) + sys.exit(1) diff --git a/scripts/xsslint/xsslint/main.py b/scripts/xsslint/xsslint/main.py deleted file mode 100644 index f8f8672b74b3..000000000000 --- a/scripts/xsslint/xsslint/main.py +++ /dev/null @@ -1,187 +0,0 @@ -""" -The main function for the XSS linter. -""" - - -import argparse -import importlib -import os -import sys -from functools import reduce - -from xsslint.reporting import SummaryResults -from xsslint.rules import RuleSet -from xsslint.utils import is_skip_dir - - -def _load_config_module(module_path): - cwd = os.getcwd() - if cwd not in sys.path: - # Enable config module to be imported relative to wherever the script was run from. - sys.path.append(cwd) - return importlib.import_module(module_path) - - -def _build_ruleset(template_linters): - """ - Combines the RuleSets from the provided template_linters into a single, aggregate RuleSet. - - Arguments: - template_linters: A list of linting objects. - - Returns: - The combined RuleSet. - """ - return reduce( - lambda combined, current: combined + current.ruleset, - template_linters, - RuleSet() - ) - - -def _process_file(full_path, template_linters, options, summary_results, out): - """ - For each linter, lints the provided file. This means finding and printing - violations. - - Arguments: - full_path: The full path of the file to lint. - template_linters: A list of linting objects. - options: A list of the options. - summary_results: A SummaryResults with a summary of the violations. - out: output file - - """ - num_violations = 0 - directory = os.path.dirname(full_path) - file_name = os.path.basename(full_path) - try: - for template_linter in template_linters: - results = template_linter.process_file(directory, file_name) - results.print_results(options, summary_results, out) - except BaseException as e: - raise Exception(f"Failed to process path: {full_path}") from e - - -def _process_os_dir(directory, files, template_linters, options, summary_results, out): - """ - Calls out to lint each file in the passed list of files. - - Arguments: - directory: Directory being linted. - files: All files in the directory to be linted. - template_linters: A list of linting objects. - options: A list of the options. - summary_results: A SummaryResults with a summary of the violations. - out: output file - - """ - for current_file in sorted(files, key=lambda s: s.lower()): - full_path = os.path.join(directory, current_file) - _process_file(full_path, template_linters, options, summary_results, out) - - -def _process_os_dirs(starting_dir, template_linters, options, summary_results, out): - """ - For each linter, lints all the directories in the starting directory. - - Arguments: - starting_dir: The initial directory to begin the walk. - template_linters: A list of linting objects. - options: A list of the options. - summary_results: A SummaryResults with a summary of the violations. - out: output file - - """ - skip_dirs = options.get('skip_dirs', ()) - for root, dirs, files in os.walk(starting_dir): - if is_skip_dir(skip_dirs, root): - del dirs - continue - dirs.sort(key=lambda s: s.lower()) - _process_os_dir(root, files, template_linters, options, summary_results, out) - - -def _lint(file_or_dir, template_linters, options, summary_results, out): - """ - For each linter, lints the provided file or directory. - - Arguments: - file_or_dir: The file or initial directory to lint. - template_linters: A list of linting objects. - options: A list of the options. - summary_results: A SummaryResults with a summary of the violations. - out: output file - - """ - - if file_or_dir is not None and os.path.isfile(file_or_dir): - _process_file(file_or_dir, template_linters, options, summary_results, out) - else: - directory = "." - if file_or_dir is not None: - if os.path.exists(file_or_dir): - directory = file_or_dir - else: - raise ValueError(f"Path [{file_or_dir}] is not a valid file or directory.") - _process_os_dirs(directory, template_linters, options, summary_results, out) - - summary_results.print_results(options, out) - - -def main(): - """ - Used to execute the linter. Use --help option for help. - - Prints all violations. - """ - epilog = "For more help using the xss linter, including details on how to\n" - epilog += "understand and fix any violations, read the docs here:\n" - epilog += "\n" - # pylint: disable=line-too-long - epilog += " https://edx.readthedocs.org/projects/edx-developer-guide/en/latest/conventions/preventing_xss.html#xss-linter\n" - - parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description='Checks that templates are safe.', - epilog=epilog, - ) - parser.add_argument( - '--list-files', dest='list_files', action='store_true', - help='Only display the filenames that contain violations.' - ) - parser.add_argument( - '--rule-totals', dest='rule_totals', action='store_true', - help='Display the totals for each rule.' - ) - parser.add_argument( - '--summary-format', dest='summary_format', - choices=['eslint', 'json'], default='eslint', - help='Choose the display format for the summary.' - ) - parser.add_argument( - '--verbose', dest='verbose', action='store_true', - help='Print multiple lines where possible for additional context of violations.' - ) - parser.add_argument( - '--config', dest='config', action='store', default='xsslint.default_config', - help='Specifies the config module to use. The config module should be in Python package syntax.' - ) - parser.add_argument('path', nargs="?", default=None, help='A file to lint or directory to recursively lint.') - - args = parser.parse_args() - config = _load_config_module(args.config) - options = { - 'list_files': args.list_files, - 'rule_totals': args.rule_totals, - 'summary_format': args.summary_format, - 'verbose': args.verbose, - 'skip_dirs': getattr(config, 'SKIP_DIRS', ()) - } - template_linters = getattr(config, 'LINTERS', ()) - if not template_linters: - raise ValueError(f"LINTERS is empty or undefined in the config module ({args.config}).") - - ruleset = _build_ruleset(template_linters) - summary_results = SummaryResults(ruleset) - _lint(args.path, template_linters, options, summary_results, out=sys.stdout) diff --git a/stylelint.config.js b/stylelint.config.js deleted file mode 100644 index bd7769911708..000000000000 --- a/stylelint.config.js +++ /dev/null @@ -1,3 +0,0 @@ -module.exports = { - extends: '@edx/stylelint-config-edx' -};