Merge pull request #178 from magento/develop

MQE-1114: Merge the release branch to master

Author: KevinBKozan

.gitignore

@@ -13,4 +13,6 @@ coverage/
 .vscode
 codeception.yml
 dev/tests/functional/MFTF.suite.yml
-dev/tests/functional/_output
+dev/tests/functional/_output
+dev/mftf.log
+dev/tests/mftf.log

CHANGELOG.md

@@ -1,6 +1,59 @@
 Magento Functional Testing Framework Changelog
 ================================================
 
+2.3.0
+-----
+### Enhancements  
+* Traceability
+    * MFTF now outputs generation run-time information, warnings, and errors to an `mftf.log` file. 
+    * Overall error messages for various generation errors have been improved. Usage of the `--debug` flag provides file-specific errors for all XML-related errors.
+    * Allure Reports now require a unique `story` and `title` combination, to prevent collisions in Allure Report generation.
+    * The `features` annotation now ignores user input and defaults to the module the test lives under (for clear Allure organization).
+    * The `<group value="skip"/>` annotation has been replaced with a `<skip>` annotation, allowing for nested `IssueId` elements.
+    * Tests now require the following annotations: `stories`, `title`, `description`, `severity`.
+        * This will be enforced in a future major release.
+* Modularity
+    * MFTF has been decoupled from MagentoCE:
+        * MFTF can now generate and run tests by itself via `bin/mftf` commands.
+        * It is now a top level MagentoCE dependency, and no longer relies on supporting files in MagentoCE.
+        * It can be used as an isolated dependency for Magento projects such as extensions.
+    * `generate:tests` now warns the user if any declared `<page>` has an inconsistent `module` (`Backend` vs `Magento_Backend`)
+    * The `--force` flag now completely ignores checking of the Magento Installation, allowing generation of tests without a Magento Instance to be running.
+* Customizability
+    * Various test materials can now be extended via an `extends="ExistingMaterial"` attribute. This allows for creation of simple copies of any `entity`, `actionGroup`, or `test`, with small modifications.
+    * `test` and `actionGroup` deltas can now be provided in bulk via a `before/after` attribute on the `test` or `actionGroup` element. Deltas provided this way do not need individual `before/after` attributes, and are inserted sequentially.
+    * Secure and sensitive test data can now be stored and used via a new `.credentials` file, with declaration and usage syntax similar to `.env` file references.
+    * A new `<generateDate>` action has been added to allow users to create and use dates according to the given `date` and `format`.
+        * See DevDocs for more information on all above `Customizability` features.
+* Maintainability
+    * New `bin/mftf` commands have been introduced with parity to existing `robo` commands.
+        * `robo` commands are still supported, but will be deprecated in a future major release.
+    * The `mftf upgrade:tests` command has been introduced, which runs all test upgrade scripts against the provided path.
+        * A new upgrade script was created to replace all test material schema paths to instead use a URN path.
+    * The `mftf generate:urn-catalog` command has been introduced to create a URN catalog in PHPStorm to support the above upgrade.
+    * A warning is now shown on generation if a page's url is referenced without specifying the url (`{{page}}` vs `{{page.url}}`).
+    * An error is now thrown if any test materials contain any overriding element (eg different `<element>`s in a `<section>` with the same `name`)
+        * This previously would cause the last read element to override the previous, causing a silent but potentially incorrect test addition.
+    * Test distribution algorithm for `--config parallel` has been enhanced to take average step length into account.
+
+### Fixes
+* `_after` hook of tests now executes if a non test-related failure causes the test to error.
+* Fixed periods in Allure Report showing up as `•`.
+* Fixed Windows incompatibility of relative paths in various files.
+* Suites will no longer generate if they do not contain any tests.
+* Fixed an issue in generation where users could not use javascript variables in `executeJS` actions.
+* Fixed an issue in generation where entity replacement in action-groups replaced all entities with the first reference found.
+* Fixed an issue in generation where `createData` actions inside `actionGroups` could not properly reference the given `createDataKey`.
+* Fixed an issue where `suites` could not generate if they included an `actionGroup` with two arguments.
+* Fixed an issue in generation where calling the same entity twice (with different parameters) would replace both calls with the first resolved value.
+* The `magentoCLI` action now correctly executes the given command if the `MAGENTO_BASE_URL` contains `index.php` after the domain (ex `https://magento.instance/index.php`)
+* The `stepKey` attribute can no longer be an empty.
+* Variable substitution has been enabled for `regex` and `command` attributes in test actions.
+
+### GitHub Issues/Pull requests:
+* [#161](https://github.com/magento/magento2-functional-testing-framework/pull/161) -- MAGETWO-46837: Implementing extension to wait for readiness metrics.
+* [#72](https://github.com/magento/magento2-functional-testing-framework/issues/72) -- declare(strict_types=1) causes static code check failure (fixed in [#154](https://github.com/magento/magento2-functional-testing-framework/pull/154))
+
 2.2.0
 -----
 ### Enhancements  

RoboFile.php

@@ -0,0 +1,8 @@
+:: Copyright © Magento, Inc. All rights reserved.
+:: See COPYING.txt for license details.
+
+@echo off
+call bin\static-checks.bat
+
+@echo off
+call bin\phpunit-checks.bat

bin/all-checks.bat

@@ -4,7 +4,6 @@
 # THIS FILE CANNOT CONTAIN BLANK LINES                            #
 ###################################################################
 bin/blacklist.txt
-dev/tests/static/Magento/Sniffs/Annotations/Helper.php
-dev/tests/static/Magento/Sniffs/Annotations/RequireAnnotatedAttributesSniff.php
-dev/tests/static/Magento/Sniffs/Annotations/RequireAnnotatedMethodsSniff.php
+dev/tests/static/Magento/Sniffs/Commenting/FunctionCommentSniff.php
+dev/tests/static/Magento/Sniffs/Commenting/VariableCommentSniff.php
 dev/tests/verification/_generated

bin/blacklist.txt

@@ -0,0 +1,41 @@
+:: Copyright © Magento, Inc. All rights reserved.
+:: See COPYING.txt for license details.
+
+@echo off
+SETLOCAL EnableDelayedExpansion
+SET BLACKLIST_FILE=bin/blacklist.txt
+SET i=0
+
+FOR /F %%x IN ('git ls-tree --full-tree -r --name-only HEAD') DO (
+    SET GOOD_EXT=
+    if "%%~xx"==".php" set GOOD_EXT=1
+    if "%%~xx"==".xml" set GOOD_EXT=1
+    if "%%~xx"==".xsd" set GOOD_EXT=1
+    IF DEFINED GOOD_EXT (
+        SET BLACKLISTED=
+        FOR /F "tokens=* skip=5" %%f IN (%BLACKLIST_FILE%) DO (
+            SET LINE=%%x
+            IF NOT "!LINE!"=="!LINE:%%f=!" (
+                SET BLACKLISTED=1
+            )
+        )
+        IF NOT DEFINED BLACKLISTED (
+            FIND "Copyright © Magento, Inc. All rights reserved." %%x >nul
+            IF ERRORLEVEL 1 (
+                SET /A i+=1
+                SET NO_COPYRIGHT_LIST[!i!]=%%x
+            )
+        )
+    )
+)
+
+IF DEFINED NO_COPYRIGHT_LIST[1] (
+    ECHO THE FOLLOWING FILES ARE MISSING THE MAGENTO COPYRIGHT:
+    ECHO.
+    ECHO     Copyright © Magento, Inc. All rights reserved.
+    ECHO     See COPYING.txt for license details.
+    ECHO.
+    FOR /L %%a IN (1,1,%i%) DO (
+        ECHO !NO_COPYRIGHT_LIST[%%a]!
+    )
+)

bin/copyright-check.bat

@@ -11,13 +11,30 @@ if (PHP_SAPI !== 'cli') {
     exit(1);
 }
 
+$autoloadPath = realpath(__DIR__ . '/../../../autoload.php');
+$testBootstrapPath = realpath(__DIR__ . '/../dev/tests/functional/_bootstrap.php');
+
+try {
+    if (file_exists($autoloadPath)) {
+        require_once $autoloadPath;
+    } else {
+        require_once $testBootstrapPath;
+    }
+} catch (\Exception $e) {
+    echo 'Autoload error: ' . $e->getMessage();
+    exit(1);
+}
+
+
 try {
-    require_once __DIR__ . '/../bootstrap.php';
     $application = new Symfony\Component\Console\Application();
     $application->setName('Magento Functional Testing Framework CLI');
-    $application->setVersion('1.0.0');
-    $application->add(new Magento\FunctionalTestingFramework\Console\SetupEnvCommand());
-    $application->add(new Magento\FunctionalTestingFramework\Console\BuildProjectCommand());
+    $application->setVersion('2.3.0');
+    /** @var \Magento\FunctionalTestingFramework\Console\CommandListInterface $commandList */
+    $commandList = new \Magento\FunctionalTestingFramework\Console\CommandList;
+    foreach ($commandList->getCommands() as $command) {
+        $application->add($command);
+    }
     $application->run();
 } catch (\Exception $e) {
     while ($e) {