Parallel cucumbers

CHANGELOG.md

@@ -3,6 +3,7 @@
     - Profiles:
       - ADDED: Use `is_sidepath:of:name` and `street:name` as fallback names for unnamed sidewalks and sidepaths in foot and bicycle profiles [#7259](https://github.com/Project-OSRM/osrm-backend/issues/7259)
     - Build:
+      - CHANGED: Cucumber tests now can run in parallel and other improvements [#7309](https://github.com/Project-OSRM/osrm-backend/issues/7309)
       - FIXED: Update Node.js binding path from `lib/binding` to `lib/binding_napi_v8` to match node-pre-gyp versioning conventions [#7272](https://github.com/Project-OSRM/osrm-backend/pull/7272)
       - FIXED: Reduce MSVC compiler warnings by suppressing informational warnings while preserving bug-indicating warnings [#7253](https://github.com/Project-OSRM/osrm-backend/issues/7253)
       - FIXED: Merge `osrm_extract` and `osrm_guidance` to avoid circular dependencies. [#7315](https://github.com/Project-OSRM/osrm-backend/pull/7315)

cucumber.mjs

@@ -1,32 +1,81 @@
-// Default profile
-export default {
-  strict: true,
-  tags: 'not @stress and not @todo and not @mld',
-  import: ['features/support', 'features/step_definitions'],
-};
+// See: https://github.com/cucumber/cucumber-js/blob/main/docs/profiles.md
 
-// Additional profiles  
-export const ch = {
-  strict: true,
-  tags: 'not @stress and not @todo and not @mld',
-  format: ['progress'],
-  import: ['features/support', 'features/step_definitions'],
-};
+export default function() {
+  const penv = process.env;
 
-export const todo = {
-  strict: true,
-  tags: '@todo',
-  import: ['features/support', 'features/step_definitions'],
-};
+  function int(s, def) {
+    return (s && parseInt(s)) || def;
+  }
 
-export const all = {
-  strict: true,
-  import: ['features/support', 'features/step_definitions'],
-};
+  function str(s, def) {
+    return s || def;
+  }
+
+  const commonWorldParameters = {
+    httpTimeout:  int(penv.CUCUMBER_HTTP_TIMEOUT,  2000), // must be less than default timeout
+    testPath:     str(penv.CUCUMBER_TEST_PATH,     'test'),
+    profilesPath: str(penv.CUCUMBER_PROFILES_PATH, 'profiles'),
+    logsPath:     str(penv.CUCUMBER_LOGS_PATH,     'test/logs'),
+    cachePath:    str(penv.CUCUMBER_CACHE_PATH,    'test/cache'),
+    buildPath:    str(penv.OSRM_BUILD_DIR,         'build'),
+    loadMethod:   str(penv.OSRM_LOAD_METHOD,       'datastore'),
+    algorithm:    str(penv.OSRM_ALGORITHM,         'ch'),
+    port:         int(penv.OSRM_PORT,              5000),
+    ip:           str(penv.OSRM_IP,                '127.0.0.1'),
+  }
+
+  const baseConfig = {
+    strict: true,
+    import: [
+      'features/support/',
+      'features/step_definitions/',
+      'features/lib/'
+    ],
+    worldParameters : commonWorldParameters,
+    tags: 'not @stress and not @todo',
+  }
+
+  function wp(worldParameters) {
+    return { worldParameters: worldParameters };
+  }
+
+  const htmlReportFilename = `test/logs/cucumber-${process.env.algorithm}-${process.env.loadmethod}.report.html`
+
+  return {
+    // Default profile
+    default: {
+      ... baseConfig,
+    },
+
+    // base configs
+    home: {
+      ... baseConfig,
+      format: [
+        'progress-bar',
+        ['html', htmlReportFilename]
+      ],
+    },
+    github: {
+      ... baseConfig,
+      format: [
+        './features/lib/github_summary_formatter.js',
+        ['html', htmlReportFilename]
+      ],
+      publish: true
+    },
+
+    // patches to base configs
+    stress: { tags: '@stress'},
+    todo:   { tags: '@todo'},
+    all:    { tags: '' },
+
+    // algorithms
+    ch:  wp({algorithm: 'ch'}),
+    mld: wp({algorithm: 'mld'}),
 
-export const mld = {
-  strict: true,
-  tags: 'not @stress and not @todo and not @ch',
-  format: ['progress'],
-  import: ['features/support', 'features/step_definitions'],
+    // data load methods
+    datastore: wp({loadMethod: 'datastore'}),
+    directly:  wp({loadMethod: 'directly'}),
+    mmap:      wp({loadMethod: 'mmap'}),
+  }
 };

docs/cucumber.md

@@ -0,0 +1,150 @@
+# Cucumber
+
+This documentation describes the technical aspects of our cucumber test suite.
+
+- See also: [on how to write Cucumber tests](testing.md).
+- See also: [about Cucumber in the OSRM wiki](https://github.com/Project-OSRM/osrm-backend/wiki/Cucumber-Test-Suite).
+- See also: [the Cucumber docs](https://github.com/cucumber/cucumber-js/tree/main/docs).
+
+## tl;dr
+
+Run the Cucumber tests with:
+
+``` bash
+$ npm test -- --parallel 16
+```
+
+## Single OSRM Configuration
+
+An OSRM configuration consists of a *routing algorithm* and a *data load method*. OSRM
+currently supports the routing algorithms:
+- `ch` (Contraction Hierarchy), and
+- `mld` (Multi-Level-Dijkstra)
+
+and the data load methods:
+
+- `directly` (load the files into memory),
+- `mmap` (use memory mapped files), and
+- `datastore` (use shared memory).
+
+To test all scenarios with a single OSRM configuration, say:
+
+``` bash
+$ npx cucumber-js -p home -p mld -p mmap --parallel 8 --fail-fast
+```
+Explanations follow:
+
+### Profiles
+
+Profiles are chosen with the `-p` commandline argument.  Cucumber profiles allow you to
+change multiple configuration items with just one commandline argument.  If you set
+more than one profile they are all merged into one configuration.
+
+Note: Cucumber profiles should not be confused with OSRM profiles.  Cucumber profiles
+are defined in `cucumber.mjs`.  OSRM profiles reside in the `profiles/*.lua` files.
+
+Our implementation offers following stock profiles. You should always use one base
+profile followed by zero or more additional profiles.
+
+| Name      |                                                                |
+| --------- | -------------------------------------------------------------- |
+| home      | Base profile to use on a developer machine                     |
+| github    | Base profile to use on the github CI server                    |
+| ch        | Additional profile that selects the CH algorithm               |
+| mld       | Additional profile that selects the MLD algorithm              |
+| mmap      | Additional profile that selects the mmap data load method      |
+| directly  | Additional profile that selects the directly data load method  |
+| datastore | Additional profile that selects the datastore data load method |
+| stress    | Additional profile that selects only @stress tests             |
+| todo      | Additional profile that selects only @todo tests               |
+| all       | Additional profile that selects all tests                      |
+
+### Arguments
+
+Here is a  [description of all
+arguments](https://github.com/cucumber/cucumber-js/blob/main/docs/configuration.md#options)
+you can pass to Cucumber. The interesting ones probably are: `--fail-fast`, `--format`,
+`--parallel`, and `--tags`.
+
+Note: when using `--parallel N` make sure there are `N` contiguous free ports at the
+configured port number (eg. at ports 5000--5000+N).
+
+## All OSRM Configurations
+
+We provide a shortcut to run all 6 configurations:
+
+``` bash
+$ npm test -- --parallel 16
+```
+This is how the tests are run on the CI server. You can pass the same arguments as
+mentioned above.
+
+## Cache
+
+To speed up subsequent runs with the same parameters, the files generated by Cucumber
+and the by the OSRM extraction chain are held in a cache directory. This cache is
+located by default in `test/cache` and should be cleaned periodically:
+
+``` bash
+$ rm -rf test/cache
+```
+
+## Configuration
+
+The whole configuration is done in `cucumber.mjs`. You can either edit `worldParameters`
+in `cucumber.mjs` or use environment variables to override single defaults.
+
+| worldParameters | Environment Variable     | Defaults to  |                         |
+| --------------- | ------------------------ | ------------ | ----------------------- |
+|                 | `CUCUMBER_TIMEOUT`       | 5000         | Scenario timeout in ms. |
+| `httpTimeout`   | `CUCUMBER_HTTP_TIMEOUT`  | 2000         | HTTP timeout in ms.     |
+| `testPath`      | `CUCUMBER_TEST_PATH`     | `test`       | The test directory      |
+| `profilesPath`  | `CUCUMBER_PROFILES_PATH` | `profiles`   | The profiles directory  |
+| `logsPath`      | `CUCUMBER_LOGS_PATH`     | `test/logs`  | The logs directory      |
+| `cachePath`     | `CUCUMBER_CACHE_PATH`    | `test/cache` | The cache directory     |
+| `buildPath`     | `OSRM_BUILD_DIR`         | `build`      | Path to the binaries    |
+| `loadMethod`    | `OSRM_LOAD_METHOD`       | datastore    | Data load method        |
+| `algorithm`     | `OSRM_ALGORITHM`         | ch           | Routing algorithm       |
+| `ip`            | `OSRM_IP`                | 127.0.0.1    | IP Address              |
+| `port`          | `OSRM_PORT`              | 5000         | IP Port                 |
+
+The default Cucumber timeout can be changed by setting the environment variable
+`CUCUMBER_TIMEOUT`. This is discouraged, because the default timeout of 5 seconds is
+plenty for the problem sizes we are dealing with. The probable reasons for a test timing
+out are that `osrm-routed` died or that sync between `osrm-datastore` and `osrm-routed`
+was lost.
+
+### Other environment variables
+
+`OSRM_RASTER_SOURCE` is set by 'Given the raster source' and is supposed to be read back
+in your `profiles/*.lua` profile by `os.getenv('OSRM_RASTER_SOURCE')`.
+
+`OSRM_PROFILE` See: [Pull Request #4516](https://github.com/Project-OSRM/osrm-backend/pull/4516)
+
+
+## Tags
+
+Single scenarios or whole feature files can be tagged. Tag names can be selected
+arbitrarily although it is best to conform to the tags already used. Eg. the tag
+`@guidance` can be used to run only those tests related to the guidance feature:
+
+``` bash
+$ npm test -- --parallel 16 --tags @guidance
+```
+
+We also support following special tags:
+
+| Tag                                 | A scenario thus tagged ...                                   |
+| ----------------------------------- | ------------------------------------------------------------ |
+| `@isolated`                         | will not run while any other scenario is running in parallel |
+| `@with_(datastore\|directly\|mmap)` | will be executed iff the load method matches                 |
+| `@no_(datastore\|directly\|mmap)`   | will be executed unless the load method matches              |
+| `@with_(ch\|mld)`                   | will be executed iff the algorithm matches                   |
+| `@no_(ch\|mld)`                     | will be executed unless the algorithm matches                |
+| `@with_(linux\|darwin\|win32)`      | will be executed iff the OS matches                          |
+| `@no_(linux\|darwin\|win32)`        | will be executed unless the OS matches                       |
+
+A test that calls `osrm-datastore --spring-clean` should not run concurrently with any
+other test, thus the tag `@isolated` should be applied. A test that runs or kills
+`osrm-routed` should not run while testing the datastore load method, and thus should be
+labeled with the tag `@no_datastore`.

features/lib/github_summary_formatter.js

@@ -0,0 +1,84 @@
+import fs from 'node:fs';
+import { SummaryFormatter } from '@cucumber/cucumber';
+
+const FAILURES = ['FAILED', 'AMBIGUOUS', 'UNDEFINED'];
+const WARNINGS = ['SKIPPED', 'PENDING', 'UNKNOWN'];
+// const SUCCESSES = ['PASSED'];
+
+function isFailure(result) {
+  return FAILURES.includes(result.status);
+}
+function isWarning(result) {
+  return WARNINGS.includes(result.status);
+}
+// function isSuccess(result) {
+//   return SUCCESSES.includes(result.status);
+// }
+function toSeconds(timeStamp) {
+  return +timeStamp.seconds + timeStamp.nanos / 1e9;
+}
+
+/**
+ * A Summary Formatter that also appends to $GITHUB_STEP_SUMMARY
+ *
+ * This formatter first calls the stock summary formatter, then appends one line of
+ * MarkDown to the file denoted by $GITHUB_STEP_SUMMARY.
+ *
+ * The hacky part is that we have to *append* to $GITHUB_STEP_SUMMARY. By using the
+ * obvious:
+ *
+ *   --format github_summary_formatter:"$GITHUB_STEP_SUMMARY"
+ *
+ * we would *overwrite* that file.
+ *
+ * Another thing to consider is that Cucumber only allows one formatter that grabs
+ * stdout.  In order to avoid the clumsy:
+ *
+ *   --format summary --format github_summary_formatter:"/dev/null"
+ *
+ * this formatter also does the duty of the stock summary formatter:
+ *
+ *   --format github_summary_formatter
+ *
+ * Override the default filename of: $GITHUB_STEP_SUMMARY
+ *
+ *   formatOptions: { 'github_summary_formatter': { filename: 'somewhere.else' }},
+ */
+
+export default class GithubSummaryFormatter extends SummaryFormatter {
+  static documentation = 'A Summary Formatter that also appends a $GITHUB_STEP_SUMMARY';
+  constructor(options) {
+    super(options);
+    this.filename = options.parsedArgvOptions?.github_summary_formatter?.filename
+        ?? process.env.GITHUB_STEP_SUMMARY;
+  }
+
+  logSummary(testRunDuration) {
+    // because Cucumber only allows one formatter thats grab stdout,
+    // we have to mimic the summary formatter first
+    super.logSummary(testRunDuration);
+    if (!this.filename)
+      return;
+
+    let failures = 0;
+    let warnings = 0;
+    let successes = 0;
+
+    for (const { worstTestStepResult } of this.eventDataCollector.getTestCaseAttempts()) {
+      if (isFailure(worstTestStepResult)) {
+        ++failures;
+      } else if (isWarning(worstTestStepResult)) {
+        ++warnings;
+      } else {
+        ++successes;
+      }
+    };
+    if (failures > 0) {
+      failures = `:x: ${failures}`;
+    }
+    const seconds = toSeconds(testRunDuration).toFixed(3);
+    const msg = `| ${successes} | ${warnings} | ${failures} | ${seconds} |\n`;
+
+    fs.appendFileSync(this.filename, msg);
+  }
+}