Handle failures and add 'progress' formatter (close #6, #16, #17)

- Failing steps are now properly handled and following steps in scenario are skipped
- Second core.feature scenario is passing
- Progress formatter is now available and used by default
- We still miss stack trace output on failure

cucumber.js

@@ -1,9 +1,30 @@
 #!/usr/bin/env node
-var fs                = require('fs');
-var Cucumber          = require('./lib/cucumber');
-var supportCodePath   = process.ARGV[3] ? process.cwd() + '/' + process.ARGV[3] : './features/step_definitions/cucumber_steps';
-var supportCode       = require(supportCodePath);
-var cucumber          = Cucumber(fs.readFileSync(process.ARGV[2]), supportCode);
-var progressFormatter = Cucumber.Listener.ProgressFormatter;
-cucumber.attachListener(progressFormatter());
-cucumber.start(function() {});
+var fs              = require('fs');
+var Cucumber        = require('./lib/cucumber');
+var supportCodePath = process.ARGV[3] ? process.cwd() + '/' + process.ARGV[3] : './features/step_definitions/cucumber_steps';
+var supportCode     = require(supportCodePath);
+var cucumber        = Cucumber(fs.readFileSync(process.ARGV[2]), supportCode);
+var formatter       = Cucumber.Listener.ProgressFormatter();
+cucumber.attachListener(formatter);
+cucumber.start(function(succeeded) {
+  var code = succeeded ? 0 : 1;
+  var exitFunction = function() {
+    process.exit(code);
+  };
+
+  // --- exit after waiting for all pending output ---
+  var waitingIO = false;
+  process.stdout.on('drain', function() {
+    if (waitingIO) {
+      // the kernel buffer is now empty
+      exitFunction();
+    }
+  });
+  if (process.stdout.write("")) {
+    // no buffer left, exit now:
+    exitFunction();
+  } else {
+    // write() returned false, kernel buffer is not empty yet...
+    waitingIO = true;
+  }
+});

features/progress_formatter.feature

@@ -0,0 +1,133 @@
+Feature: progress formatter
+  In order to get quick feedback when doing BDD
+  As a developer
+  I want to use a "progress" formatter
+
+  Scenario: one scenario, one step, passing
+    Given a step definition matching /a passing step/
+    When I run the following feature with the "progress" formatter:
+      """
+      Feature:
+        Scenario:
+          Given a passing step
+      """
+    Then the listener should output the following:
+      """
+      .
+
+      1 scenario (1 passed)
+      1 step (1 passed)
+      """
+
+  Scenario: one scenario, two steps, passing
+    Given a step definition matching /a passing step/
+    When I run the following feature with the "progress" formatter:
+      """
+      Feature:
+        Scenario:
+          Given a passing step
+          And a passing step
+      """
+    Then the listener should output the following:
+      """
+      ..
+
+      1 scenario (1 passed)
+      2 steps (2 passed)
+      """
+
+  Scenario: two scenarios, five steps, passing
+    Given a step definition matching /a passing step/
+    When I run the following feature with the "progress" formatter:
+      """
+      Feature:
+        Scenario:
+          Given a passing step
+          And a passing step
+        Scenario:
+          Given a passing step
+          And a passing step
+          When a passing step
+      """
+    Then the listener should output the following:
+      """
+      ..
+
+      2 scenarios (2 passed)
+      5 steps (5 passed)
+      """
+
+  Scenario: one scenario, one step, failing
+    Given a step definition failing with message "boom" matching /a failing step/
+    When I run the following feature with the "progress" formatter:
+      """
+      Feature:
+        Scenario:
+          Given a failing step
+      """
+    Then the listener should output the following:
+      """
+      F
+
+      1 scenario (1 failed)
+      1 step (1 failed)
+      """
+
+  Scenario: one scenario, two steps, second failing
+    Given a step definition matching /a passing step/
+    And a step definition failing with message "boom" matching /a failing step/
+    When I run the following feature with the "progress" formatter:
+      """
+      Feature:
+        Scenario:
+          Given a passing step
+          When a failing step
+      """
+    Then the listener should output the following:
+      """
+      .F
+
+      1 scenario (1 failed)
+      2 steps (1 failed, 1 passed)
+      """
+
+  Scenario: one two-step passing scenario, one two-step scenario with latest step failing
+    Given a step definition matching /a passing step/
+    And a step definition failing with message "boom" matching /a failing step/
+    When I run the following feature with the "progress" formatter:
+      """
+      Feature:
+        Scenario:
+          Given a passing step
+          When a passing step
+        Scenario:
+          Given a passing step
+          When a failing step
+      """
+    Then the listener should output the following:
+      """
+      ...F
+
+      2 scenarios (1 failed, 1 passed)
+      4 steps (1 failed, 3 passed)
+      """
+
+  Scenario: one failing scenario with a skipped step
+    Given a step definition matching /a passing step/
+    And a step definition matching /a skipped step/
+    And a step definition failing with message "boom" matching /a failing step/
+    When I run the following feature with the "progress" formatter:
+      """
+      Feature:
+        Scenario:
+          Given a passing step
+          When a failing step
+          Then a skipped step
+      """
+    Then the listener should output the following:
+      """
+      .F-
+
+      1 scenario (1 failed)
+      3 steps (1 failed, 1 skipped, 1 passed)
+      """

features/step_definitions/cucumber_js_mappings.rb

@@ -160,12 +160,12 @@ def write_failing_mapping(step_name)
 #   end
 
   def assert_passing_scenario
-    assert_partial_output("1 scenario(s) (1 passed)", all_output)
+    assert_partial_output("1 scenario (1 passed)", all_output)
     assert_success true
   end
 
   def assert_failing_scenario
-    assert_partial_output("1 scenario(s) (1 failed)", all_output)
+    assert_partial_output("1 scenario (1 failed)", all_output)
     assert_success false
   end
 

features/step_definitions/cucumber_steps.js

@@ -21,7 +21,24 @@ var stepDefinitions = function() {
   // The created step definition body should:
   //   1. Pass all the time.
   Given(/^a(?: "(Given|When|Then)")? step definition matching \/(.*)\/$/, function(keyword, name, callback) {
-    var content = function(callback) { callback() };
+    var content = function(callback) { callback(); };
+    _addStepDefinition(keyword, name, content);
+    callback();
+  });
+
+  // Creates a Given, When or Then step definition that does nothing but fails all the time.
+  //
+  // Matching groups:
+  //   1. /Given|When|Then/ Step definition keyword (optional)
+  //   2. /.*/              Step definition name
+  //   3. /.*/              Exception message
+  //
+  // Created step definition matching groups: none.
+  //
+  // The created step definition body should:
+  //   1. Fail all the time.
+  Given(/^a(?: "(Given|When|Then)")? step definition failing with message "(.*)" matching \/(.*)\/$/, function(keyword, errorMessage, name, callback) {
+    var content = function(callback) { throw(errorMessage); };
     _addStepDefinition(keyword, name, content);
     callback();
   });

lib/cucumber/ast/tree_walker.js

@@ -1,9 +1,14 @@
 var TreeWalker = function(features, supportCodeLibrary, listeners) {
   var listeners;
+  var allFeaturesSucceded = true;
+  var skippingSteps       = false;
 
   var self = {
     walk: function walk(callback) {
-      self.visitFeatures(features, callback);
+      self.visitFeatures(features, function() {
+        var featuresResult = self.didAllFeaturesSucceed();
+        callback(featuresResult);
+      });
     },
 
     visitFeatures: function visitFeatures(features, callback) {
@@ -26,6 +31,7 @@ var TreeWalker = function(features, supportCodeLibrary, listeners) {
     },
 
     visitScenario: function visitScenario(scenario, callback) {
+      self.witnessNewScenario();
       var payload = { scenario: scenario };
       var event   = TreeWalker.Event(TreeWalker.SCENARIO_EVENT_NAME, payload);
       self.broadcastEventAroundUserFunction(
@@ -36,16 +42,15 @@ var TreeWalker = function(features, supportCodeLibrary, listeners) {
     },
 
     visitStep: function visitStep(step, callback) {
-      var payload = { step: step };
-      var event   = TreeWalker.Event(TreeWalker.STEP_EVENT_NAME, payload);
-      self.broadcastEventAroundUserFunction(
-        event,
-        function(callback) { step.acceptVisitor(self, callback); },
-        callback
-      );
+      if (self.isSkippingSteps())
+        self.skipStep(step, callback);
+      else
+        self.executeStep(step, callback);
     },
 
     visitStepResult: function visitStepResult(stepResult, callback) {
+      if (!stepResult.isSuccessful())
+        self.witnessFailedStep();
       var payload = { stepResult: stepResult };
       var event   = TreeWalker.Event(TreeWalker.STEP_RESULT_EVENT_NAME, payload);
       self.broadcastEvent(event, callback);
@@ -86,6 +91,39 @@ var TreeWalker = function(features, supportCodeLibrary, listeners) {
 
     lookupStepDefinitionByName: function lookupStepDefinitionByName(stepName) {
       return supportCodeLibrary.lookupStepDefinitionByName(stepName);
+    },
+
+    didAllFeaturesSucceed: function didAllFeaturesSucceed() {
+      return allFeaturesSucceded;
+    },
+
+    witnessFailedStep: function witnessFailedStep() {
+      allFeaturesSucceded = false;
+      skippingSteps       = true;
+    },
+
+    witnessNewScenario: function witnessNewScenario() {
+      skippingSteps = false;
+    },
+
+    isSkippingSteps: function isSkippingSteps() {
+      return skippingSteps;
+    },
+
+    executeStep: function executeStep(step, callback) {
+      var payload = { step: step };
+      var event   = TreeWalker.Event(TreeWalker.STEP_EVENT_NAME, payload);
+      self.broadcastEventAroundUserFunction(
+        event,
+        function(callback) { step.acceptVisitor(self, callback); },
+        callback
+      );
+    },
+
+    skipStep: function skipStep(step, callback) {
+      var payload = { step: step };
+      var event   = TreeWalker.Event(TreeWalker.SKIPPED_STEP_EVENT_NAME, payload);
+      self.broadcastEvent(event, callback);
     }
   };
   return self;
@@ -94,6 +132,7 @@ TreeWalker.FEATURES_EVENT_NAME                 = 'Features';
 TreeWalker.FEATURE_EVENT_NAME                  = 'Feature';
 TreeWalker.SCENARIO_EVENT_NAME                 = 'Scenario';
 TreeWalker.STEP_EVENT_NAME                     = 'Step';
+TreeWalker.SKIPPED_STEP_EVENT_NAME             = 'SkippedStep';
 TreeWalker.STEP_RESULT_EVENT_NAME              = 'StepResult';
 TreeWalker.BEFORE_EVENT_NAME_PREFIX            = 'Before';
 TreeWalker.AFTER_EVENT_NAME_PREFIX             = 'After';

lib/cucumber/debug/simple_ast_listener.js

@@ -56,6 +56,9 @@ var SimpleAstListener = function(options) {
         log(currentStep.getDocString().getString(), 3);
         log('"""', 3);
       };
+      if (!stepResult.isSuccessful()) {
+        log('--- FAILED ---', 3);
+      }
       callback();
     },
 
@@ -93,4 +96,4 @@ var SimpleAstListener = function(options) {
     return indented;
   };
 };
-module.exports = SimpleAstListener;
+module.exports = SimpleAstListener;