Improve extraction of tokens from regexes

Fixed flawed extraction of tokens with optional sequences, i.e.
when quantifier could be zero.
Related issue:
- uBlockOrigin/uBlock-issues#2367

Ignore look-around sequences as suggested when normalizing into
tokenizable string.
Related issue:
- uBlockOrigin/uBlock-issues#2368

Fix regex analyzer throwing with trailing `-` in character
class sequence.
Related issue:
- AdguardTeam/AdguardFilters#134630

src/js/static-filtering-parser.js

@@ -3009,11 +3009,11 @@ Parser.utils = Parser.prototype.utils = (( ) => {
 
  class regex {
  static firstCharCodeClass(s) {
- return /^[\x01%0-9A-Za-z]/.test(s) ? 1 : 0;
+ return /^[\x01\x03%0-9A-Za-z]/.test(s) ? 1 : 0;
  }
 
  static lastCharCodeClass(s) {
- return /[\x01%0-9A-Za-z]$/.test(s) ? 1 : 0;
+ return /[\x01\x03%0-9A-Za-z]$/.test(s) ? 1 : 0;
  }
 
  static tokenizableStrFromNode(node) {
@@ -3042,18 +3042,24 @@ Parser.utils = Parser.prototype.utils = (( ) => {
  return String.fromCharCode(firstChar, lastChar);
  }
  case 4: /* T_GROUP, 'Group' */ {
- if ( node.flags.NegativeLookAhead === 1 ) { return '\x01'; }
- if ( node.flags.NegativeLookBehind === 1 ) { return '\x01'; }
+ if (
+ node.flags.LookAhead === 1 ||
+ node.flags.NegativeLookAhead === 1 ||
+ node.flags.LookBehind === 1 ||
+ node.flags.NegativeLookBehind === 1
+ ) {
+ return '';
+ }
  return this.tokenizableStrFromNode(node.val);
  }
  case 16: /* T_QUANTIFIER, 'Quantifier' */ {
  const s = this.tokenizableStrFromNode(node.val);
  const first = this.firstCharCodeClass(s);
  const last = this.lastCharCodeClass(s);
- if ( node.flags.min === 0 && first === 0 && last === 0 ) {
- return '';
+ if ( node.flags.min !== 0 ) {
+ return String.fromCharCode(first, last);
  }
- return String.fromCharCode(first, last);
+ return String.fromCharCode(first+2, last+2);
  }
  case 64: /* T_HEXCHAR, 'HexChar' */ {
  return String.fromCharCode(parseInt(node.val.slice(1), 16));
@@ -3142,13 +3148,33 @@ Parser.utils = Parser.prototype.utils = (( ) => {
 
  static toTokenizableStr(reStr) {
  if ( regexAnalyzer === null ) { return ''; }
+ let s = '';
  try {
- return this.tokenizableStrFromNode(
+ s = this.tokenizableStrFromNode(
  regexAnalyzer(reStr, false).tree()
  );
  } catch(ex) {
  }
- return '';
+ // Process optional sequences
+ const reOptional = /[\x02\x03]+/g;
+ for (;;) {
+ const match = reOptional.exec(s);
+ if ( match === null ) { break; }
+ const left = s.slice(0, match.index);
+ const middle = match[0];
+ const right = s.slice(match.index + middle.length);
+ s = left;
+ s += this.firstCharCodeClass(right) === 1 ||
+ this.firstCharCodeClass(middle) === 1
+ ? '\x01'
+ : '\x00';
+ s += this.lastCharCodeClass(left) === 1 ||
+ this.lastCharCodeClass(middle) === 1
+ ? '\x01'
+ : '\x00';
+ s += right;
+ }
+ return s;
  }
  }
 

src/lib/regexanalyzer/regex.js

@@ -1019,6 +1019,12 @@ var rnd = function( a, b ){ return Math.round((b-a)*Math.random()+a); },
  }
  }
 
+ if ( isRange && escaped === false && ']' === ch )
+ {
+ isRange = false;
+ chars.push('-');
+ }
+
  if ( isRange )
  {
  if ( chars.length )