vendor/minifier/src/js/token.rs

// Take a look at the license at the top of the repository in the LICENSE file.

use std::convert::TryFrom;
use std::fmt;
use std::str::{CharIndices, FromStr};

#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Clone, Copy)]
pub enum ReservedChar {
    Comma,
    OpenParenthese,
    CloseParenthese,
    OpenCurlyBrace,
    CloseCurlyBrace,
    OpenBracket,
    CloseBracket,
    Colon,
    SemiColon,
    Dot,
    Quote,
    DoubleQuote,
    ExclamationMark,
    QuestionMark,
    Slash,
    Modulo,
    Star,
    Minus,
    Plus,
    EqualSign,
    Backslash,
    Space,
    Tab,
    Backline,
    LessThan,
    SuperiorThan,
    Pipe,
    Ampersand,
    BackTick,
}

impl ReservedChar {
    pub fn is_white_character(&self) -> bool {
        *self == ReservedChar::Space
            || *self == ReservedChar::Tab
            || *self == ReservedChar::Backline
    }
}

impl fmt::Display for ReservedChar {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(
            f,
            "{}",
            match *self {
                ReservedChar::Comma => ',',
                ReservedChar::OpenParenthese => '(',
                ReservedChar::CloseParenthese => ')',
                ReservedChar::OpenCurlyBrace => '{',
                ReservedChar::CloseCurlyBrace => '}',
                ReservedChar::OpenBracket => '[',
                ReservedChar::CloseBracket => ']',
                ReservedChar::Colon => ':',
                ReservedChar::SemiColon => ';',
                ReservedChar::Dot => '.',
                ReservedChar::Quote => '\'',
                ReservedChar::DoubleQuote => '"',
                ReservedChar::ExclamationMark => '!',
                ReservedChar::QuestionMark => '?',
                ReservedChar::Slash => '/',
                ReservedChar::Modulo => '%',
                ReservedChar::Star => '*',
                ReservedChar::Minus => '-',
                ReservedChar::Plus => '+',
                ReservedChar::EqualSign => '=',
                ReservedChar::Backslash => '\\',
                ReservedChar::Space => ' ',
                ReservedChar::Tab => '\t',
                ReservedChar::Backline => '\n',
                ReservedChar::LessThan => '<',
                ReservedChar::SuperiorThan => '>',
                ReservedChar::Pipe => '|',
                ReservedChar::Ampersand => '&',
                ReservedChar::BackTick => '`',
            }
        )
    }
}

impl TryFrom<char> for ReservedChar {
    type Error = &'static str;

    fn try_from(value: char) -> Result<ReservedChar, Self::Error> {
        match value {
            ',' => Ok(ReservedChar::Comma),
            '(' => Ok(ReservedChar::OpenParenthese),
            ')' => Ok(ReservedChar::CloseParenthese),
            '{' => Ok(ReservedChar::OpenCurlyBrace),
            '}' => Ok(ReservedChar::CloseCurlyBrace),
            '[' => Ok(ReservedChar::OpenBracket),
            ']' => Ok(ReservedChar::CloseBracket),
            ':' => Ok(ReservedChar::Colon),
            ';' => Ok(ReservedChar::SemiColon),
            '.' => Ok(ReservedChar::Dot),
            '\'' => Ok(ReservedChar::Quote),
            '"' => Ok(ReservedChar::DoubleQuote),
            '!' => Ok(ReservedChar::ExclamationMark),
            '?' => Ok(ReservedChar::QuestionMark),
            '/' => Ok(ReservedChar::Slash),
            '%' => Ok(ReservedChar::Modulo),
            '*' => Ok(ReservedChar::Star),
            '-' => Ok(ReservedChar::Minus),
            '+' => Ok(ReservedChar::Plus),
            '=' => Ok(ReservedChar::EqualSign),
            '\\' => Ok(ReservedChar::Backslash),
            ' ' => Ok(ReservedChar::Space),
            '\t' => Ok(ReservedChar::Tab),
            '\n' | '\r' => Ok(ReservedChar::Backline),
            '<' => Ok(ReservedChar::LessThan),
            '>' => Ok(ReservedChar::SuperiorThan),
            '|' => Ok(ReservedChar::Pipe),
            '&' => Ok(ReservedChar::Ampersand),
            '`' => Ok(ReservedChar::BackTick),
            _ => Err("Unknown reserved char"),
        }
    }
}

#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)]
pub enum Keyword {
    Break,
    Case,
    Catch,
    Const,
    Continue,
    Default,
    Do,
    Else,
    False,
    Finally,
    Function,
    For,
    If,
    In,
    InstanceOf,
    Let,
    New,
    Null,
    Private,
    Protected,
    Public,
    Return,
    Switch,
    This,
    Throw,
    True,
    Try,
    Typeof,
    Static,
    Var,
    While,
}

impl Keyword {
    fn requires_before(&self) -> bool {
        matches!(*self, Keyword::In | Keyword::InstanceOf)
    }
}

impl fmt::Display for Keyword {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(
            f,
            "{}",
            match *self {
                Keyword::Break => "break",
                Keyword::Case => "case",
                Keyword::Catch => "catch",
                Keyword::Const => "const",
                Keyword::Continue => "continue",
                Keyword::Default => "default",
                Keyword::Do => "do",
                Keyword::Else => "else",
                Keyword::False => "false",
                Keyword::Finally => "finally",
                Keyword::Function => "function",
                Keyword::For => "for",
                Keyword::If => "if",
                Keyword::In => "in",
                Keyword::InstanceOf => "instanceof",
                Keyword::Let => "let",
                Keyword::New => "new",
                Keyword::Null => "null",
                Keyword::Private => "private",
                Keyword::Protected => "protected",
                Keyword::Public => "public",
                Keyword::Return => "return",
                Keyword::Switch => "switch",
                Keyword::This => "this",
                Keyword::Throw => "throw",
                Keyword::True => "true",
                Keyword::Try => "try",
                Keyword::Typeof => "typeof",
                Keyword::Static => "static",
                Keyword::Var => "var",
                Keyword::While => "while",
            }
        )
    }
}

impl<'a> TryFrom<&'a str> for Keyword {
    type Error = &'static str;

    fn try_from(value: &str) -> Result<Keyword, Self::Error> {
        match value {
            "break" => Ok(Keyword::Break),
            "case" => Ok(Keyword::Case),
            "catch" => Ok(Keyword::Catch),
            "const" => Ok(Keyword::Const),
            "continue" => Ok(Keyword::Continue),
            "default" => Ok(Keyword::Default),
            "do" => Ok(Keyword::Do),
            "else" => Ok(Keyword::Else),
            "false" => Ok(Keyword::False),
            "finally" => Ok(Keyword::Finally),
            "function" => Ok(Keyword::Function),
            "for" => Ok(Keyword::For),
            "if" => Ok(Keyword::If),
            "in" => Ok(Keyword::In),
            "instanceof" => Ok(Keyword::InstanceOf),
            "let" => Ok(Keyword::Let),
            "new" => Ok(Keyword::New),
            "null" => Ok(Keyword::Null),
            "private" => Ok(Keyword::Private),
            "protected" => Ok(Keyword::Protected),
            "public" => Ok(Keyword::Public),
            "return" => Ok(Keyword::Return),
            "switch" => Ok(Keyword::Switch),
            "this" => Ok(Keyword::This),
            "throw" => Ok(Keyword::Throw),
            "true" => Ok(Keyword::True),
            "try" => Ok(Keyword::Try),
            "typeof" => Ok(Keyword::Typeof),
            "static" => Ok(Keyword::Static),
            "var" => Ok(Keyword::Var),
            "while" => Ok(Keyword::While),
            _ => Err("Unkown keyword"),
        }
    }
}

#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Clone, Copy)]
pub enum Condition {
    And,
    Or,
    DifferentThan,
    SuperDifferentThan,
    EqualTo,
    SuperEqualTo,
    SuperiorThan,
    SuperiorOrEqualTo,
    InferiorThan,
    InferiorOrEqualTo,
}

impl fmt::Display for Condition {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(
            f,
            "{}",
            match *self {
                Condition::And => "&&",
                Condition::Or => "||",
                Condition::DifferentThan => "!=",
                Condition::SuperDifferentThan => "!==",
                Condition::EqualTo => "==",
                Condition::SuperEqualTo => "===",
                Condition::SuperiorThan => ">",
                Condition::SuperiorOrEqualTo => ">=",
                Condition::InferiorThan => "<",
                Condition::InferiorOrEqualTo => "<=",
            }
        )
    }
}

impl TryFrom<ReservedChar> for Condition {
    type Error = &'static str;

    fn try_from(value: ReservedChar) -> Result<Condition, Self::Error> {
        Ok(match value {
            ReservedChar::SuperiorThan => Condition::SuperiorThan,
            ReservedChar::LessThan => Condition::InferiorThan,
            _ => return Err("Unkown condition"),
        })
    }
}

#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Clone, Copy)]
pub enum Operation {
    Addition,
    AdditionEqual,
    Subtract,
    SubtractEqual,
    Multiply,
    MultiplyEqual,
    Divide,
    DivideEqual,
    Modulo,
    ModuloEqual,
    Equal,
}

impl Operation {
    pub fn is_assign(&self) -> bool {
        matches!(
            *self,
            Operation::AdditionEqual
                | Operation::SubtractEqual
                | Operation::MultiplyEqual
                | Operation::DivideEqual
                | Operation::ModuloEqual
                | Operation::Equal
        )
    }
}

impl fmt::Display for Operation {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(
            f,
            "{}",
            match *self {
                Operation::Addition => "+",
                Operation::AdditionEqual => "+=",
                Operation::Subtract => "-",
                Operation::SubtractEqual => "-=",
                Operation::Multiply => "*",
                Operation::MultiplyEqual => "*=",
                Operation::Divide => "/",
                Operation::DivideEqual => "/=",
                Operation::Modulo => "%",
                Operation::ModuloEqual => "%=",
                Operation::Equal => "=",
            }
        )
    }
}

impl TryFrom<ReservedChar> for Operation {
    type Error = &'static str;

    fn try_from(value: ReservedChar) -> Result<Operation, Self::Error> {
        Ok(match value {
            ReservedChar::Plus => Operation::Addition,
            ReservedChar::Minus => Operation::Subtract,
            ReservedChar::Slash => Operation::Divide,
            ReservedChar::Star => Operation::Multiply,
            ReservedChar::Modulo => Operation::Modulo,
            ReservedChar::EqualSign => Operation::Equal,
            _ => return Err("Unkown operation"),
        })
    }
}

#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Hash)]
pub enum Token<'a> {
    Keyword(Keyword),
    Char(ReservedChar),
    String(&'a str),
    Comment(&'a str),
    License(&'a str),
    Other(&'a str),
    Regex {
        regex: &'a str,
        is_global: bool,
        is_interactive: bool,
    },
    Condition(Condition),
    Operation(Operation),
    CreatedVarDecl(String),
    CreatedVar(String),
    Number(usize),
    FloatingNumber(&'a str),
}

impl<'a> fmt::Display for Token<'a> {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match *self {
            Token::Keyword(x) => write!(f, "{}", x),
            Token::Char(x) => write!(f, "{}", x),
            Token::String(x) | Token::Comment(x) | Token::Other(x) => write!(f, "{}", x),
            Token::License(x) => write!(f, "/*!{}*/", x),
            Token::Regex {
                regex,
                is_global,
                is_interactive,
            } => {
                let x = write!(f, "/{}/", regex);
                if is_global {
                    write!(f, "g")?;
                }
                if is_interactive {
                    write!(f, "i")?;
                }
                x
            }
            Token::Condition(x) => write!(f, "{}", x),
            Token::Operation(x) => write!(f, "{}", x),
            Token::CreatedVarDecl(ref x) => write!(f, "{}", x),
            Token::CreatedVar(ref x) => write!(f, "{}", x),
            Token::Number(x) => write!(f, "{}", x),
            Token::FloatingNumber(ref x) => write!(f, "{}", x),
        }
    }
}

impl<'a> Token<'a> {
    pub fn is_comment(&self) -> bool {
        matches!(*self, Token::Comment(_))
    }

    pub fn is_license(&self) -> bool {
        matches!(*self, Token::License(_))
    }

    pub fn is_reserved_char(&self) -> bool {
        matches!(*self, Token::Char(_))
    }

    pub fn get_char(&self) -> Option<ReservedChar> {
        match *self {
            Token::Char(c) => Some(c),
            _ => None,
        }
    }

    pub fn eq_char(&self, rc: ReservedChar) -> bool {
        match *self {
            Token::Char(c) => c == rc,
            _ => false,
        }
    }

    pub fn eq_operation(&self, ope: Operation) -> bool {
        match *self {
            Token::Operation(o) => o == ope,
            _ => false,
        }
    }

    pub fn is_operation(&self) -> bool {
        matches!(*self, Token::Operation(_))
    }

    pub fn eq_condition(&self, cond: Condition) -> bool {
        match *self {
            Token::Condition(c) => c == cond,
            _ => false,
        }
    }

    pub fn is_condition(&self) -> bool {
        matches!(*self, Token::Condition(_))
    }

    pub fn is_other(&self) -> bool {
        matches!(*self, Token::Other(_))
    }

    pub fn get_other(&self) -> Option<&str> {
        match *self {
            Token::Other(s) => Some(s),
            _ => None,
        }
    }

    pub fn is_white_character(&self) -> bool {
        match *self {
            Token::Char(c) => c.is_white_character(),
            _ => false,
        }
    }

    pub fn is_keyword(&self) -> bool {
        matches!(*self, Token::Keyword(_))
    }

    pub fn get_keyword(&self) -> Option<Keyword> {
        match *self {
            Token::Keyword(k) => Some(k),
            _ => None,
        }
    }

    pub fn is_string(&self) -> bool {
        matches!(*self, Token::String(_))
    }

    pub fn get_string(&self) -> Option<&str> {
        match *self {
            Token::String(s) => Some(s),
            _ => None,
        }
    }

    pub fn is_regex(&self) -> bool {
        matches!(*self, Token::Regex { .. })
    }

    pub fn is_created_var_decl(&self) -> bool {
        matches!(*self, Token::CreatedVarDecl(_))
    }

    pub fn is_created_var(&self) -> bool {
        matches!(*self, Token::CreatedVar(_))
    }

    pub fn is_number(&self) -> bool {
        matches!(*self, Token::Number(_))
    }

    pub fn is_floating_number(&self) -> bool {
        matches!(*self, Token::FloatingNumber(_))
    }

    fn get_required(&self) -> Option<char> {
        match *self {
            Token::Keyword(_)
            | Token::Other(_)
            | Token::CreatedVarDecl(_)
            | Token::Number(_)
            | Token::FloatingNumber(_) => Some(' '),
            _ => None,
        }
    }

    fn requires_before(&self) -> bool {
        match *self {
            Token::Keyword(k) => k.requires_before(),
            _ => false,
        }
    }
}

fn get_line_comment<'a>(
    source: &'a str,
    iterator: &mut MyPeekable<'_>,
    start_pos: &mut usize,
) -> Option<Token<'a>> {
    *start_pos += 1;
    for (pos, c) in iterator {
        if let Ok(c) = ReservedChar::try_from(c) {
            if c == ReservedChar::Backline {
                let ret = Some(Token::Comment(&source[*start_pos..pos]));
                *start_pos = pos;
                return ret;
            }
        }
    }
    None
}

fn get_regex<'a>(
    source: &'a str,
    iterator: &mut MyPeekable<'_>,
    start_pos: &mut usize,
    v: &[Token<'_>],
) -> Option<Token<'a>> {
    let mut back = v.len();
    while back > 0 {
        back -= 1;
        if v[back].is_white_character() || v[back].is_comment() || v[back].is_license() {
            continue;
        }
        match &v[back] {
            Token::Char(ReservedChar::SemiColon)
            | Token::Char(ReservedChar::Colon)
            | Token::Char(ReservedChar::Comma)
            | Token::Char(ReservedChar::OpenBracket)
            | Token::Char(ReservedChar::OpenParenthese)
            | Token::Char(ReservedChar::ExclamationMark)
            | Token::Char(ReservedChar::OpenCurlyBrace)
            | Token::Char(ReservedChar::QuestionMark)
            | Token::Char(ReservedChar::Backline)
            | Token::Char(ReservedChar::Pipe)
            | Token::Char(ReservedChar::Ampersand) => break,
            t if t.is_operation() || t.is_condition() => break,
            _ => return None,
        }
    }
    iterator.start_save();
    while let Some((pos, c)) = iterator.next() {
        if c == '\\' {
            // we skip next character
            iterator.next();
            continue;
        }
        if let Ok(c) = ReservedChar::try_from(c) {
            if c == ReservedChar::Slash {
                let mut is_global = false;
                let mut is_interactive = false;
                let mut add = 0;
                loop {
                    match iterator.peek() {
                        Some((_, 'i')) => is_interactive = true,
                        Some((_, 'g')) => is_global = true,
                        _ => break,
                    };
                    iterator.next();
                    add += 1;
                }
                let ret = Some(Token::Regex {
                    regex: &source[*start_pos + 1..pos],
                    is_interactive,
                    is_global,
                });
                *start_pos = pos + add;
                iterator.drop_save();
                return ret;
            } else if c == ReservedChar::Backline {
                break;
            }
        }
    }
    iterator.stop_save();
    None
}

fn get_comment<'a>(
    source: &'a str,
    iterator: &mut MyPeekable<'_>,
    start_pos: &mut usize,
) -> Token<'a> {
    let mut prev = ReservedChar::Quote;
    *start_pos += 1;
    let builder = if let Some((_, c)) = iterator.next() {
        if c == '!' {
            *start_pos += 1;
            Token::License
        } else {
            if let Ok(c) = ReservedChar::try_from(c) {
                prev = c;
            }
            Token::Comment
        }
    } else {
        Token::Comment
    };

    let mut current_pos = *start_pos;
    for (pos, c) in iterator {
        current_pos = pos;
        if let Ok(c) = ReservedChar::try_from(c) {
            if c == ReservedChar::Slash && prev == ReservedChar::Star {
                current_pos -= 2;
                break;
            }
            prev = c;
        } else {
            prev = ReservedChar::Space;
        }
    }
    // Unclosed comment so returning it anyway...
    let ret = builder(&source[*start_pos..=current_pos]);
    *start_pos = current_pos + 2;
    ret
}

fn get_string<'a>(
    source: &'a str,
    iterator: &mut MyPeekable<'_>,
    start_pos: &mut usize,
    start: ReservedChar,
) -> Option<Token<'a>> {
    while let Some((pos, c)) = iterator.next() {
        if c == '\\' {
            // we skip next character
            iterator.next();
            continue;
        }
        if let Ok(c) = ReservedChar::try_from(c) {
            if c == start {
                let ret = Some(Token::String(&source[*start_pos..pos + 1]));
                *start_pos = pos;
                return ret;
            }
        }
    }
    None
}

fn get_backtick_string<'a>(
    source: &'a str,
    iterator: &mut MyPeekable<'_>,
    start_pos: &mut usize,
) -> Option<Token<'a>> {
    while let Some((pos, c)) = iterator.next() {
        if c == '\\' {
            // we skip next character
            iterator.next();
            continue;
        }
        if c == '$' && iterator.peek().map(|(_, c)| c == '{').unwrap_or(false) {
            let mut count = 0;

            loop {
                if let Some((mut pos, c)) = iterator.next() {
                    if c == '\\' {
                        // we skip next character
                        iterator.next();
                        continue;
                    } else if c == '"' || c == '\'' {
                        // We don't care about the result
                        get_string(
                            source,
                            iterator,
                            &mut pos,
                            ReservedChar::try_from(c)
                                .expect("ReservedChar::try_from unexpectedly failed..."),
                        );
                    } else if c == '`' {
                        get_backtick_string(source, iterator, &mut pos);
                    } else if c == '{' {
                        count += 1;
                    } else if c == '}' {
                        count -= 1;
                        if count == 0 {
                            break;
                        }
                    }
                } else {
                    return None;
                }
            }
        } else if c == '`' {
            let ret = Some(Token::String(&source[*start_pos..pos + 1]));
            *start_pos = pos;
            return ret;
        }
    }
    None
}

fn first_useful<'a>(v: &'a [Token<'a>]) -> Option<&'a Token<'a>> {
    for x in v.iter().rev() {
        if x.is_white_character() {
            continue;
        }
        return Some(x);
    }
    None
}

fn fill_other<'a>(source: &'a str, v: &mut Vec<Token<'a>>, start: usize, pos: usize) {
    if start < pos {
        if let Ok(w) = Keyword::try_from(&source[start..pos]) {
            v.push(Token::Keyword(w));
        } else if let Ok(n) = usize::from_str(&source[start..pos]) {
            v.push(Token::Number(n))
        } else if f64::from_str(&source[start..pos]).is_ok() {
            v.push(Token::FloatingNumber(&source[start..pos]))
        } else {
            v.push(Token::Other(&source[start..pos]));
        }
    }
}

fn handle_equal_sign(v: &mut Vec<Token<'_>>, c: ReservedChar) -> bool {
    if c != ReservedChar::EqualSign {
        return false;
    }
    match v.last().unwrap_or(&Token::Other("")) {
        Token::Operation(Operation::Equal) => {
            v.pop();
            v.push(Token::Condition(Condition::EqualTo));
        }
        Token::Condition(Condition::EqualTo) => {
            v.pop();
            v.push(Token::Condition(Condition::SuperEqualTo));
        }
        Token::Char(ReservedChar::ExclamationMark) => {
            v.pop();
            v.push(Token::Condition(Condition::DifferentThan));
        }
        Token::Condition(Condition::DifferentThan) => {
            v.pop();
            v.push(Token::Condition(Condition::SuperDifferentThan));
        }
        Token::Operation(Operation::Divide) => {
            v.pop();
            v.push(Token::Operation(Operation::DivideEqual));
        }
        Token::Operation(Operation::Multiply) => {
            v.pop();
            v.push(Token::Operation(Operation::MultiplyEqual));
        }
        Token::Operation(Operation::Addition) => {
            v.pop();
            v.push(Token::Operation(Operation::AdditionEqual));
        }
        Token::Operation(Operation::Subtract) => {
            v.pop();
            v.push(Token::Operation(Operation::SubtractEqual));
        }
        Token::Operation(Operation::Modulo) => {
            v.pop();
            v.push(Token::Operation(Operation::ModuloEqual));
        }
        Token::Condition(Condition::SuperiorThan) => {
            v.pop();
            v.push(Token::Condition(Condition::SuperiorOrEqualTo));
        }
        Token::Condition(Condition::InferiorThan) => {
            v.pop();
            v.push(Token::Condition(Condition::InferiorOrEqualTo));
        }
        _ => {
            return false;
        }
    }
    true
}

fn check_if_number(iterator: &mut MyPeekable<'_>, start: usize, pos: usize, source: &str) -> bool {
    if source[start..pos].find('.').is_some() {
        return false;
    } else if u64::from_str(&source[start..pos]).is_ok() {
        return true;
    } else if let Some((_, x)) = iterator.peek() {
        return x as u8 >= b'0' && x as u8 <= b'9';
    }
    false
}

struct MyPeekable<'a> {
    inner: CharIndices<'a>,
    saved: Vec<(usize, char)>,
    peeked: Option<(usize, char)>,
    is_saving: bool,
}

impl<'a> MyPeekable<'a> {
    fn new(indices: CharIndices<'a>) -> MyPeekable<'a> {
        MyPeekable {
            inner: indices,
            saved: Vec::with_capacity(500),
            peeked: None,
            is_saving: false,
        }
    }

    fn start_save(&mut self) {
        self.is_saving = true;
        if let Some(p) = self.peeked {
            self.saved.push(p);
        }
    }

    fn drop_save(&mut self) {
        self.is_saving = false;
        self.saved.clear();
    }

    fn stop_save(&mut self) {
        self.is_saving = false;
        if let Some(p) = self.peeked {
            self.saved.push(p);
        }
        self.peeked = None;
    }

    /// Returns None if saving.
    fn peek(&mut self) -> Option<(usize, char)> {
        if self.peeked.is_none() {
            self.peeked = self.inner.next();
            if self.is_saving {
                if let Some(p) = self.peeked {
                    self.saved.push(p);
                }
            }
        }
        self.peeked
    }
}

impl<'a> Iterator for MyPeekable<'a> {
    type Item = (usize, char);

    fn next(&mut self) -> Option<Self::Item> {
        if self.peeked.is_some() {
            self.peeked.take()
        } else {
            if !self.is_saving && !self.saved.is_empty() {
                return Some(self.saved.remove(0));
            }
            match self.inner.next() {
                Some(r) if self.is_saving => {
                    self.saved.push(r);
                    Some(r)
                }
                r => r,
            }
        }
    }
}

pub fn tokenize(source: &str) -> Tokens<'_> {
    let mut v = Vec::with_capacity(1000);
    let mut start = 0;
    let mut iterator = MyPeekable::new(source.char_indices());

    loop {
        let (mut pos, c) = match iterator.next() {
            Some(x) => x,
            None => {
                fill_other(source, &mut v, start, source.len());
                break;
            }
        };
        if let Ok(c) = ReservedChar::try_from(c) {
            if c == ReservedChar::Dot && check_if_number(&mut iterator, start, pos, source) {
                let mut cont = true;
                if let Some(x) = iterator.peek() {
                    if !"0123456789,; \t\n<>/*&|{}[]-+=~%^:!".contains(x.1) {
                        fill_other(source, &mut v, start, pos);
                        start = pos;
                        cont = false;
                    }
                }
                if cont {
                    continue;
                }
            }
            fill_other(source, &mut v, start, pos);
            match c {
                ReservedChar::Quote | ReservedChar::DoubleQuote => {
                    if let Some(s) = get_string(source, &mut iterator, &mut pos, c) {
                        v.push(s);
                    }
                }
                ReservedChar::BackTick => {
                    if let Some(s) = get_backtick_string(source, &mut iterator, &mut pos) {
                        v.push(s);
                    }
                }
                ReservedChar::Slash
                    if v.last()
                        .unwrap_or(&Token::Other(""))
                        .eq_operation(Operation::Divide) =>
                {
                    v.pop();
                    if let Some(s) = get_line_comment(source, &mut iterator, &mut pos) {
                        v.push(s);
                    }
                }
                ReservedChar::Slash
                    if iterator.peek().is_some()
                        && iterator.peek().unwrap().1 != '/'
                        && iterator.peek().unwrap().1 != '*'
                        && !first_useful(&v).unwrap_or(&Token::String("")).is_other() =>
                {
                    if let Some(r) = get_regex(source, &mut iterator, &mut pos, &v) {
                        v.push(r);
                    } else {
                        v.push(Token::Operation(Operation::Divide));
                    }
                }
                ReservedChar::Star
                    if v.last()
                        .unwrap_or(&Token::Other(""))
                        .eq_operation(Operation::Divide) =>
                {
                    v.pop();
                    v.push(get_comment(source, &mut iterator, &mut pos));
                }
                ReservedChar::Pipe
                    if v.last()
                        .unwrap_or(&Token::Other(""))
                        .eq_char(ReservedChar::Pipe) =>
                {
                    v.pop();
                    v.push(Token::Condition(Condition::Or));
                }
                ReservedChar::Ampersand
                    if v.last()
                        .unwrap_or(&Token::Other(""))
                        .eq_char(ReservedChar::Ampersand) =>
                {
                    v.pop();
                    v.push(Token::Condition(Condition::And));
                }
                _ if handle_equal_sign(&mut v, c) => {}
                _ => {
                    if let Ok(o) = Operation::try_from(c) {
                        v.push(Token::Operation(o));
                    } else if let Ok(o) = Condition::try_from(c) {
                        v.push(Token::Condition(o));
                    } else {
                        v.push(Token::Char(c));
                    }
                }
            }
            start = pos + 1;
        }
    }
    Tokens(v)
}

#[derive(Debug, PartialEq, Eq)]
pub struct Tokens<'a>(pub Vec<Token<'a>>);

macro_rules! tokens_writer {
    ($self:ident, $w:ident) => {
        let tokens = &$self.0;
        for i in 0..tokens.len() {
            if i > 0
                && tokens[i].requires_before()
                && !tokens[i - 1].is_keyword()
                && !tokens[i - 1].is_other()
                && !tokens[i - 1].is_reserved_char()
                && !tokens[i - 1].is_string()
            {
                write!($w, " ")?;
            }
            write!($w, "{}", tokens[i])?;
            if let Some(c) = match tokens[i] {
                Token::Keyword(_) | Token::Other(_) if i + 1 < tokens.len() => {
                    tokens[i + 1].get_required()
                }
                _ => None,
            } {
                write!($w, "{}", c)?;
            }
        }
    };
}

impl<'a> Tokens<'a> {
    pub(super) fn write<W: std::io::Write>(self, mut w: W) -> std::io::Result<()> {
        tokens_writer!(self, w);
        Ok(())
    }
}

impl<'a> fmt::Display for Tokens<'a> {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        tokens_writer!(self, f);
        Ok(())
    }
}

impl<'a> Tokens<'a> {
    #[must_use]
    pub fn apply<F>(self, func: F) -> Tokens<'a>
    where
        F: Fn(Tokens<'a>) -> Tokens<'a>,
    {
        func(self)
    }
}

impl<'a> IntoIterator for Tokens<'a> {
    type Item = Token<'a>;
    type IntoIter = std::vec::IntoIter<Token<'a>>;

    fn into_iter(self) -> Self::IntoIter {
        self.0.into_iter()
    }
}

impl<'a> std::ops::Deref for Tokens<'a> {
    type Target = Vec<Token<'a>>;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

impl<'a> From<Vec<Token<'a>>> for Tokens<'a> {
    fn from(v: Vec<Token<'a>>) -> Self {
        Tokens(v)
    }
}

impl<'a> From<&[Token<'a>]> for Tokens<'a> {
    fn from(v: &[Token<'a>]) -> Self {
        Tokens(v.to_vec())
    }
}

#[test]
fn check_regex() {
    let source = r#"var x = /"\.x/g;"#;
    let expected_result = r#"var x=/"\.x/g"#;
    assert_eq!(crate::js::minify(source).to_string(), expected_result);

    let v = tokenize(source).apply(crate::js::clean_tokens);
    assert_eq!(
        v.0[3],
        Token::Regex {
            regex: "\"\\.x",
            is_global: true,
            is_interactive: false,
        }
    );

    let source = r#"var x = /"\.x/gigigigig;var x = "hello";"#;
    let expected_result = r#"var x=/"\.x/gi;var x="hello""#;
    assert_eq!(crate::js::minify(source).to_string(), expected_result);

    let v = tokenize(source).apply(crate::js::clean_tokens);
    assert_eq!(
        v.0[3],
        Token::Regex {
            regex: "\"\\.x",
            is_global: true,
            is_interactive: true,
        }
    );
}

#[test]
fn more_regex() {
    let source = r#"var x = /"\.x\/a/i;"#;
    let expected_result = r#"var x=/"\.x\/a/i"#;
    assert_eq!(crate::js::minify(source).to_string(), expected_result);

    let v = tokenize(source).apply(crate::js::clean_tokens);
    assert_eq!(
        v.0[3],
        Token::Regex {
            regex: "\"\\.x\\/a",
            is_global: false,
            is_interactive: true,
        }
    );

    let source = r#"var x = /\\/i;"#;
    let expected_result = r#"var x=/\\/i"#;
    assert_eq!(crate::js::minify(source).to_string(), expected_result);

    let v = tokenize(source).apply(crate::js::clean_tokens);
    assert_eq!(
        v.0[3],
        Token::Regex {
            regex: "\\\\",
            is_global: false,
            is_interactive: true,
        }
    );
}

#[test]
fn even_more_regex() {
    let source = r#"var x = /a-z /;"#;

    let v = tokenize(source).apply(crate::js::clean_tokens);
    assert_eq!(
        v.0[3],
        Token::Regex {
            regex: "a-z ",
            is_global: false,
            is_interactive: false,
        }
    );
}

#[test]
fn not_regex_test() {
    let source = "( x ) / 2; x / y;x /= y";

    let v = tokenize(source).apply(crate::js::clean_tokens);
    assert_eq!(
        &v.0,
        &[
            Token::Char(ReservedChar::OpenParenthese),
            Token::Other("x"),
            Token::Char(ReservedChar::CloseParenthese),
            Token::Operation(Operation::Divide),
            Token::Number(2),
            Token::Char(ReservedChar::SemiColon),
            Token::Other("x"),
            Token::Operation(Operation::Divide),
            Token::Other("y"),
            Token::Char(ReservedChar::SemiColon),
            Token::Other("x"),
            Token::Operation(Operation::DivideEqual),
            Token::Other("y")
        ]
    );

    let source = "let x = /x\ny/;";

    let v = tokenize(source).apply(crate::js::clean_tokens);
    assert_eq!(
        &v.0,
        &[
            Token::Keyword(Keyword::Let),
            Token::Other("x"),
            Token::Operation(Operation::Equal),
            Token::Operation(Operation::Divide),
            Token::Other("x"),
            Token::Other("y"),
            Token::Operation(Operation::Divide)
        ]
    );
}

#[test]
fn test_tokens_parsing() {
    let source = "true = == 2.3 === 32";

    let v = tokenize(source).apply(crate::js::clean_tokens);
    assert_eq!(
        &v.0,
        &[
            Token::Keyword(Keyword::True),
            Token::Operation(Operation::Equal),
            Token::Condition(Condition::EqualTo),
            Token::FloatingNumber("2.3"),
            Token::Condition(Condition::SuperEqualTo),
            Token::Number(32)
        ]
    );
}

#[test]
fn test_string_parsing() {
    let source = "var x = 'hello people!'";

    let v = tokenize(source).apply(crate::js::clean_tokens);
    assert_eq!(
        &v.0,
        &[
            Token::Keyword(Keyword::Var),
            Token::Other("x"),
            Token::Operation(Operation::Equal),
            Token::String("\'hello people!\'")
        ]
    );
}

#[test]
fn test_number_parsing() {
    let source = "var x = .12; let y = 4.; var z = 12; .3 4. 'a' let u = 12.2";

    let v = tokenize(source).apply(crate::js::clean_tokens);
    assert_eq!(
        &v.0,
        &[
            Token::Keyword(Keyword::Var),
            Token::Other("x"),
            Token::Operation(Operation::Equal),
            Token::FloatingNumber(".12"),
            Token::Char(ReservedChar::SemiColon),
            Token::Keyword(Keyword::Let),
            Token::Other("y"),
            Token::Operation(Operation::Equal),
            Token::FloatingNumber("4."),
            Token::Char(ReservedChar::SemiColon),
            Token::Keyword(Keyword::Var),
            Token::Other("z"),
            Token::Operation(Operation::Equal),
            Token::Number(12),
            Token::Char(ReservedChar::SemiColon),
            Token::FloatingNumber(".3"),
            Token::FloatingNumber("4."),
            Token::String("'a'"),
            Token::Keyword(Keyword::Let),
            Token::Other("u"),
            Token::Operation(Operation::Equal),
            Token::FloatingNumber("12.2")
        ]
    );
}

#[test]
fn test_number_parsing2() {
    let source = "var x = 12.a;";

    let v = tokenize(source).apply(crate::js::clean_tokens);
    assert_eq!(
        &v.0,
        &[
            Token::Keyword(Keyword::Var),
            Token::Other("x"),
            Token::Operation(Operation::Equal),
            Token::Number(12),
            Token::Char(ReservedChar::Dot),
            Token::Other("a")
        ]
    );
}

#[test]
fn tokens_spaces() {
    let source = "t in e";

    let v = tokenize(source).apply(crate::js::clean_tokens);
    assert_eq!(
        &v.0,
        &[
            Token::Other("t"),
            Token::Keyword(Keyword::In),
            Token::Other("e")
        ]
    );
}

#[test]
fn division_by_id() {
    let source = "100/abc";

    let v = tokenize(source).apply(crate::js::clean_tokens);
    assert_eq!(
        &v.0,
        &[
            Token::Number(100),
            Token::Operation(Operation::Divide),
            Token::Other("abc")
        ]
    );
}

#[test]
fn weird_regex() {
    let source = "if (!/\\/(contact|legal)\\//.test(a)) {}";

    let v = tokenize(source).apply(crate::js::clean_tokens);
    assert_eq!(
        &v.0,
        &[
            Token::Keyword(Keyword::If),
            Token::Char(ReservedChar::OpenParenthese),
            Token::Char(ReservedChar::ExclamationMark),
            Token::Regex {
                regex: "\\/(contact|legal)\\/",
                is_global: false,
                is_interactive: false
            },
            Token::Char(ReservedChar::Dot),
            Token::Other("test"),
            Token::Char(ReservedChar::OpenParenthese),
            Token::Other("a"),
            Token::Char(ReservedChar::CloseParenthese),
            Token::Char(ReservedChar::CloseParenthese),
            Token::Char(ReservedChar::OpenCurlyBrace),
            Token::Char(ReservedChar::CloseCurlyBrace),
        ]
    );
}

#[test]
fn test_regexes() {
    let source = "/\\/(contact|legal)\\//.test";

    let v = tokenize(source).apply(crate::js::clean_tokens);
    assert_eq!(
        &v.0,
        &[
            Token::Regex {
                regex: "\\/(contact|legal)\\/",
                is_global: false,
                is_interactive: false
            },
            Token::Char(ReservedChar::Dot),
            Token::Other("test"),
        ]
    );

    let source = "/\\*(contact|legal)/.test";

    let v = tokenize(source).apply(crate::js::clean_tokens);
    assert_eq!(
        &v.0,
        &[
            Token::Regex {
                regex: "\\*(contact|legal)",
                is_global: false,
                is_interactive: false
            },
            Token::Char(ReservedChar::Dot),
            Token::Other("test"),
        ]
    );
}

#[test]
fn test_comments() {
    let source = "/*(contact|legal)/.test";

    let v = tokenize(source);
    assert_eq!(&v.0, &[Token::Comment("(contact|legal)/.test"),],);

    let source = "/*(contact|legal)/.test*/ a";

    let v = tokenize(source);
    assert_eq!(
        &v.0,
        &[
            Token::Comment("(contact|legal)/.test"),
            Token::Char(ReservedChar::Space),
            Token::Other("a"),
        ],
    );
}