compliance-scanner-agent/compliance-agent/src/pipeline/plc/lexer.rs

//! Lexer for IEC 61131-3 Structured Text.
//!
//! Tokenizes ST source into a flat token stream with 1-based line numbers.
//! Keywords are case-insensitive. Handles `(* *)` and `//` comments, `'..'` and
//! `".."` strings (with `''`/`""` escapes), based integers (`16#FF`, `2#1010`),
//! and duration/date literals (`T#5s`, `DT#...`) kept as raw text.

/// A lexed token with its source line.
#[derive(Debug, Clone)]
pub struct Token {
    pub kind: Tok,
    pub line: u32,
}

#[derive(Debug, Clone, PartialEq)]
pub enum Tok {
    Int(i64),
    Real(f64),
    Str(String),
    Time(String),
    Bool(bool),
    Ident(String),
    Kw(Keyword),
    Assign, // :=
    Plus,   // +
    Minus,  // -
    Star,   // *
    Slash,  // /
    Power,  // **
    LParen, // (
    RParen, // )
    LBrack, // [
    RBrack, // ]
    Dot,    // .
    DotDot, // ..
    Comma,  // ,
    Semi,   // ;
    Colon,  // :
    Lt,     // <
    Le,     // <=
    Gt,     // >
    Ge,     // >=
    Eq,     // =
    Ne,     // <>
    Amp,    // &
    Eof,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Keyword {
    Program,
    EndProgram,
    Function,
    EndFunction,
    FunctionBlock,
    EndFunctionBlock,
    Var,
    VarInput,
    VarOutput,
    VarInOut,
    VarGlobal,
    VarTemp,
    VarExternal,
    Constant,
    EndVar,
    Array,
    Of,
    If,
    Then,
    Elsif,
    Else,
    EndIf,
    Case,
    EndCase,
    For,
    To,
    By,
    Do,
    EndFor,
    While,
    EndWhile,
    Repeat,
    Until,
    EndRepeat,
    Return,
    Exit,
    Jmp,
    Not,
    And,
    Or,
    Xor,
    Mod,
    Type,
    EndType,
    Struct,
    EndStruct,
}

fn keyword_from(word: &str) -> Option<Keyword> {
    use Keyword::*;
    Some(match word.to_ascii_uppercase().as_str() {
        "PROGRAM" => Program,
        "END_PROGRAM" => EndProgram,
        "FUNCTION" => Function,
        "END_FUNCTION" => EndFunction,
        "FUNCTION_BLOCK" => FunctionBlock,
        "END_FUNCTION_BLOCK" => EndFunctionBlock,
        "VAR" => Var,
        "VAR_INPUT" => VarInput,
        "VAR_OUTPUT" => VarOutput,
        "VAR_IN_OUT" => VarInOut,
        "VAR_GLOBAL" => VarGlobal,
        "VAR_TEMP" => VarTemp,
        "VAR_EXTERNAL" => VarExternal,
        "CONSTANT" => Constant,
        "END_VAR" => EndVar,
        "ARRAY" => Array,
        "OF" => Of,
        "IF" => If,
        "THEN" => Then,
        "ELSIF" => Elsif,
        "ELSE" => Else,
        "END_IF" => EndIf,
        "CASE" => Case,
        "END_CASE" => EndCase,
        "FOR" => For,
        "TO" => To,
        "BY" => By,
        "DO" => Do,
        "END_FOR" => EndFor,
        "WHILE" => While,
        "END_WHILE" => EndWhile,
        "REPEAT" => Repeat,
        "UNTIL" => Until,
        "END_REPEAT" => EndRepeat,
        "RETURN" => Return,
        "EXIT" => Exit,
        "JMP" => Jmp,
        "NOT" => Not,
        "AND" => And,
        "OR" => Or,
        "XOR" => Xor,
        "MOD" => Mod,
        "TYPE" => Type,
        "END_TYPE" => EndType,
        "STRUCT" => Struct,
        "END_STRUCT" => EndStruct,
        _ => return None,
    })
}

/// Tokenize `src`. Unknown characters are skipped (best-effort — a scanner must
/// not die on odd input).
pub fn lex(src: &str) -> Vec<Token> {
    let chars: Vec<char> = src.chars().collect();
    let mut i = 0usize;
    let mut line = 1u32;
    let mut out = Vec::new();

    let bump_line = |c: char, line: &mut u32| {
        if c == '\n' {
            *line += 1;
        }
    };

    while i < chars.len() {
        let c = chars[i];

        // Whitespace.
        if c.is_whitespace() {
            bump_line(c, &mut line);
            i += 1;
            continue;
        }

        // Line comment: //
        if c == '/' && i + 1 < chars.len() && chars[i + 1] == '/' {
            while i < chars.len() && chars[i] != '\n' {
                i += 1;
            }
            continue;
        }

        // Block comment: (* ... *)
        if c == '(' && i + 1 < chars.len() && chars[i + 1] == '*' {
            i += 2;
            while i + 1 < chars.len() && !(chars[i] == '*' && chars[i + 1] == ')') {
                bump_line(chars[i], &mut line);
                i += 1;
            }
            i = (i + 2).min(chars.len());
            continue;
        }

        let tok_line = line;

        // String literal: '...' or "..."
        if c == '\'' || c == '"' {
            let quote = c;
            i += 1;
            let mut s = String::new();
            while i < chars.len() {
                let ch = chars[i];
                if ch == quote {
                    // Doubled quote is an escaped quote.
                    if i + 1 < chars.len() && chars[i + 1] == quote {
                        s.push(quote);
                        i += 2;
                        continue;
                    }
                    i += 1;
                    break;
                }
                bump_line(ch, &mut line);
                s.push(ch);
                i += 1;
            }
            out.push(Token {
                kind: Tok::Str(s),
                line: tok_line,
            });
            continue;
        }

        // Identifier / keyword / time literal / boolean.
        if c.is_ascii_alphabetic() || c == '_' {
            let start = i;
            while i < chars.len() && (chars[i].is_ascii_alphanumeric() || chars[i] == '_') {
                i += 1;
            }
            let word: String = chars[start..i].iter().collect();

            // Duration/date/time literal prefix: T#, TIME#, DT#, D#, TOD#, LT# ...
            if i < chars.len() && chars[i] == '#' {
                let up = word.to_ascii_uppercase();
                if matches!(
                    up.as_str(),
                    "T" | "TIME" | "DT" | "D" | "TOD" | "LT" | "DATE"
                ) {
                    let lit_start = start;
                    i += 1; // consume '#'
                    while i < chars.len()
                        && (chars[i].is_ascii_alphanumeric()
                            || chars[i] == '.'
                            || chars[i] == '_'
                            || chars[i] == ':')
                    {
                        i += 1;
                    }
                    let lit: String = chars[lit_start..i].iter().collect();
                    out.push(Token {
                        kind: Tok::Time(lit),
                        line: tok_line,
                    });
                    continue;
                }
            }

            let kind = match word.to_ascii_uppercase().as_str() {
                "TRUE" => Tok::Bool(true),
                "FALSE" => Tok::Bool(false),
                _ => match keyword_from(&word) {
                    Some(kw) => Tok::Kw(kw),
                    None => Tok::Ident(word),
                },
            };
            out.push(Token {
                kind,
                line: tok_line,
            });
            continue;
        }

        // Number: decimal, real, or based (16#..., 2#...).
        if c.is_ascii_digit() {
            let start = i;
            while i < chars.len() && (chars[i].is_ascii_digit() || chars[i] == '_') {
                i += 1;
            }
            // Based literal: <base>#<digits>
            if i < chars.len() && chars[i] == '#' {
                let base_str: String = chars[start..i].iter().filter(|c| **c != '_').collect();
                i += 1;
                let dstart = i;
                while i < chars.len() && (chars[i].is_ascii_alphanumeric() || chars[i] == '_') {
                    i += 1;
                }
                let digits: String = chars[dstart..i].iter().filter(|c| **c != '_').collect();
                let radix = base_str.parse::<u32>().unwrap_or(10);
                let val = i64::from_str_radix(&digits, radix.clamp(2, 36)).unwrap_or(0);
                out.push(Token {
                    kind: Tok::Int(val),
                    line: tok_line,
                });
                continue;
            }
            // Real: has a '.' (not '..') or exponent.
            let is_real =
                i < chars.len() && chars[i] == '.' && !(i + 1 < chars.len() && chars[i + 1] == '.');
            if is_real {
                i += 1;
                while i < chars.len() && (chars[i].is_ascii_digit() || chars[i] == '_') {
                    i += 1;
                }
                let raw: String = chars[start..i].iter().filter(|c| **c != '_').collect();
                out.push(Token {
                    kind: Tok::Real(raw.parse().unwrap_or(0.0)),
                    line: tok_line,
                });
                continue;
            }
            let raw: String = chars[start..i].iter().filter(|c| **c != '_').collect();
            out.push(Token {
                kind: Tok::Int(raw.parse().unwrap_or(0)),
                line: tok_line,
            });
            continue;
        }

        // Operators / punctuation (longest match first).
        let two: String = chars[i..(i + 2).min(chars.len())].iter().collect();
        let kind = match two.as_str() {
            ":=" => Some(Tok::Assign),
            "<=" => Some(Tok::Le),
            ">=" => Some(Tok::Ge),
            "<>" => Some(Tok::Ne),
            ".." => Some(Tok::DotDot),
            "**" => Some(Tok::Power),
            _ => None,
        };
        if let Some(k) = kind {
            out.push(Token {
                kind: k,
                line: tok_line,
            });
            i += 2;
            continue;
        }
        let one = match c {
            '+' => Some(Tok::Plus),
            '-' => Some(Tok::Minus),
            '*' => Some(Tok::Star),
            '/' => Some(Tok::Slash),
            '(' => Some(Tok::LParen),
            ')' => Some(Tok::RParen),
            '[' => Some(Tok::LBrack),
            ']' => Some(Tok::RBrack),
            '.' => Some(Tok::Dot),
            ',' => Some(Tok::Comma),
            ';' => Some(Tok::Semi),
            ':' => Some(Tok::Colon),
            '<' => Some(Tok::Lt),
            '>' => Some(Tok::Gt),
            '=' => Some(Tok::Eq),
            '&' => Some(Tok::Amp),
            _ => None,
        };
        if let Some(k) = one {
            out.push(Token {
                kind: k,
                line: tok_line,
            });
        }
        i += 1;
    }

    out.push(Token {
        kind: Tok::Eof,
        line,
    });
    out
}