Scanner Skeleton PreviousNext

note

    description: "General lexical analyzers"
    library: "Gobo Eiffel Lexical Library"
    copyright: "Copyright (c) 2001-2019, Eric Bezault and others"
    license: "MIT License"

deferred class YY_SCANNER

create

    make
            -- Create a new scanner with
            -- standard input as input file.

    make_with_file (a_file: KI_CHARACTER_INPUT_STREAM)
            -- Create a new scanner with
            -- a_file as input file.
            -- To be used when a_file contains ISO-8859-1 characters,
            -- or when it is using the UTF-8 encoding and the scanner is
            -- either using the "%option utf8" or has been manually written
            -- to expect sequences of UTF-8 bytes.
        require
            a_file_not_void: a_file /= Void
            a_file_open_read: a_file.is_open_read

    make_with_unicode_file (a_file: KI_CHARACTER_INPUT_STREAM)
            -- Create a new scanner with a_file as input file.
            -- a_file is expected to be encoded in UTF-8
            -- or ISO-8859-1, and the input buffer will handle
            -- the corresponding Unicode characters.
        require
            a_file_not_void: a_file /= Void
            a_file_open_read: a_file.is_open_read

    make_with_buffer (a_buffer: like input_buffer)
            -- Create a new scanner with
            -- a_buffer as input buffer.
        require
            a_buffer_not_void: a_buffer /= Void
        ensure
            input_buffer_set: input_buffer = a_buffer

feature -- Initialization

    reset
            -- Reset scanner before scanning next input source.
            -- (This routine can be called in wrap before scanning
            -- another input buffer.)

    reset_with_file (a_file: KI_CHARACTER_INPUT_STREAM)
            -- Reset scanner before scanning next input source.
            -- Then reuse input_buffer and set it to a_file
            -- if it was a file buffer, create a new file input buffer
            -- with a_file otherwise.
        require
            a_file_not_void: a_file /= Void
            a_file_open_read: a_file.is_open_read

    reset_with_string (a_string: STRING_8)
            -- Reset scanner before scanning next input source.
            -- Then reuse input_buffer and set it to a_string
            -- if it was not Empty_buffer, create a new input buffer
            -- with a_string otherwise.
        require
            a_string_not_void: a_string /= Void

    reset_start_condition
            -- Clear pushed start conditions and set start_condition
            -- to the "INITIAL" start condition.
        ensure
            pushed_start_conditions_cleared: pushed_start_condition_count = 0

feature -- Access

    last_token: INTEGER
            -- Code of last token read
            -- (0 means that the end-of-input has been reached,
            -- non-positive values mean that an error occurred
            -- (see header-comment of scanning_error.))

    text: STRING_8
            -- Text of last token read
            -- (Create a new string at each call.)
            --
            -- Note that if input_buffer contains Unicode characters
            -- which cannot be represented as 8-bit characters, they
            -- will be replaced by a replacement character specified
            -- in the buffer.
        ensure
            text_not_void: Result /= Void
            correct_count: Result.count = text_count

    unicode_text: STRING_32
            -- Unicode text of last token read
            -- (Create a new string at each call.)
            --
            -- Note that if the scanner is written to receive sequences
            -- of UTF-8 bytes, unicode_text will treat each single
            -- byte as a character. It will not try to decode the UTF-8 bytes
            -- into Unicode characters.
            --
            -- Note that unicode_text does not contain surrogate
            -- or invalid Unicode characters.
        ensure
            unicode_text_not_void: Result /= Void
            correct_count: Result.count = text_count

    utf8_text: STRING_8
            -- UTF-8 representation of last token read
            -- (Create a new string at each call.)
            --
            -- Note that unicode_text does not contain surrogate
            -- or invalid Unicode characters, therefore the resulting
            -- string is valid UTF-8.
        ensure
            utf8_text_not_void: Result /= Void
            utf8_text_is_string_8: Result.same_type ({STRING_8} "")
            valid_utf8: {UC_UTF8_ROUTINES}.valid_utf8 (Result)
            correct_count: Result.count = {UC_UTF8_ROUTINES}.string_byte_count (unicode_text)
            definition: Result.is_equal ({UC_UTF8_ROUTINES}.string_to_utf8 (unicode_text))

    text_item (i: INTEGER): CHARACTER_8
            -- i-th character of last token read
        require
            i_large_enough: i >= 1
            i_small_enough: i <= text_count
        ensure
            definition: Result = text.item (i)

    unicode_text_item (i: INTEGER): CHARACTER_32
            -- i-th Unicode character of last token read
            --
            -- Note that unicode_text does not contain surrogate
            -- or invalid Unicode characters.
        require
            i_large_enough: i >= 1
            i_small_enough: i <= text_count
        ensure
            definition: Result = unicode_text.item (i)

    text_substring (s, e: INTEGER): STRING
            -- Substring of last token read
            -- (Create a new string at each call.)
            -- (For efficiency reason, this function can bypass the
            -- call to text and create the substring directly from
            -- the input buffer.)
        require
            meaningful_start: 1 <= s
            meaningful_interval: s <= e + 1
            meaningful_end: e <= text_count
        ensure
            text_substring_not_void: Result /= Void
            text_substring_empty: (s > e) implies Result.is_empty
            definition: Result.is_equal (text.substring (s, e))

    unicode_text_substring (s, e: INTEGER): STRING_32
            -- Unicode substring of last token read
            -- (Create a new string at each call.)
            -- (For efficiency reason, this function can bypass the
            -- call to unicode_text and create the substring directly from
            -- the input buffer.)
            --
            -- Note that unicode_text does not contain surrogate
            -- or invalid Unicode characters.
        require
            meaningful_start: 1 <= s
            meaningful_interval: s <= e + 1
            meaningful_end: e <= text_count
        ensure
            unicode_text_substring_not_void: Result /= Void
            unicode_text_substring_empty: (s > e) implies Result.is_empty
            definition: Result.is_equal (unicode_text.substring (s, e))

    utf8_text_substring (s, e: INTEGER): STRING_8
            -- UTF-8 representation of substring of last token read
            -- (Create a new string at each call.)
            -- (For efficiency reason, this function can bypass the
            -- call to unicode_text and create the substring directly from
            -- the input buffer.)
            --
            -- Note that unicode_text does not contain surrogate
            -- or invalid Unicode characters, therefore the resulting
            -- string is valid UTF-8.
        require
            meaningful_start: 1 <= s
            meaningful_interval: s <= e + 1
            meaningful_end: e <= text_count
        ensure
            utf8_text_not_void: Result /= Void
            utf8_text_is_string_8: Result.same_type ({STRING_8} "")
            valid_utf8: {UC_UTF8_ROUTINES}.valid_utf8 (Result)
            utf8_text_substring_empty: (s > e) implies Result.is_empty
            definition: Result.is_equal ({UC_UTF8_ROUTINES}.string_to_utf8 (unicode_text.substring (s, e)))
            correct_count: Result.count = {UC_UTF8_ROUTINES}.string_byte_count (unicode_text.substring (s, e))

    start_condition: INTEGER
            -- Start condition

feature -- Measurement

    text_count: INTEGER
            -- Length of last token read
        ensure
            text_count_not_negative: Result >= 0

    line: INTEGER
            -- Line number of last token read when
            -- '%option line' has been specified
        ensure
            line_positive: Result >= 1

    column: INTEGER
            -- Column number of last token read when
            -- '%option line' has been specified
        ensure
            column_positive: Result >= 1

    position: INTEGER
            -- Position of last token read (i.e. number of
            -- characters from the start of the input source)
            -- when '%option position' has been specified
        ensure
            position_positive: Result >= 1

    pushed_start_condition_count: INTEGER
            -- Number of start conditions already pushed (and not popped yet)
        ensure
            pushed_start_condition_count_not_negative: Result >= 0

feature -- Status report

    end_of_file: BOOLEAN
            -- Has the end of input buffer been reached?
            -- This means that last_token has been set
            -- to 0 indicating "all done".

    scanning_error: BOOLEAN
            -- Has an error occurred during scanning?
            -- This can occur when too many reject are called (and hence
            -- nothing can be matched anymore) or when the option "nodefault"
            -- (or option -s) has been specified but the default rule is
            -- matched nevertheless.

    valid_start_condition (sc: INTEGER): BOOLEAN
            -- Is sc a valid start condition?

feature -- Setting

    set_last_token (a_token: INTEGER)
            -- Set last_token to a_token.
        ensure
            last_token_set: last_token = a_token

    set_start_condition (a_start_condition: INTEGER)
            -- Set start_condition to a_start_condition.
        require
            valid_start_condition: valid_start_condition (a_start_condition)
        ensure
            start_condition_set: start_condition = a_start_condition

    push_start_condition (a_start_condition: INTEGER)
            -- Set start condition and add previous to stack.
        require
            valid_start_condition: valid_start_condition (a_start_condition)
        ensure
            start_condition_set: start_condition = a_start_condition
            one_more: pushed_start_condition_count = old pushed_start_condition_count + 1

    pop_start_condition
            -- Restore previous start condition.
        require
            has_pushed_start_conditions: pushed_start_condition_count > 0
        ensure
            one_less: pushed_start_condition_count = old pushed_start_condition_count - 1

feature -- Scanning

    scan
            -- Scan input_buffer until end of file is found
            -- or an error occurs.
        ensure
            end_of_file: not scanning_error implies end_of_file

    read_token
            -- Read a token from input_buffer.
            -- Make result available in last_token.

feature -- Element change

    append_text_to_string (a_string: STRING_8)
            -- Append text at end of a_string.
            -- (For efficiency reason, this feature can bypass the
            -- call to text and directly copy the characters from
            -- the input buffer.)
        require
            a_string_not_void: a_string /= Void
        ensure
            count_set: a_string.count = old (a_string.count) + text_count
            definition: a_string.substring (old (a_string.count) + 1, a_string.count).same_string (text)

    append_unicode_text_to_string (a_string: STRING_32)
            -- Append unicode_text at end of a_string.
            -- (For efficiency reason, this feature can bypass the
            -- call to unicode_text and directly copy the characters from
            -- the input buffer.)
        require
            a_string_not_void: a_string /= Void
        ensure
            count_set: a_string.count = old (a_string.count) + text_count
            definition: a_string.substring (old (a_string.count) + 1, a_string.count).same_string (unicode_text)

    append_utf8_text_to_string (a_string: STRING_8)
            -- Append utf8_text at end of a_string.
            -- (For efficiency reason, this feature can bypass the
            -- call to utf8_text and directly copy the characters from
            -- the input buffer.)
        require
            a_string_not_void: a_string /= Void
            a_string_is_string_8: a_string.same_type ({STRING_8} "")
        ensure
            count_set: a_string.count = old (a_string.count) + utf8_text.count
            definition: a_string.substring (old (a_string.count) + 1, a_string.count).same_string (utf8_text)

    append_text_substring_to_string (s, e: INTEGER; a_string: STRING_8)
            -- Append text_substring at end of a_string.
            -- (For efficiency reason, this feature can bypass
            -- the call to text_substring and directly copy
            -- the characters from the input buffer.)
        require
            a_string_not_void: a_string /= Void
            s_large_enough: 1 <= s
            valid_interval: s <= e + 1
            e_small_enough: e <= text_count
        ensure
            count_set: a_string.count = old (a_string.count) + (e - s + 1)
            definition: a_string.substring (old (a_string.count) + 1, a_string.count).same_string (text_substring (s, e))

    append_unicode_text_substring_to_string (s, e: INTEGER; a_string: STRING_32)
            -- Append unicode_text_substring at end of a_string.
            -- (For efficiency reason, this feature can bypass
            -- the call to unicode_text_substring and directly copy
            -- the characters from the input buffer.)
        require
            a_string_not_void: a_string /= Void
            s_large_enough: 1 <= s
            valid_interval: s <= e + 1
            e_small_enough: e <= text_count
        ensure
            count_set: a_string.count = old (a_string.count) + (e - s + 1)
            definition: a_string.substring (old (a_string.count) + 1, a_string.count).same_string (unicode_text_substring (s, e))

    append_utf8_text_substring_to_string (s, e: INTEGER; a_string: STRING_8)
            -- Append utf8_text_substring at end of a_string.
            -- (For efficiency reason, this feature can bypass the
            -- call to utf8_text_substring and directly copy the characters from
            -- the input buffer.)
        require
            a_string_not_void: a_string /= Void
            a_string_is_string_8: a_string.same_type ({STRING_8} "")
            s_large_enough: 1 <= s
            valid_interval: s <= e + 1
            e_small_enough: e <= text_count
        ensure
            count_set: a_string.count = old (a_string.count) + utf8_text_substring (s, e).count
            definition: a_string.substring (old (a_string.count) + 1, a_string.count).same_string (utf8_text_substring (s, e))

    terminate
            -- Terminate scanner and set last_token
            -- to 0 indicating "all done".

    wrap: BOOLEAN
            -- Should current scanner terminate when end of file is reached?
            -- This function can be redefined to switch to another input
            -- buffer (but don't forget to update start_condition).
            -- (Default: True.)

    more
            -- Tell scanner to append the next matched token
            -- to current value of text instead of
            -- replacing it.

    less (n: INTEGER)
            -- Return all but the first n matched
            -- characters back to input_buffer.
        require
            n_large_enough: n >= 0
            n_small_enough: n <= text_count
        ensure
            text_count_set: text_count = n

    unread_character (c: CHARACTER_8)
            -- Put c back to input_buffer. This will alter both
            -- text and the content of input_buffer.

    unread_unicode_character (c: CHARACTER_32)
            -- Put c back to input_buffer. This will alter both
            -- unicode_text and the content of input_buffer.
            -- The behavior is undefined if c is too large to fit into input_buffer.

    read_character
            -- Read a character from input_buffer.
            -- Make result available in last_character and last_unicode_character.
            --
            -- Note that if input_buffer contains Unicode characters
            -- which cannot be represented as 8-bit characters, they
            -- will be replaced by a replacement character specified
            -- in the buffer.

    last_character: CHARACTER_8
            -- Last character read by read_character

    last_unicode_character: CHARACTER_32
            -- Last Unicode character read by read_character

feature -- Input

    input_buffer: YY_BUFFER
            -- Input buffer

    set_input_buffer (a_buffer: like input_buffer)
            -- Set input_buffer to a_buffer.
        require
            a_buffer_not_void: a_buffer /= Void
        ensure
            input_buffer_set: input_buffer = a_buffer

    flush_input_buffer
            -- Flush input_buffer. input_buffer will be automatically
            -- refilled unless end of file has been found.
        ensure
            flushed: input_buffer.count = 0

    new_file_buffer (a_file: KI_CHARACTER_INPUT_STREAM): YY_FILE_BUFFER
            -- New input buffer for a_file.
            -- To be used when a_file contains ISO-8859-1 characters,
            -- or when it is using the UTF-8 encoding and the scanner is
            -- either using the "%option utf8" or has been manually written
            -- to expect sequences of UTF-8 bytes.
        require
            a_file_not_void: a_file /= Void
            a_file_open_read: a_file.is_open_read
        ensure
            new_buffer_not_void: Result /= Void

    new_unicode_file_buffer (a_file: KI_CHARACTER_INPUT_STREAM): YY_UNICODE_FILE_BUFFER
            -- New Unicode input buffer for a_file.
            -- a_file is expected to be encoded in UTF-8
            -- or ISO-8859-1.
            -- The scanner will receive Unicode characters,
            -- not sequences of UTF-8 bytes.
        require
            a_file_not_void: a_file /= Void
            a_file_open_read: a_file.is_open_read
        ensure
            new_unicode_file_buffer_not_void: Result /= Void

    new_utf8_file_buffer (a_file: KI_CHARACTER_INPUT_STREAM): YY_UTF8_FILE_BUFFER
            -- New UTF-8 input buffer for a_file.
            -- To be used when a_file contains ISO-8859-1 characters or when it
            -- is using the UTF-8 encoding, and the scanner is either using the
            -- "%option utf8" or has been manually written to expect sequences
            -- of UTF-8 bytes.
            -- The scanner will receive sequences of UTF-8 bytes.
        require
            a_file_not_void: a_file /= Void
            a_file_open_read: a_file.is_open_read
        ensure
            new_utf8_file_buffer_not_void: Result /= Void

    new_string_buffer (a_string: STRIN_8): YY_BUFFER
            -- New input buffer for a_string.
            -- To be used when a_string contains ISO-8859-1 characters,
            -- or when it is using the UTF-8 encoding and the scanner is
            -- either using the "%option utf8" or has been manually written
            -- to expect sequences of UTF-8 bytes.
        require
            a_string_not_void: a_string /= Void
            a_string_is_string: a_string.same_type ({STRING_8} "")
        ensure
            new_buffer_not_void: Result /= Void

    new_unicode_string_buffer (a_string: READABLE_STRING_GENERAL): YY_UNICODE_BUFFER
            -- New Unicode input buffer for a_string.
            -- To be used when a_string contains ISO-8859-1 or Unicode characters.
            -- The scanner will receive Unicode characters, not sequences of UTF-8 bytes.
        require
            a_string_not_void: a_string /= Void
        ensure
            new_unicode_string_buffer_not_void: Result /= Void

    new_utf8_string_buffer (a_string: READABLE_STRING_GENERAL): YY_UTF8_BUFFER
            -- New UTF-8 input buffer for a_string.
            -- To be used when a_string contains ISO-8859-1 or Unicode characters,
            -- and the scanner is either using the "%option utf8" or has been
            -- manually written to expect sequences of UTF-8 bytes.
            -- a_string is expected to contain valid non-surrogate Unicode
            -- characters. Invalid or surrogate Unicode characters are encoded
            -- with one byte 0xFF (which is an invalid byte in UTF-8).
            -- The scanner will receive sequences of UTF-8 bytes.
        require
            a_string_not_void: a_string /= Void
        ensure
            new_utf8_string_buffer_not_void: Result /= Void

    Empty_buffer: YY_BUFFER
            -- Empty input buffer
        ensure
            empty_buffer_not_void: Result /= Void

feature -- Output

    output (a_text: like text)
            -- Output a_text.
            -- (Note: this routine can be redefined in descendant
            -- classes. Default: print a_text to standard output.)
        require
            a_text_not_void: a_text /= Void

    echo
            -- Output text using feature output.

feature -- Action

    pre_action
            -- Action executed before every semantic action
            -- when '%option pre-action' has been specified.
            -- (Note: this routine can be redefined in descendant
            -- classes. Default: do nothing.)

    post_action
            -- Action executed after every semantic action
            -- when '%option post-action' has been specified.
            -- (Note: this routine can be redefined in descendant
            -- classes. Default: do nothing.)

    pre_eof_action
            -- Action executed before every end-of-file semantic action
            -- (i.e. <<EOF>>) when '%option pre-eof-action' has been specified.
            -- (Note: this routine can be redefined in descendant classes.
            -- Default: do nothing.)

    post_eof_action
            -- Action executed after every end-of-file semantic action
            -- (i.e. <<EOF>>) when '%option post-eof-action' has been specified.
            -- (Note: this routine can be redefined in descendant classes.
            -- Default: do nothing.)

    default_action
            -- Action executed when default rule is matched.
            -- (Note: this routine can be redefined in descendant classes.
            -- Default: print last character read to standard output.)

feature -- Error handling

    fatal_error (a_message: STRING_8)
            -- A fatal error occurred.
            -- Print a_message.
        require
            a_message_not_void: a_message /= Void

    report_invalid_unicode_character_error (a_code: NATURAL_32)
            -- Report that the surrogate or invalid Unicode character
            -- with code a_code has been read from the input
            -- buffer and caused the scanner to fail.

feature -- Debugging

    print_last_token
            -- Print to standard error debug information
            -- about the last token read. Can be redefined
            -- in descendant classes to print more information.
            -- (Called at the end of read_token when compiled
            -- with 'debug ("GELEX")' enabled).

invariant

    input_buffer_not_void: input_buffer /= Void
    valid_start_condition: valid_start_condition (start_condition)

end

Copyright 2001-2019, Eric Bezault and others
mailto:
ericb@gobosoft.com
http:
//www.gobosoft.com
Last Updated: 28 September 2019

HomeTocPreviousNext