cs432_2024_08/p3-docs/token_8h_source.html

#ifndef __TOKENS_H

#define __TOKENS_H


#include "common.h"


typedef regex_t Regex;


Regex* Regex_new (const char* regex);


bool Regex_match (Regex *regex, const char *text, char *match);


void Regex_free (Regex* regex);


typedef enum TokenType {

    ID, DECLIT, HEXLIT, STRLIT, KEY, SYM

} TokenType;


typedef struct Token

{

    TokenType type;


    char text[MAX_TOKEN_LEN];


    int line;


    struct Token* next;


} Token;


const char* TokenType_to_string(TokenType type);


bool token_str_eq(const char* str1, const char* str2);


Token* Token_new (TokenType type, const char* text, int line);


void Token_free (Token* token);


typedef struct TokenQueue

{

    Token* head;


    Token* tail;


} TokenQueue;


TokenQueue* TokenQueue_new (void);


void TokenQueue_add (TokenQueue* queue, Token* token);


Token* TokenQueue_peek (TokenQueue* queue);


Token* TokenQueue_remove (TokenQueue* queue);


bool TokenQueue_is_empty (TokenQueue* queue);


size_t TokenQueue_size (TokenQueue* queue);


void TokenQueue_print (TokenQueue* queue, FILE* out);


void TokenQueue_free (TokenQueue* queue);


#endif

common.h
Includes, constants, declarations, and macros used across the compiler.

MAX_TOKEN_LEN
#define MAX_TOKEN_LEN
Maximum length (in characters) of any single token.
Definition common.h:47

Token
Single token.
Definition token.h:79

Token::next
struct Token * next
Pointer to next token (used to store in a list)
Definition token.h:98

Token::line
int line
Source line number.
Definition token.h:93

Token::type
TokenType type
Type of the token.
Definition token.h:83

Token::text
char text[256]
Raw text of the token.
Definition token.h:88

TokenQueue
Linked list of tokens.
Definition token.h:153

TokenQueue::tail
Token * tail
Back of list (or NULL if list is empty)
Definition token.h:162

TokenQueue::head
Token * head
Front of list (or NULL if list is empty)
Definition token.h:157

TokenQueue_print
void TokenQueue_print(TokenQueue *queue, FILE *out)
Print a queue to the given file descriptor (debug output)
Definition token.c:120

TokenQueue_new
TokenQueue * TokenQueue_new(void)
Allocate and initialize a new, empty queue of tokens.
Definition token.c:65

TokenQueue_size
size_t TokenQueue_size(TokenQueue *queue)
Calculate size of the queue.
Definition token.c:111

TokenType_to_string
const char * TokenType_to_string(TokenType type)
Convert a token type to a string for output.
Definition token.c:31

Token_free
void Token_free(Token *token)
Deallocate a token.
Definition token.c:60

Regex_new
Regex * Regex_new(const char *regex)
Allocate and compile a new regular expression.
Definition token.c:3

Regex_match
bool Regex_match(Regex *regex, const char *text, char *match)
Match a regular expression against some text.
Definition token.c:12

TokenQueue_free
void TokenQueue_free(TokenQueue *queue)
Deallocate a token queue.
Definition token.c:129

token_str_eq
bool token_str_eq(const char *str1, const char *str2)
Check string equality for tokens. Limits comparison to MAX_TOKEN_LEN for safety.
Definition token.c:44

TokenQueue_is_empty
bool TokenQueue_is_empty(TokenQueue *queue)
Check whether a queue is empty.
Definition token.c:106

TokenType
TokenType
Valid token types.
Definition token.h:69

Token_new
Token * Token_new(TokenType type, const char *text, int line)
Allocate and initialize a new token.
Definition token.c:49

Regex
regex_t Regex
Compiled regular expression.
Definition token.h:25

Regex_free
void Regex_free(Regex *regex)
Deallocate a regular expression.
Definition token.c:25

TokenQueue_peek
Token * TokenQueue_peek(TokenQueue *queue)
Return the next token from a queue without removing it (first-in-first-out)
Definition token.c:85

TokenQueue_add
void TokenQueue_add(TokenQueue *queue, Token *token)
Add a token to a queue.
Definition token.c:72

TokenQueue_remove
Token * TokenQueue_remove(TokenQueue *queue)
Remove a token from a queue (first-in-first-out)
Definition token.c:90