mirror of
https://github.com/blackboxprogramming/lucidia.git
synced 2026-03-17 06:57:10 -05:00
15 lines
442 B
Python
15 lines
442 B
Python
"""Tokenizer wrapper for Lucidia."""
|
|
|
|
|
|
class TokenizerWrapper:
|
|
def __init__(self, tokenizer):
|
|
self.tokenizer = tokenizer
|
|
|
|
def encode(self, text):
|
|
"""Encode text into token ids (placeholder)."""
|
|
raise NotImplementedError("Encode method not implemented")
|
|
|
|
def decode(self, token_ids):
|
|
"""Decode token ids into text (placeholder)."""
|
|
raise NotImplementedError("Decode method not implemented")
|