dimgigov
805dbc8ad6
Critical fixes: - B-Tree splitChild: fixed out-of-bounds on child.children.len (mid+1..len → mid+1..<len) - Wire protocol: all deserializeValue cases now properly assign to result - Wire protocol: fkBool/fkInt8 now increment pos after reading - MVCC: write-write conflict now detected for completed transactions - MVCC: commit only marks visible versions as deleted (not concurrent uncommitted) - Lexer: buffer overrun on backslash escape at end of input — added bounds check - Bulgarian/Russian stemmers: fixed UTF-8 byte offsets for Cyrillic suffixes (2 bytes per char) High fixes: - WebSocket: added missing std/tables import - Raft: added randomize() call for election timeouts - MemTable.get: binary search O(log n) instead of linear scan O(n) Medium fixes: - Zerocopy: string length now written as 4-byte int32 (was single byte) - Raft appendEntries: guard against nextIdx underflow - Columnar: null tracking with nulls seq on ColumnPtr - Removed dead code in lexer readIdent (redundant true/false checks) - Updated Bulgarian stemming test to match corrected byte offsets All 214 tests pass across 48 suites with 0 failures.
214 lines
7.9 KiB
Nim
214 lines
7.9 KiB
Nim
## Multi-Language FTS — tokenizers for different languages
|
||
import std/tables
|
||
import std/unicode
|
||
import std/strutils
|
||
import std/sets
|
||
|
||
type
|
||
Language* = enum
|
||
langEnglish = "en"
|
||
langSpanish = "es"
|
||
langFrench = "fr"
|
||
langGerman = "de"
|
||
langRussian = "ru"
|
||
langBulgarian = "bg"
|
||
langChinese = "zh"
|
||
langJapanese = "ja"
|
||
langArabic = "ar"
|
||
langAuto = "auto"
|
||
|
||
Stemmer* = proc(word: string): string {.gcsafe.}
|
||
StopWords* = HashSet[string]
|
||
|
||
LanguageConfig* = object
|
||
language*: Language
|
||
stemmer*: Stemmer
|
||
stopWords*: StopWords
|
||
tokenizer*: proc(text: string): seq[string] {.gcsafe.}
|
||
|
||
const
|
||
stopWordsEn* = [
|
||
"a", "an", "the", "is", "it", "in", "on", "at", "to", "for",
|
||
"of", "with", "by", "from", "as", "into", "through", "during",
|
||
"and", "or", "not", "but", "if", "then", "else", "when",
|
||
"be", "was", "were", "been", "being", "have", "has", "had",
|
||
"do", "does", "did", "will", "would", "could", "should", "may",
|
||
"this", "that", "these", "those", "i", "you", "he", "she", "we", "they",
|
||
]
|
||
|
||
stopWordsBg* = [
|
||
"и", "в", "на", "за", "от", "да", "се", "е", "са", "по",
|
||
"не", "че", "с", "към", "но", "или", "ако", "при", "до",
|
||
"как", "какво", "кой", "коя", "кое", "кои", "този", "тази",
|
||
"това", "тези", "със", "между", "след", "преди", "без",
|
||
"още", "вече", "само", "може", "трябва", "има", "няма",
|
||
]
|
||
|
||
stopWordsDe* = [
|
||
"der", "die", "das", "ein", "eine", "und", "oder", "aber", "nicht",
|
||
"ist", "sind", "war", "hat", "haben", "werden", "wird", "kann",
|
||
"mit", "von", "für", "auf", "in", "an", "zu", "bei", "nach",
|
||
"über", "unter", "vor", "zwischen", "durch", "gegen", "ohne",
|
||
]
|
||
|
||
stopWordsFr* = [
|
||
"le", "la", "les", "un", "une", "des", "et", "ou", "mais", "pas",
|
||
"est", "sont", "était", "a", "ont", "sera", "peut", "avec",
|
||
"de", "du", "pour", "sur", "dans", "par", "entre", "sous", "chez",
|
||
"je", "tu", "il", "elle", "nous", "vous", "ils", "elles",
|
||
]
|
||
|
||
stopWordsRu* = [
|
||
"и", "в", "на", "не", "что", "он", "я", "с", "а", "как",
|
||
"это", "по", "но", "они", "мы", "за", "от", "из", "у", "к",
|
||
"бы", "ты", "его", "её", "их", "её", "мой", "твой", "наш",
|
||
"ваш", "свой", "который", "этот", "тот", "такой", "каждый",
|
||
]
|
||
|
||
proc stemEnglish*(word: string): string =
|
||
if word.len <= 3: return word
|
||
if word.endsWith("ing"): return word[0..^4]
|
||
if word.endsWith("tion"): return word[0..^5]
|
||
if word.endsWith("ness"): return word[0..^5]
|
||
if word.endsWith("ment"): return word[0..^5]
|
||
if word.endsWith("able"): return word[0..^5]
|
||
if word.endsWith("ies"): return word[0..^4] & "y"
|
||
if word.endsWith("es") and word.len > 4: return word[0..^3]
|
||
if word.endsWith("ed") and word.len > 4: return word[0..^3]
|
||
if word.endsWith("ly") and word.len > 4: return word[0..^3]
|
||
if word.endsWith("s") and not word.endsWith("ss") and word.len > 3: return word[0..^2]
|
||
return word
|
||
|
||
proc stemBulgarian*(word: string): string =
|
||
if word.len <= 3: return word
|
||
# Bulgarian suffixes (all Cyrillic chars are 2 bytes)
|
||
# 3 Cyrillic chars = 6 bytes -> ^7
|
||
if word.endsWith("ища"): return word[0..^7]
|
||
if word.endsWith("ище"): return word[0..^7]
|
||
if word.endsWith("ция"): return word[0..^7]
|
||
if word.endsWith("ние"): return word[0..^7]
|
||
if word.endsWith("ост"): return word[0..^7]
|
||
if word.endsWith("ски"): return word[0..^7]
|
||
# 4 Cyrillic chars = 8 bytes -> ^9
|
||
if word.endsWith("ство"): return word[0..^9]
|
||
# 2 Cyrillic chars = 4 bytes -> ^5
|
||
if word.endsWith("на"): return word[0..^5]
|
||
if word.endsWith("та"): return word[0..^5]
|
||
return word
|
||
|
||
proc stemGerman*(word: string): string =
|
||
if word.len <= 3: return word
|
||
if word.endsWith("ung"): return word[0..^4]
|
||
if word.endsWith("heit"): return word[0..^5]
|
||
if word.endsWith("keit"): return word[0..^5]
|
||
if word.endsWith("lich"): return word[0..^5]
|
||
if word.endsWith("isch"): return word[0..^5]
|
||
if word.endsWith("chen"): return word[0..^5]
|
||
if word.endsWith("schaft"): return word[0..^7]
|
||
if word.endsWith("en"): return word[0..^3]
|
||
if word.endsWith("er"): return word[0..^3]
|
||
if word.endsWith("es"): return word[0..^3]
|
||
return word
|
||
|
||
proc stemFrench*(word: string): string =
|
||
if word.len <= 3: return word
|
||
if word.endsWith("ement"): return word[0..^6]
|
||
if word.endsWith("ment"): return word[0..^5]
|
||
if word.endsWith("tion"): return word[0..^5]
|
||
if word.endsWith("eur"): return word[0..^4]
|
||
if word.endsWith("euse"): return word[0..^5]
|
||
if word.endsWith("ique"): return word[0..^5]
|
||
if word.endsWith("esse"): return word[0..^5]
|
||
if word.endsWith("eux"): return word[0..^4]
|
||
if word.endsWith("er"): return word[0..^3]
|
||
if word.endsWith("es"): return word[0..^3]
|
||
return word
|
||
|
||
proc stemRussian*(word: string): string =
|
||
if word.len <= 3: return word
|
||
# Russian suffixes (all Cyrillic chars are 2 bytes)
|
||
# 4 Cyrillic chars = 8 bytes -> ^9
|
||
if word.endsWith("ость"): return word[0..^9]
|
||
if word.endsWith("ение"): return word[0..^9]
|
||
if word.endsWith("ание"): return word[0..^9]
|
||
if word.endsWith("тель"): return word[0..^9]
|
||
if word.endsWith("ский"): return word[0..^9]
|
||
# 3 Cyrillic chars = 6 bytes -> ^7
|
||
if word.endsWith("ция"): return word[0..^7]
|
||
if word.endsWith("ние"): return word[0..^7]
|
||
if word.endsWith("ать"): return word[0..^7]
|
||
if word.endsWith("ить"): return word[0..^7]
|
||
if word.endsWith("ыть"): return word[0..^7]
|
||
return word
|
||
|
||
proc getLanguageConfig*(lang: Language): LanguageConfig =
|
||
case lang
|
||
of langEnglish:
|
||
LanguageConfig(language: lang, stemmer: stemEnglish,
|
||
stopWords: stopWordsEn.toHashSet())
|
||
of langBulgarian:
|
||
LanguageConfig(language: lang, stemmer: stemBulgarian,
|
||
stopWords: stopWordsBg.toHashSet())
|
||
of langGerman:
|
||
LanguageConfig(language: lang, stemmer: stemGerman,
|
||
stopWords: stopWordsDe.toHashSet())
|
||
of langFrench:
|
||
LanguageConfig(language: lang, stemmer: stemFrench,
|
||
stopWords: stopWordsFr.toHashSet())
|
||
of langRussian:
|
||
LanguageConfig(language: lang, stemmer: stemRussian,
|
||
stopWords: stopWordsRu.toHashSet())
|
||
else:
|
||
LanguageConfig(language: lang, stemmer: stemEnglish,
|
||
stopWords: stopWordsEn.toHashSet())
|
||
|
||
proc tokenize*(text: string, config: LanguageConfig): seq[string] =
|
||
result = @[]
|
||
var word = ""
|
||
for ch in text:
|
||
let cp = ord(ch)
|
||
# Accept ASCII alphanumeric, Cyrillic, CJK, Arabic, and common separators
|
||
if (cp >= 0x30 and cp <= 0x39) or # digits
|
||
(cp >= 0x41 and cp <= 0x5A) or # A-Z
|
||
(cp >= 0x61 and cp <= 0x7A) or # a-z
|
||
(cp >= 0xC0 and cp <= 0xFF) or # Latin Extended + Cyrillic start
|
||
(cp >= 0x400 and cp <= 0x4FF) or # Cyrillic
|
||
ch == '_' or ch == '-':
|
||
word &= ch
|
||
else:
|
||
if word.len > 0:
|
||
var token = word.toLower()
|
||
if config.stemmer != nil:
|
||
token = config.stemmer(token)
|
||
if token.len >= 2 and token notin config.stopWords:
|
||
result.add(token)
|
||
word = ""
|
||
if word.len > 0:
|
||
var token = word.toLower()
|
||
if config.stemmer != nil:
|
||
token = config.stemmer(token)
|
||
if token.len >= 2 and token notin config.stopWords:
|
||
result.add(token)
|
||
|
||
proc detectLanguage*(text: string): Language =
|
||
# Simple heuristic based on byte patterns
|
||
# UTF-8 Cyrillic: bytes 0xD0-0xD1 followed by 0x80-0xBF
|
||
var cyrillicCount = 0
|
||
var latinCount = 0
|
||
var i = 0
|
||
let bytes = cast[seq[byte]](text)
|
||
while i < bytes.len:
|
||
let b = bytes[i]
|
||
if b >= 0xD0'u8 and b <= 0xD1'u8 and i + 1 < bytes.len and bytes[i+1] >= 0x80'u8:
|
||
inc cyrillicCount
|
||
i += 2
|
||
elif b >= 0x41'u8 and b <= 0x7A'u8:
|
||
inc latinCount
|
||
i += 1
|
||
else:
|
||
i += 1
|
||
|
||
if cyrillicCount > latinCount:
|
||
return langRussian # Could be BG or RU — default to RU
|
||
return langEnglish
|