+(defun ez-encode (string)
+ "Encodes a string as a 2-word 5-padded Z-string.
+Used for matching input with dictionary entries when tokenizing."
+ (let ((chars
+ (mapcan
+ (lambda (c)
+ (cond
+ ((seq-contains-p (elt ez-zstring-alphabets 0) c)
+ (list (seq-position (elt ez-zstring-alphabets 0) c)))
+ ((seq-contains-p (elt ez-zstring-alphabets 1) c)
+ (list 4 (seq-position (elt ez-zstring-alphabets 1) c)))
+ ((seq-contains-p (elt ez-zstring-alphabets 2) c)
+ (list 5 (seq-position (elt ez-zstring-alphabets 2) c)))
+ (t
+ (list 5 6 (lsh c -5) (logand c #b11111)))))
+ string)))
+ (if (< (length chars) 6)
+ (append chars (make-list (- 6 (length chars)) 5))
+ (take 6 chars))))
+
+;; Dictionary
+
+(defun ez-parse-dictionary ()
+ (let* ((nseps (ez-mem-ref-byte ez-dict-base))
+ (separators
+ (mapcar (lambda (i) (ez-mem-ref-byte (+ ez-dict-base i)))
+ (number-sequence 1 nseps)))
+ (bytes-per-entry (ez-mem-ref-byte (+ ez-dict-base 1 nseps)))
+ (nentries (ez-mem-ref-word (+ ez-dict-base 2 nseps)))
+ (entries-base (+ ez-dict-base nseps 4))
+ (entries nil))
+ (dotimes (i nentries)
+ (let ((this-base (+ entries-base (* 7 i))))
+ (setq entries (cons (cons (ez-get-zstring this-base)
+ this-base)
+ entries))))
+ (setq ez-dict-entries (reverse entries))
+ (setq ez-dict-separators separators)))
+