From 1f455afedf7225660eeca567d85e4453e268e0cc Mon Sep 17 00:00:00 2001
From: Ziyang Hu <hu.ziyang@cantab.net>
Date: Mon, 22 May 2023 13:29:56 +0800
Subject: [PATCH] clear tokenizer hash when removing indices

---
 cozo-core/src/fts/mod.rs          | 4 ++--
 cozo-core/src/runtime/relation.rs | 8 ++++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/cozo-core/src/fts/mod.rs b/cozo-core/src/fts/mod.rs
index c0973b16..046bbadb 100644
--- a/cozo-core/src/fts/mod.rs
+++ b/cozo-core/src/fts/mod.rs
@@ -245,8 +245,8 @@ pub(crate) struct FtsIndexConfig {
 
 #[derive(Default)]
 pub(crate) struct TokenizerCache {
-    named_cache: RwLock<HashMap<SmartString<LazyCompact>, Arc<TextAnalyzer>>>,
-    hashed_cache: RwLock<HashMap<Vec<u8>, Arc<TextAnalyzer>>>,
+    pub(crate) named_cache: RwLock<HashMap<SmartString<LazyCompact>, Arc<TextAnalyzer>>>,
+    pub(crate) hashed_cache: RwLock<HashMap<Vec<u8>, Arc<TextAnalyzer>>>,
 }
 
 impl TokenizerCache {
diff --git a/cozo-core/src/runtime/relation.rs b/cozo-core/src/runtime/relation.rs
index d4abf6dd..187c014f 100644
--- a/cozo-core/src/runtime/relation.rs
+++ b/cozo-core/src/runtime/relation.rs
@@ -32,8 +32,8 @@ use crate::query::compile::IndexPositionUse;
 use crate::runtime::hnsw::HnswIndexManifest;
 use crate::runtime::minhash_lsh::{HashPermutations, LshParams, MinHashLshIndexManifest, Weights};
 use crate::runtime::transact::SessionTx;
-use crate::{NamedRows, StoreTx};
 use crate::utils::TempCollector;
+use crate::{NamedRows, StoreTx};
 
 #[derive(
     Copy,
@@ -833,7 +833,6 @@ impl<'a> SessionTx<'a> {
 
         let mut stack = vec![];
 
-
         let hash_perms = manifest.get_hash_perms();
         let mut existing = TempCollector::default();
         for tuple in rel_handle.scan_all(self) {
@@ -1379,6 +1378,11 @@ impl<'a> SessionTx<'a> {
     ) -> Result<Vec<(Vec<u8>, Vec<u8>)>> {
         let mut rel = self.get_relation(rel_name, true)?;
         let is_lsh = rel.lsh_indices.contains_key(&idx_name.name);
+        let is_fts = rel.fts_indices.contains_key(&idx_name.name);
+        if is_lsh || is_fts {
+            self.tokenizers.named_cache.write().unwrap().clear();
+            self.tokenizers.hashed_cache.write().unwrap().clear();
+        }
         if rel.indices.remove(&idx_name.name).is_none()
             && rel.hnsw_indices.remove(&idx_name.name).is_none()
             && rel.lsh_indices.remove(&idx_name.name).is_none()