added original_content better

anansyTypes/structures/lexeme.py

@@ -13,6 +13,12 @@ class Lexeme(ABC):
     def content(self) -> Union[str, List[str]]:
         """Text content of the lexeme."""
         pass
+    
+    @property
+    @abstractmethod
+    def original_content(self) -> Union[str, List[str]]:
+        """Original text content of the lexeme."""
+        pass
 
     @property
     @abstractmethod
@@ -43,6 +49,7 @@ class Lexeme(ABC):
                 lambda x: {
                     "id": x.id,
                     "content": x.content,
+                    "original_content": x.original_content,
                     "embedding": {"data": x.embedding.data.tolist()},
                 },
                 when_used="json",
@@ -53,6 +60,7 @@ class Lexeme(ABC):
             json_schema=core_schema.typed_dict_schema(
                 {
                     "id": core_schema.typed_dict_field(core_schema.int_schema()),
+                    "original_content": core_schema.typed_dict_field(core_schema.str_schema()),
                     "content": core_schema.typed_dict_field(core_schema.str_schema()),
                     "embedding": core_schema.typed_dict_field(
                         handler.generate_schema(Embedding)

anansyTypes/structures/phrase.py

@@ -15,6 +15,11 @@ class NoEmbeddingPhrase(Lexeme):
     @property
     def content(self) -> str:
         return self._content
+    
+    @property
+    def original_content(self) -> str:
+        #temporarily return content as original_content
+        return self._content
 
     @property
     def embedding(self) -> Embedding: