From f75b887abbc8eb40e347467d4b36a372cdbe376a Mon Sep 17 00:00:00 2001 From: stuartgo Date: Tue, 20 Oct 2026 09:20:18 +0200 Subject: [PATCH] get rid of old embedding and word based system --- anansyTypes/__init__.py | 36 +++------ anansyTypes/enums.py | 26 ++----- anansyTypes/schemas.py | 112 +++++++++++----------------- anansyTypes/structures/__init__.py | 8 -- anansyTypes/structures/embedding.py | 37 --------- anansyTypes/structures/lexeme.py | 26 ------- anansyTypes/structures/phrase.py | 20 ----- anansyTypes/structures/symbol.py | 9 +-- anansyTypes/structures/word.py | 8 -- pyproject.toml | 2 +- 10 files changed, 63 insertions(+), 221 deletions(-) delete mode 100644 anansyTypes/structures/embedding.py delete mode 100644 anansyTypes/structures/lexeme.py delete mode 100644 anansyTypes/structures/phrase.py delete mode 100644 anansyTypes/structures/word.py diff --git a/anansyTypes/__init__.py b/anansyTypes/__init__.py index 2ed458e..44c8d7c 100644 --- a/anansyTypes/__init__.py +++ b/anansyTypes/__init__.py @@ -1,32 +1,20 @@ from .enums import * from .schemas import * -from .structures import Embedding, Word, Symbol, Phrase,Lexeme +from .structures import Symbol __all__ = [ "enums", "schemas", - "Embedding", - "Word", "Symbol", - "Phrase", - "Lexeme", - "EmbedRequest", - "EmbeddingResponse", "ResponseRequest", "PreprocessRequest", - "PreprocessResponse", - "GetSymbolRequest", - "GetSymbolResponse", - "GetWordRequest", - "GetWordResponse", - "GetSimilarWordRequest", - "GetAllPhrasesResponse", "GetSimilarSymbolsRequest", - "GetEmbeddingSimilarSymbolsRequest", "SearchSymbolsRequest", "GetSymbolsResponse", "ReplaceFinishedRequest", + "TextToSymbolRequest", + "TextToSymbolResponse", "SymbolLeaf", "SentenceWord", "WordsPreprocessResponse", @@ -34,11 +22,14 @@ __all__ = [ "WordLookup", "LookupRequest", "LookupResponse", - "JevQuestion", - "JevRequest", - "JevOption", - "JevAnswer", - "JevResponse", + "SomQuestion", + "SomRequest", + "SomOption", + "SomAnswer", + "SomCall", + "SomResponse", + "SomStep", + "TranslationTrace", "GetLeafSymbolsRequest", "UsageStatusResponse", "ModelType", @@ -50,8 +41,3 @@ __all__ = [ "Currency", "SubscriptionType" ] - - - - - diff --git a/anansyTypes/enums.py b/anansyTypes/enums.py index 248bd70..2c601de 100644 --- a/anansyTypes/enums.py +++ b/anansyTypes/enums.py @@ -1,10 +1,7 @@ -import anansyTypes from enum import Enum -from typing import Union class ModelType(Enum): - EMBEDDING = "embedding" TRANSCRIPTION = "transcription" RESPONSE = "response" @@ -13,10 +10,6 @@ class ModelName(Enum): pass -class EmbeddingModelName(ModelName): - NB_BERT = "nb_bert" - - class TranscriptionModelName(ModelName): NB_WHISPER_SMALL = "nb_whisper_small" @@ -26,15 +19,12 @@ class ResponseModelName(ModelName): class PreprocessorType(Enum): - NORWEGIAN = "norwegian" - DEFAULT = "norwegian" - NORWEGIAN_WORDLIST="norwegian_wordlist" - ENGLISH="english" + NORWEGIAN_WORDLIST = "norwegian_wordlist" + ENGLISH = "english" class SymbolTranslatorType(Enum): - EMBEDDING = "embedding" - JEV = "jev" + SOM = "som" # the system one model, choosing symbols from the hierarchy class SymbolProvider(Enum): @@ -47,20 +37,16 @@ class Language(Enum): DEFAULT = "en" -class ResourceType(Enum): - WORD = "word" - SYMBOL = "symbol" - class SubscriptionType(Enum): FREE = "free" PAID_MONTH = "paid_month" PAID_YEAR = "paid_year" - + class Currency(Enum): EUR = "EUR" USD = "USD" NOK = "NOK" - + class DiscountType(Enum): PERCENTAGE = "percentage" - ABSOLUTE = "absolute" \ No newline at end of file + ABSOLUTE = "absolute" diff --git a/anansyTypes/schemas.py b/anansyTypes/schemas.py index 9c5902b..7a61deb 100644 --- a/anansyTypes/schemas.py +++ b/anansyTypes/schemas.py @@ -1,24 +1,14 @@ from pydantic import BaseModel as PydanticBaseModel -from typing import List, Any, Dict, Optional,Union +from typing import Any, List, Dict, Optional -from .structures import Embedding,Word, Symbol,Phrase -from pydantic import field_validator, ConfigDict -from pydantic_tensor import Tensor -import torch -from typing import Any, Literal +from .enums import SymbolProvider +from .structures import Symbol +from pydantic import ConfigDict +from typing import Literal class ModifiedBaseModel(PydanticBaseModel): pass - #model_config = ConfigDict(arbitrary_types_allowed=True) -class EmbeddingRequest(ModifiedBaseModel): - stemmed_words: List[str] - tokens: dict - word_ids: List[int] - -class EmbeddingResponse(ModifiedBaseModel): - embeddings: List[Embedding] - class ResponseRequest(ModifiedBaseModel): text: str context: Dict[str, Any] @@ -26,40 +16,9 @@ class ResponseRequest(ModifiedBaseModel): class PreprocessRequest(ModifiedBaseModel): text: str -class PreprocessResponse(ModifiedBaseModel): - raw_words: List[str] - stemmed_words: List[str] - tokens: Dict[str, Any] - word_ids: List[int] - -class GetSymbolRequest(ModifiedBaseModel): - lexeme_id: int - class GetSymbolsResponse(ModifiedBaseModel): symbols: List[Symbol] -class GetSymbolResponse(ModifiedBaseModel): - symbol: Optional[Symbol] - -class GetWordRequest(ModifiedBaseModel): - word_text: str - stemmed: bool = False - -class GetWordResponse(ModifiedBaseModel): - word: Optional[Word] - -class GetSimilarWordRequest(ModifiedBaseModel): - embedding: Embedding - -class GetAllPhrasesResponse(ModifiedBaseModel): - phrases: Optional[List[Phrase]] - -class GetFullPhrasesRequest(ModifiedBaseModel): - phrases: List[Phrase] - -class GetFullPhrasesResponse(ModifiedBaseModel): - phrases: Optional[List[Phrase]] - class GetSimilarSymbolsRequest(ModifiedBaseModel): """/replace: symbols to offer instead of `symbol`, which was shown for `word` in `sentence`.""" limit: int = 3 @@ -68,13 +27,6 @@ class GetSimilarSymbolsRequest(ModifiedBaseModel): sentence:str language: Optional[str] = None # fallback language for a caller with no account config (see ConfigFetcher) -class GetEmbeddingSimilarSymbolsRequest(ModifiedBaseModel): - """Symbols nearest an embedding, leaving out the given words and symbols.""" - embedding: Embedding - limit: int = 3 - exclude_word_ids: List[int] = [] - exclude_symbol_ids: List[int] = [] - class SearchSymbolsRequest(ModifiedBaseModel): query: str limit: int = 8 @@ -94,13 +46,11 @@ class AudioToSymbolRequest(ModifiedBaseModel): class AudioToSymbolResponse(ModifiedBaseModel): symbols: List[Symbol] sentence: str - + class TextToSymbolRequest(ModifiedBaseModel): text: str - language:str - -class TextToSymbolResponse(ModifiedBaseModel): - symbols: List[Symbol] + language: Optional[str] = None # fallback language for a caller with no account config (see ConfigFetcher) + trace: bool = False # also return the SOM requests and responses, with timings (not in production) class ReplaceFinishedRequest(ModifiedBaseModel): """A correction: the sentence as heard, the word, and the symbols before and after (images left empty).""" @@ -112,8 +62,8 @@ class ReplaceFinishedRequest(ModifiedBaseModel): rank: int # the replacement's position in the list it was picked from, from 0 query: Optional[str] = None # the search text, for source "search" language: Optional[str] = None # fallback language for a caller with no account config (see ConfigFetcher) - -#### Symbol hierarchy (jev translator) #### + +#### Symbol hierarchy (SOM translator) #### class SymbolLeaf(ModifiedBaseModel): """One leaf of the symbol hierarchy. A label with several parts ("bat|club") gives one leaf per part.""" @@ -147,7 +97,7 @@ class LookupResponse(ModifiedBaseModel): words: List[WordLookup] # one per requested word people: List[SymbolLeaf] # the generic person symbols offered for names -class JevQuestion(ModifiedBaseModel): +class SomQuestion(ModifiedBaseModel): """choose: which of `leaves` matches the word. search: which child of `category_id` (None: the top) does.""" name: str kind: Literal["choose", "search"] @@ -157,22 +107,48 @@ class JevQuestion(ModifiedBaseModel): leaves: List[SymbolLeaf] = [] category_id: Optional[str] = None -class JevRequest(ModifiedBaseModel): +class SomRequest(ModifiedBaseModel): sentence: str - questions: List[JevQuestion] + questions: List[SomQuestion] + trace: bool = False # return the calls made to the SOM API -class JevOption(ModifiedBaseModel): +class SomOption(ModifiedBaseModel): """A symbol or a category the model gave probability p.""" p: float leaf: Optional[SymbolLeaf] = None category_id: Optional[str] = None -class JevAnswer(ModifiedBaseModel): - options: List[JevOption] = [] +class SomAnswer(ModifiedBaseModel): + options: List[SomOption] = [] p_none: float = 0.0 # probability that none of the options matches -class JevResponse(ModifiedBaseModel): - answers: Dict[str, JevAnswer] # question name -> answer +class SomCall(ModifiedBaseModel): + """One HTTP call to the SOM API, as the backend made it.""" + request: dict # the payload sent: model, state (the sentence) and questions + response: dict # the API's answer, as received + status: int + attempts: int # 1 unless it was retried + duration_ms: float # including retries + +class SomResponse(ModifiedBaseModel): + answers: Dict[str, SomAnswer] # question name -> answer + calls: Optional[List[SomCall]] = None # with trace: one per batch of questions + +class SomStep(ModifiedBaseModel): + """One step of a translation: the questions asked together, their answers, and the API calls behind them.""" + questions: List[SomQuestion] + answers: Dict[str, SomAnswer] + calls: List[SomCall] = [] + duration_ms: float # the round trip from anansyLogic, through the backend, to the SOM API + +class TranslationTrace(ModifiedBaseModel): + provider: SymbolProvider + timings_ms: Dict[str, float] = {} # preprocess, lookup, som (every step), symbols, total + steps: List[SomStep] = [] + +class TextToSymbolResponse(ModifiedBaseModel): + symbols: List[Symbol] + trace: Optional[TranslationTrace] = None # only when asked for, and allowed (ALLOW_TRACE) class GetLeafSymbolsRequest(ModifiedBaseModel): leaves: List[SymbolLeaf] diff --git a/anansyTypes/structures/__init__.py b/anansyTypes/structures/__init__.py index 7ac324a..dca16cc 100644 --- a/anansyTypes/structures/__init__.py +++ b/anansyTypes/structures/__init__.py @@ -1,14 +1,6 @@ -from .embedding import Embedding -from .lexeme import Lexeme -from .word import Word from .symbol import Symbol -from .phrase import Phrase __all__ = [ - "Embedding", - "Lexeme", - "Word", "Symbol", - "Phrase", ] diff --git a/anansyTypes/structures/embedding.py b/anansyTypes/structures/embedding.py deleted file mode 100644 index b396863..0000000 --- a/anansyTypes/structures/embedding.py +++ /dev/null @@ -1,37 +0,0 @@ -from typing import Any -import torch -from pydantic import BaseModel -from pydantic_core import core_schema -from typing_extensions import Annotated -import numpy as np - -class TensorPydantic: - @classmethod - def __get_pydantic_core_schema__(cls, source, handler): - def validate(v: Any): - if isinstance(v, torch.Tensor): - return v - if isinstance(v, list): - return torch.tensor(v, dtype=torch.float32) - raise TypeError(f"Cannot convert {type(v)} to torch.Tensor") - - return core_schema.json_or_python_schema( - json_schema=core_schema.list_schema(core_schema.any_schema()), - python_schema=core_schema.no_info_plain_validator_function(validate), - serialization=core_schema.plain_serializer_function_ser_schema( - lambda t: t.detach().cpu().tolist() - ), - ) - - -Tensor = Annotated[torch.Tensor, TensorPydantic] - - -class Embedding(BaseModel): - data: Tensor - - @classmethod - def from_dict(cls, data: dict) -> "Embedding": - return cls(data=torch.tensor(data["data"], dtype=torch.float32)) - - \ No newline at end of file diff --git a/anansyTypes/structures/lexeme.py b/anansyTypes/structures/lexeme.py deleted file mode 100644 index f460b33..0000000 --- a/anansyTypes/structures/lexeme.py +++ /dev/null @@ -1,26 +0,0 @@ -from abc import ABC, abstractmethod -from typing import List, Union, Any -from dataclasses import dataclass - -from pydantic import BaseModel -from .embedding import Embedding - -class Lexeme(BaseModel,ABC): - id: int - content: Union[str, List[str]] - original_content: Union[str, List[str]] - embedding: Embedding - - - @classmethod - def from_dict(cls, data: dict) -> "Lexeme": - return cls( - id=data["id"], - content=data["content"], - embedding=Embedding.from_dict(data["embedding"]), - original_content=data["original_content"], - ) - def __new__(cls, *args, **kwargs): - if cls is Lexeme: - raise TypeError("Lexeme cannot be instantiated, use a subclass instead") - return super().__new__(cls) \ No newline at end of file diff --git a/anansyTypes/structures/phrase.py b/anansyTypes/structures/phrase.py deleted file mode 100644 index bb4333f..0000000 --- a/anansyTypes/structures/phrase.py +++ /dev/null @@ -1,20 +0,0 @@ -from typing import List, Any -from pydantic import model_validator -from pydantic_core import core_schema -import torch - -from .lexeme import Lexeme -from .embedding import Embedding -import numpy as np - - -class Phrase(Lexeme): - - @classmethod - def from_dict(cls, data: dict) -> "Phrase": - return cls( - id=data["id"], - content=data["content"], - original_content=data["original_content"], - embedding=Embedding.from_dict(data["embedding"]), - ) diff --git a/anansyTypes/structures/symbol.py b/anansyTypes/structures/symbol.py index 28d7e44..2ddb2b6 100644 --- a/anansyTypes/structures/symbol.py +++ b/anansyTypes/structures/symbol.py @@ -1,11 +1,4 @@ -from dataclasses import dataclass -from typing import Any -import numpy as np from pydantic import BaseModel -from pydantic_tensor import Tensor -from typing import Any, Literal -from pydantic_core import core_schema -import base64 class Symbol(BaseModel): @@ -19,4 +12,4 @@ class Symbol(BaseModel): id=data["id"], label=data["label"], image=data["image"], - ) \ No newline at end of file + ) diff --git a/anansyTypes/structures/word.py b/anansyTypes/structures/word.py deleted file mode 100644 index b4dc073..0000000 --- a/anansyTypes/structures/word.py +++ /dev/null @@ -1,8 +0,0 @@ -from .lexeme import Lexeme -from typing import Any -from .embedding import Embedding -from pydantic_core import core_schema - - -class Word(Lexeme): - pass \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 1445105..1d2afae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ classifiers = [ "Operating System :: OS Independent" ] dependencies=[ - "pydantic-tensor" + "pydantic" ] [build-system] requires = ["setuptools>=65.5.1", "wheel" ]