Added refunded to responses

anansyTypes/__init__.py

@@ -1,32 +1,20 @@
 from .enums import *
 from .schemas import *
-from .structures import Embedding, Word, Symbol, Phrase,Lexeme
+from .structures import Symbol
 
 
 __all__ = [
     "enums",
     "schemas",
-    "Embedding",
-    "Word",
     "Symbol",
-    "Phrase",
-    "Lexeme",
-    "EmbedRequest",
-    "EmbeddingResponse",
     "ResponseRequest",
     "PreprocessRequest",
-    "PreprocessResponse",
-    "GetSymbolRequest",
-    "GetSymbolResponse",
-    "GetWordRequest",
-    "GetWordResponse",
-    "GetSimilarWordRequest",
-    "GetAllPhrasesResponse",
     "GetSimilarSymbolsRequest",
-    "GetEmbeddingSimilarSymbolsRequest",
     "SearchSymbolsRequest",
     "GetSymbolsResponse",
     "ReplaceFinishedRequest",
+    "TextToSymbolRequest",
+    "TextToSymbolResponse",
     "SymbolLeaf",
     "SentenceWord",
     "WordsPreprocessResponse",
@@ -34,11 +22,14 @@ __all__ = [
     "WordLookup",
     "LookupRequest",
     "LookupResponse",
-    "JevQuestion",
-    "JevRequest",
-    "JevOption",
-    "JevAnswer",
-    "JevResponse",
+    "SomQuestion",
+    "SomRequest",
+    "SomOption",
+    "SomAnswer",
+    "SomCall",
+    "SomResponse",
+    "SomStep",
+    "TranslationTrace",
     "GetLeafSymbolsRequest",
     "UsageStatusResponse",
     "ModelType",
@@ -50,8 +41,3 @@ __all__ = [
     "Currency",
     "SubscriptionType"
 ]
-
-
-
-
-

anansyTypes/enums.py

@@ -1,10 +1,7 @@
-import anansyTypes
 from enum import Enum
-from typing import Union
 
 
 class ModelType(Enum):
-    EMBEDDING = "embedding"
     TRANSCRIPTION = "transcription"
     RESPONSE = "response"
 
@@ -13,10 +10,6 @@ class ModelName(Enum):
     pass
 
 
-class EmbeddingModelName(ModelName):
-    NB_BERT = "nb_bert"
-
-
 class TranscriptionModelName(ModelName):
     NB_WHISPER_SMALL = "nb_whisper_small"
 
@@ -26,15 +19,13 @@ class ResponseModelName(ModelName):
 
 
 class PreprocessorType(Enum):
-    NORWEGIAN = "norwegian"
-    DEFAULT = "norwegian"
-    NORWEGIAN_WORDLIST="norwegian_wordlist"
-    ENGLISH="english"
+    NORWEGIAN_WORDLIST = "norwegian_wordlist"
+    ENGLISH = "english"
+    GERMAN = "german"
 
 
 class SymbolTranslatorType(Enum):
-    EMBEDDING = "embedding"
-    JEV = "jev"
+    SOM = "som"     # the system one model, choosing symbols from the hierarchy
 
 
 class SymbolProvider(Enum):
@@ -45,22 +36,19 @@ class Language(Enum):
     NORWEGIAN = "no"
     ENGLISH = "en"
     DEFAULT = "en"
+    GERMAN = "de"
 
 
-class ResourceType(Enum):
-    WORD = "word"
-    SYMBOL = "symbol"
-
 class SubscriptionType(Enum):
     FREE = "free"
     PAID_MONTH = "paid_month"
     PAID_YEAR = "paid_year"
-    
+
 class Currency(Enum):
     EUR = "EUR"
     USD = "USD"
     NOK = "NOK"
-    
+
 class DiscountType(Enum):
     PERCENTAGE = "percentage"
-    ABSOLUTE = "absolute"
+    ABSOLUTE = "absolute"

anansyTypes/schemas.py

@@ -1,24 +1,14 @@
 from pydantic import BaseModel as PydanticBaseModel
-from typing import List, Any, Dict, Optional,Union
+from typing import Any, List, Dict, Optional
 
-from .structures import Embedding,Word, Symbol,Phrase
-from pydantic import field_validator, ConfigDict
-from pydantic_tensor import Tensor
-import torch
-from typing import Any, Literal
+from .enums import SymbolProvider
+from .structures import Symbol
+from pydantic import ConfigDict
+from typing import Literal
 class ModifiedBaseModel(PydanticBaseModel):
     pass
-    #model_config = ConfigDict(arbitrary_types_allowed=True)
 
 
-class EmbeddingRequest(ModifiedBaseModel):
-    stemmed_words: List[str]
-    tokens: dict
-    word_ids: List[int]
-
-class EmbeddingResponse(ModifiedBaseModel):
-    embeddings: List[Embedding]
-
 class ResponseRequest(ModifiedBaseModel):
     text: str
     context: Dict[str, Any]
@@ -26,40 +16,9 @@ class ResponseRequest(ModifiedBaseModel):
 class PreprocessRequest(ModifiedBaseModel):
     text: str
 
-class PreprocessResponse(ModifiedBaseModel):
-    raw_words: List[str]
-    stemmed_words: List[str]
-    tokens: Dict[str, Any]
-    word_ids: List[int]
-
-class GetSymbolRequest(ModifiedBaseModel):
-    lexeme_id: int
-
 class GetSymbolsResponse(ModifiedBaseModel):
     symbols: List[Symbol]
 
-class GetSymbolResponse(ModifiedBaseModel):
-    symbol: Optional[Symbol]
-
-class GetWordRequest(ModifiedBaseModel):
-    word_text: str
-    stemmed: bool = False
-
-class GetWordResponse(ModifiedBaseModel):
-    word: Optional[Word]
-
-class GetSimilarWordRequest(ModifiedBaseModel):
-    embedding: Embedding
-
-class GetAllPhrasesResponse(ModifiedBaseModel):
-    phrases: Optional[List[Phrase]]
-    
-class GetFullPhrasesRequest(ModifiedBaseModel):
-    phrases: List[Phrase]
-
-class GetFullPhrasesResponse(ModifiedBaseModel):
-    phrases: Optional[List[Phrase]]
-
 class GetSimilarSymbolsRequest(ModifiedBaseModel):
     """/replace: symbols to offer instead of `symbol`, which was shown for `word` in `sentence`."""
     limit: int = 3
@@ -68,13 +27,6 @@ class GetSimilarSymbolsRequest(ModifiedBaseModel):
     sentence:str
     language: Optional[str] = None  # fallback language for a caller with no account config (see ConfigFetcher)
 
-class GetEmbeddingSimilarSymbolsRequest(ModifiedBaseModel):
-    """Symbols nearest an embedding, leaving out the given words and symbols."""
-    embedding: Embedding
-    limit: int = 3
-    exclude_word_ids: List[int] = []
-    exclude_symbol_ids: List[int] = []
-
 class SearchSymbolsRequest(ModifiedBaseModel):
     query: str
     limit: int = 8
@@ -94,13 +46,12 @@ class AudioToSymbolRequest(ModifiedBaseModel):
 class AudioToSymbolResponse(ModifiedBaseModel):
     symbols: List[Symbol]
     sentence: str
-    
+    refunded: bool = False                      # the translation was too poor to use up a free translation
+
 class TextToSymbolRequest(ModifiedBaseModel):
     text: str
-    language:str
-    
-class TextToSymbolResponse(ModifiedBaseModel):
-    symbols: List[Symbol]
+    language: Optional[str] = None  # fallback language for a caller with no account config (see ConfigFetcher)
+    trace: bool = False             # also return the SOM requests and responses, with timings (not in production)
 
 class ReplaceFinishedRequest(ModifiedBaseModel):
     """A correction: the sentence as heard, the word, and the symbols before and after (images left empty)."""
@@ -112,8 +63,8 @@ class ReplaceFinishedRequest(ModifiedBaseModel):
     rank: int                       # the replacement's position in the list it was picked from, from 0
     query: Optional[str] = None     # the search text, for source "search"
     language: Optional[str] = None  # fallback language for a caller with no account config (see ConfigFetcher)
-    
-#### Symbol hierarchy (jev translator) ####
+
+#### Symbol hierarchy (SOM translator) ####
 
 class SymbolLeaf(ModifiedBaseModel):
     """One leaf of the symbol hierarchy. A label with several parts ("bat|club") gives one leaf per part."""
@@ -128,6 +79,7 @@ class SentenceWord(ModifiedBaseModel):
     name: bool = False              # a person's name: gets a generic person symbol
     gender: Optional[str] = None    # "fem" or "mask" for a name, if known
     grammar: bool = False           # a common grammar word ("the", "og"): skipped if the dictionary has no match
+    punctuation: bool = False       # "?" or "!", shown as its own symbol (only sent where the provider has one)
 
 class WordsPreprocessResponse(ModifiedBaseModel):
     words: List[SentenceWord]
@@ -147,7 +99,7 @@ class LookupResponse(ModifiedBaseModel):
     words: List[WordLookup]         # one per requested word
     people: List[SymbolLeaf]        # the generic person symbols offered for names
 
-class JevQuestion(ModifiedBaseModel):
+class SomQuestion(ModifiedBaseModel):
     """choose: which of `leaves` matches the word. search: which child of `category_id` (None: the top) does."""
     name: str
     kind: Literal["choose", "search"]
@@ -157,22 +109,49 @@ class JevQuestion(ModifiedBaseModel):
     leaves: List[SymbolLeaf] = []
     category_id: Optional[str] = None
 
-class JevRequest(ModifiedBaseModel):
+class SomRequest(ModifiedBaseModel):
     sentence: str
-    questions: List[JevQuestion]
+    questions: List[SomQuestion]
+    trace: bool = False             # return the calls made to the SOM API
 
-class JevOption(ModifiedBaseModel):
+class SomOption(ModifiedBaseModel):
     """A symbol or a category the model gave probability p."""
     p: float
     leaf: Optional[SymbolLeaf] = None
     category_id: Optional[str] = None
 
-class JevAnswer(ModifiedBaseModel):
-    options: List[JevOption] = []
+class SomAnswer(ModifiedBaseModel):
+    options: List[SomOption] = []
     p_none: float = 0.0             # probability that none of the options matches
 
-class JevResponse(ModifiedBaseModel):
-    answers: Dict[str, JevAnswer]   # question name -> answer
+class SomCall(ModifiedBaseModel):
+    """One HTTP call to the SOM API, as the backend made it."""
+    request: dict                   # the payload sent: model, state (the sentence) and questions
+    response: dict                  # the API's answer, as received
+    status: int
+    attempts: int                   # 1 unless it was retried
+    duration_ms: float              # including retries
+
+class SomResponse(ModifiedBaseModel):
+    answers: Dict[str, SomAnswer]   # question name -> answer
+    calls: Optional[List[SomCall]] = None   # with trace: one per batch of questions
+
+class SomStep(ModifiedBaseModel):
+    """One step of a translation: the questions asked together, their answers, and the API calls behind them."""
+    questions: List[SomQuestion]
+    answers: Dict[str, SomAnswer]
+    calls: List[SomCall] = []
+    duration_ms: float              # the round trip from anansyLogic, through the backend, to the SOM API
+
+class TranslationTrace(ModifiedBaseModel):
+    provider: SymbolProvider
+    timings_ms: Dict[str, float] = {}   # preprocess, lookup, som (every step), symbols, total
+    steps: List[SomStep] = []
+
+class TextToSymbolResponse(ModifiedBaseModel):
+    symbols: List[Symbol]
+    trace: Optional[TranslationTrace] = None    # only when asked for, and allowed (ALLOW_TRACE)
+    refunded: bool = False                      # the translation was too poor to use up a free translation
 
 class GetLeafSymbolsRequest(ModifiedBaseModel):
     leaves: List[SymbolLeaf]

anansyTypes/structures/__init__.py

@@ -1,14 +1,6 @@
-from .embedding import Embedding
-from .lexeme import Lexeme
-from .word import Word
 from .symbol import Symbol
-from .phrase import Phrase
 
 
 __all__ = [
-    "Embedding",
-    "Lexeme",
-    "Word",
     "Symbol",
-    "Phrase",
 ]

anansyTypes/structures/embedding.py

@@ -1,37 +0,0 @@
-from typing import Any
-import torch
-from pydantic import BaseModel
-from pydantic_core import core_schema
-from typing_extensions import Annotated
-import numpy as np
-
-class TensorPydantic:
-    @classmethod
-    def __get_pydantic_core_schema__(cls, source, handler):
-        def validate(v: Any):
-            if isinstance(v, torch.Tensor):
-                return v
-            if isinstance(v, list):
-                return torch.tensor(v, dtype=torch.float32)
-            raise TypeError(f"Cannot convert {type(v)} to torch.Tensor")
-
-        return core_schema.json_or_python_schema(
-            json_schema=core_schema.list_schema(core_schema.any_schema()),
-            python_schema=core_schema.no_info_plain_validator_function(validate),
-            serialization=core_schema.plain_serializer_function_ser_schema(
-                lambda t: t.detach().cpu().tolist()
-            ),
-        )
-
-
-Tensor = Annotated[torch.Tensor, TensorPydantic]
-
-
-class Embedding(BaseModel):
-    data: Tensor
-    
-    @classmethod
-    def from_dict(cls, data: dict) -> "Embedding":
-        return cls(data=torch.tensor(data["data"], dtype=torch.float32))
-    
-    

anansyTypes/structures/lexeme.py

@@ -1,26 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import List, Union, Any
-from dataclasses import dataclass
-
-from pydantic import BaseModel
-from .embedding import Embedding
-
-class Lexeme(BaseModel,ABC):
-    id: int
-    content: Union[str, List[str]]
-    original_content: Union[str, List[str]]
-    embedding: Embedding
-    
-
-    @classmethod
-    def from_dict(cls, data: dict) -> "Lexeme":
-        return cls(
-            id=data["id"],
-            content=data["content"],
-            embedding=Embedding.from_dict(data["embedding"]),
-            original_content=data["original_content"],
-        ) 
-    def __new__(cls, *args, **kwargs):
-        if cls is Lexeme:
-            raise TypeError("Lexeme cannot be instantiated, use a subclass instead")
-        return super().__new__(cls)

anansyTypes/structures/phrase.py

@@ -1,20 +0,0 @@
-from typing import List, Any
-from pydantic import model_validator
-from pydantic_core import core_schema
-import torch
-
-from .lexeme import Lexeme
-from .embedding import Embedding
-import numpy as np
-
-
-class Phrase(Lexeme):
-
-    @classmethod
-    def from_dict(cls, data: dict) -> "Phrase":
-        return cls(
-            id=data["id"],
-            content=data["content"],
-            original_content=data["original_content"],
-            embedding=Embedding.from_dict(data["embedding"]),
-        )

anansyTypes/structures/symbol.py

@@ -1,11 +1,4 @@
-from dataclasses import dataclass
-from typing import Any
-import numpy as np
 from pydantic import BaseModel
-from pydantic_tensor import Tensor
-from typing import Any, Literal
-from pydantic_core import core_schema
-import base64
 
 
 class Symbol(BaseModel):
@@ -19,4 +12,4 @@ class Symbol(BaseModel):
             id=data["id"],
             label=data["label"],
             image=data["image"],
-        )
+        )

anansyTypes/structures/word.py

@@ -1,8 +0,0 @@
-from .lexeme import Lexeme
-from typing import Any
-from .embedding import Embedding
-from pydantic_core import core_schema
-
-
-class Word(Lexeme):
-    pass

pyproject.toml

@@ -15,7 +15,7 @@ classifiers = [
     "Operating System :: OS Independent"
 ]
 dependencies=[
-    "pydantic-tensor"
+    "pydantic"
 ]
 [build-system]
 requires = ["setuptools>=65.5.1", "wheel" ]