From 0a07ea6d39a96e6a8ac9f99b8297857de969a76d Mon Sep 17 00:00:00 2001 From: Christoph Eder Date: Sat, 13 Dec 2025 18:09:07 +0100 Subject: [PATCH 1/6] Changes in schemas.py Trying to use proper types now Changes in embedding.py Implemented serialisation --- anansyTypes/schemas.py | 36 +++++++---------------------- anansyTypes/structures/embedding.py | 13 +++++++++++ 2 files changed, 21 insertions(+), 28 deletions(-) diff --git a/anansyTypes/schemas.py b/anansyTypes/schemas.py index 9140fe2..0b10c9b 100644 --- a/anansyTypes/schemas.py +++ b/anansyTypes/schemas.py @@ -1,33 +1,13 @@ from pydantic import BaseModel as PydanticBaseModel -from typing import List, Any, Dict,Union +from typing import List, Any, Dict, Optional,Union from .structures import Embedding,Word, Symbol,Phrase from pydantic import field_validator, ConfigDict from pydantic_tensor import Tensor import torch from typing import Any, Literal class ModifiedBaseModel(PydanticBaseModel): - model_config = ConfigDict(arbitrary_types_allowed=True) - - @field_validator('*', mode='before') - @classmethod - def convert_custom_types(cls, v): - if isinstance(v, Word): - return { - 'id': v.id, - 'content': v.content, - 'embedding': {'data': v.embedding.data.tolist()} - } - if isinstance(v, Phrase): - return { - 'id': v.id, - 'content': v.content, - 'embedding': {'data': v.embedding.data.tolist()} - } - if isinstance(v, Embedding): - return {'data': v.data.tolist()} - if isinstance(v, list): - return [cls.convert_custom_types(item) for item in v] - return v + pass + #model_config = ConfigDict(arbitrary_types_allowed=True) class EmbeddingRequest(ModifiedBaseModel): @@ -36,7 +16,7 @@ class EmbeddingRequest(ModifiedBaseModel): word_ids: List[int] class EmbeddingResponse(ModifiedBaseModel): - embeddings: List[dict] + embeddings: List[Embedding] class ResponseRequest(ModifiedBaseModel): text: str @@ -54,20 +34,20 @@ class GetSymbolRequest(ModifiedBaseModel): lexeme_id: int class GetSymbolResponse(ModifiedBaseModel): - symbol: Union[Symbol, None] + symbol: Optional[Symbol] class GetWordRequest(ModifiedBaseModel): word_text: str stemmed: bool = False class GetWordResponse(ModifiedBaseModel): - word: Union[dict, None] + word: Optional[Word] class GetSimilarWordRequest(ModifiedBaseModel): - embedding: dict + embedding: Embedding class GetAllPhrasesResponse(ModifiedBaseModel): - phrases: List[dict] + phrases: List[Phrase] class GetTranscriptionRequest(ModifiedBaseModel): audio_data: bytes diff --git a/anansyTypes/structures/embedding.py b/anansyTypes/structures/embedding.py index 5ea6050..3cbffd0 100644 --- a/anansyTypes/structures/embedding.py +++ b/anansyTypes/structures/embedding.py @@ -26,3 +26,16 @@ class Embedding: @classmethod def from_dict(cls, data: dict) -> "Embedding": return cls(data=np.array(data["data"], dtype=np.float32)) + + @classmethod + def __get_pydantic_core_schema__(cls, source_type: Any, handler): + from pydantic_core import core_schema + + return core_schema.with_info_plain_validator_function( + lambda value, _: cls.from_dict(value) if isinstance(value, dict) else value, + serialization=core_schema.plain_serializer_function_ser_schema( + lambda x: {"data": x.data.tolist()}, + when_used="json", + ), + ) + From 43064b1572e8218d49d85a36cad6af8dfcd193ee Mon Sep 17 00:00:00 2001 From: Christoph Eder Date: Sat, 13 Dec 2025 18:14:08 +0100 Subject: [PATCH 2/6] Changes in embedding.py, phrase.py, word.py Adopted serialisation schema to last commit --- anansyTypes/structures/embedding.py | 14 +++++++--- anansyTypes/structures/phrase.py | 41 ++++++++++++++--------------- anansyTypes/structures/word.py | 24 +++++++---------- 3 files changed, 40 insertions(+), 39 deletions(-) diff --git a/anansyTypes/structures/embedding.py b/anansyTypes/structures/embedding.py index 3cbffd0..9553cdb 100644 --- a/anansyTypes/structures/embedding.py +++ b/anansyTypes/structures/embedding.py @@ -26,16 +26,22 @@ class Embedding: @classmethod def from_dict(cls, data: dict) -> "Embedding": return cls(data=np.array(data["data"], dtype=np.float32)) - + @classmethod def __get_pydantic_core_schema__(cls, source_type: Any, handler): from pydantic_core import core_schema - + + def validate_embedding(value): + if isinstance(value, cls): + return value + if isinstance(value, dict): + return cls.from_dict(value) + raise ValueError(f"Cannot convert {type(value)} to Embedding") + return core_schema.with_info_plain_validator_function( - lambda value, _: cls.from_dict(value) if isinstance(value, dict) else value, + lambda value, _: validate_embedding(value), serialization=core_schema.plain_serializer_function_ser_schema( lambda x: {"data": x.data.tolist()}, when_used="json", ), ) - diff --git a/anansyTypes/structures/phrase.py b/anansyTypes/structures/phrase.py index 9252afe..a5dba06 100644 --- a/anansyTypes/structures/phrase.py +++ b/anansyTypes/structures/phrase.py @@ -14,39 +14,38 @@ class Phrase(Lexeme): @property def content(self) -> str: return self._content - + @property def embedding(self) -> Embedding: return self._embedding - + @classmethod - def from_dict(cls, data: dict) -> 'Phrase': + def from_dict(cls, data: dict) -> "Phrase": return cls( - id=data['id'], - content=data['content'], - embedding=Embedding.from_dict(data['embedding']) + id=data["id"], + content=data["content"], + embedding=Embedding.from_dict(data["embedding"]), ) @classmethod def __get_pydantic_core_schema__(cls, source_type: Any, handler): from pydantic_core import core_schema - - return core_schema.no_info_after_validator_function( - lambda x: x if isinstance(x, cls) else cls(**x), - core_schema.typed_dict_schema( - { - 'id': core_schema.typed_dict_field(handler.generate_schema(int)), - 'content': core_schema.typed_dict_field(handler.generate_schema(str)), - 'embedding': core_schema.typed_dict_field(handler.generate_schema(Embedding)), - } - ), + + def validate_phrase(value): + if isinstance(value, cls): + return value + if isinstance(value, dict): + return cls.from_dict(value) + raise ValueError(f"Cannot convert {type(value)} to Phrase") + + return core_schema.with_info_plain_validator_function( + lambda value, _: validate_phrase(value), serialization=core_schema.plain_serializer_function_ser_schema( lambda x: { - 'id': x.id, - 'content': x.content, - 'embedding': x.embedding + "id": x.id, + "content": x.content, + "embedding": {"data": x.embedding.data.tolist()}, }, - when_used='json', + when_used="json", ), ) - diff --git a/anansyTypes/structures/word.py b/anansyTypes/structures/word.py index abea127..9f1128c 100644 --- a/anansyTypes/structures/word.py +++ b/anansyTypes/structures/word.py @@ -36,24 +36,20 @@ class Word(Lexeme): def __get_pydantic_core_schema__(cls, source_type: Any, handler): from pydantic_core import core_schema - return core_schema.no_info_after_validator_function( - lambda x: x if isinstance(x, cls) else cls(**x), - core_schema.typed_dict_schema( - { - "id": core_schema.typed_dict_field(handler.generate_schema(int)), - "content": core_schema.typed_dict_field( - handler.generate_schema(str) - ), - "embedding": core_schema.typed_dict_field( - handler.generate_schema(Embedding) - ), - } - ), + def validate_word(value): + if isinstance(value, cls): + return value + if isinstance(value, dict): + return cls.from_dict(value) + raise ValueError(f"Cannot convert {type(value)} to Word") + + return core_schema.with_info_plain_validator_function( + lambda value, _: validate_word(value), serialization=core_schema.plain_serializer_function_ser_schema( lambda x: { "id": x.id, "content": x.content, - "embedding": x.embedding, # Let Pydantic serialize Embedding + "embedding": {"data": x.embedding.data.tolist()}, }, when_used="json", ), From d7fe47b71c1c5de466f958d6186c146a06473956 Mon Sep 17 00:00:00 2001 From: Christoph Eder Date: Sat, 13 Dec 2025 18:23:26 +0100 Subject: [PATCH 3/6] Changes in embedding.py, lexeme.py, phrase.py, word.py Defines typed_schema for seralisation Moved serialisation method to lexeme --- anansyTypes/structures/embedding.py | 14 +++++++-- anansyTypes/structures/lexeme.py | 45 +++++++++++++++++++++++++++-- anansyTypes/structures/phrase.py | 25 +--------------- anansyTypes/structures/word.py | 23 --------------- 4 files changed, 54 insertions(+), 53 deletions(-) diff --git a/anansyTypes/structures/embedding.py b/anansyTypes/structures/embedding.py index 9553cdb..ef50d69 100644 --- a/anansyTypes/structures/embedding.py +++ b/anansyTypes/structures/embedding.py @@ -30,18 +30,26 @@ class Embedding: @classmethod def __get_pydantic_core_schema__(cls, source_type: Any, handler): from pydantic_core import core_schema - + def validate_embedding(value): if isinstance(value, cls): return value if isinstance(value, dict): return cls.from_dict(value) raise ValueError(f"Cannot convert {type(value)} to Embedding") - - return core_schema.with_info_plain_validator_function( + + python_schema = core_schema.with_info_plain_validator_function( lambda value, _: validate_embedding(value), serialization=core_schema.plain_serializer_function_ser_schema( lambda x: {"data": x.data.tolist()}, when_used="json", ), ) + + return core_schema.json_or_python_schema( + json_schema=core_schema.typed_dict_schema({ + 'data': core_schema.typed_dict_field(core_schema.list_schema()) + }), + python_schema=python_schema, + ) + diff --git a/anansyTypes/structures/lexeme.py b/anansyTypes/structures/lexeme.py index 2d27e8d..b26edf6 100644 --- a/anansyTypes/structures/lexeme.py +++ b/anansyTypes/structures/lexeme.py @@ -19,6 +19,45 @@ class Lexeme(ABC): def embedding(self) -> Embedding: """Embedding(s) associated with the lexeme.""" pass - - - + + @classmethod + @abstractmethod + def from_dict(cls, data: dict) -> "Lexeme": + """Create instance from dictionary.""" + pass + + @classmethod + def __get_pydantic_core_schema__(cls, source_type: Any, handler): + from pydantic_core import core_schema + + def validate_lexeme(value): + if isinstance(value, cls): + return value + if isinstance(value, dict): + return cls.from_dict(value) + raise ValueError(f"Cannot convert {type(value)} to {cls.__name__}") + + python_schema = core_schema.with_info_plain_validator_function( + lambda value, _: validate_lexeme(value), + serialization=core_schema.plain_serializer_function_ser_schema( + lambda x: { + "id": x.id, + "content": x.content, + "embedding": {"data": x.embedding.data.tolist()}, + }, + when_used="json", + ), + ) + + return core_schema.json_or_python_schema( + json_schema=core_schema.typed_dict_schema( + { + "id": core_schema.typed_dict_field(core_schema.int_schema()), + "content": core_schema.typed_dict_field(core_schema.str_schema()), + "embedding": core_schema.typed_dict_field( + handler.generate_schema(Embedding) + ), + } + ), + python_schema=python_schema, + ) diff --git a/anansyTypes/structures/phrase.py b/anansyTypes/structures/phrase.py index a5dba06..92d56ed 100644 --- a/anansyTypes/structures/phrase.py +++ b/anansyTypes/structures/phrase.py @@ -25,27 +25,4 @@ class Phrase(Lexeme): id=data["id"], content=data["content"], embedding=Embedding.from_dict(data["embedding"]), - ) - - @classmethod - def __get_pydantic_core_schema__(cls, source_type: Any, handler): - from pydantic_core import core_schema - - def validate_phrase(value): - if isinstance(value, cls): - return value - if isinstance(value, dict): - return cls.from_dict(value) - raise ValueError(f"Cannot convert {type(value)} to Phrase") - - return core_schema.with_info_plain_validator_function( - lambda value, _: validate_phrase(value), - serialization=core_schema.plain_serializer_function_ser_schema( - lambda x: { - "id": x.id, - "content": x.content, - "embedding": {"data": x.embedding.data.tolist()}, - }, - when_used="json", - ), - ) + ) \ No newline at end of file diff --git a/anansyTypes/structures/word.py b/anansyTypes/structures/word.py index 9f1128c..f95b2df 100644 --- a/anansyTypes/structures/word.py +++ b/anansyTypes/structures/word.py @@ -31,26 +31,3 @@ class Word(Lexeme): content=data["content"], embedding=Embedding.from_dict(data["embedding"]), ) - - @classmethod - def __get_pydantic_core_schema__(cls, source_type: Any, handler): - from pydantic_core import core_schema - - def validate_word(value): - if isinstance(value, cls): - return value - if isinstance(value, dict): - return cls.from_dict(value) - raise ValueError(f"Cannot convert {type(value)} to Word") - - return core_schema.with_info_plain_validator_function( - lambda value, _: validate_word(value), - serialization=core_schema.plain_serializer_function_ser_schema( - lambda x: { - "id": x.id, - "content": x.content, - "embedding": {"data": x.embedding.data.tolist()}, - }, - when_used="json", - ), - ) From edeaf3aae94dcbf1e450fb65bf25ae3b9d09ef8a Mon Sep 17 00:00:00 2001 From: Christoph Eder Date: Sat, 13 Dec 2025 18:24:25 +0100 Subject: [PATCH 4/6] Changes in lexeme.py Fixed missing import --- anansyTypes/structures/lexeme.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anansyTypes/structures/lexeme.py b/anansyTypes/structures/lexeme.py index b26edf6..cfc1852 100644 --- a/anansyTypes/structures/lexeme.py +++ b/anansyTypes/structures/lexeme.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import List, Union +from typing import List, Union, Any from dataclasses import dataclass from .embedding import Embedding From 01ace5b7cc1974a5652fbf5a03acdeaa60ab0b03 Mon Sep 17 00:00:00 2001 From: Christoph Eder Date: Sat, 13 Dec 2025 18:26:36 +0100 Subject: [PATCH 5/6] Changes in symbol.py Adopted changes from embedding.py --- anansyTypes/structures/symbol.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/anansyTypes/structures/symbol.py b/anansyTypes/structures/symbol.py index c412b52..abe2a27 100644 --- a/anansyTypes/structures/symbol.py +++ b/anansyTypes/structures/symbol.py @@ -40,7 +40,7 @@ class Symbol: return cls.from_dict(value) raise ValueError(f"Cannot convert {type(value)} to Symbol") - return core_schema.with_info_plain_validator_function( + python_schema = core_schema.with_info_plain_validator_function( lambda value, _: validate_symbol(value), serialization=core_schema.plain_serializer_function_ser_schema( lambda x: { @@ -53,3 +53,14 @@ class Symbol: when_used="json", ), ) + + return core_schema.json_or_python_schema( + json_schema=core_schema.typed_dict_schema( + { + "id": core_schema.typed_dict_field(core_schema.int_schema()), + "label": core_schema.typed_dict_field(core_schema.str_schema()), + "image": core_schema.typed_dict_field(core_schema.str_schema()), + } + ), + python_schema=python_schema, + ) From ddfdd41e6ab8b1af74e50e6f67900a932e87c4f1 Mon Sep 17 00:00:00 2001 From: Christoph Eder Date: Sat, 13 Dec 2025 19:10:25 +0100 Subject: [PATCH 6/6] Changes in enums.py Fixed generalisation of enums --- anansyTypes/enums.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/anansyTypes/enums.py b/anansyTypes/enums.py index 4e80e8f..cf6c1dd 100644 --- a/anansyTypes/enums.py +++ b/anansyTypes/enums.py @@ -1,3 +1,4 @@ +import anansyTypes from enum import Enum from typing import Union @@ -8,21 +9,22 @@ class ModelType(Enum): RESPONSE = "response" -class EmbeddingModelName(Enum): +class ModelName(Enum): + pass + + +class EmbeddingModelName(ModelName): NB_BERT = "nb_bert" -class TranscriptionModelName(Enum): +class TranscriptionModelName(ModelName): NB_WHISPER_SMALL = "nb_whisper_small" -class ResponseModelName(Enum): +class ResponseModelName(ModelName): GENERIC = "generic" -ModelName= Union[EmbeddingModelName, TranscriptionModelName, ResponseModelName] - - class PreprocessorType(Enum): NORWEGIAN = "norwegian" DEFAULT = "norwegian"