Skip to content

Commit 5aa191f

Browse files
Use <djblob> for automatic serialization, fix is_blob detection
Type system changes: - Core type `blob` stores raw bytes without serialization - Built-in type `<djblob>` handles automatic serialization/deserialization - Update jobs table to use <djblob> for key and error_stack columns - Remove enable_python_native_blobs config check (always enabled) Bug fixes: - Fix is_blob detection to include NATIVE_BLOB types (longblob, mediumblob, etc.) - Fix original_type fallback when None - Fix test_type_aliases to use lowercase keys for CORE_TYPE_SQL lookup - Allow None context for built-in types in heading initialization - Update native type warning message wording 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent 4edf5ed commit 5aa191f

File tree

7 files changed

+29
-41
lines changed

7 files changed

+29
-41
lines changed

src/datajoint/blob.py

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
import numpy as np
1414

1515
from .errors import DataJointError
16-
from .settings import config
1716

1817
deserialize_lookup = {
1918
0: {"dtype": None, "scalar_type": "UNKNOWN"},
@@ -89,12 +88,6 @@ def __init__(self, squeeze=False):
8988
self.protocol = None
9089

9190
def set_dj0(self):
92-
if not config.get("enable_python_native_blobs"):
93-
raise DataJointError(
94-
"""v0.12+ python native blobs disabled.
95-
See also: https://github.com/datajoint/datajoint-python#python-native-blobs"""
96-
)
97-
9891
self.protocol = b"dj0\0" # when using new blob features
9992

10093
def squeeze(self, array, convert_to_scalar=True):

src/datajoint/declare.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -530,9 +530,9 @@ def compile_attribute(line, in_key, foreign_key_sql, context):
530530
match["comment"] = ":{type}:{comment}".format(**match)
531531
substitute_special_type(match, category, foreign_key_sql, context)
532532
elif category in NATIVE_TYPES:
533-
# Non-standard native type - warn user
533+
# Native type - warn user
534534
logger.warning(
535-
f"Non-standard native type '{match['type']}' in attribute '{match['name']}'. "
535+
f"Native type '{match['type']}' is used in attribute '{match['name']}'. "
536536
"Consider using a core DataJoint type for better portability."
537537
)
538538

src/datajoint/fetch.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ def _get(connection, attr, data, squeeze, download_path):
8383
return uuid_module.UUID(bytes=data)
8484

8585
if attr.is_blob:
86-
return data # raw bytes
86+
return data # raw bytes (use <djblob> for automatic deserialization)
8787

8888
# Native types - pass through unchanged
8989
return data

src/datajoint/heading.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,7 @@ def _init_from_database(self):
286286
autoincrement=bool(re.search(r"auto_increment", attr["Extra"], flags=re.I)),
287287
numeric=any(TYPE_PATTERN[t].match(attr["type"]) for t in ("DECIMAL", "INTEGER", "FLOAT")),
288288
string=any(TYPE_PATTERN[t].match(attr["type"]) for t in ("ENUM", "TEMPORAL", "STRING")),
289-
is_blob=bool(TYPE_PATTERN["BLOB"].match(attr["type"])),
289+
is_blob=any(TYPE_PATTERN[t].match(attr["type"]) for t in ("BLOB", "NATIVE_BLOB")),
290290
uuid=False,
291291
json=bool(TYPE_PATTERN["JSON"].match(attr["type"])),
292292
adapter=None,
@@ -316,7 +316,7 @@ def _init_from_database(self):
316316

317317
# process AttributeTypes (adapted types in angle brackets)
318318
if special and TYPE_PATTERN["ADAPTED"].match(attr["type"]):
319-
assert context is not None, "Declaration context is not set"
319+
# Context can be None for built-in types that are globally registered
320320
adapter_name = special["type"]
321321
try:
322322
adapter_result = get_adapter(context, adapter_name)
@@ -338,7 +338,7 @@ def _init_from_database(self):
338338
# Handle core type aliases (uuid, float32, etc.)
339339
if special:
340340
# Check original_type for core type detection (not attr["type"] which is now db type)
341-
original_type = attr.get("original_type", attr["type"])
341+
original_type = attr["original_type"] or attr["type"]
342342
try:
343343
category = next(c for c in SPECIAL_TYPES if TYPE_PATTERN[c].match(original_type))
344344
except StopIteration:

src/datajoint/jobs.py

Lines changed: 20 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
from .errors import DuplicateError
55
from .hash import key_hash
66
from .heading import Heading
7-
from .settings import config
87
from .table import Table
98

109
ERROR_MESSAGE_LENGTH = 2047
@@ -27,9 +26,9 @@ def __init__(self, conn, database):
2726
key_hash :char(32) # key hash
2827
---
2928
status :enum('reserved','error','ignore') # if tuple is missing, the job is available
30-
key=null :blob # structure containing the key
29+
key=null :<djblob> # structure containing the key
3130
error_message="" :varchar({error_message_length}) # error message returned if failed
32-
error_stack=null :mediumblob # error stack if failed
31+
error_stack=null :<djblob> # error stack if failed
3332
user="" :varchar(255) # database user
3433
host="" :varchar(255) # system hostname
3534
pid=0 :int unsigned # system process id
@@ -76,8 +75,7 @@ def reserve(self, table_name, key):
7675
user=self._user,
7776
)
7877
try:
79-
with config.override(enable_python_native_blobs=True):
80-
self.insert1(job, ignore_extra_fields=True)
78+
self.insert1(job, ignore_extra_fields=True)
8179
except DuplicateError:
8280
return False
8381
return True
@@ -107,8 +105,7 @@ def ignore(self, table_name, key):
107105
user=self._user,
108106
)
109107
try:
110-
with config.override(enable_python_native_blobs=True):
111-
self.insert1(job, ignore_extra_fields=True)
108+
self.insert1(job, ignore_extra_fields=True)
112109
except DuplicateError:
113110
return False
114111
return True
@@ -135,20 +132,19 @@ def error(self, table_name, key, error_message, error_stack=None):
135132
"""
136133
if len(error_message) > ERROR_MESSAGE_LENGTH:
137134
error_message = error_message[: ERROR_MESSAGE_LENGTH - len(TRUNCATION_APPENDIX)] + TRUNCATION_APPENDIX
138-
with config.override(enable_python_native_blobs=True):
139-
self.insert1(
140-
dict(
141-
table_name=table_name,
142-
key_hash=key_hash(key),
143-
status="error",
144-
host=platform.node(),
145-
pid=os.getpid(),
146-
connection_id=self.connection.connection_id,
147-
user=self._user,
148-
key=key,
149-
error_message=error_message,
150-
error_stack=error_stack,
151-
),
152-
replace=True,
153-
ignore_extra_fields=True,
154-
)
135+
self.insert1(
136+
dict(
137+
table_name=table_name,
138+
key_hash=key_hash(key),
139+
status="error",
140+
host=platform.node(),
141+
pid=os.getpid(),
142+
connection_id=self.connection.connection_id,
143+
user=self._user,
144+
key=key,
145+
error_message=error_message,
146+
error_stack=error_stack,
147+
),
148+
replace=True,
149+
ignore_extra_fields=True,
150+
)

src/datajoint/table.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -975,8 +975,7 @@ def __make_placeholder(self, name, value, ignore_extra_fields=False, row=None):
975975
# Numeric - convert to string
976976
elif attr.numeric:
977977
value = str(int(value) if isinstance(value, bool) else value)
978-
# Blob - pass through as bytes (adapters handle serialization)
979-
# elif attr.is_blob: pass through unchanged
978+
# Blob - pass through as bytes (use <djblob> for automatic serialization)
980979

981980
return name, placeholder, value
982981

tests/test_type_aliases.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ def test_type_alias_pattern_matching(self, alias, expected_category):
3333
category = match_type(alias)
3434
assert category == expected_category
3535
assert category in SPECIAL_TYPES
36-
assert category in CORE_TYPE_SQL
36+
assert category.lower() in CORE_TYPE_SQL # CORE_TYPE_SQL uses lowercase keys
3737

3838
@pytest.mark.parametrize(
3939
"alias,expected_mysql_type",
@@ -54,7 +54,7 @@ def test_type_alias_pattern_matching(self, alias, expected_category):
5454
def test_type_alias_mysql_mapping(self, alias, expected_mysql_type):
5555
"""Test that type aliases map to correct MySQL types."""
5656
category = match_type(alias)
57-
mysql_type = CORE_TYPE_SQL[category]
57+
mysql_type = CORE_TYPE_SQL[category.lower()] # CORE_TYPE_SQL uses lowercase keys
5858
assert mysql_type == expected_mysql_type
5959

6060
@pytest.mark.parametrize(

0 commit comments

Comments
 (0)