Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 17 additions & 17 deletions src/explanation/custom-codecs.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ class MyCodec(dj.Codec):
"""Store custom objects."""
name = "mytype" # Used as <mytype> in definitions

def get_dtype(self, is_external: bool) -> str:
def get_dtype(self, is_store: bool) -> str:
"""Return storage type."""
return "<blob>" # Chain to blob serialization

Expand Down Expand Up @@ -78,9 +78,9 @@ class GraphCodec(dj.Codec):
"""Store NetworkX graphs as adjacency data."""
name = "graph"

def get_dtype(self, is_external: bool) -> str:
# Store as blob (internal) or hash-addressed (external)
return "<hash>" if is_external else "<blob>"
def get_dtype(self, is_store: bool) -> str:
# Store as blob (internal) or blob@ (external)
return "<blob@>" if is_store else "<blob>"

def encode(self, graph, *, key=None, store_name=None):
"""Serialize graph to dict."""
Expand Down Expand Up @@ -136,10 +136,10 @@ class BamCodec(dj.Codec):
"""Store BAM alignments."""
name = "bam"

def get_dtype(self, is_external: bool) -> str:
if not is_external:
def get_dtype(self, is_store: bool) -> str:
if not is_store:
raise dj.DataJointError("<bam> requires external storage: use <bam@>")
return "<object>" # Path-addressed storage for file structure
return "<object@>" # Path-addressed storage for file structure

def encode(self, alignments, *, key=None, store_name=None):
"""Write alignments to BAM format."""
Expand All @@ -162,8 +162,8 @@ class MedicalImageCodec(dj.Codec):
"""Store medical images with metadata."""
name = "medimg"

def get_dtype(self, is_external: bool) -> str:
return "<hash>" if is_external else "<blob>"
def get_dtype(self, is_store: bool) -> str:
return "<blob@>" if is_store else "<blob>"

def encode(self, image, *, key=None, store_name=None):
"""Serialize SimpleITK image."""
Expand Down Expand Up @@ -197,7 +197,7 @@ graph LR
class CompressedGraphCodec(dj.Codec):
name = "cgraph"

def get_dtype(self, is_external: bool) -> str:
def get_dtype(self, is_store: bool) -> str:
return "<graph>" # Chain to graph codec

def encode(self, graph, *, key=None, store_name=None):
Expand All @@ -216,8 +216,8 @@ class CompressedGraphCodec(dj.Codec):
class SmallDataCodec(dj.Codec):
name = "small"

def get_dtype(self, is_external: bool) -> str:
if is_external:
def get_dtype(self, is_store: bool) -> str:
if is_store:
raise dj.DataJointError("<small> is internal-only")
return "json"
```
Expand All @@ -228,10 +228,10 @@ class SmallDataCodec(dj.Codec):
class LargeDataCodec(dj.Codec):
name = "large"

def get_dtype(self, is_external: bool) -> str:
if not is_external:
def get_dtype(self, is_store: bool) -> str:
if not is_store:
raise dj.DataJointError("<large> requires @: use <large@>")
return "<object>"
return "<object@>"
```

### Both Modes
Expand All @@ -240,8 +240,8 @@ class LargeDataCodec(dj.Codec):
class FlexibleCodec(dj.Codec):
name = "flex"

def get_dtype(self, is_external: bool) -> str:
return "<hash>" if is_external else "<blob>"
def get_dtype(self, is_store: bool) -> str:
return "<blob@>" if is_store else "<blob>"
```

## Validation
Expand Down
10 changes: 5 additions & 5 deletions src/how-to/create-custom-codec.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,16 +58,16 @@ Return the storage type:
```python
def get_dtype(self, is_store: bool) -> str:
if is_store:
return "<hash>" # Hash-addressed storage
return "bytes" # Inline database blob
return "<blob@>" # Blob in object storage
return "bytes" # Inline database blob
```

Common return values:

- `"bytes"` — Binary in database
- `"json"` — JSON in database
- `"<blob>"` — Chain to blob codec (hash-addressed when `@`)
- `"<hash>"` — Hash-addressed storage
- `"<blob>"` — Chain to blob codec (internal storage)
- `"<blob@>"` — Blob in object storage

### `encode(value, *, key=None, store_name=None)`

Expand Down Expand Up @@ -135,7 +135,7 @@ class ZarrCodec(dj.Codec):
def get_dtype(self, is_store: bool) -> str:
if not is_store:
raise DataJointError("<zarr> requires @ (store only)")
return "<object>" # Schema-addressed storage
return "<object@>" # Schema-addressed storage

def encode(self, path, *, key=None, store_name=None):
return path # Path to zarr directory
Expand Down
6 changes: 3 additions & 3 deletions src/tutorials/advanced/custom-codecs.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@
"@schema\n",
"class Connectivity(dj.Manual):\n",
" definition = \"\"\"\n",
" conn_id : int\n",
" conn_id : uint16\n",
" ---\n",
" network : <graph>\n",
" \"\"\""
Expand Down Expand Up @@ -253,7 +253,7 @@
"@schema\n",
"class Unit(dj.Manual):\n",
" definition = \"\"\"\n",
" unit_id : int\n",
" unit_id : uint16\n",
" ---\n",
" spikes : <spike_train>\n",
" \"\"\"\n",
Expand Down Expand Up @@ -304,4 +304,4 @@
},
"nbformat": 4,
"nbformat_minor": 5
}
}
4 changes: 2 additions & 2 deletions src/tutorials/advanced/sql-comparison.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -1235,7 +1235,7 @@
"# Restriction: filters Session to rows matching the Subject query \n",
"Session & (Subject & {'species': 'mouse'})\n",
"\n",
"# Antijoin: Session rows NOT matching any Subject (none here, all subjects exist)\n",
"# Anti-restriction: Session rows NOT matching any Subject (none here, all subjects exist)\n",
"Session - Subject"
]
},
Expand Down Expand Up @@ -2583,7 +2583,7 @@
"| `WHERE condition` | `& {'col': value}` or `& 'expr'` | Restriction |\n",
"| `JOIN ... USING` | `Table1 * Table2` | Natural join |\n",
"| `GROUP BY ... AGG()` | `.aggr(Table, alias='agg()')` | Aggregation |\n",
"| `NOT IN (subquery)` | `Table1 - Table2` | Antijoin |\n",
"| `NOT IN (subquery)` | `Table1 - Table2` | Anti-restriction |\n",
"| `UNION` | `Table1 + Table2` | Union |"
]
},
Expand Down