docarray · samsja · May 5, 2023 · Apr 27, 2023 · Apr 28, 2023 · May 2, 2023
diff --git a/docarray/array/any_array.py b/docarray/array/any_array.py
@@ -106,7 +106,7 @@ def __getattr__(self, item: str):
     def _get_data_column(
         self: T,
         field: str,
-    ) -> Union[MutableSequence, T, 'AbstractTensor']:
+    ) -> Union[MutableSequence, T, 'AbstractTensor', None]:
         """Return all values of the fields from all docs this array contains
 
         :param field: name of the fields to extract

diff --git a/docarray/array/doc_list/doc_list.py b/docarray/array/doc_list/doc_list.py
@@ -102,6 +102,13 @@ class Image(BaseDoc):
     del docs[0:5]  # remove elements for 0 to 5 from DocList
     ```
 
+    !!! note
+        If the DocList is homogeneous and its schema contains nested BaseDoc
+        (i.e, BaseDoc inside a BaseDoc) where the nested Document is `Optional`, calling
+        `docs.nested_doc` will return a List of the nested BaseDoc instead of DocList.
+        This is because the nested field could be None and therefore could not fit into
+        a DocList.
+
     :param docs: iterable of Document
 
     """
@@ -200,6 +207,7 @@ def __class_getitem__(cls, item: Union[Type[BaseDoc], TypeVar, str]):
 
         if (
             not is_union_type(field_type)
+            and self.__class__.doc_type.__fields__[field].required
             and isinstance(field_type, type)
             and issubclass(field_type, BaseDoc)
         ):

diff --git a/docarray/array/doc_vec/column_storage.py b/docarray/array/doc_vec/column_storage.py
@@ -5,6 +5,7 @@
     Dict,
     Iterable,
     MutableMapping,
+    Optional,
     Type,
     TypeVar,
     Union,
@@ -37,9 +38,9 @@ class ColumnStorage:
 
     def __init__(
         self,
-        tensor_columns: Dict[str, AbstractTensor],
-        doc_columns: Dict[str, 'DocVec'],
-        docs_vec_columns: Dict[str, ListAdvancedIndexing['DocVec']],
+        tensor_columns: Dict[str, Optional[AbstractTensor]],
+        doc_columns: Dict[str, Optional['DocVec']],
+        docs_vec_columns: Dict[str, Optional[ListAdvancedIndexing['DocVec']]],
         any_columns: Dict[str, ListAdvancedIndexing],
         tensor_type: Type[AbstractTensor] = NdArray,
     ):
@@ -63,12 +64,22 @@ def __len__(self) -> int:
     def __getitem__(self: T, item: IndexIterType) -> T:
         if isinstance(item, tuple):
             item = list(item)
-        tensor_columns = {key: col[item] for key, col in self.tensor_columns.items()}
-        doc_columns = {key: col[item] for key, col in self.doc_columns.items()}
+        tensor_columns = {
+            key: col[item] if col is not None else None
+            for key, col in self.tensor_columns.items()
+        }
+        doc_columns = {
+            key: col[item] if col is not None else None
+            for key, col in self.doc_columns.items()
+        }
         docs_vec_columns = {
-            key: col[item] for key, col in self.docs_vec_columns.items()
+            key: col[item] if col is not None else None
+            for key, col in self.docs_vec_columns.items()
+        }
+        any_columns = {
+            key: col[item] if col is not None else None
+            for key, col in self.any_columns.items()
         }
-        any_columns = {key: col[item] for key, col in self.any_columns.items()}
 
         return self.__class__(
             tensor_columns,
@@ -91,15 +102,34 @@ def __init__(self, index: int, storage: ColumnStorage):
     def __getitem__(self, name: str) -> Any:
         if name in self.storage.tensor_columns.keys():
             tensor = self.storage.tensor_columns[name]
+            if tensor is None:
+                return None
             if tensor.get_comp_backend().n_dim(tensor) == 1:
                 # to ensure consistensy between numpy and pytorch
                 # we wrap the scalr in a tensor of ndim = 1
                 # otherwise numpy pass by value whereas torch by reference
-                return self.storage.tensor_columns[name][self.index : self.index + 1]
+                col = self.storage.tensor_columns[name]
 
-        return self.storage.columns[name][self.index]
+                if col is not None:
+                    return col[self.index : self.index + 1]
+                else:
+                    return None
+
+        col = self.storage.columns[name]
+
+        if col is None:
+            return None
+        return col[self.index]
 
     def __setitem__(self, name, value) -> None:
+        if self.storage.columns[name] is None:
+            raise ValueError(
+                f'Cannot set an item to a None column. This mean that '
+                f'the DocVec that encapsulate this doc has the field '
+                f'{name} set to None. If you want to modify that you need to do it at the'
+                f'DocVec level. `docs.field = np.zeros(10)`'
+            )
+
         self.storage.columns[name][self.index] = value
 
     def __delitem__(self, key):