Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions src/ifcopenshell-python/ifcopenshell/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -194,12 +194,19 @@ def open(
readonly: bool = False,
mmap: bool = False,
bypass_types: Optional[Sequence[str]] = None,
lazy: bool = False,
logger: Optional[logger] = None,
) -> Union[file, sqlite, _stream]:
"""Loads an IFC dataset from a filepath

:param should_stream: Whether to open the file in streaming mode. Could be useful
for reading large files.
:param lazy: Index the file with one quick scan and parse each instance's
attributes only when they are first read. Opening is then nearly
instant and memory stays proportional to what is accessed; reading
every attribute of every instance costs the same as a normal open,
just spread over the reads. Falls back to a normal open if the file
uses syntax the scanner does not handle.
:param logger: Logger that receives native parser messages.

You can specify a file format. If no format is given, it is guessed from
Expand Down Expand Up @@ -242,10 +249,12 @@ def open(
return stream(path)
if readonly: # Temporary conditional see #7131. Remove once newer builds don't segfault on Linux.
f = ifcopenshell_wrapper.open(str(path.absolute()), readonly, *optional_logger_args(logger))
elif bypass_types:
elif bypass_types or lazy:
f = ifcopenshell_wrapper.file.create_uninitialized(*optional_logger_args(logger))
for ty in bypass_types:
for ty in bypass_types or ():
f.bypass_type(ty)
if lazy:
f.lazy_loading(True)
if mmap:
# mmap parameter is only available for builds with USE_MMAP, not used in our main builds
f.initialize(str(path.absolute()), mmap=mmap) # ty: ignore[unknown-argument]
Expand Down
13 changes: 13 additions & 0 deletions src/ifcopenshell-python/ifcopenshell/ifcopenshell_wrapper.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,7 @@ class instance_streamer:
def push_page(self, page_data): ...
def read_instance_py(self, type_as_declaration_instance=False): ...
def references(self, *args): ...
def resolve_references_in_place(self, value): ...
def schema(self): ...
def semicolon_count(self): ...
def status(self): ...
Expand Down Expand Up @@ -865,6 +866,18 @@ class file(file_mixin):
def bypass_type(self, type_name: str) -> None:
"""Skip loading instances of ``type_name``."""
...
def lazy_loading(self, *args: bool) -> bool:
"""Get, or with an argument set, whether ``initialize()`` indexes the file with one scan and parses each instance's attributes on first access. Set before ``initialize()``."""
...
def parse_threads(self, *args: int) -> int:
"""Get, or with an argument set, the number of threads ``initialize()`` parses instances with; 0 uses one per core (capped at 16) or honours ``IFCOPENSHELL_PARSE_THREADS``."""
...
def effective_parse_threads(self) -> int:
"""The thread count ``initialize()`` will use given ``parse_threads()`` and the environment."""
...
def paged_reading(self, *args: bool) -> bool:
"""Get, or with an argument set, whether ``initialize()`` reads the file through the paged reader instead of loading it whole. Set before ``initialize()``."""
...
# NOTE: inaccurate `*args` - not all args are `str`.
def initialize(self, *args: str) -> bool:
"""Parse a file on a ``create_uninitialized()`` instance.
Expand Down
7 changes: 5 additions & 2 deletions src/ifcopenshell-python/test/test_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,10 +197,13 @@ def test_getting_an_element_by_id(self):
self.file.by_id("id")

def test_getting_an_element_by_guid(self):
element = self.file.createIfcWall("id")
# Only a 22-character GlobalId is indexed.
element = self.file.createIfcWall("0YvctVUKr0kugbFTf53O9L")
with pytest.raises(TypeError):
self.file.by_guid(1)
assert self.file.by_guid("id") == element
assert self.file.by_guid("0YvctVUKr0kugbFTf53O9L") == element
with pytest.raises(RuntimeError):
self.file.by_guid("id")

def test_adding_an_element(self):
g = ifcopenshell.file()
Expand Down
78 changes: 78 additions & 0 deletions src/ifcopenshell-python/test/test_lazy.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
# IfcOpenShell - IFC toolkit and geometry engine
# Copyright (C) 2026 IfcOpenShell contributors
#
# This file is part of IfcOpenShell.
#
# IfcOpenShell is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# IfcOpenShell is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with IfcOpenShell. If not, see <http://www.gnu.org/licenses/>.

# This file was generated with the assistance of an AI coding tool.

import os

import pytest

import ifcopenshell
import ifcopenshell.api.pset
import ifcopenshell.api.root
import ifcopenshell.util.element

FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures")
FILES = ["ColumnPSetsOfSets.ifc"]


@pytest.mark.parametrize("name", FILES)
def test_lazy_open_matches_full_parse(name):
ifcopenshell.get_log() # the log buffer is global: drop what earlier modules logged
strict = ifcopenshell.open(os.path.join(FIXTURES, name))
lazy = ifcopenshell.open(os.path.join(FIXTURES, name), lazy=True)
assert lazy.schema == strict.schema
strict_ids = sorted(e.id() for e in strict)
assert sorted(e.id() for e in lazy) == strict_ids
for e in strict:
l = lazy.by_id(e.id())
assert l.is_a() == e.is_a()
assert str(l) == str(e)
assert len(lazy.get_inverse(l)) == len(strict.get_inverse(e))
for e in strict.by_type("IfcRoot"):
assert lazy.by_guid(e.GlobalId).id() == e.id()
assert len(lazy.by_type("IfcWall")) == len(strict.by_type("IfcWall"))
assert ifcopenshell.get_log() == ""


def test_lazy_file_can_be_edited_and_written(tmp_path):
lazy = ifcopenshell.open(os.path.join(FIXTURES, FILES[0]), lazy=True)
existing = lazy.by_type("IfcProduct")[0]
existing.Name = "Renamed"
wall = ifcopenshell.api.root.create_entity(lazy, ifc_class="IfcWall")
pset = ifcopenshell.api.pset.add_pset(lazy, product=wall, name="Pset_LazyTest")
ifcopenshell.api.pset.edit_pset(lazy, pset=pset, properties={"Answer": 42})
out = tmp_path / "lazy.ifc"
lazy.write(str(out))
reread = ifcopenshell.open(str(out))
assert reread.by_id(existing.id()).Name == "Renamed"
assert ifcopenshell.util.element.get_psets(reread.by_id(wall.id()))["Pset_LazyTest"]["Answer"] == 42
assert len(list(reread)) == len(list(ifcopenshell.open(os.path.join(FIXTURES, FILES[0])))) + len(list(lazy)) - len(
list(ifcopenshell.open(os.path.join(FIXTURES, FILES[0])))
)


def test_lazy_open_falls_back_on_unsupported_syntax(tmp_path):
src = open(os.path.join(FIXTURES, FILES[0]), "rb").read()
data_at = src.index(b"DATA;")
patched = src[: data_at + 5] + b"\nSTRAY;" + src[data_at + 5 :]
path = tmp_path / "stray.ifc"
path.write_bytes(patched)
f = ifcopenshell.open(str(path), lazy=True)
assert not f.lazy_loading()
assert len(f.by_type("IfcRoot")) > 0
6 changes: 6 additions & 0 deletions src/ifcparse/argument_type.h
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,12 @@ enum argument_type {
Argument_AGGREGATE_OF_AGGREGATE_OF_DOUBLE,
Argument_AGGREGATE_OF_AGGREGATE_OF_ENTITY_INSTANCE,

// Parse-time only: an entity reference (#name) that has not been
// resolved to an instance yet. Never visible once a file is loaded.
Argument_UNRESOLVED_REFERENCE,
Argument_UNRESOLVED_REFERENCE_LIST,
Argument_UNRESOLVED_REFERENCE_LIST_LIST,

Argument_UNKNOWN
};

Expand Down
3 changes: 3 additions & 0 deletions src/ifcparse/entity_instance_data.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@ class size_visitor {
int operator()(const express::base& /*i*/) const { return -1; }
int operator()(const std::vector<express::base>& i) const { return (int)i.size(); }
int operator()(const std::vector<std::vector<express::base>>& i) const { return (int)i.size(); }
int operator()(const unresolved_reference& /*i*/) const { return -1; }
int operator()(const unresolved_reference_list& i) const { return (int)i.names.size(); }
int operator()(const unresolved_reference_list_list& i) const { return (int)i.names.size(); }
};

namespace {
Expand Down
28 changes: 28 additions & 0 deletions src/ifcparse/file.h
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,10 @@ class IFC_PARSE_API instance_streamer {
return good_;
}

void resolve_references_in_place(bool value) {
storage_.resolve_references_in_place = value;
}

const ifcopenshell::unresolved_references& references() const {
return references_to_resolve_;
}
Expand Down Expand Up @@ -233,6 +237,9 @@ class IFC_PARSE_API file {
batch_deletion_ids_t;
batch_deletion_ids_t batch_deletion_ids_;
bool batch_mode_ = false;
bool lazy_loading_ = false;
unsigned parse_threads_ = 0;
bool paged_reading_ = false;
void process_deletion_(const express::base& entity);

public:
Expand Down Expand Up @@ -287,6 +294,27 @@ class IFC_PARSE_API file {
/// @param type_name case insensitive name of the type to bypass
void bypass_type(const std::string& type_name);

// Index the file with one scan and parse each instance's attributes on
// first access instead of parsing everything up front. Set before
// initialize(). Falls back to the full parse if the scan finds anything
// it does not handle.
void lazy_loading(bool value) { lazy_loading_ = value; }
bool lazy_loading() const { return lazy_loading_; }

// Threads used to parse instances; 0 (the default) picks one per core,
// capped at 16, or honours IFCOPENSHELL_PARSE_THREADS. Set before
// initialize().
void parse_threads(unsigned value) { parse_threads_ = value; }
unsigned parse_threads() const { return parse_threads_; }
unsigned effective_parse_threads() const;

// Read the file through the paged reader (64 KB pages, 4 MB cache)
// instead of loading it into memory as a whole. Set before
// initialize(). Applies to the full parse; lazy loading always reads
// in pages.
void paged_reading(bool value) { paged_reading_ = value; }
bool paged_reading() const { return paged_reading_; }

~file();

ifcopenshell::file_open_status good() const { return good_; }
Expand Down
57 changes: 1 addition & 56 deletions src/ifcparse/file_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -75,33 +75,6 @@ full_buffer_impl::full_buffer_impl(const std::string& content, const caller_fed_
, size_(content.size()) {
}

size_t full_buffer_impl::size() const { return size_; }

char full_buffer_impl::get(size_t pos) const {
if (pos >= buf_.size()) {
throw std::out_of_range("get out of range");
}
return buf_[pos];
}

uint64_t full_buffer_impl::get_u64(size_t pos) const {
if (pos + sizeof(uint64_t) > buf_.size()) {
throw std::out_of_range("get_u64 out of range");
}
uint64_t value;
std::memcpy(&value, buf_.data() + pos, sizeof(value));
return value;
}

uint32_t full_buffer_impl::get_u32(size_t pos) const {
if (pos + sizeof(uint32_t) > buf_.size()) {
throw std::out_of_range("get_u32 out of range");
}
uint32_t value;
std::memcpy(&value, buf_.data() + pos, sizeof(value));
return value;
}

void full_buffer_impl::push_next_page(const std::string& data) {
buf_.insert(buf_.end(), data.begin(), data.end());
size_ = buf_.size();
Expand Down Expand Up @@ -136,8 +109,6 @@ paged_file_impl::~paged_file_impl() {
fp_ = nullptr;
}

size_t paged_file_impl::size() const { return file_size_; }

char paged_file_impl::get(size_t pos) const {
if (pos >= file_size_) {
throw std::out_of_range("get out of range");
Expand Down Expand Up @@ -236,6 +207,7 @@ void paged_file_impl::evict_() const {
const size_t victim = lru_.back();
lru_.pop_back();
map_.erase(victim);
++evictions_;
}

#ifdef USE_MMAP
Expand All @@ -247,33 +219,6 @@ mmap_impl::mmap_impl(const std::string& fn) {
size_ = static_cast<size_t>(map_.size());
}

size_t mmap_impl::size() const { return size_; }

char mmap_impl::get(size_t pos) const {
if (pos >= size_) {
throw std::out_of_range("get out of range");
}
return map_.data()[pos];
}

uint64_t mmap_impl::get_u64(size_t pos) const {
if (pos + sizeof(uint64_t) > size_) {
throw std::out_of_range("get_u64 out of range");
}
uint64_t value;
std::memcpy(&value, map_.data() + pos, sizeof(value));
return value;
}

uint32_t mmap_impl::get_u32(size_t pos) const {
if (pos + sizeof(uint32_t) > size_) {
throw std::out_of_range("get_u32 out of range");
}
uint32_t value;
std::memcpy(&value, map_.data() + pos, sizeof(value));
return value;
}

void mmap_impl::push_next_page(const std::string&) {
throw std::logic_error("push_next_page: backend does not support pushed mode");
}
Expand Down
Loading
Loading