Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 34 additions & 5 deletions src/main/java/org/lmdbjava/ByteBufProxy.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import static io.netty.buffer.PooledByteBufAllocator.DEFAULT;
import static java.lang.Class.forName;
import static java.util.Objects.requireNonNull;
import static org.lmdbjava.Library.RUNTIME;
import static org.lmdbjava.UnsafeAccess.UNSAFE;

import io.netty.buffer.ByteBuf;
Expand All @@ -26,6 +27,7 @@
import java.nio.ByteOrder;
import java.util.Comparator;
import jnr.ffi.Pointer;
import jnr.ffi.provider.MemoryManager;

/**
* A buffer proxy backed by Netty's {@link ByteBuf}.
Expand All @@ -45,6 +47,7 @@ public final class ByteBufProxy extends BufferProxy<ByteBuf> {
private static final String FIELD_NAME_ADDRESS = "memoryAddress";
private static final String FIELD_NAME_LENGTH = "length";
private static final String NAME = "io.netty.buffer.PooledUnsafeDirectByteBuf";
private static final MemoryManager MEM_MGR = RUNTIME.getMemoryManager();
private final long lengthOffset;
private final long addressOffset;

Expand Down Expand Up @@ -195,22 +198,48 @@ protected byte[] getBytes(final ByteBuf buffer) {

@Override
protected Pointer in(final ByteBuf buffer, final Pointer ptr) {
final long ptrAddr = ptr.address();
UNSAFE.putLong(ptrAddr + STRUCT_FIELD_OFFSET_SIZE, buffer.writerIndex() - buffer.readerIndex());
UNSAFE.putLong(
ptrAddr + STRUCT_FIELD_OFFSET_DATA, buffer.memoryAddress() + buffer.readerIndex());
return null;
final int size = buffer.writerIndex() - buffer.readerIndex();
if (buffer.hasMemoryAddress()) {
// Fast path: the buffer is direct, so point the MDB_val straight at its memory (zero copy).
final long ptrAddr = ptr.address();
UNSAFE.putLong(ptrAddr + STRUCT_FIELD_OFFSET_SIZE, size);
UNSAFE.putLong(
ptrAddr + STRUCT_FIELD_OFFSET_DATA, buffer.memoryAddress() + buffer.readerIndex());
return null;
}
// Address-less buffer (any heap ByteBuf, incl. a heap-backed Netty 4.2 AdaptiveByteBuf):
// buffer.memoryAddress() would throw UnsupportedOperationException (lmdbjava#261). Copy the
// readable bytes into native scratch and point the MDB_val there — the same approach
// ByteArrayProxy uses. The returned Pointer keeps that scratch reachable for the native call.
return copyToNative(buffer, size, ptr);
}

@Override
protected Pointer in(final ByteBuf buffer, final int size, final Pointer ptr) {
if (!buffer.hasMemoryAddress()) {
// The reserve path repoints the caller's buffer at LMDB-owned memory via out()'s field swap,
// which only works on a direct PooledUnsafeDirectByteBuf. A heap buffer cannot be repointed,
// so fail with a clear message instead of an opaque UnsupportedOperationException.
throw new LmdbException(
"ByteBuf reserve requires a direct buffer (hasMemoryAddress()==true)");
}
final long ptrAddr = ptr.address();
UNSAFE.putLong(ptrAddr + STRUCT_FIELD_OFFSET_SIZE, size);
UNSAFE.putLong(
ptrAddr + STRUCT_FIELD_OFFSET_DATA, buffer.memoryAddress() + buffer.readerIndex());
return null;
}

private static Pointer copyToNative(final ByteBuf buffer, final int size, final Pointer ptr) {
final Pointer pointer = MEM_MGR.allocateDirect(size);
final byte[] bytes = new byte[size];
buffer.getBytes(buffer.readerIndex(), bytes);
pointer.put(0, bytes, 0, size);
ptr.putLong(STRUCT_FIELD_OFFSET_SIZE, size);
ptr.putAddress(STRUCT_FIELD_OFFSET_DATA, pointer.address());
return pointer;
}

@Override
protected ByteBuf out(final ByteBuf buffer, final Pointer ptr) {
final long ptrAddr = ptr.address();
Expand Down
111 changes: 111 additions & 0 deletions src/test/java/org/lmdbjava/ByteBufHeapBufferTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
/*
* Copyright © 2016-2025 The LmdbJava Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.lmdbjava;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.assertj.core.api.Assertions.assertThat;
import static org.lmdbjava.DbiFlags.MDB_CREATE;

import io.netty.buffer.ByteBuf;
import io.netty.buffer.PooledByteBufAllocator;
import java.nio.file.Path;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

/**
* Covers heap (address-less) buffers through {@link ByteBufProxy#PROXY_NETTY}. Previously any
* buffer with {@code hasMemoryAddress() == false} made {@code in()} throw {@code
* UnsupportedOperationException} from {@code ByteBuf.memoryAddress()}. This is the
* version-independent root of lmdbjava#261 (Netty 4.2's adaptive allocator can hand back heap
* buffers); it reproduces on 4.1 with any heap buffer.
*/
final class ByteBufHeapBufferTest {

private static final String VALUE = "Hello World";
private static final byte[] VALUE_BYTES = VALUE.getBytes(UTF_8);

private TempDir tempDir;

@BeforeEach
void beforeEach() {
tempDir = new TempDir();
}

@AfterEach
void afterEach() {
tempDir.cleanup();
}

private Env<ByteBuf> openEnv() {
final Path dir = tempDir.createTempDir();
return Env.create(ByteBufProxy.PROXY_NETTY).setMapSize(10_485_760).setMaxDbs(1).open(dir);
}

/** A heap value must round-trip (key direct, value heap). */
@Test
void putGet_withHeapValue() {
try (Env<ByteBuf> env = openEnv()) {
final Dbi<ByteBuf> db =
env.createDbi().setDbName("db").withDefaultComparator().addDbiFlag(MDB_CREATE).open();
final ByteBuf key = PooledByteBufAllocator.DEFAULT.directBuffer(env.getMaxKeySize());
final ByteBuf value = PooledByteBufAllocator.DEFAULT.heapBuffer(64);
try {
assertThat(value.hasMemoryAddress()).isFalse(); // sanity: exercising the heap path
key.writeCharSequence("greeting", UTF_8);
value.writeCharSequence(VALUE, UTF_8);
db.put(key, value);
try (Txn<ByteBuf> txn = env.txnRead()) {
final ByteBuf found = db.get(txn, key);
assertThat(found).isNotNull();
final byte[] got = new byte[found.readableBytes()];
found.getBytes(found.readerIndex(), got);
assertThat(got).isEqualTo(VALUE_BYTES);
}
} finally {
key.release();
value.release();
}
}
}

/** A heap key must work for both put and lookup. */
@Test
void putGet_withHeapKeyAndValue() {
try (Env<ByteBuf> env = openEnv()) {
final Dbi<ByteBuf> db =
env.createDbi().setDbName("db").withDefaultComparator().addDbiFlag(MDB_CREATE).open();
final ByteBuf key = PooledByteBufAllocator.DEFAULT.heapBuffer(env.getMaxKeySize());
final ByteBuf value = PooledByteBufAllocator.DEFAULT.heapBuffer(64);
try {
assertThat(key.hasMemoryAddress()).isFalse();
key.writeCharSequence("greeting", UTF_8);
value.writeCharSequence(VALUE, UTF_8);
db.put(key, value);
try (Txn<ByteBuf> txn = env.txnRead()) {
final ByteBuf found = db.get(txn, key); // heap key lookup
assertThat(found).isNotNull();
final byte[] got = new byte[found.readableBytes()];
found.getBytes(found.readerIndex(), got);
assertThat(got).isEqualTo(VALUE_BYTES);
}
} finally {
key.release();
value.release();
}
}
}
}