Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
## Unreleased

* Send a unique `hostname:pid:uuid` identifier in the `workerid` gRPC header for each worker instance, matching Python. Retain the ID across requests and reconnects while keeping IDs distinct for workers sharing a channel or builder.
* Write self-describing v2 blob payload tokens by default while retaining v1 and v2 reads from the configured storage endpoint and container. Require complete token formats for direct downloads and account for URL token length in worker action-size estimates. Upgrade readers before deploying v2 writers; changing storage locations requires explicit migration or a store configured for the original location.
* Add explicit primary-account-wide and additional-account read scopes for Azure Blob payloads while retaining configured-container access by default. Additional accounts require identity authentication and exact HTTPS account service endpoints. Preserve named and nested v1/v2 references and ignore legacy v2 URL query and fragment suffixes.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,29 @@
import com.google.protobuf.Empty;
import com.google.protobuf.Message;
import com.google.protobuf.StringValue;
import com.microsoft.durabletask.DurableTaskGrpcWorker;
import com.microsoft.durabletask.DurableTaskGrpcWorkerBuilder;
import com.microsoft.durabletask.EntityInstanceId;
import com.microsoft.durabletask.EntityOperationFailedException;
import com.microsoft.durabletask.FailureDetails;
import com.microsoft.durabletask.OrchestrationRunner;
import com.microsoft.durabletask.TaskOrchestration;
import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.*;
import com.microsoft.durabletask.implementation.protobuf.TaskHubSidecarServiceGrpc;
import io.grpc.CallOptions;
import io.grpc.Channel;
import io.grpc.ClientCall;
import io.grpc.ManagedChannel;
import io.grpc.Metadata;
import io.grpc.MethodDescriptor;
import io.grpc.Server;
import io.grpc.ServerCall;
import io.grpc.ServerCallHandler;
import io.grpc.ServerInterceptor;
import io.grpc.inprocess.InProcessChannelBuilder;
import io.grpc.inprocess.InProcessServerBuilder;
import io.grpc.protobuf.ProtoUtils;
import io.grpc.stub.StreamObserver;

import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
Expand All @@ -27,6 +39,11 @@

import java.io.IOException;
import java.io.UncheckedIOException;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;

import static org.junit.jupiter.api.Assertions.*;
Expand Down Expand Up @@ -625,6 +642,67 @@ void isPermanentStorageFailure_runtimeException_returnsFalse() {
new RuntimeException("test")));
}

@Test
void permanentPayloadFailureCompletionsRetainWorkerId() throws Exception {
String token = "blob:v1:durabletask-payloads:missing";
BlobStorageException failure = storageFailure(404);
when(mockStore.download(token)).thenThrow(failure);
List<String> workerIds = Collections.synchronizedList(new ArrayList<>());
AtomicReference<ActivityResponse> completion = new AtomicReference<>();
String serverName = InProcessServerBuilder.generateName();
Server server = InProcessServerBuilder.forName(serverName).directExecutor()
.intercept(new ServerInterceptor() {
@Override
public <ReqT, RespT> ServerCall.Listener<ReqT> interceptCall(
ServerCall<ReqT, RespT> call, Metadata headers, ServerCallHandler<ReqT, RespT> next) {
workerIds.add(headers.get(Metadata.Key.of("workerid", Metadata.ASCII_STRING_MARSHALLER)));
return next.startCall(call, headers);
}
})
.addService(new TaskHubSidecarServiceGrpc.TaskHubSidecarServiceImplBase() {
@Override
public void getWorkItems(GetWorkItemsRequest request, StreamObserver<WorkItem> observer) {
observer.onNext(WorkItem.newBuilder().setCompletionToken("failure-token")
.setActivityRequest(ActivityRequest.newBuilder().setName("Echo").setTaskId(1)
.setOrchestrationInstance(OrchestrationInstance.newBuilder().setInstanceId("test"))
.setInput(StringValue.of(token))).build());
observer.onCompleted();
}

@Override
public void completeActivityTask(ActivityResponse request, StreamObserver<CompleteTaskResponse> observer) {
completion.set(request);
observer.onNext(CompleteTaskResponse.getDefaultInstance());
observer.onCompleted();
}
}).build().start();
ManagedChannel channel = InProcessChannelBuilder.forName(serverName).directExecutor().build();
try (DurableTaskGrpcWorker worker = new DurableTaskGrpcWorkerBuilder()
.grpcChannel(channel).addInterceptor(interceptor).build()) {
Field field = DurableTaskGrpcWorker.class.getDeclaredField("sidecarClient");
field.setAccessible(true);
TaskHubSidecarServiceGrpc.TaskHubSidecarServiceBlockingStub stub =
(TaskHubSidecarServiceGrpc.TaskHubSidecarServiceBlockingStub) field.get(worker);
assertFalse(stub.withDeadlineAfter(5, TimeUnit.SECONDS)
.getWorkItems(GetWorkItemsRequest.getDefaultInstance()).hasNext(),
"The permanently unresolvable work item must be failed and removed from the stream");

assertNotNull(completion.get());
assertTrue(completion.get().getFailureDetails().getIsNonRetriable());
assertEquals("failure-token", completion.get().getCompletionToken());
assertEquals(2, workerIds.size());
assertNotNull(workerIds.get(0));
assertFalse(workerIds.get(0).isEmpty());
assertEquals(workerIds.get(0), workerIds.get(1),
"The interceptor's downstream failure completion must carry the polling worker's ID");
} finally {
channel.shutdownNow();
server.shutdownNow();
assertTrue(channel.awaitTermination(5, TimeUnit.SECONDS));
assertTrue(server.awaitTermination(5, TimeUnit.SECONDS));
}
}

// ==================== Helpers ====================

private static BlobStorageException storageFailure(int status) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@
import com.google.protobuf.Empty;
import com.microsoft.durabletask.DurableTaskClient;
import com.microsoft.durabletask.DurableTaskGrpcWorker;
import com.microsoft.durabletask.DurableTaskGrpcWorkerBuilder;
import com.microsoft.durabletask.implementation.protobuf.TaskHubSidecarServiceGrpc.TaskHubSidecarServiceBlockingStub;
import io.grpc.CallOptions;
import io.grpc.Channel;
import io.grpc.Grpc;
import io.grpc.ManagedChannel;
import io.grpc.Metadata;
Expand All @@ -23,6 +26,9 @@
import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.MockedStatic;

import java.lang.management.ManagementFactory;
import java.lang.management.RuntimeMXBean;
import java.lang.reflect.Field;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.util.Arrays;
Expand All @@ -39,8 +45,9 @@
class AudienceTransportTest {
@ParameterizedTest
@ValueSource(booleans = {false, true})
void localGrpcReconnectRetainsScopeAndTokenCache(boolean worker) throws Exception {
void localGrpcReconnectRetainsScopeTokenCacheAndWorkerId(boolean worker) throws Exception {
List<String> authorizations = Collections.synchronizedList(new ArrayList<>());
List<String> workerIds = Collections.synchronizedList(new ArrayList<>());
Set<SocketAddress> connections = Collections.synchronizedSet(new HashSet<>());
Server server = NettyServerBuilder.forAddress(new InetSocketAddress("127.0.0.1", 0))
.addService(ServerServiceDefinition.builder("test.Service")
Expand All @@ -54,12 +61,14 @@ void localGrpcReconnectRetainsScopeAndTokenCache(boolean worker) throws Exceptio
public <ReqT, RespT> ServerCall.Listener<ReqT> interceptCall(
ServerCall<ReqT, RespT> call, Metadata headers, ServerCallHandler<ReqT, RespT> next) {
authorizations.add(headers.get(Metadata.Key.of("Authorization", Metadata.ASCII_STRING_MARSHALLER)));
workerIds.add(headers.get(Metadata.Key.of("workerid", Metadata.ASCII_STRING_MARSHALLER)));
connections.add(call.getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR));
return next.startCall(call, headers);
}
})
.build().start();
ManagedChannel channel = null;
DurableTaskGrpcWorker taskWorker = null;
try (MockedStatic<ResourceId> defaults = mockStatic(ResourceId.class, CALLS_REAL_METHODS)) {
defaults.when(ResourceId::getDefault).thenReturn("https://durabletask.azure.us");
ResourceIdTest.RecordingCredential credential = new ResourceIdTest.RecordingCredential();
Expand All @@ -71,16 +80,32 @@ public <ReqT, RespT> ServerCall.Listener<ReqT> interceptCall(
: new DurableTaskSchedulerClientOptions().setEndpointAddress(endpoint)
.setAllowInsecureCredentials(true).setTaskHubName("test-hub")
.setCredential(credential).createGrpcChannel());
Channel requestChannel = channel;
if (worker) {
taskWorker = new DurableTaskGrpcWorkerBuilder().grpcChannel(channel).build();
requestChannel = workerChannel(taskWorker);
}
assertTrue(credential.scopes.isEmpty());
call(channel);
call(requestChannel);
defaults.when(ResourceId::getDefault).thenReturn("https://durabletask.io");
channel.enterIdle();
call(channel);
call(channel);
call(requestChannel);
call(requestChannel);
assertEquals(Arrays.asList("Bearer token-1", "Bearer token-2", "Bearer token-2"), authorizations);
assertEquals(Collections.nCopies(2, "https://durabletask.azure.us/.default"), credential.scopes);
assertEquals(2, connections.size(), "The second call must use a new transport connection");
if (worker) {
assertNotNull(workerIds.get(0));
assertFalse(workerIds.get(0).isEmpty());
assertEquals(Collections.nCopies(3, workerIds.get(0)), workerIds,
"The worker ID must survive token refresh and transport reconnection");
} else {
assertEquals(Collections.nCopies(3, null), workerIds);
}
} finally {
if (taskWorker != null) {
taskWorker.close();
}
if (channel != null) {
channel.shutdownNow();
}
Expand All @@ -92,6 +117,19 @@ public <ReqT, RespT> ServerCall.Listener<ReqT> interceptCall(
}
}

@Test
void invalidWorkerProcessIdFailsWithAnExplicitError() {
RuntimeMXBean runtime = mock(RuntimeMXBean.class);
when(runtime.getName()).thenReturn("not-a-pid@host");
try (MockedStatic<ManagementFactory> management = mockStatic(ManagementFactory.class)) {
management.when(ManagementFactory::getRuntimeMXBean).thenReturn(runtime);
IllegalStateException error = assertThrows(IllegalStateException.class,
() -> new DurableTaskGrpcWorkerBuilder().build());
assertEquals("Cannot determine the worker process ID.", error.getMessage());
assertInstanceOf(NumberFormatException.class, error.getCause());
}
}

@Test
void callerSuppliedChannelsRemainCallerOwnedAndDoNotAcquireTokensAtBuildTime() {
ResourceIdTest.RecordingCredential credential = new ResourceIdTest.RecordingCredential();
Expand All @@ -110,7 +148,13 @@ void callerSuppliedChannelsRemainCallerOwnedAndDoNotAcquireTokensAtBuildTime() {
assertTrue(credential.scopes.isEmpty());
}

private static void call(ManagedChannel channel) {
private static Channel workerChannel(DurableTaskGrpcWorker worker) throws ReflectiveOperationException {
Field field = DurableTaskGrpcWorker.class.getDeclaredField("sidecarClient");
field.setAccessible(true);
return ((TaskHubSidecarServiceBlockingStub) field.get(worker)).getChannel();
}

private static void call(Channel channel) {
ClientCalls.blockingUnaryCall(channel, ResourceIdTest.METHOD,
CallOptions.DEFAULT.withDeadlineAfter(10, TimeUnit.SECONDS), Empty.getDefaultInstance());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,15 @@

import io.grpc.*;
import io.grpc.ClientInterceptors;
import io.grpc.stub.MetadataUtils;
import io.opentelemetry.api.trace.Span;
import io.opentelemetry.api.trace.SpanKind;
import io.opentelemetry.api.trace.StatusCode;
import io.opentelemetry.context.Scope;

import java.lang.management.ManagementFactory;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.time.Duration;
import java.time.Instant;
import java.util.*;
Expand All @@ -32,6 +36,8 @@

/**
* Task hub worker that connects to a sidecar process over gRPC to execute orchestrator and activity events.
* Each instance sends a unique {@code hostname:pid:uuid} identifier in the {@code workerid}
* gRPC header, retained across requests and reconnections even when the channel is shared.
*/
public final class DurableTaskGrpcWorker implements AutoCloseable {
private static final int DEFAULT_PORT = 4001;
Expand Down Expand Up @@ -62,6 +68,9 @@ public final class DurableTaskGrpcWorker implements AutoCloseable {
private final int largePayloadTokenSizeBytes;

DurableTaskGrpcWorker(DurableTaskGrpcWorkerBuilder builder, WorkItemFilter workItemFilter) {
Metadata workerHeaders = new Metadata();
workerHeaders.put(Metadata.Key.of("workerid", Metadata.ASCII_STRING_MARSHALLER), createWorkerId());

this.orchestrationFactories.putAll(builder.orchestrationFactories);
this.activityFactories.putAll(builder.activityFactories);
this.entityFactories.putAll(builder.entityFactories);
Expand All @@ -87,6 +96,10 @@ public final class DurableTaskGrpcWorker implements AutoCloseable {
sidecarGrpcChannel = this.managedSidecarChannel;
}

// Include worker metadata on downstream calls made by registered interceptors.
sidecarGrpcChannel = ClientInterceptors.intercept(
sidecarGrpcChannel, MetadataUtils.newAttachHeadersInterceptor(workerHeaders));

// Apply any registered interceptors (e.g., large payload externalization)
if (!builder.interceptors.isEmpty()) {
sidecarGrpcChannel = ClientInterceptors.intercept(sidecarGrpcChannel, builder.interceptors);
Expand Down Expand Up @@ -116,6 +129,18 @@ public final class DurableTaskGrpcWorker implements AutoCloseable {
this.getWorkItemsRequest = buildGetWorkItemsRequest();
}

private static String createWorkerId() {
try {
// Java 8 has no ProcessHandle API; the runtime MXBean exposes the PID as pid@hostname.
long processId = Long.parseLong(ManagementFactory.getRuntimeMXBean().getName().split("@", 2)[0]);
return InetAddress.getLocalHost().getHostName() + ":" + processId + ":" + UUID.randomUUID();
} catch (NumberFormatException e) {
throw new IllegalStateException("Cannot determine the worker process ID.", e);
} catch (UnknownHostException e) {
throw new IllegalStateException("Cannot determine the hostname for the worker ID.", e);
}
}

/**
* Establishes a gRPC connection to the sidecar and starts processing work-items in the background.
* <p>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.UUID;

import static org.junit.jupiter.api.Assertions.*;

Expand All @@ -38,6 +40,7 @@ public class OrchestratorChunkingTest {
private final List<OrchestratorResponse> capturedResponses =
Collections.synchronizedList(new ArrayList<>());
private final List<String> abandonedTokens = new ArrayList<>();
private final List<String> capturedWorkerIds = new ArrayList<>();

private Server inProcessServer;
private ManagedChannel inProcessChannel;
Expand All @@ -48,6 +51,14 @@ void setUp() throws Exception {

inProcessServer = InProcessServerBuilder.forName(serverName)
.directExecutor()
.intercept(new ServerInterceptor() {
@Override
public <ReqT, RespT> ServerCall.Listener<ReqT> interceptCall(
ServerCall<ReqT, RespT> call, Metadata headers, ServerCallHandler<ReqT, RespT> next) {
capturedWorkerIds.add(headers.get(Metadata.Key.of("workerid", Metadata.ASCII_STRING_MARSHALLER)));
return next.startCall(call, headers);
}
})
.addService(new TaskHubSidecarServiceGrpc.TaskHubSidecarServiceImplBase() {
@Override
public void completeOrchestratorTask(
Expand Down Expand Up @@ -87,6 +98,35 @@ void tearDown() {

// ==================== validateActionsSize tests ====================

@Test
void workerIdsMatchPythonFormatAndRemainUniqueWhenSharingBuilderAndChannel() throws Exception {
DurableTaskGrpcWorkerBuilder builder = new DurableTaskGrpcWorkerBuilder().grpcChannel(inProcessChannel);
OrchestratorResponse response = OrchestratorResponse.getDefaultInstance();
try (DurableTaskGrpcWorker first = builder.build();
DurableTaskGrpcWorker second = builder.build()) {
invokeChunkingMethod(first, response);
invokeChunkingMethod(second, response);
invokeChunkingMethod(first, response);
}
TaskHubSidecarServiceGrpc.newBlockingStub(inProcessChannel).completeOrchestratorTask(response);

assertEquals(4, capturedWorkerIds.size());
String firstId = capturedWorkerIds.get(0);
String secondId = capturedWorkerIds.get(1);
assertNotNull(firstId);
assertNotNull(secondId);
assertNotEquals(firstId, secondId, "Workers must not share an ID even when their channel and builder are shared");
assertEquals(firstId, capturedWorkerIds.get(2), "A worker must retain its ID across requests");
assertNull(capturedWorkerIds.get(3), "The underlying shared channel must not acquire a worker ID");
for (String id : new String[] {firstId, secondId}) {
String[] parts = id.split(":", -1);
assertEquals(3, parts.length);
assertEquals(InetAddress.getLocalHost().getHostName(), parts[0]);
assertEquals(Long.toString(ProcessHandle.current().pid()), parts[1]);
assertEquals(4, UUID.fromString(parts[2]).version());
}
}

@Test
void workerBuilder_defaultLargePayloadThreshold_matchesStorageDefault() {
DurableTaskGrpcWorkerBuilder builder = new DurableTaskGrpcWorkerBuilder();
Expand Down
Loading