Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.activemq.network;
018
019import java.io.IOException;
020import java.security.GeneralSecurityException;
021import java.security.cert.X509Certificate;
022import java.util.Arrays;
023import java.util.Collection;
024import java.util.Collections;
025import java.util.Iterator;
026import java.util.List;
027import java.util.Properties;
028import java.util.Set;
029import java.util.concurrent.ConcurrentHashMap;
030import java.util.concurrent.ConcurrentMap;
031import java.util.concurrent.CountDownLatch;
032import java.util.concurrent.ExecutionException;
033import java.util.concurrent.ExecutorService;
034import java.util.concurrent.Executors;
035import java.util.concurrent.Future;
036import java.util.concurrent.TimeUnit;
037import java.util.concurrent.TimeoutException;
038import java.util.concurrent.atomic.AtomicBoolean;
039import java.util.regex.Pattern;
040
041import javax.management.ObjectName;
042
043import org.apache.activemq.DestinationDoesNotExistException;
044import org.apache.activemq.Service;
045import org.apache.activemq.advisory.AdvisoryBroker;
046import org.apache.activemq.advisory.AdvisorySupport;
047import org.apache.activemq.broker.BrokerService;
048import org.apache.activemq.broker.BrokerServiceAware;
049import org.apache.activemq.broker.ConnectionContext;
050import org.apache.activemq.broker.TransportConnection;
051import org.apache.activemq.broker.region.AbstractRegion;
052import org.apache.activemq.broker.region.DurableTopicSubscription;
053import org.apache.activemq.broker.region.Region;
054import org.apache.activemq.broker.region.RegionBroker;
055import org.apache.activemq.broker.region.Subscription;
056import org.apache.activemq.broker.region.policy.PolicyEntry;
057import org.apache.activemq.command.ActiveMQDestination;
058import org.apache.activemq.command.ActiveMQMessage;
059import org.apache.activemq.command.ActiveMQTempDestination;
060import org.apache.activemq.command.ActiveMQTopic;
061import org.apache.activemq.command.BrokerId;
062import org.apache.activemq.command.BrokerInfo;
063import org.apache.activemq.command.BrokerSubscriptionInfo;
064import org.apache.activemq.command.Command;
065import org.apache.activemq.command.CommandTypes;
066import org.apache.activemq.command.ConnectionError;
067import org.apache.activemq.command.ConnectionId;
068import org.apache.activemq.command.ConnectionInfo;
069import org.apache.activemq.command.ConsumerId;
070import org.apache.activemq.command.ConsumerInfo;
071import org.apache.activemq.command.DataStructure;
072import org.apache.activemq.command.DestinationInfo;
073import org.apache.activemq.command.ExceptionResponse;
074import org.apache.activemq.command.KeepAliveInfo;
075import org.apache.activemq.command.Message;
076import org.apache.activemq.command.MessageAck;
077import org.apache.activemq.command.MessageDispatch;
078import org.apache.activemq.command.MessageId;
079import org.apache.activemq.command.NetworkBridgeFilter;
080import org.apache.activemq.command.ProducerInfo;
081import org.apache.activemq.command.RemoveInfo;
082import org.apache.activemq.command.RemoveSubscriptionInfo;
083import org.apache.activemq.command.Response;
084import org.apache.activemq.command.SessionInfo;
085import org.apache.activemq.command.ShutdownInfo;
086import org.apache.activemq.command.SubscriptionInfo;
087import org.apache.activemq.command.WireFormatInfo;
088import org.apache.activemq.filter.DestinationFilter;
089import org.apache.activemq.filter.NonCachedMessageEvaluationContext;
090import org.apache.activemq.security.SecurityContext;
091import org.apache.activemq.transport.DefaultTransportListener;
092import org.apache.activemq.transport.FutureResponse;
093import org.apache.activemq.transport.ResponseCallback;
094import org.apache.activemq.transport.Transport;
095import org.apache.activemq.transport.TransportDisposedIOException;
096import org.apache.activemq.transport.TransportFilter;
097import org.apache.activemq.transport.failover.FailoverTransport;
098import org.apache.activemq.transport.tcp.TcpTransport;
099import org.apache.activemq.util.IdGenerator;
100import org.apache.activemq.util.IntrospectionSupport;
101import org.apache.activemq.util.LongSequenceGenerator;
102import org.apache.activemq.util.MarshallingSupport;
103import org.apache.activemq.util.NetworkBridgeUtils;
104import org.apache.activemq.util.ServiceStopper;
105import org.apache.activemq.util.ServiceSupport;
106import org.apache.activemq.util.StringToListOfActiveMQDestinationConverter;
107import org.slf4j.Logger;
108import org.slf4j.LoggerFactory;
109
110/**
111 * A useful base class for implementing demand forwarding bridges.
112 */
113public abstract class DemandForwardingBridgeSupport implements NetworkBridge, BrokerServiceAware {
114    private static final Logger LOG = LoggerFactory.getLogger(DemandForwardingBridgeSupport.class);
115    protected static final String DURABLE_SUB_PREFIX = "NC-DS_";
116    protected final Transport localBroker;
117    protected final Transport remoteBroker;
118    protected IdGenerator idGenerator = new IdGenerator();
119    protected final LongSequenceGenerator consumerIdGenerator = new LongSequenceGenerator();
120    protected ConnectionInfo localConnectionInfo;
121    protected ConnectionInfo remoteConnectionInfo;
122    protected SessionInfo localSessionInfo;
123    protected ProducerInfo producerInfo;
124    protected String remoteBrokerName = "Unknown";
125    protected String localClientId;
126    protected ConsumerInfo demandConsumerInfo;
127    protected int demandConsumerDispatched;
128    protected final AtomicBoolean localBridgeStarted = new AtomicBoolean(false);
129    protected final AtomicBoolean remoteBridgeStarted = new AtomicBoolean(false);
130    protected final AtomicBoolean bridgeFailed = new AtomicBoolean();
131    protected final AtomicBoolean disposed = new AtomicBoolean();
132    protected BrokerId localBrokerId;
133    protected ActiveMQDestination[] excludedDestinations;
134    protected ActiveMQDestination[] dynamicallyIncludedDestinations;
135    protected ActiveMQDestination[] staticallyIncludedDestinations;
136    protected ActiveMQDestination[] durableDestinations;
137    protected final ConcurrentMap<ConsumerId, DemandSubscription> subscriptionMapByLocalId = new ConcurrentHashMap<>();
138    protected final ConcurrentMap<ConsumerId, DemandSubscription> subscriptionMapByRemoteId = new ConcurrentHashMap<>();
139    protected final Set<ConsumerId> forcedDurableRemoteId = Collections.newSetFromMap(new ConcurrentHashMap<ConsumerId, Boolean>());
140    protected final BrokerId localBrokerPath[] = new BrokerId[]{null};
141    protected final CountDownLatch startedLatch = new CountDownLatch(2);
142    protected final CountDownLatch localStartedLatch = new CountDownLatch(1);
143    protected final CountDownLatch staticDestinationsLatch = new CountDownLatch(1);
144    protected final AtomicBoolean lastConnectSucceeded = new AtomicBoolean(false);
145    protected NetworkBridgeConfiguration configuration;
146    protected final NetworkBridgeFilterFactory defaultFilterFactory = new DefaultNetworkBridgeFilterFactory();
147
148    protected final BrokerId remoteBrokerPath[] = new BrokerId[]{null};
149    protected BrokerId remoteBrokerId;
150
151    protected final NetworkBridgeStatistics networkBridgeStatistics = new NetworkBridgeStatistics();
152
153    private NetworkBridgeListener networkBridgeListener;
154    private boolean createdByDuplex;
155    private BrokerInfo localBrokerInfo;
156    private BrokerInfo remoteBrokerInfo;
157
158    private final FutureBrokerInfo futureRemoteBrokerInfo = new FutureBrokerInfo(remoteBrokerInfo, disposed);
159    private final FutureBrokerInfo futureLocalBrokerInfo = new FutureBrokerInfo(localBrokerInfo, disposed);
160
161    private final AtomicBoolean started = new AtomicBoolean();
162    private TransportConnection duplexInitiatingConnection;
163    private final AtomicBoolean duplexInitiatingConnectionInfoReceived = new AtomicBoolean();
164    protected BrokerService brokerService = null;
165    private ObjectName mbeanObjectName;
166    private final ExecutorService serialExecutor = Executors.newSingleThreadExecutor();
167    //Use a new executor for processing BrokerSubscriptionInfo so we don't block other threads
168    private final ExecutorService syncExecutor = Executors.newSingleThreadExecutor();
169    private Transport duplexInboundLocalBroker = null;
170    private ProducerInfo duplexInboundLocalProducerInfo;
171
172    public DemandForwardingBridgeSupport(NetworkBridgeConfiguration configuration, Transport localBroker, Transport remoteBroker) {
173        this.configuration = configuration;
174        this.localBroker = localBroker;
175        this.remoteBroker = remoteBroker;
176    }
177
178    public void duplexStart(TransportConnection connection, BrokerInfo localBrokerInfo, BrokerInfo remoteBrokerInfo) throws Exception {
179        this.localBrokerInfo = localBrokerInfo;
180        this.remoteBrokerInfo = remoteBrokerInfo;
181        this.duplexInitiatingConnection = connection;
182        start();
183        serviceRemoteCommand(remoteBrokerInfo);
184    }
185
186    @Override
187    public void start() throws Exception {
188        if (started.compareAndSet(false, true)) {
189
190            if (brokerService == null) {
191                throw new IllegalArgumentException("BrokerService is null on " + this);
192            }
193
194            networkBridgeStatistics.setEnabled(brokerService.isEnableStatistics());
195
196            if (isDuplex()) {
197                duplexInboundLocalBroker = NetworkBridgeFactory.createLocalAsyncTransport(brokerService.getBroker().getVmConnectorURI());
198                duplexInboundLocalBroker.setTransportListener(new DefaultTransportListener() {
199
200                    @Override
201                    public void onCommand(Object o) {
202                        Command command = (Command) o;
203                        serviceLocalCommand(command);
204                    }
205
206                    @Override
207                    public void onException(IOException error) {
208                        serviceLocalException(error);
209                    }
210                });
211                duplexInboundLocalBroker.start();
212            }
213
214            localBroker.setTransportListener(new DefaultTransportListener() {
215
216                @Override
217                public void onCommand(Object o) {
218                    Command command = (Command) o;
219                    serviceLocalCommand(command);
220                }
221
222                @Override
223                public void onException(IOException error) {
224                    if (!futureLocalBrokerInfo.isDone()) {
225                        LOG.info("Error with pending local brokerInfo on: {} ({})", localBroker, error.getMessage());
226                        LOG.debug("Peer error: ", error);
227                        futureLocalBrokerInfo.cancel(true);
228                        return;
229                    }
230                    serviceLocalException(error);
231                }
232            });
233
234            remoteBroker.setTransportListener(new DefaultTransportListener() {
235
236                @Override
237                public void onCommand(Object o) {
238                    Command command = (Command) o;
239                    serviceRemoteCommand(command);
240                }
241
242                @Override
243                public void onException(IOException error) {
244                    if (!futureRemoteBrokerInfo.isDone()) {
245                        LOG.info("Error with pending remote brokerInfo on: {} ({})", remoteBroker, error.getMessage());
246                        LOG.debug("Peer error: ", error);
247                        futureRemoteBrokerInfo.cancel(true);
248                        return;
249                    }
250                    serviceRemoteException(error);
251                }
252            });
253
254            remoteBroker.start();
255            localBroker.start();
256
257            if (!disposed.get()) {
258                try {
259                    triggerStartAsyncNetworkBridgeCreation();
260                } catch (IOException e) {
261                    LOG.warn("Caught exception from remote start", e);
262                }
263            } else {
264                LOG.warn("Bridge was disposed before the start() method was fully executed.");
265                throw new TransportDisposedIOException();
266            }
267        }
268    }
269
270    @Override
271    public void stop() throws Exception {
272        if (started.compareAndSet(true, false)) {
273            if (disposed.compareAndSet(false, true)) {
274                LOG.debug(" stopping {} bridge to {}", configuration.getBrokerName(), remoteBrokerName);
275
276                futureRemoteBrokerInfo.cancel(true);
277                futureLocalBrokerInfo.cancel(true);
278
279                NetworkBridgeListener l = this.networkBridgeListener;
280                if (l != null) {
281                    l.onStop(this);
282                }
283                try {
284                    // local start complete
285                    if (startedLatch.getCount() < 2) {
286                        LOG.trace("{} unregister bridge ({}) to {}",
287                                configuration.getBrokerName(), this, remoteBrokerName);
288                        brokerService.getBroker().removeBroker(null, remoteBrokerInfo);
289                        brokerService.getBroker().networkBridgeStopped(remoteBrokerInfo);
290                    }
291
292                    remoteBridgeStarted.set(false);
293                    final CountDownLatch sendShutdown = new CountDownLatch(1);
294
295                    brokerService.getTaskRunnerFactory().execute(new Runnable() {
296                        @Override
297                        public void run() {
298                            try {
299                                serialExecutor.shutdown();
300                                if (!serialExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
301                                    List<Runnable> pendingTasks = serialExecutor.shutdownNow();
302                                    LOG.info("pending tasks on stop {}", pendingTasks);
303                                }
304                                //Shutdown the syncExecutor, call countDown to make sure a thread can
305                                //terminate if it is waiting
306                                staticDestinationsLatch.countDown();
307                                syncExecutor.shutdown();
308                                if (!syncExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
309                                    List<Runnable> pendingTasks = syncExecutor.shutdownNow();
310                                    LOG.info("pending tasks on stop {}", pendingTasks);
311                                }
312                                localBroker.oneway(new ShutdownInfo());
313                                remoteBroker.oneway(new ShutdownInfo());
314                            } catch (Throwable e) {
315                                LOG.debug("Caught exception sending shutdown", e);
316                            } finally {
317                                sendShutdown.countDown();
318                            }
319
320                        }
321                    }, "ActiveMQ ForwardingBridge StopTask");
322
323                    if (!sendShutdown.await(10, TimeUnit.SECONDS)) {
324                        LOG.info("Network Could not shutdown in a timely manner");
325                    }
326                } finally {
327                    ServiceStopper ss = new ServiceStopper();
328                    stopFailoverTransport(remoteBroker);
329                    ss.stop(remoteBroker);
330                    ss.stop(localBroker);
331                    ss.stop(duplexInboundLocalBroker);
332                    // Release the started Latch since another thread could be
333                    // stuck waiting for it to start up.
334                    startedLatch.countDown();
335                    startedLatch.countDown();
336                    localStartedLatch.countDown();
337                    staticDestinationsLatch.countDown();
338
339                    ss.throwFirstException();
340                }
341            }
342
343            LOG.info("{} bridge to {} stopped", configuration.getBrokerName(), remoteBrokerName);
344        }
345    }
346
347    private void stopFailoverTransport(Transport transport) {
348        FailoverTransport failoverTransport = transport.narrow(FailoverTransport.class);
349        if (failoverTransport != null) {
350            // may be blocked on write, in which case stop will block
351            try {
352                failoverTransport.handleTransportFailure(new IOException("Bridge stopped"));
353            } catch (InterruptedException ignored) {}
354        }
355    }
356
357    protected void triggerStartAsyncNetworkBridgeCreation() throws IOException {
358        brokerService.getTaskRunnerFactory().execute(new Runnable() {
359            @Override
360            public void run() {
361                final String originalName = Thread.currentThread().getName();
362                Thread.currentThread().setName("triggerStartAsyncNetworkBridgeCreation: " +
363                        "remoteBroker=" + remoteBroker + ", localBroker= " + localBroker);
364
365                try {
366                    // First we collect the info data from both the local and remote ends
367                    collectBrokerInfos();
368
369                    // Once we have all required broker info we can attempt to start
370                    // the local and then remote sides of the bridge.
371                    doStartLocalAndRemoteBridges();
372                } finally {
373                    Thread.currentThread().setName(originalName);
374                }
375            }
376        });
377    }
378
379    private void collectBrokerInfos() {
380        int timeout = 30000;
381        TcpTransport tcpTransport = remoteBroker.narrow(TcpTransport.class);
382        if (tcpTransport != null) {
383           timeout = tcpTransport.getConnectionTimeout();
384        }
385
386        // First wait for the remote to feed us its BrokerInfo, then we can check on
387        // the LocalBrokerInfo and decide is this is a loop.
388        try {
389            remoteBrokerInfo = futureRemoteBrokerInfo.get(timeout, TimeUnit.MILLISECONDS);
390            if (remoteBrokerInfo == null) {
391                serviceLocalException(new Throwable("remoteBrokerInfo is null"));
392                return;
393            }
394        } catch (Exception e) {
395            serviceRemoteException(e);
396            return;
397        }
398
399        try {
400            localBrokerInfo = futureLocalBrokerInfo.get(timeout, TimeUnit.MILLISECONDS);
401            if (localBrokerInfo == null) {
402                serviceLocalException(new Throwable("localBrokerInfo is null"));
403                return;
404            }
405
406            // Before we try and build the bridge lets check if we are in a loop
407            // and if so just stop now before registering anything.
408            remoteBrokerId = remoteBrokerInfo.getBrokerId();
409            if (localBrokerId.equals(remoteBrokerId)) {
410                LOG.trace("{} disconnecting remote loop back connector for: {}, with id: {}",
411                        configuration.getBrokerName(), remoteBrokerName, remoteBrokerId);
412                ServiceSupport.dispose(localBroker);
413                ServiceSupport.dispose(remoteBroker);
414                // the bridge is left in a bit of limbo, but it won't get retried
415                // in this state.
416                return;
417            }
418
419            // Fill in the remote broker's information now.
420            remoteBrokerPath[0] = remoteBrokerId;
421            remoteBrokerName = remoteBrokerInfo.getBrokerName();
422            if (configuration.isUseBrokerNamesAsIdSeed()) {
423                idGenerator = new IdGenerator(brokerService.getBrokerName() + "->" + remoteBrokerName);
424            }
425        } catch (Throwable e) {
426            serviceLocalException(e);
427        }
428    }
429
430    private void doStartLocalAndRemoteBridges() {
431
432        if (disposed.get()) {
433            return;
434        }
435
436        if (isCreatedByDuplex()) {
437            // apply remote (propagated) configuration to local duplex bridge before start
438            Properties props = null;
439            try {
440                props = MarshallingSupport.stringToProperties(remoteBrokerInfo.getNetworkProperties());
441                IntrospectionSupport.getProperties(configuration, props, null);
442                if (configuration.getExcludedDestinations() != null) {
443                    excludedDestinations = configuration.getExcludedDestinations().toArray(
444                            new ActiveMQDestination[configuration.getExcludedDestinations().size()]);
445                }
446                if (configuration.getStaticallyIncludedDestinations() != null) {
447                    staticallyIncludedDestinations = configuration.getStaticallyIncludedDestinations().toArray(
448                            new ActiveMQDestination[configuration.getStaticallyIncludedDestinations().size()]);
449                }
450                if (configuration.getDynamicallyIncludedDestinations() != null) {
451                    dynamicallyIncludedDestinations = configuration.getDynamicallyIncludedDestinations().toArray(
452                            new ActiveMQDestination[configuration.getDynamicallyIncludedDestinations().size()]);
453                }
454            } catch (Throwable t) {
455                LOG.error("Error mapping remote configuration: {}", props, t);
456            }
457        }
458
459        try {
460            startLocalBridge();
461        } catch (Throwable e) {
462            serviceLocalException(e);
463            return;
464        }
465
466        try {
467            startRemoteBridge();
468        } catch (Throwable e) {
469            serviceRemoteException(e);
470            return;
471        }
472
473        try {
474            if (safeWaitUntilStarted()) {
475                setupStaticDestinations();
476                staticDestinationsLatch.countDown();
477            }
478        } catch (Throwable e) {
479            serviceLocalException(e);
480        }
481    }
482
483    private void startLocalBridge() throws Throwable {
484        if (!bridgeFailed.get() && localBridgeStarted.compareAndSet(false, true)) {
485            synchronized (this) {
486                LOG.trace("{} starting local Bridge, localBroker={}", configuration.getBrokerName(), localBroker);
487                if (!disposed.get()) {
488
489                    if (idGenerator == null) {
490                        throw new IllegalStateException("Id Generator cannot be null");
491                    }
492
493                    localConnectionInfo = new ConnectionInfo();
494                    localConnectionInfo.setConnectionId(new ConnectionId(idGenerator.generateId()));
495                    localClientId = configuration.getName() + configuration.getClientIdToken() + remoteBrokerName + configuration.getClientIdToken() + "inbound" + configuration.getClientIdToken() + configuration.getBrokerName();
496                    localConnectionInfo.setClientId(localClientId);
497                    localConnectionInfo.setUserName(configuration.getUserName());
498                    localConnectionInfo.setPassword(configuration.getPassword());
499                    Transport originalTransport = remoteBroker;
500                    while (originalTransport instanceof TransportFilter) {
501                        originalTransport = ((TransportFilter) originalTransport).getNext();
502                    }
503                    if (originalTransport instanceof TcpTransport) {
504                        X509Certificate[] peerCerts = originalTransport.getPeerCertificates();
505                        localConnectionInfo.setTransportContext(peerCerts);
506                    }
507                    // sync requests that may fail
508                    Object resp = localBroker.request(localConnectionInfo);
509                    if (resp instanceof ExceptionResponse) {
510                        throw ((ExceptionResponse) resp).getException();
511                    }
512                    localSessionInfo = new SessionInfo(localConnectionInfo, 1);
513                    localBroker.oneway(localSessionInfo);
514
515                    if (configuration.isDuplex()) {
516                        // separate in-bound channel for forwards so we don't
517                        // contend with out-bound dispatch on same connection
518                        remoteBrokerInfo.setNetworkConnection(true);
519                        duplexInboundLocalBroker.oneway(remoteBrokerInfo);
520
521                        ConnectionInfo duplexLocalConnectionInfo = new ConnectionInfo();
522                        duplexLocalConnectionInfo.setConnectionId(new ConnectionId(idGenerator.generateId()));
523                        duplexLocalConnectionInfo.setClientId(configuration.getName() + configuration.getClientIdToken() + remoteBrokerName + configuration.getClientIdToken() + "inbound" + configuration.getClientIdToken() + "duplex"
524                                + configuration.getClientIdToken() + configuration.getBrokerName());
525                        duplexLocalConnectionInfo.setUserName(configuration.getUserName());
526                        duplexLocalConnectionInfo.setPassword(configuration.getPassword());
527
528                        if (originalTransport instanceof TcpTransport) {
529                            X509Certificate[] peerCerts = originalTransport.getPeerCertificates();
530                            duplexLocalConnectionInfo.setTransportContext(peerCerts);
531                        }
532                        // sync requests that may fail
533                        resp = duplexInboundLocalBroker.request(duplexLocalConnectionInfo);
534                        if (resp instanceof ExceptionResponse) {
535                            throw ((ExceptionResponse) resp).getException();
536                        }
537                        SessionInfo duplexInboundSession = new SessionInfo(duplexLocalConnectionInfo, 1);
538                        duplexInboundLocalProducerInfo = new ProducerInfo(duplexInboundSession, 1);
539                        duplexInboundLocalBroker.oneway(duplexInboundSession);
540                        duplexInboundLocalBroker.oneway(duplexInboundLocalProducerInfo);
541                    }
542                    brokerService.getBroker().networkBridgeStarted(remoteBrokerInfo, this.createdByDuplex, remoteBroker.toString());
543                    NetworkBridgeListener l = this.networkBridgeListener;
544                    if (l != null) {
545                        l.onStart(this);
546                    }
547
548                    // Let the local broker know the remote broker's ID.
549                    localBroker.oneway(remoteBrokerInfo);
550                    // new peer broker (a consumer can work with remote broker also)
551                    brokerService.getBroker().addBroker(null, remoteBrokerInfo);
552
553                    LOG.info("Network connection between {} and {} ({}) has been established.",
554                            localBroker, remoteBroker, remoteBrokerName);
555                    LOG.trace("{} register bridge ({}) to {}",
556                            configuration.getBrokerName(), this, remoteBrokerName);
557                } else {
558                    LOG.warn("Bridge was disposed before the startLocalBridge() method was fully executed.");
559                }
560                startedLatch.countDown();
561                localStartedLatch.countDown();
562            }
563        }
564    }
565
566    protected void startRemoteBridge() throws Exception {
567        if (!bridgeFailed.get() && remoteBridgeStarted.compareAndSet(false, true)) {
568            LOG.trace("{} starting remote Bridge, remoteBroker={}", configuration.getBrokerName(), remoteBroker);
569            synchronized (this) {
570                if (!isCreatedByDuplex()) {
571                    BrokerInfo brokerInfo = new BrokerInfo();
572                    brokerInfo.setBrokerName(configuration.getBrokerName());
573                    brokerInfo.setBrokerURL(configuration.getBrokerURL());
574                    brokerInfo.setNetworkConnection(true);
575                    brokerInfo.setDuplexConnection(configuration.isDuplex());
576                    // set our properties
577                    Properties props = new Properties();
578                    IntrospectionSupport.getProperties(configuration, props, null);
579
580                    String dynamicallyIncludedDestinationsKey = "dynamicallyIncludedDestinations";
581                    String staticallyIncludedDestinationsKey = "staticallyIncludedDestinations";
582
583                    if (!configuration.getDynamicallyIncludedDestinations().isEmpty()) {
584                        props.put(dynamicallyIncludedDestinationsKey,
585                                StringToListOfActiveMQDestinationConverter.
586                                convertFromActiveMQDestination(configuration.getDynamicallyIncludedDestinations(), true));
587                    }
588                    if (!configuration.getStaticallyIncludedDestinations().isEmpty()) {
589                        props.put(staticallyIncludedDestinationsKey,
590                                StringToListOfActiveMQDestinationConverter.
591                                convertFromActiveMQDestination(configuration.getStaticallyIncludedDestinations(), true));
592                    }
593
594                    props.remove("networkTTL");
595                    String str = MarshallingSupport.propertiesToString(props);
596                    brokerInfo.setNetworkProperties(str);
597                    brokerInfo.setBrokerId(this.localBrokerId);
598                    remoteBroker.oneway(brokerInfo);
599                    if (configuration.isSyncDurableSubs() &&
600                            remoteBroker.getWireFormat().getVersion() >= CommandTypes.PROTOCOL_VERSION_DURABLE_SYNC) {
601                        remoteBroker.oneway(NetworkBridgeUtils.getBrokerSubscriptionInfo(brokerService,
602                                configuration));
603                    }
604                }
605                if (remoteConnectionInfo != null) {
606                    remoteBroker.oneway(remoteConnectionInfo.createRemoveCommand());
607                }
608                remoteConnectionInfo = new ConnectionInfo();
609                remoteConnectionInfo.setConnectionId(new ConnectionId(idGenerator.generateId()));
610                remoteConnectionInfo.setClientId(configuration.getName() + configuration.getClientIdToken() + configuration.getBrokerName() + configuration.getClientIdToken() + "outbound");
611                
612                if(configuration.getRemoteUserName() != null) {
613                    remoteConnectionInfo.setUserName(configuration.getRemoteUserName());
614                    remoteConnectionInfo.setPassword(configuration.getRemotePassword());
615                } else {
616                    remoteConnectionInfo.setUserName(configuration.getUserName());
617                    remoteConnectionInfo.setPassword(configuration.getPassword());
618                }
619                remoteBroker.oneway(remoteConnectionInfo);
620
621                SessionInfo remoteSessionInfo = new SessionInfo(remoteConnectionInfo, 1);
622                remoteBroker.oneway(remoteSessionInfo);
623                producerInfo = new ProducerInfo(remoteSessionInfo, 1);
624                producerInfo.setResponseRequired(false);
625                remoteBroker.oneway(producerInfo);
626                // Listen to consumer advisory messages on the remote broker to determine demand.
627                if (!configuration.isStaticBridge()) {
628                    demandConsumerInfo = new ConsumerInfo(remoteSessionInfo, 1);
629                    // always dispatch advisory message asynchronously so that
630                    // we never block the producer broker if we are slow
631                    demandConsumerInfo.setDispatchAsync(true);
632                    String advisoryTopic = configuration.getDestinationFilter();
633                    if (configuration.isBridgeTempDestinations()) {
634                        advisoryTopic += "," + AdvisorySupport.TEMP_DESTINATION_COMPOSITE_ADVISORY_TOPIC;
635                    }
636                    demandConsumerInfo.setDestination(new ActiveMQTopic(advisoryTopic));
637                    configureConsumerPrefetch(demandConsumerInfo);
638                    remoteBroker.oneway(demandConsumerInfo);
639                }
640                startedLatch.countDown();
641            }
642        }
643    }
644
645    @Override
646    public void serviceRemoteException(Throwable error) {
647        if (!disposed.get()) {
648            if (error instanceof SecurityException || error instanceof GeneralSecurityException) {
649                LOG.error("Network connection between {} and {} shutdown due to a remote error: {}", localBroker, remoteBroker, error.toString());
650            } else {
651                LOG.warn("Network connection between {} and {} shutdown due to a remote error: {}", localBroker, remoteBroker, error.toString());
652            }
653            LOG.debug("The remote Exception was: {}", error, error);
654            brokerService.getTaskRunnerFactory().execute(new Runnable() {
655                @Override
656                public void run() {
657                    ServiceSupport.dispose(getControllingService());
658                }
659            });
660            fireBridgeFailed(error);
661        }
662    }
663
664    /**
665     * Checks whether or not this consumer is a direct bridge network subscription
666     * @param info
667     * @return
668     */
669    protected boolean isDirectBridgeConsumer(ConsumerInfo info) {
670        return (info.getSubscriptionName() != null && info.getSubscriptionName().startsWith(DURABLE_SUB_PREFIX)) &&
671                (info.getClientId() == null || info.getClientId().startsWith(configuration.getName()));
672    }
673
674    protected boolean isProxyBridgeSubscription(String clientId, String subName) {
675        if (subName != null && clientId != null) {
676            if (subName.startsWith(DURABLE_SUB_PREFIX) && !clientId.startsWith(configuration.getName())) {
677                return true;
678            }
679        }
680        return false;
681    }
682
683    /**
684     * This scenaior is primarily used for durable sync on broker restarts
685     *
686     * @param sub
687     * @param clientId
688     * @param subName
689     */
690    protected void addProxyNetworkSubscriptionClientId(final DemandSubscription sub, final String clientId, String subName) {
691        if (clientId != null && sub != null && subName != null) {
692                String newClientId = getProxyBridgeClientId(clientId);
693                final SubscriptionInfo newSubInfo = new SubscriptionInfo(newClientId, subName);
694                sub.getDurableRemoteSubs().add(newSubInfo);
695                LOG.debug("Adding proxy network subscription {} to demand subscription", newSubInfo);
696
697        } else {
698            LOG.debug("Skipping addProxyNetworkSubscription");
699        }
700    }
701
702    /**
703     * Add a durable remote proxy subscription when we can generate via the BrokerId path
704     * This is the most common scenario
705     *
706     * @param sub
707     * @param path
708     * @param subName
709     */
710    protected void addProxyNetworkSubscriptionBrokerPath(final DemandSubscription sub, final BrokerId[] path, String subName) {
711        if (sub != null && path.length > 1 && subName != null) {
712            String b1 = path[path.length-1].toString();
713            String b2 = path[path.length-2].toString();
714            final SubscriptionInfo newSubInfo = new SubscriptionInfo(b2 + configuration.getClientIdToken() + "inbound" + configuration.getClientIdToken() + b1, subName);
715            sub.getDurableRemoteSubs().add(newSubInfo);
716        }
717    }
718
719    private String getProxyBridgeClientId(String clientId) {
720        String newClientId = clientId;
721        String[] clientIdTokens = newClientId != null ? newClientId.split(Pattern.quote(configuration.getClientIdToken())) : null;
722        if (clientIdTokens != null && clientIdTokens.length > 2) {
723            newClientId = clientIdTokens[clientIdTokens.length - 3] +  configuration.getClientIdToken() + "inbound"
724                    + configuration.getClientIdToken() +  clientIdTokens[clientIdTokens.length -1];
725        }
726        return newClientId;
727    }
728
729    protected boolean isProxyNSConsumerBrokerPath(ConsumerInfo info) {
730        return info.getBrokerPath() != null && info.getBrokerPath().length > 1;
731    }
732
733    protected boolean isProxyNSConsumerClientId(String clientId) {
734        return clientId != null && clientId.split(Pattern.quote(configuration.getClientIdToken())).length > 3;
735    }
736
737    protected void serviceRemoteCommand(Command command) {
738        if (!disposed.get()) {
739            try {
740                if (command.isMessageDispatch()) {
741                    safeWaitUntilStarted();
742                    MessageDispatch md = (MessageDispatch) command;
743                    serviceRemoteConsumerAdvisory(md.getMessage().getDataStructure());
744                    ackAdvisory(md.getMessage());
745                } else if (command.isBrokerInfo()) {
746                    futureRemoteBrokerInfo.set((BrokerInfo) command);
747                } else if (command instanceof BrokerSubscriptionInfo) {
748                    final BrokerSubscriptionInfo brokerSubscriptionInfo = (BrokerSubscriptionInfo) command;
749
750                    //Start in a new thread so we don't block the transport waiting for staticDestinations
751                    syncExecutor.execute(new Runnable() {
752
753                        @Override
754                        public void run() {
755                            try {
756                                staticDestinationsLatch.await();
757                                //Make sure after the countDown of staticDestinationsLatch we aren't stopping
758                                if (!disposed.get()) {
759                                    BrokerSubscriptionInfo subInfo = brokerSubscriptionInfo;
760                                    LOG.debug("Received Remote BrokerSubscriptionInfo on {} from {}",
761                                            brokerService.getBrokerName(), subInfo.getBrokerName());
762
763                                    if (configuration.isSyncDurableSubs() && configuration.isConduitSubscriptions()
764                                            && !configuration.isDynamicOnly()) {
765                                        if (started.get()) {
766                                            if (subInfo.getSubscriptionInfos() != null) {
767                                                for (ConsumerInfo info : subInfo.getSubscriptionInfos()) {
768                                                    //re-add any process any non-NC consumers that match the
769                                                    //dynamicallyIncludedDestinations list
770                                                    //Also re-add network consumers that are not part of this direct
771                                                    //bridge (proxy of proxy bridges)
772                                                    if((info.getSubscriptionName() == null || !isDirectBridgeConsumer(info)) &&
773                                                            NetworkBridgeUtils.matchesDestinations(dynamicallyIncludedDestinations, info.getDestination())) {
774                                                        serviceRemoteConsumerAdvisory(info);
775                                                    }
776                                                }
777                                            }
778
779                                            //After re-added, clean up any empty durables
780                                            for (Iterator<DemandSubscription> i = subscriptionMapByLocalId.values().iterator(); i.hasNext(); ) {
781                                                DemandSubscription ds = i.next();
782                                                if (NetworkBridgeUtils.matchesDestinations(dynamicallyIncludedDestinations, ds.getLocalInfo().getDestination())) {
783                                                    cleanupDurableSub(ds, i);
784                                                }
785                                            }
786                                        }
787                                    }
788                                }
789                            } catch (Exception e) {
790                                LOG.warn("Error processing BrokerSubscriptionInfo: {}", e.getMessage(), e);
791                                LOG.debug(e.getMessage(), e);
792                            }
793                        }
794                    });
795
796                } else if (command.getClass() == ConnectionError.class) {
797                    ConnectionError ce = (ConnectionError) command;
798                    serviceRemoteException(ce.getException());
799                } else {
800                    if (isDuplex()) {
801                        LOG.trace("{} duplex command type: {}", configuration.getBrokerName(), command.getDataStructureType());
802                        if (command.isMessage()) {
803                            final ActiveMQMessage message = (ActiveMQMessage) command;
804                            if (NetworkBridgeFilter.isAdvisoryInterpretedByNetworkBridge(message)) {
805                                serviceRemoteConsumerAdvisory(message.getDataStructure());
806                                ackAdvisory(message);
807                            } else {
808                                if (!isPermissableDestination(message.getDestination(), true)) {
809                                    return;
810                                }
811                                safeWaitUntilStarted();
812                                // message being forwarded - we need to
813                                // propagate the response to our local send
814                                if (canDuplexDispatch(message)) {
815                                    message.setProducerId(duplexInboundLocalProducerInfo.getProducerId());
816                                    if (message.isResponseRequired() || configuration.isAlwaysSyncSend()) {
817                                        duplexInboundLocalBroker.asyncRequest(message, new ResponseCallback() {
818                                            final int correlationId = message.getCommandId();
819
820                                            @Override
821                                            public void onCompletion(FutureResponse resp) {
822                                                try {
823                                                    Response reply = resp.getResult();
824                                                    reply.setCorrelationId(correlationId);
825                                                    remoteBroker.oneway(reply);
826                                                    //increment counter when messages are received in duplex mode
827                                                    networkBridgeStatistics.getReceivedCount().increment();
828                                                } catch (IOException error) {
829                                                    LOG.error("Exception: {} on duplex forward of: {}", error, message);
830                                                    serviceRemoteException(error);
831                                                }
832                                            }
833                                        });
834                                    } else {
835                                        duplexInboundLocalBroker.oneway(message);
836                                        networkBridgeStatistics.getReceivedCount().increment();
837                                    }
838                                    serviceInboundMessage(message);
839                                } else {
840                                    if (message.isResponseRequired() || configuration.isAlwaysSyncSend()) {
841                                        Response reply = new Response();
842                                        reply.setCorrelationId(message.getCommandId());
843                                        remoteBroker.oneway(reply);
844                                    }
845                                }
846                            }
847                        } else {
848                            switch (command.getDataStructureType()) {
849                                case ConnectionInfo.DATA_STRUCTURE_TYPE:
850                                    if (duplexInitiatingConnection != null && duplexInitiatingConnectionInfoReceived.compareAndSet(false, true)) {
851                                        // end of initiating connection setup - propogate to initial connection to get mbean by clientid
852                                        duplexInitiatingConnection.processAddConnection((ConnectionInfo) command);
853                                    } else {
854                                        localBroker.oneway(command);
855                                    }
856                                    break;
857                                case SessionInfo.DATA_STRUCTURE_TYPE:
858                                    localBroker.oneway(command);
859                                    break;
860                                case ProducerInfo.DATA_STRUCTURE_TYPE:
861                                    // using duplexInboundLocalProducerInfo
862                                    break;
863                                case MessageAck.DATA_STRUCTURE_TYPE:
864                                    MessageAck ack = (MessageAck) command;
865                                    DemandSubscription localSub = subscriptionMapByRemoteId.get(ack.getConsumerId());
866                                    if (localSub != null) {
867                                        ack.setConsumerId(localSub.getLocalInfo().getConsumerId());
868                                        localBroker.oneway(ack);
869                                    } else {
870                                        LOG.warn("Matching local subscription not found for ack: {}", ack);
871                                    }
872                                    break;
873                                case ConsumerInfo.DATA_STRUCTURE_TYPE:
874                                    localStartedLatch.await();
875                                    if (started.get()) {
876                                        final ConsumerInfo consumerInfo = (ConsumerInfo) command;
877                                        if (isDuplicateSuppressionOff(consumerInfo)) {
878                                            addConsumerInfo(consumerInfo);
879                                        } else {
880                                            synchronized (brokerService.getVmConnectorURI()) {
881                                                addConsumerInfo(consumerInfo);
882                                            }
883                                        }
884                                    } else {
885                                        // received a subscription whilst stopping
886                                        LOG.warn("Stopping - ignoring ConsumerInfo: {}", command);
887                                    }
888                                    break;
889                                case ShutdownInfo.DATA_STRUCTURE_TYPE:
890                                    // initiator is shutting down, controlled case
891                                    // abortive close dealt with by inactivity monitor
892                                    LOG.info("Stopping network bridge on shutdown of remote broker");
893                                    serviceRemoteException(new IOException(command.toString()));
894                                    break;
895                                default:
896                                    LOG.debug("Ignoring remote command: {}", command);
897                            }
898                        }
899                    } else {
900                        switch (command.getDataStructureType()) {
901                            case KeepAliveInfo.DATA_STRUCTURE_TYPE:
902                            case WireFormatInfo.DATA_STRUCTURE_TYPE:
903                            case ShutdownInfo.DATA_STRUCTURE_TYPE:
904                                break;
905                            default:
906                                LOG.warn("Unexpected remote command: {}", command);
907                        }
908                    }
909                }
910            } catch (Throwable e) {
911                LOG.debug("Exception processing remote command: {}", command, e);
912                serviceRemoteException(e);
913            }
914        }
915    }
916
917    private void ackAdvisory(Message message) throws IOException {
918        demandConsumerDispatched++;
919        if (demandConsumerDispatched > (demandConsumerInfo.getPrefetchSize() *
920                (configuration.getAdvisoryAckPercentage() / 100f))) {
921            final MessageAck ack = new MessageAck(message, MessageAck.STANDARD_ACK_TYPE, demandConsumerDispatched);
922            ack.setConsumerId(demandConsumerInfo.getConsumerId());
923            brokerService.getTaskRunnerFactory().execute(new Runnable() {
924                @Override
925                public void run() {
926                    try {
927                        remoteBroker.oneway(ack);
928                    } catch (IOException e) {
929                        LOG.warn("Failed to send advisory ack {}", ack, e);
930                    }
931                }
932            });
933            demandConsumerDispatched = 0;
934        }
935    }
936
937    private void serviceRemoteConsumerAdvisory(DataStructure data) throws IOException {
938        final int networkTTL = configuration.getConsumerTTL();
939        if (data.getClass() == ConsumerInfo.class) {
940            // Create a new local subscription
941            ConsumerInfo info = (ConsumerInfo) data;
942            BrokerId[] path = info.getBrokerPath();
943
944            if (info.isBrowser()) {
945                LOG.debug("{} Ignoring sub from {}, browsers explicitly suppressed", configuration.getBrokerName(), remoteBrokerName);
946                return;
947            }
948
949            if (path != null && networkTTL > -1 && path.length >= networkTTL) {
950                LOG.debug("{} Ignoring sub from {}, restricted to {} network hops only: {}",
951                        configuration.getBrokerName(), remoteBrokerName, networkTTL, info);
952                return;
953            }
954
955            if (contains(path, localBrokerPath[0])) {
956                // Ignore this consumer as it's a consumer we locally sent to the broker.
957                LOG.debug("{} Ignoring sub from {}, already routed through this broker once: {}",
958                        configuration.getBrokerName(), remoteBrokerName, info);
959                return;
960            }
961
962            if (!isPermissableDestination(info.getDestination())) {
963                // ignore if not in the permitted or in the excluded list
964                LOG.debug("{} Ignoring sub from {}, destination {} is not permitted: {}",
965                        configuration.getBrokerName(), remoteBrokerName, info.getDestination(), info);
966                return;
967            }
968
969            // in a cyclic network there can be multiple bridges per broker that can propagate
970            // a network subscription so there is a need to synchronize on a shared entity
971            // if duplicate suppression is required
972            if (isDuplicateSuppressionOff(info)) {
973                addConsumerInfo(info);
974            } else {
975                synchronized (brokerService.getVmConnectorURI()) {
976                    addConsumerInfo(info);
977                }
978            }
979        } else if (data.getClass() == DestinationInfo.class) {
980            final DestinationInfo destInfo = (DestinationInfo) data;
981            BrokerId[] path = destInfo.getBrokerPath();
982            if (path != null && networkTTL > -1 && path.length >= networkTTL) {
983                LOG.debug("{} Ignoring destination {} restricted to {} network hops only",
984                        configuration.getBrokerName(), destInfo, networkTTL);
985                return;
986            }
987            if (contains(destInfo.getBrokerPath(), localBrokerPath[0])) {
988                LOG.debug("{} Ignoring destination {} already routed through this broker once", configuration.getBrokerName(), destInfo);
989                return;
990            }
991            destInfo.setConnectionId(localConnectionInfo.getConnectionId());
992            if (destInfo.getDestination() instanceof ActiveMQTempDestination) {
993                // re-set connection id so comes from here
994                ActiveMQTempDestination tempDest = (ActiveMQTempDestination) destInfo.getDestination();
995                tempDest.setConnectionId(localSessionInfo.getSessionId().getConnectionId());
996            }
997            destInfo.setBrokerPath(appendToBrokerPath(destInfo.getBrokerPath(), getRemoteBrokerPath()));
998            LOG.trace("{} bridging {} destination on {} from {}, destination: {}",
999                    configuration.getBrokerName(), (destInfo.isAddOperation() ? "add" : "remove"), localBroker, remoteBrokerName, destInfo);
1000            if (destInfo.isRemoveOperation()) {
1001                // not synced with addSubs so we will need to ignore any potential new subs with a timeout!=0
1002                destInfo.setTimeout(1);
1003            }
1004            // Serialize both add/remove dest with removeSub operations such that all removeSub advisories are generated
1005            serialExecutor.execute(new Runnable() {
1006                @Override
1007                public void run() {
1008                    try {
1009                        localBroker.oneway(destInfo);
1010                    } catch (IOException e) {
1011                        LOG.warn("failed to deliver remove command for destination: {}", destInfo.getDestination(), e);
1012                    }
1013                }
1014            });
1015
1016        } else if (data.getClass() == RemoveInfo.class) {
1017            ConsumerId id = (ConsumerId) ((RemoveInfo) data).getObjectId();
1018            removeDemandSubscription(id);
1019
1020            if (forcedDurableRemoteId.remove(id)) {
1021                for (Iterator<DemandSubscription> i = subscriptionMapByLocalId.values().iterator(); i.hasNext(); ) {
1022                    DemandSubscription ds = i.next();
1023                    boolean removed = ds.removeForcedDurableConsumer(id);
1024                    if (removed) {
1025                        cleanupDurableSub(ds, i);
1026                    }
1027                }
1028           }
1029
1030        } else if (data.getClass() == RemoveSubscriptionInfo.class) {
1031            final RemoveSubscriptionInfo info = ((RemoveSubscriptionInfo) data);
1032            final SubscriptionInfo subscriptionInfo = new SubscriptionInfo(info.getClientId(), info.getSubscriptionName());
1033            final boolean proxyBridgeSub = isProxyBridgeSubscription(subscriptionInfo.getClientId(),
1034                    subscriptionInfo.getSubscriptionName());
1035            for (Iterator<DemandSubscription> i = subscriptionMapByLocalId.values().iterator(); i.hasNext(); ) {
1036                DemandSubscription ds = i.next();
1037                boolean removed = ds.getDurableRemoteSubs().remove(subscriptionInfo);
1038
1039                //If this is a proxy bridge subscription we need to try changing the clientId
1040                if (!removed && proxyBridgeSub){
1041                    subscriptionInfo.setClientId(getProxyBridgeClientId(subscriptionInfo.getClientId()));
1042                    if (ds.getDurableRemoteSubs().contains(subscriptionInfo)) {
1043                        ds.getDurableRemoteSubs().remove(subscriptionInfo);
1044                        removed = true;
1045                    }
1046                }
1047
1048                if (removed) {
1049                    cleanupDurableSub(ds, i);
1050                }
1051            }
1052        }
1053    }
1054
1055    private void cleanupDurableSub(final DemandSubscription ds,
1056            Iterator<DemandSubscription> i) throws IOException {
1057
1058        if (ds != null && ds.getLocalDurableSubscriber() != null && ds.getDurableRemoteSubs().isEmpty()
1059                && ds.getForcedDurableConsumersSize() == 0) {
1060            // deactivate subscriber
1061            RemoveInfo removeInfo = new RemoveInfo(ds.getLocalInfo().getConsumerId());
1062            localBroker.oneway(removeInfo);
1063
1064            // remove subscriber
1065            RemoveSubscriptionInfo sending = new RemoveSubscriptionInfo();
1066            sending.setClientId(localClientId);
1067            sending.setSubscriptionName(ds.getLocalDurableSubscriber().getSubscriptionName());
1068            sending.setConnectionId(this.localConnectionInfo.getConnectionId());
1069            localBroker.oneway(sending);
1070
1071            //remove subscriber from local map
1072            i.remove();
1073
1074            //need to remove the mapping from the remote map as well
1075            subscriptionMapByRemoteId.remove(ds.getRemoteInfo().getConsumerId());
1076        }
1077    }
1078
1079    @Override
1080    public void serviceLocalException(Throwable error) {
1081        serviceLocalException(null, error);
1082    }
1083
1084    public void serviceLocalException(MessageDispatch messageDispatch, Throwable error) {
1085        LOG.trace("serviceLocalException: disposed {} ex", disposed.get(), error);
1086        if (!disposed.get()) {
1087            if (error instanceof DestinationDoesNotExistException && ((DestinationDoesNotExistException) error).isTemporary()) {
1088                // not a reason to terminate the bridge - temps can disappear with
1089                // pending sends as the demand sub may outlive the remote dest
1090                if (messageDispatch != null) {
1091                    LOG.warn("PoisonAck of {} on forwarding error: {}", messageDispatch.getMessage().getMessageId(), error);
1092                    try {
1093                        MessageAck poisonAck = new MessageAck(messageDispatch, MessageAck.POISON_ACK_TYPE, 1);
1094                        poisonAck.setPoisonCause(error);
1095                        localBroker.oneway(poisonAck);
1096                    } catch (IOException ioe) {
1097                        LOG.error("Failed to poison ack message following forward failure: ", ioe);
1098                    }
1099                    fireFailedForwardAdvisory(messageDispatch, error);
1100                } else {
1101                    LOG.warn("Ignoring exception on forwarding to non existent temp dest: ", error);
1102                }
1103                return;
1104            }
1105
1106            LOG.info("Network connection between {} and {} shutdown due to a local error: {}", localBroker, remoteBroker, error);
1107            LOG.debug("The local Exception was: {}", error, error);
1108
1109            brokerService.getTaskRunnerFactory().execute(new Runnable() {
1110                @Override
1111                public void run() {
1112                    ServiceSupport.dispose(getControllingService());
1113                }
1114            });
1115            fireBridgeFailed(error);
1116        }
1117    }
1118
1119    private void fireFailedForwardAdvisory(MessageDispatch messageDispatch, Throwable error) {
1120        if (configuration.isAdvisoryForFailedForward()) {
1121            AdvisoryBroker advisoryBroker = null;
1122            try {
1123                advisoryBroker = (AdvisoryBroker) brokerService.getBroker().getAdaptor(AdvisoryBroker.class);
1124
1125                if (advisoryBroker != null) {
1126                    ConnectionContext context = new ConnectionContext();
1127                    context.setSecurityContext(SecurityContext.BROKER_SECURITY_CONTEXT);
1128                    context.setBroker(brokerService.getBroker());
1129
1130                    ActiveMQMessage advisoryMessage = new ActiveMQMessage();
1131                    advisoryMessage.setStringProperty("cause", error.getLocalizedMessage());
1132                    advisoryBroker.fireAdvisory(context, AdvisorySupport.getNetworkBridgeForwardFailureAdvisoryTopic(), messageDispatch.getMessage(), null,
1133                            advisoryMessage);
1134
1135                }
1136            } catch (Exception e) {
1137                LOG.warn("failed to fire forward failure advisory, cause: {}", e);
1138                LOG.debug("detail", e);
1139            }
1140        }
1141    }
1142
1143    protected Service getControllingService() {
1144        return duplexInitiatingConnection != null ? duplexInitiatingConnection : DemandForwardingBridgeSupport.this;
1145    }
1146
1147    protected void addSubscription(DemandSubscription sub) throws IOException {
1148        if (sub != null) {
1149            localBroker.oneway(sub.getLocalInfo());
1150        }
1151    }
1152
1153    protected void removeSubscription(final DemandSubscription sub) throws IOException {
1154        if (sub != null) {
1155            LOG.trace("{} remove local subscription: {} for remote {}", configuration.getBrokerName(), sub.getLocalInfo().getConsumerId(), sub.getRemoteInfo().getConsumerId());
1156
1157            // ensure not available for conduit subs pending removal
1158            subscriptionMapByLocalId.remove(sub.getLocalInfo().getConsumerId());
1159            subscriptionMapByRemoteId.remove(sub.getRemoteInfo().getConsumerId());
1160
1161            // continue removal in separate thread to free up this thread for outstanding responses
1162            // Serialize with removeDestination operations so that removeSubs are serialized with
1163            // removeDestinations such that all removeSub advisories are generated
1164            serialExecutor.execute(new Runnable() {
1165                @Override
1166                public void run() {
1167                    sub.waitForCompletion();
1168                    try {
1169                        localBroker.oneway(sub.getLocalInfo().createRemoveCommand());
1170                    } catch (IOException e) {
1171                        LOG.warn("failed to deliver remove command for local subscription, for remote {}", sub.getRemoteInfo().getConsumerId(), e);
1172                    }
1173                }
1174            });
1175        }
1176    }
1177
1178    protected Message configureMessage(MessageDispatch md) throws IOException {
1179        Message message = md.getMessage().copy();
1180        // Update the packet to show where it came from.
1181        message.setBrokerPath(appendToBrokerPath(message.getBrokerPath(), localBrokerPath));
1182        message.setProducerId(producerInfo.getProducerId());
1183        message.setDestination(md.getDestination());
1184        message.setMemoryUsage(null);
1185        if (message.getOriginalTransactionId() == null) {
1186            message.setOriginalTransactionId(message.getTransactionId());
1187        }
1188        message.setTransactionId(null);
1189        if (configuration.isUseCompression()) {
1190            message.compress();
1191        }
1192        return message;
1193    }
1194
1195    protected void serviceLocalCommand(Command command) {
1196        if (!disposed.get()) {
1197            try {
1198                if (command.isMessageDispatch()) {
1199                    safeWaitUntilStarted();
1200                    networkBridgeStatistics.getEnqueues().increment();
1201                    final MessageDispatch md = (MessageDispatch) command;
1202                    final DemandSubscription sub = subscriptionMapByLocalId.get(md.getConsumerId());
1203                    if (sub != null && md.getMessage() != null && sub.incrementOutstandingResponses()) {
1204
1205                        if (suppressMessageDispatch(md, sub)) {
1206                            LOG.debug("{} message not forwarded to {} because message came from there or fails TTL, brokerPath: {}, message: {}",
1207                                    configuration.getBrokerName(), remoteBrokerName, Arrays.toString(md.getMessage().getBrokerPath()), md.getMessage());
1208                            // still ack as it may be durable
1209                            try {
1210                                localBroker.oneway(new MessageAck(md, MessageAck.INDIVIDUAL_ACK_TYPE, 1));
1211                            } finally {
1212                                sub.decrementOutstandingResponses();
1213                            }
1214                            return;
1215                        }
1216
1217                        Message message = configureMessage(md);
1218                        LOG.debug("bridging ({} -> {}), consumer: {}, destination: {}, brokerPath: {}, message: {}",
1219                                configuration.getBrokerName(), remoteBrokerName, md.getConsumerId(), message.getDestination(), Arrays.toString(message.getBrokerPath()), (LOG.isTraceEnabled() ? message : message.getMessageId()));
1220                        if (isDuplex() && NetworkBridgeFilter.isAdvisoryInterpretedByNetworkBridge(message)) {
1221                            try {
1222                                // never request b/c they are eventually                     acked async
1223                                remoteBroker.oneway(message);
1224                            } finally {
1225                                sub.decrementOutstandingResponses();
1226                            }
1227                            return;
1228                        }
1229                        if (isPermissableDestination(md.getDestination())) {
1230                           if (message.isPersistent() || configuration.isAlwaysSyncSend()) {
1231
1232                              // The message was not sent using async send, so we should only
1233                              // ack the local broker when we get confirmation that the remote
1234                              // broker has received the message.
1235                              remoteBroker.asyncRequest(message, new ResponseCallback() {
1236                                 @Override
1237                                 public void onCompletion(FutureResponse future) {
1238                                    try {
1239                                       Response response = future.getResult();
1240                                       if (response.isException()) {
1241                                          ExceptionResponse er = (ExceptionResponse) response;
1242                                          serviceLocalException(md, er.getException());
1243                                       } else {
1244                                          localBroker.oneway(new MessageAck(md, MessageAck.INDIVIDUAL_ACK_TYPE, 1));
1245                                          networkBridgeStatistics.getDequeues().increment();
1246                                       }
1247                                    } catch (IOException e) {
1248                                       serviceLocalException(md, e);
1249                                    } finally {
1250                                       sub.decrementOutstandingResponses();
1251                                    }
1252                                 }
1253                              });
1254
1255                           } else {
1256                              // If the message was originally sent using async send, we will
1257                              // preserve that QOS by bridging it using an async send (small chance
1258                              // of message loss).
1259                              try {
1260                                 remoteBroker.oneway(message);
1261                                 localBroker.oneway(new MessageAck(md, MessageAck.INDIVIDUAL_ACK_TYPE, 1));
1262                                 networkBridgeStatistics.getDequeues().increment();
1263                              } finally {
1264                                 sub.decrementOutstandingResponses();
1265                              }
1266                           }
1267                           serviceOutbound(message);
1268                        }
1269                    } else {
1270                        LOG.debug("No subscription registered with this network bridge for consumerId: {} for message: {}", md.getConsumerId(), md.getMessage());
1271                    }
1272                } else if (command.isBrokerInfo()) {
1273                    futureLocalBrokerInfo.set((BrokerInfo) command);
1274                } else if (command.isShutdownInfo()) {
1275                    LOG.info("{} Shutting down {}", configuration.getBrokerName(), configuration.getName());
1276                    stop();
1277                } else if (command.getClass() == ConnectionError.class) {
1278                    ConnectionError ce = (ConnectionError) command;
1279                    serviceLocalException(ce.getException());
1280                } else {
1281                    switch (command.getDataStructureType()) {
1282                        case WireFormatInfo.DATA_STRUCTURE_TYPE:
1283                            break;
1284                        case BrokerSubscriptionInfo.DATA_STRUCTURE_TYPE:
1285                            break;
1286                        default:
1287                            LOG.warn("Unexpected local command: {}", command);
1288                    }
1289                }
1290            } catch (Throwable e) {
1291                LOG.warn("Caught an exception processing local command", e);
1292                serviceLocalException(e);
1293            }
1294        }
1295    }
1296
1297    private boolean suppressMessageDispatch(MessageDispatch md, DemandSubscription sub) throws Exception {
1298        boolean suppress = false;
1299        // for durable subs, suppression via filter leaves dangling acks so we
1300        // need to check here and allow the ack irrespective
1301        if (sub.getLocalInfo().isDurable()) {
1302            NonCachedMessageEvaluationContext messageEvalContext = new NonCachedMessageEvaluationContext();
1303            messageEvalContext.setMessageReference(md.getMessage());
1304            messageEvalContext.setDestination(md.getDestination());
1305            suppress = !sub.getNetworkBridgeFilter().matches(messageEvalContext);
1306        }
1307        return suppress;
1308    }
1309
1310    public static boolean contains(BrokerId[] brokerPath, BrokerId brokerId) {
1311        if (brokerPath != null) {
1312            for (BrokerId id : brokerPath) {
1313                if (brokerId.equals(id)) {
1314                    return true;
1315                }
1316            }
1317        }
1318        return false;
1319    }
1320
1321    protected BrokerId[] appendToBrokerPath(BrokerId[] brokerPath, BrokerId[] pathsToAppend) {
1322        if (brokerPath == null || brokerPath.length == 0) {
1323            return pathsToAppend;
1324        }
1325        BrokerId rc[] = new BrokerId[brokerPath.length + pathsToAppend.length];
1326        System.arraycopy(brokerPath, 0, rc, 0, brokerPath.length);
1327        System.arraycopy(pathsToAppend, 0, rc, brokerPath.length, pathsToAppend.length);
1328        return rc;
1329    }
1330
1331    protected BrokerId[] appendToBrokerPath(BrokerId[] brokerPath, BrokerId idToAppend) {
1332        if (brokerPath == null || brokerPath.length == 0) {
1333            return new BrokerId[]{idToAppend};
1334        }
1335        BrokerId rc[] = new BrokerId[brokerPath.length + 1];
1336        System.arraycopy(brokerPath, 0, rc, 0, brokerPath.length);
1337        rc[brokerPath.length] = idToAppend;
1338        return rc;
1339    }
1340
1341    protected boolean isPermissableDestination(ActiveMQDestination destination) {
1342        return isPermissableDestination(destination, false);
1343    }
1344
1345    protected boolean isPermissableDestination(ActiveMQDestination destination, boolean allowTemporary) {
1346        // Are we not bridging temporary destinations?
1347        if (destination.isTemporary()) {
1348            if (allowTemporary) {
1349                return true;
1350            } else {
1351                return configuration.isBridgeTempDestinations();
1352            }
1353        }
1354
1355        ActiveMQDestination[] dests = excludedDestinations;
1356        if (dests != null && dests.length > 0) {
1357            for (ActiveMQDestination dest : dests) {
1358                DestinationFilter exclusionFilter = DestinationFilter.parseFilter(dest);
1359                if (dest != null && exclusionFilter.matches(destination) && dest.getDestinationType() == destination.getDestinationType()) {
1360                    return false;
1361                }
1362            }
1363        }
1364
1365        dests = staticallyIncludedDestinations;
1366        if (dests != null && dests.length > 0) {
1367            for (ActiveMQDestination dest : dests) {
1368                DestinationFilter inclusionFilter = DestinationFilter.parseFilter(dest);
1369                if (dest != null && inclusionFilter.matches(destination) && dest.getDestinationType() == destination.getDestinationType()) {
1370                    return true;
1371                }
1372            }
1373        }
1374
1375        dests = dynamicallyIncludedDestinations;
1376        if (dests != null && dests.length > 0) {
1377            for (ActiveMQDestination dest : dests) {
1378                DestinationFilter inclusionFilter = DestinationFilter.parseFilter(dest);
1379                if (dest != null && inclusionFilter.matches(destination) && dest.getDestinationType() == destination.getDestinationType()) {
1380                    return true;
1381                }
1382            }
1383
1384            return false;
1385        }
1386
1387        return true;
1388    }
1389
1390    /**
1391     * Subscriptions for these destinations are always created
1392     */
1393    protected void setupStaticDestinations() {
1394        ActiveMQDestination[] dests = staticallyIncludedDestinations;
1395        if (dests != null) {
1396            for (ActiveMQDestination dest : dests) {
1397                if (isPermissableDestination(dest)) {
1398                    DemandSubscription sub = createDemandSubscription(dest, null);
1399                    if (sub != null) {
1400                        sub.setStaticallyIncluded(true);
1401                        try {
1402                            addSubscription(sub);
1403                        } catch (IOException e) {
1404                            LOG.error("Failed to add static destination {}", dest, e);
1405                        }
1406                        LOG.trace("{}, bridging messages for static destination: {}", configuration.getBrokerName(), dest);
1407                    } else {
1408                        LOG.info("{}, static destination excluded: {}, demand already exists", configuration.getBrokerName(), dest);
1409                    }
1410                } else {
1411                    LOG.info("{}, static destination excluded: {}", configuration.getBrokerName(), dest);
1412                }
1413            }
1414        }
1415    }
1416
1417    protected void addConsumerInfo(final ConsumerInfo consumerInfo) throws IOException {
1418        ConsumerInfo info = consumerInfo.copy();
1419        addRemoteBrokerToBrokerPath(info);
1420        DemandSubscription sub = createDemandSubscription(info);
1421        if (sub != null) {
1422            if (duplicateSuppressionIsRequired(sub)) {
1423                undoMapRegistration(sub);
1424            } else {
1425                if (consumerInfo.isDurable()) {
1426                    //Handle the demand generated by proxy network subscriptions
1427                    //The broker path is case is normal
1428                    if (isProxyNSConsumerBrokerPath(sub.getRemoteInfo()) &&
1429                            info.getSubscriptionName() != null && info.getSubscriptionName().startsWith(DURABLE_SUB_PREFIX)) {
1430                        final BrokerId[] path = info.getBrokerPath();
1431                        addProxyNetworkSubscriptionBrokerPath(sub, path, consumerInfo.getSubscriptionName());
1432                    //This is the durable sync case on broker restart
1433                    } else if (isProxyNSConsumerClientId(sub.getRemoteInfo().getClientId()) &&
1434                            isProxyBridgeSubscription(info.getClientId(), info.getSubscriptionName())) {
1435                                addProxyNetworkSubscriptionClientId(sub, sub.getRemoteInfo().getClientId(), consumerInfo.getSubscriptionName());
1436                        } else {
1437                                sub.getDurableRemoteSubs().add(new SubscriptionInfo(sub.getRemoteInfo().getClientId(), consumerInfo.getSubscriptionName()));
1438                        }
1439                }
1440                addSubscription(sub);
1441                LOG.debug("{} new demand subscription: {}", configuration.getBrokerName(), sub);
1442            }
1443        }
1444    }
1445
1446    private void undoMapRegistration(DemandSubscription sub) {
1447        subscriptionMapByLocalId.remove(sub.getLocalInfo().getConsumerId());
1448        subscriptionMapByRemoteId.remove(sub.getRemoteInfo().getConsumerId());
1449    }
1450
1451    /*
1452     * check our existing subs networkConsumerIds against the list of network
1453     * ids in this subscription A match means a duplicate which we suppress for
1454     * topics and maybe for queues
1455     */
1456    private boolean duplicateSuppressionIsRequired(DemandSubscription candidate) {
1457        final ConsumerInfo consumerInfo = candidate.getRemoteInfo();
1458        boolean suppress = false;
1459
1460        if (isDuplicateSuppressionOff(consumerInfo)) {
1461            return suppress;
1462        }
1463
1464        List<ConsumerId> candidateConsumers = consumerInfo.getNetworkConsumerIds();
1465        Collection<Subscription> currentSubs = getRegionSubscriptions(consumerInfo.getDestination());
1466        for (Subscription sub : currentSubs) {
1467            List<ConsumerId> networkConsumers = sub.getConsumerInfo().getNetworkConsumerIds();
1468            if (!networkConsumers.isEmpty()) {
1469                if (matchFound(candidateConsumers, networkConsumers)) {
1470                    if (isInActiveDurableSub(sub)) {
1471                        suppress = false;
1472                    } else {
1473                        suppress = hasLowerPriority(sub, candidate.getLocalInfo());
1474                    }
1475                    break;
1476                }
1477            }
1478        }
1479        return suppress;
1480    }
1481
1482    private boolean isDuplicateSuppressionOff(final ConsumerInfo consumerInfo) {
1483        return !configuration.isSuppressDuplicateQueueSubscriptions() && !configuration.isSuppressDuplicateTopicSubscriptions()
1484                || consumerInfo.getDestination().isQueue() && !configuration.isSuppressDuplicateQueueSubscriptions()
1485                || consumerInfo.getDestination().isTopic() && !configuration.isSuppressDuplicateTopicSubscriptions();
1486    }
1487
1488    private boolean isInActiveDurableSub(Subscription sub) {
1489        return (sub.getConsumerInfo().isDurable() && sub instanceof DurableTopicSubscription && !((DurableTopicSubscription) sub).isActive());
1490    }
1491
1492    private boolean hasLowerPriority(Subscription existingSub, ConsumerInfo candidateInfo) {
1493        boolean suppress = false;
1494
1495        if (existingSub.getConsumerInfo().getPriority() >= candidateInfo.getPriority()) {
1496            LOG.debug("{} Ignoring duplicate subscription from {}, sub: {} is duplicate by network subscription with equal or higher network priority: {}, networkConsumerIds: {}",
1497                    configuration.getBrokerName(), remoteBrokerName, candidateInfo, existingSub, existingSub.getConsumerInfo().getNetworkConsumerIds());
1498            suppress = true;
1499        } else {
1500            // remove the existing lower priority duplicate and allow this candidate
1501            try {
1502                removeDuplicateSubscription(existingSub);
1503
1504                LOG.debug("{} Replacing duplicate subscription {} with sub from {}, which has a higher priority, new sub: {}, networkConsumerIds: {}",
1505                        configuration.getBrokerName(), existingSub.getConsumerInfo(), remoteBrokerName, candidateInfo, candidateInfo.getNetworkConsumerIds());
1506            } catch (IOException e) {
1507                LOG.error("Failed to remove duplicated sub as a result of sub with higher priority, sub: {}", existingSub, e);
1508            }
1509        }
1510        return suppress;
1511    }
1512
1513    private void removeDuplicateSubscription(Subscription existingSub) throws IOException {
1514        for (NetworkConnector connector : brokerService.getNetworkConnectors()) {
1515            if (connector.removeDemandSubscription(existingSub.getConsumerInfo().getConsumerId())) {
1516                break;
1517            }
1518        }
1519    }
1520
1521    private boolean matchFound(List<ConsumerId> candidateConsumers, List<ConsumerId> networkConsumers) {
1522        boolean found = false;
1523        for (ConsumerId aliasConsumer : networkConsumers) {
1524            if (candidateConsumers.contains(aliasConsumer)) {
1525                found = true;
1526                break;
1527            }
1528        }
1529        return found;
1530    }
1531
1532    protected final Collection<Subscription> getRegionSubscriptions(ActiveMQDestination dest) {
1533        RegionBroker region_broker = (RegionBroker) brokerService.getRegionBroker();
1534        Region region;
1535        Collection<Subscription> subs;
1536
1537        region = null;
1538        switch (dest.getDestinationType()) {
1539            case ActiveMQDestination.QUEUE_TYPE:
1540                region = region_broker.getQueueRegion();
1541                break;
1542            case ActiveMQDestination.TOPIC_TYPE:
1543                region = region_broker.getTopicRegion();
1544                break;
1545            case ActiveMQDestination.TEMP_QUEUE_TYPE:
1546                region = region_broker.getTempQueueRegion();
1547                break;
1548            case ActiveMQDestination.TEMP_TOPIC_TYPE:
1549                region = region_broker.getTempTopicRegion();
1550                break;
1551        }
1552
1553        if (region instanceof AbstractRegion) {
1554            subs = ((AbstractRegion) region).getSubscriptions().values();
1555        } else {
1556            subs = null;
1557        }
1558
1559        return subs;
1560    }
1561
1562    protected DemandSubscription createDemandSubscription(ConsumerInfo info) throws IOException {
1563        // add our original id to ourselves
1564        info.addNetworkConsumerId(info.getConsumerId());
1565        return doCreateDemandSubscription(info);
1566    }
1567
1568    protected DemandSubscription doCreateDemandSubscription(ConsumerInfo info) throws IOException {
1569        DemandSubscription result = new DemandSubscription(info);
1570        result.getLocalInfo().setConsumerId(new ConsumerId(localSessionInfo.getSessionId(), consumerIdGenerator.getNextSequenceId()));
1571        if (info.getDestination().isTemporary()) {
1572            // reset the local connection Id
1573            ActiveMQTempDestination dest = (ActiveMQTempDestination) result.getLocalInfo().getDestination();
1574            dest.setConnectionId(localConnectionInfo.getConnectionId().toString());
1575        }
1576
1577        if (configuration.isDecreaseNetworkConsumerPriority()) {
1578            byte priority = (byte) configuration.getConsumerPriorityBase();
1579            if (info.getBrokerPath() != null && info.getBrokerPath().length > 1) {
1580                // The longer the path to the consumer, the less it's consumer priority.
1581                priority -= info.getBrokerPath().length + 1;
1582            }
1583            result.getLocalInfo().setPriority(priority);
1584            LOG.debug("{} using priority: {} for subscription: {}", configuration.getBrokerName(), priority, info);
1585        }
1586        configureDemandSubscription(info, result);
1587        return result;
1588    }
1589
1590    final protected DemandSubscription createDemandSubscription(ActiveMQDestination destination, final String subscriptionName) {
1591        ConsumerInfo info = new ConsumerInfo();
1592        info.setNetworkSubscription(true);
1593        info.setDestination(destination);
1594
1595        if (subscriptionName != null) {
1596            info.setSubscriptionName(subscriptionName);
1597        }
1598
1599        // Indicate that this subscription is being made on behalf of the remote broker.
1600        info.setBrokerPath(new BrokerId[]{remoteBrokerId});
1601
1602        // the remote info held by the DemandSubscription holds the original
1603        // consumerId, the local info get's overwritten
1604        info.setConsumerId(new ConsumerId(localSessionInfo.getSessionId(), consumerIdGenerator.getNextSequenceId()));
1605        DemandSubscription result = null;
1606        try {
1607            result = createDemandSubscription(info);
1608        } catch (IOException e) {
1609            LOG.error("Failed to create DemandSubscription ", e);
1610        }
1611        return result;
1612    }
1613
1614    protected void configureDemandSubscription(ConsumerInfo info, DemandSubscription sub) throws IOException {
1615        if (AdvisorySupport.isConsumerAdvisoryTopic(info.getDestination()) ||
1616                AdvisorySupport.isVirtualDestinationConsumerAdvisoryTopic(info.getDestination())) {
1617            sub.getLocalInfo().setDispatchAsync(true);
1618        } else {
1619            sub.getLocalInfo().setDispatchAsync(configuration.isDispatchAsync());
1620        }
1621        configureConsumerPrefetch(sub.getLocalInfo());
1622        subscriptionMapByLocalId.put(sub.getLocalInfo().getConsumerId(), sub);
1623        subscriptionMapByRemoteId.put(sub.getRemoteInfo().getConsumerId(), sub);
1624
1625        sub.setNetworkBridgeFilter(createNetworkBridgeFilter(info));
1626        if (!info.isDurable()) {
1627            // This works for now since we use a VM connection to the local broker.
1628            // may need to change if we ever subscribe to a remote broker.
1629            sub.getLocalInfo().setAdditionalPredicate(sub.getNetworkBridgeFilter());
1630        } else {
1631            sub.setLocalDurableSubscriber(new SubscriptionInfo(info.getClientId(), info.getSubscriptionName()));
1632        }
1633    }
1634
1635    protected void removeDemandSubscription(ConsumerId id) throws IOException {
1636        DemandSubscription sub = subscriptionMapByRemoteId.remove(id);
1637        LOG.debug("{} remove request on {} from {}, consumer id: {}, matching sub: {}",
1638                configuration.getBrokerName(), localBroker, remoteBrokerName, id, sub);
1639        if (sub != null) {
1640            removeSubscription(sub);
1641            LOG.debug("{} removed sub on {} from {}: {}",
1642                    configuration.getBrokerName(), localBroker, remoteBrokerName, sub.getRemoteInfo());
1643        }
1644    }
1645
1646    protected boolean removeDemandSubscriptionByLocalId(ConsumerId consumerId) {
1647        boolean removeDone = false;
1648        DemandSubscription sub = subscriptionMapByLocalId.get(consumerId);
1649        if (sub != null) {
1650            try {
1651                removeDemandSubscription(sub.getRemoteInfo().getConsumerId());
1652                removeDone = true;
1653            } catch (IOException e) {
1654                LOG.debug("removeDemandSubscriptionByLocalId failed for localId: {}", consumerId, e);
1655            }
1656        }
1657        return removeDone;
1658    }
1659
1660    /**
1661     * Performs a timed wait on the started latch and then checks for disposed
1662     * before performing another wait each time the the started wait times out.
1663     */
1664    protected boolean safeWaitUntilStarted() throws InterruptedException {
1665        while (!disposed.get()) {
1666            if (startedLatch.await(1, TimeUnit.SECONDS)) {
1667                break;
1668            }
1669        }
1670        return !disposed.get();
1671    }
1672
1673    protected NetworkBridgeFilter createNetworkBridgeFilter(ConsumerInfo info) throws IOException {
1674        NetworkBridgeFilterFactory filterFactory = defaultFilterFactory;
1675        if (brokerService != null && brokerService.getDestinationPolicy() != null) {
1676            PolicyEntry entry = brokerService.getDestinationPolicy().getEntryFor(info.getDestination());
1677            if (entry != null && entry.getNetworkBridgeFilterFactory() != null) {
1678                filterFactory = entry.getNetworkBridgeFilterFactory();
1679            }
1680        }
1681        return filterFactory.create(info, getRemoteBrokerPath(), configuration.getMessageTTL(), configuration.getConsumerTTL());
1682    }
1683
1684    protected void addRemoteBrokerToBrokerPath(ConsumerInfo info) throws IOException {
1685        info.setBrokerPath(appendToBrokerPath(info.getBrokerPath(), getRemoteBrokerPath()));
1686    }
1687
1688    protected BrokerId[] getRemoteBrokerPath() {
1689        return remoteBrokerPath;
1690    }
1691
1692    @Override
1693    public void setNetworkBridgeListener(NetworkBridgeListener listener) {
1694        this.networkBridgeListener = listener;
1695    }
1696
1697    private void fireBridgeFailed(Throwable reason) {
1698        LOG.trace("fire bridge failed, listener: {}", this.networkBridgeListener, reason);
1699        NetworkBridgeListener l = this.networkBridgeListener;
1700        if (l != null && this.bridgeFailed.compareAndSet(false, true)) {
1701            l.bridgeFailed();
1702        }
1703    }
1704
1705    /**
1706     * @return Returns the dynamicallyIncludedDestinations.
1707     */
1708    public ActiveMQDestination[] getDynamicallyIncludedDestinations() {
1709        return dynamicallyIncludedDestinations;
1710    }
1711
1712    /**
1713     * @param dynamicallyIncludedDestinations
1714     *         The dynamicallyIncludedDestinations to set.
1715     */
1716    public void setDynamicallyIncludedDestinations(ActiveMQDestination[] dynamicallyIncludedDestinations) {
1717        this.dynamicallyIncludedDestinations = dynamicallyIncludedDestinations;
1718    }
1719
1720    /**
1721     * @return Returns the excludedDestinations.
1722     */
1723    public ActiveMQDestination[] getExcludedDestinations() {
1724        return excludedDestinations;
1725    }
1726
1727    /**
1728     * @param excludedDestinations The excludedDestinations to set.
1729     */
1730    public void setExcludedDestinations(ActiveMQDestination[] excludedDestinations) {
1731        this.excludedDestinations = excludedDestinations;
1732    }
1733
1734    /**
1735     * @return Returns the staticallyIncludedDestinations.
1736     */
1737    public ActiveMQDestination[] getStaticallyIncludedDestinations() {
1738        return staticallyIncludedDestinations;
1739    }
1740
1741    /**
1742     * @param staticallyIncludedDestinations The staticallyIncludedDestinations to set.
1743     */
1744    public void setStaticallyIncludedDestinations(ActiveMQDestination[] staticallyIncludedDestinations) {
1745        this.staticallyIncludedDestinations = staticallyIncludedDestinations;
1746    }
1747
1748    /**
1749     * @return Returns the durableDestinations.
1750     */
1751    public ActiveMQDestination[] getDurableDestinations() {
1752        return durableDestinations;
1753    }
1754
1755    /**
1756     * @param durableDestinations The durableDestinations to set.
1757     */
1758    public void setDurableDestinations(ActiveMQDestination[] durableDestinations) {
1759        this.durableDestinations = durableDestinations;
1760    }
1761
1762    /**
1763     * @return Returns the localBroker.
1764     */
1765    public Transport getLocalBroker() {
1766        return localBroker;
1767    }
1768
1769    /**
1770     * @return Returns the remoteBroker.
1771     */
1772    public Transport getRemoteBroker() {
1773        return remoteBroker;
1774    }
1775
1776    /**
1777     * @return the createdByDuplex
1778     */
1779    public boolean isCreatedByDuplex() {
1780        return this.createdByDuplex;
1781    }
1782
1783    /**
1784     * @param createdByDuplex the createdByDuplex to set
1785     */
1786    public void setCreatedByDuplex(boolean createdByDuplex) {
1787        this.createdByDuplex = createdByDuplex;
1788    }
1789
1790    @Override
1791    public String getRemoteAddress() {
1792        return remoteBroker.getRemoteAddress();
1793    }
1794
1795    @Override
1796    public String getLocalAddress() {
1797        return localBroker.getRemoteAddress();
1798    }
1799
1800    @Override
1801    public String getRemoteBrokerName() {
1802        return remoteBrokerInfo == null ? null : remoteBrokerInfo.getBrokerName();
1803    }
1804
1805    @Override
1806    public String getRemoteBrokerId() {
1807        return (remoteBrokerInfo == null || remoteBrokerInfo.getBrokerId() == null) ? null : remoteBrokerInfo.getBrokerId().toString();
1808    }
1809
1810    @Override
1811    public String getLocalBrokerName() {
1812        return localBrokerInfo == null ? null : localBrokerInfo.getBrokerName();
1813    }
1814
1815    @Override
1816    public long getDequeueCounter() {
1817        return networkBridgeStatistics.getDequeues().getCount();
1818    }
1819
1820    @Override
1821    public long getEnqueueCounter() {
1822        return networkBridgeStatistics.getEnqueues().getCount();
1823    }
1824
1825    @Override
1826    public NetworkBridgeStatistics getNetworkBridgeStatistics() {
1827        return networkBridgeStatistics;
1828    }
1829
1830    protected boolean isDuplex() {
1831        return configuration.isDuplex() || createdByDuplex;
1832    }
1833
1834    public ConcurrentMap<ConsumerId, DemandSubscription> getLocalSubscriptionMap() {
1835        return subscriptionMapByRemoteId;
1836    }
1837
1838    @Override
1839    public void setBrokerService(BrokerService brokerService) {
1840        this.brokerService = brokerService;
1841        this.localBrokerId = brokerService.getRegionBroker().getBrokerId();
1842        localBrokerPath[0] = localBrokerId;
1843    }
1844
1845    @Override
1846    public void setMbeanObjectName(ObjectName objectName) {
1847        this.mbeanObjectName = objectName;
1848    }
1849
1850    @Override
1851    public ObjectName getMbeanObjectName() {
1852        return mbeanObjectName;
1853    }
1854
1855    @Override
1856    public void resetStats() {
1857        networkBridgeStatistics.reset();
1858    }
1859
1860    /*
1861     * Used to allow for async tasks to await receipt of the BrokerInfo from the local and
1862     * remote sides of the network bridge.
1863     */
1864    private static class FutureBrokerInfo implements Future<BrokerInfo> {
1865
1866        private final CountDownLatch slot = new CountDownLatch(1);
1867        private final AtomicBoolean disposed;
1868        private volatile BrokerInfo info = null;
1869
1870        public FutureBrokerInfo(BrokerInfo info, AtomicBoolean disposed) {
1871            this.info = info;
1872            this.disposed = disposed;
1873        }
1874
1875        @Override
1876        public boolean cancel(boolean mayInterruptIfRunning) {
1877            slot.countDown();
1878            return true;
1879        }
1880
1881        @Override
1882        public boolean isCancelled() {
1883            return slot.getCount() == 0 && info == null;
1884        }
1885
1886        @Override
1887        public boolean isDone() {
1888            return info != null;
1889        }
1890
1891        @Override
1892        public BrokerInfo get() throws InterruptedException, ExecutionException {
1893            try {
1894                if (info == null) {
1895                    while (!disposed.get()) {
1896                        if (slot.await(1, TimeUnit.SECONDS)) {
1897                            break;
1898                        }
1899                    }
1900                }
1901                return info;
1902            } catch (InterruptedException e) {
1903                Thread.currentThread().interrupt();
1904                LOG.debug("Operation interrupted: {}", e, e);
1905                throw new InterruptedException("Interrupted.");
1906            }
1907        }
1908
1909        @Override
1910        public BrokerInfo get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
1911            try {
1912                if (info == null) {
1913                    long deadline = System.currentTimeMillis() + unit.toMillis(timeout);
1914
1915                    while (!disposed.get() || System.currentTimeMillis() - deadline < 0) {
1916                        if (slot.await(1, TimeUnit.MILLISECONDS)) {
1917                            break;
1918                        }
1919                    }
1920                    if (info == null) {
1921                        throw new TimeoutException();
1922                    }
1923                }
1924                return info;
1925            } catch (InterruptedException e) {
1926                throw new InterruptedException("Interrupted.");
1927            }
1928        }
1929
1930        public void set(BrokerInfo info) {
1931            this.info = info;
1932            this.slot.countDown();
1933        }
1934    }
1935
1936    protected void serviceOutbound(Message message) {
1937        NetworkBridgeListener l = this.networkBridgeListener;
1938        if (l != null) {
1939            l.onOutboundMessage(this, message);
1940        }
1941    }
1942
1943    protected void serviceInboundMessage(Message message) {
1944        NetworkBridgeListener l = this.networkBridgeListener;
1945        if (l != null) {
1946            l.onInboundMessage(this, message);
1947        }
1948    }
1949
1950    protected boolean canDuplexDispatch(Message message) {
1951        boolean result = true;
1952        if (configuration.isCheckDuplicateMessagesOnDuplex()){
1953            final long producerSequenceId = message.getMessageId().getProducerSequenceId();
1954            //  messages are multiplexed on this producer so we need to query the persistenceAdapter
1955            long lastStoredForMessageProducer = getStoredSequenceIdForMessage(message.getMessageId());
1956            if (producerSequenceId <= lastStoredForMessageProducer) {
1957                result = false;
1958                LOG.debug("suppressing duplicate message send [{}] from network producer with producerSequence [{}] less than last stored: {}",
1959                        (LOG.isTraceEnabled() ? message : message.getMessageId()), producerSequenceId, lastStoredForMessageProducer);
1960            }
1961        }
1962        return result;
1963    }
1964
1965    protected long getStoredSequenceIdForMessage(MessageId messageId) {
1966        try {
1967            return brokerService.getPersistenceAdapter().getLastProducerSequenceId(messageId.getProducerId());
1968        } catch (IOException ignored) {
1969            LOG.debug("Failed to determine last producer sequence id for: {}", messageId, ignored);
1970        }
1971        return -1;
1972    }
1973
1974    protected void configureConsumerPrefetch(ConsumerInfo consumerInfo) {
1975        //If a consumer on an advisory topic and advisoryPrefetchSize has been explicitly
1976        //set then use it, else default to the prefetchSize setting
1977        if (AdvisorySupport.isAdvisoryTopic(consumerInfo.getDestination()) &&
1978                configuration.getAdvisoryPrefetchSize() > 0) {
1979            consumerInfo.setPrefetchSize(configuration.getAdvisoryPrefetchSize());
1980        } else {
1981            consumerInfo.setPrefetchSize(configuration.getPrefetchSize());
1982        }
1983    }
1984
1985}