diff --git a/README.md b/README.md index dcf47607..2b752600 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # TAK Server Development *Requires Java 11* -* Linux / MacOS is recommended for development. If using Windows, replace "gradlew" with "gradlew.bat" in commands below. +* Linux or MacOS is recommended for development. If using Windows, replace "gradlew" with "gradlew.bat" in commands below. An x86-64 architecture CPU is required to build from source, including on MacOS. M1 or M2 Apple silicon is not supported. Links: * [Test Execution](src/takserver-takcl-core/docs/testing.md) @@ -33,7 +33,7 @@ docker run -it -d --rm --name TakserverServer0DB \ --env POSTGRES_HOST_AUTH_METHOD=trust \ --env POSTGRES_USER=martiuser \ --env POSTGRES_DB=cot \ - -p 5432 postgis/postgis:10-3.1 + -p 5432 postgis/postgis:15-3.3 echo SQL SERVER IP: `docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' TakserverServer0DB` ``` @@ -48,7 +48,9 @@ Setup Local Database. If the postgis container was used, only the last two lines ``` Configure Local CoreConfig and Certs -```cd takserver-core/example``` +``` +cd takserver-core/example +``` This is the CoreConfig that takserver war will look for when running from the takserver-core/example directory. From this point, just follow the instructions at takserver/src/docs/TAK_Server_Configuration_Guide.pdf to set up the CoreConfig and Certs. Make sure that the CoreConfig now points to the directory where the certs were generated locally. @@ -113,16 +115,24 @@ java -Xmx -Dspring.profiles.active=messaging -jar ../build/libs/takserver ``` turn down log level of all logs: -```java -jar takserver.war $@ --logging.level.root=ERROR``` +``` +java -jar takserver.war $@ --logging.level.root=ERROR +``` turn down log level for subscriptions: -```java -jar takserver.war $@ --logging.level.com.bbn.marti.service.Subscription=ERROR``` +``` +java -jar takserver.war $@ --logging.level.com.bbn.marti.service.Subscription=ERROR +``` turn off logs just for subscriptions: -```java -jar takserver.war $@ --logging.level.com.bbn.marti.service.Subscription=OFF``` +``` +java -jar takserver.war $@ --logging.level.com.bbn.marti.service.Subscription=OFF +``` entirely disable most logging: -```java -jar takserver.war $@ --logging.level.root=OFF``` +``` +java -jar takserver.war $@ --logging.level.root=OFF +``` The default log level for most things is INFO. Possible levels are INFO, WARN, ERROR, OFF (in order of decreasing log frequency) @@ -135,6 +145,13 @@ i.e. ```--logging.level.root=ERROR``` +The TAK Server log files can be found in the _logs_ subdirectory: + +1. _takserver-messaging.log_ - Execution-level information about the messaging process, including client connection events, error messages and warnings. +2. _takserver-api.log_ - Execution-level information about the API process, including error messages and warnings. +3. _takserver-messaging-console.log_ - Java Virtual Machine (JVM) informational messages and errors, for the messaging process. +4. _takserver-api-console.log_ - Java Virtual Machine (JVM) informational messages and errors, for the API process. + ## Swagger https://localhost:8443/swagger-ui.html diff --git a/src/.licenses/config/allowed-licenses.json b/src/.licenses/config/allowed-licenses.json new file mode 100644 index 00000000..acb71e07 --- /dev/null +++ b/src/.licenses/config/allowed-licenses.json @@ -0,0 +1,35 @@ +{ + "allowedLicenses": [ + {"moduleLicense": "Apache License, Version 2.0"}, + {"moduleLicense": "Apache License"}, + {"moduleLicense": "Apache Software License, version 1.1"}, + {"moduleLicense": "The MIT License"}, + {"moduleLicense": "The 3-Clause BSD License"}, + {"moduleLicense": "The 2-Clause BSD License"}, + + {"moduleLicense": "BSD"}, + {"moduleLicense": "New BSD License"}, + {"moduleLicense": "BSD New license"}, + {"moduleLicense": "The New BSD License"}, + {"moduleLicense": "The BSD License"}, + {"moduleLicense": "Revised BSD"}, + {"moduleLicense": "BSD-style"}, + + {"moduleLicense": "Public Domain"}, + {"moduleLicense": "PUBLIC DOMAIN"}, + {"moduleLicense": "Common Public License Version 1.0"}, + + {"moduleLicense": "CDDL + GPLv2 with classpath exception"}, + {"moduleLicense": "LGPL, version 2.1"}, + {"moduleLicense": "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0"}, + {"moduleLicense": "GPL2 w/ CPE"}, + {"moduleLicense": "CDDL 1.1"}, + {"moduleLicense": "LGPL"}, + {"moduleLicense": "Google Cloud Software License"}, + {"moduleLicense": "GNU Lesser General Public License"}, + {"moduleLicense": "CDDL+GPL License"}, + {"moduleLicense": "ASL"}, + {"moduleLicense": "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.1"}, + {"moduleLicense": null} + ] +} \ No newline at end of file diff --git a/src/.quality/config/pmd-ruleset.xml b/src/.quality/config/pmd-ruleset.xml new file mode 100644 index 00000000..2e441a86 --- /dev/null +++ b/src/.quality/config/pmd-ruleset.xml @@ -0,0 +1,110 @@ + + + + + My custom rules + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Static final fields should have names in all upper case + + + + + + + + + + + + + + + + Normal variables should start with lower case letters + + + + + + + + + + + + + + + + Non-final variables should not have underscores in the name + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/build.gradle b/src/build.gradle index 6cf80e75..7e10b870 100644 --- a/src/build.gradle +++ b/src/build.gradle @@ -1,3 +1,7 @@ +//import se.bjurr.violations.gradle.plugin.ViolationsTask +//import com.github.jk1.license.render.InventoryHtmlReportRenderer +//import com.github.jk1.license.render.XmlReportRenderer + buildscript { repositories { @@ -9,12 +13,13 @@ buildscript { dependencies { classpath 'org.ajoberstar:grgit:1.7.2' +// classpath 'gradle.plugin.se.bjurr.violations:violations-gradle-plugin:1.52.2' +// classpath 'com.github.jk1:gradle-license-report:1.17' } } apply plugin: 'eclipse' apply plugin: 'idea' - apply from: 'gradle/license.gradle', to: project ext { @@ -35,7 +40,7 @@ ext { takreleaserpm = takrel1 + takrel2 def (tvmaj, tvmin) = tv.tokenize('.') - + verMajor = tvmaj verMinor = tvmin verPatch = takrel2 @@ -49,6 +54,8 @@ allprojects { apply plugin: 'java' apply plugin: 'eclipse' apply plugin: 'idea' +// apply plugin: 'pmd' + sourceCompatibility = 1.8 targetCompatibility = 1.8 @@ -94,13 +101,54 @@ subprojects { dependencies { classpath 'com.github.jengelman.gradle.plugins:shadow:' + gradle_shadow_version classpath 'gradle.plugin.org.openrepose:gradle-jaxb-plugin:2.5.0' +// classpath 'gradle.plugin.se.bjurr.violations:violations-gradle-plugin:1.52.2' +// classpath 'com.github.jk1:gradle-license-report:1.17' } } - clean { - delete 'bin/' - } +// apply plugin: 'se.bjurr.violations.violations-gradle-plugin' // Normalizes quality reports for Gitlab CI +// apply plugin: 'pmd' +// // Define a new task to merge quality reports and convert them into a format GitLab can consume +// tasks.create('violations', ViolationsTask) { +// maxReporterColumnWidth = 0 // 0 means "no limit" +// maxRuleColumnWidth = 60 +// maxSeverityColumnWidth = 0 +// maxLineColumnWidth = 0 +// maxMessageColumnWidth = 50 +// codeClimateFile = new File(project.buildDir.path + '/code-climate-file.json') +// // Will create a CodeClimate JSON report. +// violations = [ +// ["PMD", buildDir.path, ".*/pmd/.*\\.xml\$", "pmd"] +// ] +// } +// pmd { +// ignoreFailures = true +// toolVersion = '6.47.0' +// ruleSetFiles = files("${rootDir}/.quality/config/pmd-ruleset.xml") +// // Note: The ruleSets array is explicitly set to empty to avoid using the default configuration. +// ruleSets = [] +// } +// +// apply plugin: 'com.github.jk1.dependency-license-report' +// licenseReport { +// // Don't include artifacts of project's own group in the report +// excludeOwnGroup = true +// +// // Don't exclude bom dependencies. +// // If set to true, then all boms will be excluded from the report +// excludeBoms = false +// +// // Set custom report renderer, implementing ReportRenderer. +// // Yes, you can write your own to support any format necessary. +// renderers = [new XmlReportRenderer()] +// renderers = [new InventoryHtmlReportRenderer()] +// +// // This is for the allowed-licenses-file in checkLicense Task +// // Accepts File, URL or String path to local or remote file +// allowedLicensesFile = new File("${rootDir}/.licenses/config/allowed-licenses.json") +// } +// +// clean { +// delete 'bin/' +// } } - - - diff --git a/src/docs/INSTALL.txt b/src/docs/INSTALL.txt deleted file mode 100644 index 4722dff6..00000000 --- a/src/docs/INSTALL.txt +++ /dev/null @@ -1,25 +0,0 @@ -TAK Server Installation Quick Start -=================================== -Version 1.3.6 - ----- -Distribution Statement A: Approved for public release; distribution is unlimited. ----- - -Perform a fresh install of CentOS 7 - -sudo yum install epel-release - -sudo yum install takserver--.noarch.rpm - -sudo /opt/tak/db-utils/takserver-setup-db.sh - -Start TAK Server: -sudo systemctl start takserver - -Set TAK Server to run at startup: -sudo systemctl enable takserver - -Refer to the configuration guide for additional information, including generating certificates, and setting up admin users. - -/opt/tak/docs/TAK_Server_Configuration_Guide.pdf diff --git a/src/docs/TAK_Server_Configuration_Guide.odt b/src/docs/TAK_Server_Configuration_Guide.odt index 645f9d35..59a1b923 100644 Binary files a/src/docs/TAK_Server_Configuration_Guide.odt and b/src/docs/TAK_Server_Configuration_Guide.odt differ diff --git a/src/docs/TAK_Server_Configuration_Guide.pdf b/src/docs/TAK_Server_Configuration_Guide.pdf index cb17a1f0..5ef5a276 100644 Binary files a/src/docs/TAK_Server_Configuration_Guide.pdf and b/src/docs/TAK_Server_Configuration_Guide.pdf differ diff --git a/src/docs/TAK_Server_Installation_QuickStart_CentOS_RHEL_6.docx b/src/docs/TAK_Server_Installation_QuickStart_CentOS_RHEL_6.docx deleted file mode 100644 index 5a763dae..00000000 Binary files a/src/docs/TAK_Server_Installation_QuickStart_CentOS_RHEL_6.docx and /dev/null differ diff --git a/src/docs/TAK_protobuf_protocol.txt b/src/docs/TAK_protobuf_protocol.txt deleted file mode 100644 index 02c82f99..00000000 --- a/src/docs/TAK_protobuf_protocol.txt +++ /dev/null @@ -1,269 +0,0 @@ -*** Traditional Protocol - "Protocol Version 0" - -Clients send and receive XML CoT messages. -"Mesh" network participants announce via "SA" messages via UDP datagrams -over multicast to a well known address and port. Each UDP datagram contains -one (and only one) CoT XML message as its payload. - -Messages directed only to specific network participants are send by making -TCP connection to the remote recipient, sending the CoT XML, then closing -the connection. - - -Streaming connections (to TAK servers) send the same XML-based CoT payloads -over TCP sockets. The TCP stream is comprised of one CoT after -another. Messages are delimited and broken apart by searching for the token -"" and breaking apart immediately after that token. -When sending, messages must be prefaced by XML header (), -followed by a newline, followed by the complete XML . TAK servers -require that no arbitrary newlines follow the end of message and -that the next character immediate commences the next header. - - - -*** TAK Protocol - Design Goals - -The goal of the new TAK Protocol design is to allow interoperation with -other legacy clients and TAK server, as well as to strongly identify -what rendition of communication will be used in a session. This is to allow -for future expansion or complete revision of the protocol while allowing -an opportunity to support mixed client versions (and varying versions of TAK -servers). - - - -*** TAK Protocol - Generic Framework - Mesh Networks - -Mesh networks broadcasts (SA announces, etc) will reuse the existing UDP -datagram-based networking already in place. Directed (unicasted) TCP -messages will reuse the existing connect, send 1 message, disconnect -networking. - -For both TCP and UDP, instead of sending CoT as XML, clients will send data -packets whose payloads contain one message complying with the new "TAK -Protocol". -Both types of messages will utilize a data payload that begins with the "TAK -Protocol Header" followed by the "TAK Protocol Payload". The header -serves to self-identify as a TAK Protocol message, as well as indicate a -particular version number to which the subsequent Payload comforms. - -TAK Protocol Message: - - -The "TAK Protocol Header" is nothing more than a set of "magic numbers" to -identify the message header as such, and a version identifier to indicate -what TAK Protocol version the remainder of the payload is comprised of. - -TAK Protocol Header: -Where.... - is the single byte 0xbf - is the version number of the TAK Protocol the payload -in the remainder of the message conforms to. This is encoded as a "varint". -See "TAK Protocol Varint Encoding". - - - -*** TAK Protocol - Generic Framework - Streaming Connections - -Steaming connections (TAK server connections) use a different message style -as the repeating protocol version information in every message that is done -in mesh TAK Protocol Messages would be a waste of resources in the streaming -environment (since all messages will use the same Version). -The TAK Protocol Stream Message is instead defined to provide the length of -the streaming message (necessary to break apart the message from its -neighbors to avoid need to scan for special tokens). - -In a streaming connection, "TAK Protocol Streaming Messages" are sent one -after another (with no intervening data) over the streaming connection. - -TAK Protocol Stream Message: - - -Important to note here is that the "TAK Protocol Payload" is precisely the -same in form and content to that which is used for mesh network messages for -a given protocol version. - - - -The "TAK Protocol Streaming Header" is as follows: - -TAK Protocol Streaming Header: - -Where... - is the single byte 0xbf - is the number of bytes in the "TAK Protocol Payload" which - follows the header. This is encoded as a "varint". - -As mentioned prior, the version identification for the message's payload -format is omitted from the streaming header. Protocol version negotiation -is expected to occur outside of core TAK Protocol message exchange. -See "Streaming Connection Protocol Negotiation". - - - - -*** TAK Protocol Payload - Version 1 - -Version 1 of the TAK Protocol Payload is a Google Protocol Buffer based -payload. Each Payload consists of one (and only one) -atakmap::commoncommo::v1::TakMessage message which is serialized using -Google protocol buffers version 3. - -See the .proto files for more information on the specific messages and their -fields, as well as the mapping to/from CoT XML. - -Revising the messages used by Version 1 may be done in accordance with the -following rules: - -1. Additional message fields MAY be added to the end of existing messages - following normal google protobuf rules if and only if - ignorance of the new fields on decoding is 100% irrelevant to correct - semantic operation at the TAK application level of ALL TAK applications. -2. Otherwise, any and all changes must be tied to a protocol version change. - - -This version of TAK Protocol does not define any additional attributes to be -used during Streaming Connection Protocol Negotiation. - - - -*** Streaming Connection Protocol Negotiation - -TAK clients often connect to a variety of TAK servers, each of which may be -a different version of software capable of different versions of the TAK -Protocol (or indeed not capable of the TAK Protocol and simply only -supporting traditional streaming CoT as XML). - -Because of the desire to allow operation of various client and server -versions, and the desire to keep the traditional XML encoding available, the -following negotiation is performed when connecting to a TAK server with a -client that supports the TAK Protocol. - -1. Once the connection is established, if authentication is required for - this server, the authentication message is sent by the client to the server. - 1a. If the server accepts the auth, proceed to 2. - 1b. If the server denies the auth, the connection is closed. -2. Client and server expect to exchange traditional CoT XML messages per - "Traditional Protocol" section. -3. A server which supports the TAK Protocol MAY send the following CoT XML - message to indicate this support (whitespace added, xml header omitted): - - - - - - - - - - ... where the version attribute is an integer number specifying - a version of the TAK Protocol the server supports. This message - may contain one or more TakProtocolSupport elements inside the single - detail, each specifying a supported version. - The TAK server MUST send this message no more than once per connection. - - To allow for ancillary information in the negotiation, the - TakProtocolSupport element MAY contain additional attributes compliant - with the Protocol version indicated. -4. Client and server continue to expect to exchange traditional CoT XML - messages per "Traditional Protocol" section. -5. If the client wishes to initiate a transfer to TAK Protocol encoding, it - selects one of the supported versions advertised in the server's message - from step 3. It then sends the following CoT XML: - - - - - - - - - - ... where the version attribute is the integer version chosen above. - Only ONE TakRequest element is allowed. - - To allow for ancillary information in the negotiation, the - TakRequest element MAY contain additional attributes compliant - with the Protocol version indicated. - - Clients SHALL NOT send this message unless they have observed the - message from step #3, above, first. - -6. Once the client sends the message in #5, it MUST NOT send additional - CoT XML to the server. Client also MUST still process incoming CoT XML - from the server. The client MUST wait in this state for a response per - the following for at least 30 seconds. - The server MAY still send CoT XML messages up until it notices the - control request from the client (from step #5) and is ready to respond. - The server MUST then respond as soon as possible to the client with the - following message to indicate either acceptance or denial of the request: - - - - - - - - - - ... where the status attribute is either true (to indicate the server - accepts the requested version) or false (to indicate that the server - denies the request). - Only ONE TakResponse element is allowed. - - To allow for ancillary information in the negotiation, the - TakResponse element MAY contain additional attributes compliant - with the Protocol version selected in the request that this response - applies to. - - If no response is received by the client before its timeout elapses, - the client SHALL disconnect as the entire negotiation is in an - indeterminate state. The client SHOULD reconnect and begin again at step - 1, possibly with a longer timeout or alternate protocol version choice. - -7. Operation at this point depends on the response send in #6: -7a. If status was true: The server MUST NOT send additiona CoT XML after the - "true" response in #6. Instead, the server SHALL send all future data - in accordance with the TAK Protocol Streaming Connection framework - and containing TAK Payloads of the negotiated version. - The client MAY resume sending messages at this time but MUST immediately - send said messages in accordance with the TAK Protocol Streaming - Connection framework and containing TAK Payloads of the negotiated - version. NOTE: the negotiated version SHALL be the same for both - directions of the streaming connection! -7b. If status was false: Both client and server resume operation as though - they were back at step #4. The client may attempt a new negotiation - if it wishes, or may simply continue to exchange traditional XML-based - CoT messages. - -In the messages in 3, 5, and 6 above, the following common rules apply: -a. "protouid" is any valid UID representing the negotiation transaction. - The server generates this when offering protocol versions. The client - re-uses it when placing request(s) and the server re-uses it when - issuing the response to a request. - The UID SHALL be unique from UIDs used for other messages and purposes. -b. "TIME" is filled with a valid time representation per the CoT schemas. - The TIME values may be different from each other as needed. - - -*** TAK Protocol Varint Encoding - -The varints used in the headers of the TAK Protocol are encoded in -accordance with the UNSIGNED varint rules for Google protocol buffers. -This encoding is summarized here: - -1. The value must be UNSIGNED. Only values equal to or greater than zero - are allowed. -2. The value to be encoded is taken 7 bits at a time, starting with the - least significant 7 bits (bits 7 -> 0), then the next least significant bits - (14 -> 8), etc. This repeats over all 7 bit values that are significant - (that is, up to and including the most significant '1' bit). -3. For each 7 bit group: - 3a. Let S = 0 if this is the the last 7 bit group, else let S = 1 - 3b. Output a byte that is (S << 7) | (the 7 bits) - -The TAK Protocol use of Varints limits use to 64-bit values. This -effectively limits the range as [ 0, (2^63 - 1) ] and the varint coded value -to be limited to 10 bytes. - - diff --git a/src/docs/WebCop-API.docx b/src/docs/WebCop-API.docx deleted file mode 100644 index 7212b2a4..00000000 Binary files a/src/docs/WebCop-API.docx and /dev/null differ diff --git a/src/federation-common/docker/Dockerfile.fedhub b/src/federation-common/docker/Dockerfile.fedhub new file mode 100644 index 00000000..9348eddb --- /dev/null +++ b/src/federation-common/docker/Dockerfile.fedhub @@ -0,0 +1,5 @@ +FROM openjdk:11-jdk-bullseye +RUN apt update && \ + apt-get install -y emacs-nox net-tools netcat vim + +ENTRYPOINT ["/bin/bash", "-c", "/opt/tak/federation-hub/scripts/configureInDocker.sh init"] diff --git a/src/federation-common/scripts/configureInDocker.sh b/src/federation-common/scripts/configureInDocker.sh new file mode 100755 index 00000000..2c855255 --- /dev/null +++ b/src/federation-common/scripts/configureInDocker.sh @@ -0,0 +1,20 @@ +!/bin/sh +if [ $# -eq 0 ] + then + ps -ef | grep 'federation-hub-policy' | grep -v grep | awk '{print $2}' | xargs kill + ps -ef | grep 'federation-hub-ui' | grep -v grep | awk '{print $2}' | xargs kill + ps -ef | grep 'federation-hub-broker' | grep -v grep | awk '{print $2}' | xargs kill +fi + +cd /opt/tak/federation-hub/scripts/ +sleep 1 +sh federation-hub-policy.sh & +sleep 2 +sh federation-hub-broker.sh & +sleep 3 +sh federation-hub-ui.sh & + +if ! [ $# -eq 0 ] + then + tail -f /dev/null +fi \ No newline at end of file diff --git a/src/federation-common/src/main/java/io/grpc/internal/ServerCallImpl.java b/src/federation-common/src/main/java/io/grpc/internal/ServerCallImpl.java new file mode 100644 index 00000000..e9cb6ce5 --- /dev/null +++ b/src/federation-common/src/main/java/io/grpc/internal/ServerCallImpl.java @@ -0,0 +1,413 @@ +/* + * Copyright 2015 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.internal; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; +import static io.grpc.internal.GrpcAttributes.ATTR_SECURITY_LEVEL; +import static io.grpc.internal.GrpcUtil.ACCEPT_ENCODING_SPLITTER; +import static io.grpc.internal.GrpcUtil.CONTENT_LENGTH_KEY; +import static io.grpc.internal.GrpcUtil.MESSAGE_ACCEPT_ENCODING_KEY; +import static io.grpc.internal.GrpcUtil.MESSAGE_ENCODING_KEY; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Throwables; +import com.google.common.util.concurrent.MoreExecutors; +import io.grpc.Attributes; +import io.grpc.Codec; +import io.grpc.Compressor; +import io.grpc.CompressorRegistry; +import io.grpc.Context; +import io.grpc.DecompressorRegistry; +import io.grpc.InternalDecompressorRegistry; +import io.grpc.Metadata; +import io.grpc.MethodDescriptor; +import io.grpc.SecurityLevel; +import io.grpc.ServerCall; +import io.grpc.Status; +import io.perfmark.PerfMark; +import io.perfmark.Tag; +import java.io.InputStream; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * TAKServer Update instructions: + * + * See Method: sendMessageInternal for notes about what versions work properly, + * and what functionality we need within TAK Server + * + */ + +final class ServerCallImpl extends ServerCall { + + private static final Logger log = Logger.getLogger(ServerCallImpl.class.getName()); + + @VisibleForTesting + static final String TOO_MANY_RESPONSES = "Too many responses"; + @VisibleForTesting + static final String MISSING_RESPONSE = "Completed without a response"; + + private final ServerStream stream; + private final MethodDescriptor method; + private final Tag tag; + private final Context.CancellableContext context; + private final byte[] messageAcceptEncoding; + private final DecompressorRegistry decompressorRegistry; + private final CompressorRegistry compressorRegistry; + private CallTracer serverCallTracer; + + // state + private volatile boolean cancelled; + private boolean sendHeadersCalled; + private boolean closeCalled; + private Compressor compressor; + private boolean messageSent; + + ServerCallImpl(ServerStream stream, MethodDescriptor method, + Metadata inboundHeaders, Context.CancellableContext context, + DecompressorRegistry decompressorRegistry, CompressorRegistry compressorRegistry, + CallTracer serverCallTracer, Tag tag) { + this.stream = stream; + this.method = method; + this.context = context; + this.messageAcceptEncoding = inboundHeaders.get(MESSAGE_ACCEPT_ENCODING_KEY); + this.decompressorRegistry = decompressorRegistry; + this.compressorRegistry = compressorRegistry; + this.serverCallTracer = serverCallTracer; + this.serverCallTracer.reportCallStarted(); + this.tag = tag; + } + + @Override + public void request(int numMessages) { + PerfMark.startTask("ServerCall.request", tag); + try { + stream.request(numMessages); + } finally { + PerfMark.stopTask("ServerCall.request", tag); + } + } + + @Override + public void sendHeaders(Metadata headers) { + PerfMark.startTask("ServerCall.sendHeaders", tag); + try { + sendHeadersInternal(headers); + } finally { + PerfMark.stopTask("ServerCall.sendHeaders", tag); + } + } + + private void sendHeadersInternal(Metadata headers) { + checkState(!sendHeadersCalled, "sendHeaders has already been called"); + checkState(!closeCalled, "call is closed"); + + headers.discardAll(CONTENT_LENGTH_KEY); + headers.discardAll(MESSAGE_ENCODING_KEY); + if (compressor == null) { + compressor = Codec.Identity.NONE; + } else { + if (messageAcceptEncoding != null) { + // TODO(carl-mastrangelo): remove the string allocation. + if (!GrpcUtil.iterableContains( + ACCEPT_ENCODING_SPLITTER.split(new String(messageAcceptEncoding, GrpcUtil.US_ASCII)), + compressor.getMessageEncoding())) { + // resort to using no compression. + compressor = Codec.Identity.NONE; + } + } else { + compressor = Codec.Identity.NONE; + } + } + + // Always put compressor, even if it's identity. + headers.put(MESSAGE_ENCODING_KEY, compressor.getMessageEncoding()); + + stream.setCompressor(compressor); + + headers.discardAll(MESSAGE_ACCEPT_ENCODING_KEY); + byte[] advertisedEncodings = + InternalDecompressorRegistry.getRawAdvertisedMessageEncodings(decompressorRegistry); + if (advertisedEncodings.length != 0) { + headers.put(MESSAGE_ACCEPT_ENCODING_KEY, advertisedEncodings); + } + + // Don't check if sendMessage has been called, since it requires that sendHeaders was already + // called. + sendHeadersCalled = true; + stream.writeHeaders(headers); + } + + @Override + public void sendMessage(RespT message) { + PerfMark.startTask("ServerCall.sendMessage", tag); + try { + sendMessageInternal(message); + } finally { + PerfMark.stopTask("ServerCall.sendMessage", tag); + } + } + + private void sendMessageInternal(RespT message) { + checkState(sendHeadersCalled, "sendHeaders has not been called"); + checkState(!closeCalled, "call is closed"); + + if (method.getType().serverSendsOneMessage() && messageSent) { + internalClose(Status.INTERNAL.withDescription(TOO_MANY_RESPONSES)); + return; + } + + + messageSent = true; + + // GRPC BUG FIX: version 1.48-1.50 (current as of this fix) allows 0 messages through for client streaming. + // this means we can never send a return value.. to fix that, allow message flow the same as version <= 1.47. + try { + InputStream resp = method.streamResponse(message); + stream.writeMessage(resp); +// if (!getMethodDescriptor().getType().serverSendsOneMessage()) { +// stream.flush(); +// } + stream.flush(); + } catch (RuntimeException e) { + close(Status.fromThrowable(e), new Metadata()); + } catch (Error e) { + close( + Status.CANCELLED.withDescription("Server sendMessage() failed with Error"), + new Metadata()); + throw e; + } + } + + @Override + public void setMessageCompression(boolean enable) { + stream.setMessageCompression(enable); + } + + @Override + public void setCompression(String compressorName) { + // Added here to give a better error message. + checkState(!sendHeadersCalled, "sendHeaders has been called"); + + compressor = compressorRegistry.lookupCompressor(compressorName); + checkArgument(compressor != null, "Unable to find compressor by name %s", compressorName); + } + + @Override + public boolean isReady() { + if (closeCalled) { + return false; + } + return stream.isReady(); + } + + @Override + public void close(Status status, Metadata trailers) { + PerfMark.startTask("ServerCall.close", tag); + try { + closeInternal(status, trailers); + } finally { + PerfMark.stopTask("ServerCall.close", tag); + } + } + + private void closeInternal(Status status, Metadata trailers) { + checkState(!closeCalled, "call already closed"); + try { + closeCalled = true; + + if (status.isOk() && method.getType().serverSendsOneMessage() && !messageSent) { + internalClose(Status.INTERNAL.withDescription(MISSING_RESPONSE)); + return; + } + + stream.close(status, trailers); + } finally { + serverCallTracer.reportCallEnded(status.isOk()); + } + } + + @Override + public boolean isCancelled() { + return cancelled; + } + + ServerStreamListener newServerStreamListener(ServerCall.Listener listener) { + return new ServerStreamListenerImpl<>(this, listener, context); + } + + @Override + public Attributes getAttributes() { + return stream.getAttributes(); + } + + @Override + public String getAuthority() { + return stream.getAuthority(); + } + + @Override + public MethodDescriptor getMethodDescriptor() { + return method; + } + + @Override + public SecurityLevel getSecurityLevel() { + final Attributes attributes = getAttributes(); + if (attributes == null) { + return super.getSecurityLevel(); + } + final SecurityLevel securityLevel = attributes.get(ATTR_SECURITY_LEVEL); + return securityLevel == null ? super.getSecurityLevel() : securityLevel; + } + + /** + * Close the {@link ServerStream} because an internal error occurred. Allow the application to + * run until completion, but silently ignore interactions with the {@link ServerStream} from now + * on. + */ + private void internalClose(Status internalError) { + log.log(Level.WARNING, "Cancelling the stream with status {0}", new Object[] {internalError}); + stream.cancel(internalError); + serverCallTracer.reportCallEnded(internalError.isOk()); // error so always false + } + + /** + * All of these callbacks are assumed to called on an application thread, and the caller is + * responsible for handling thrown exceptions. + */ + @VisibleForTesting + static final class ServerStreamListenerImpl implements ServerStreamListener { + private final ServerCallImpl call; + private final ServerCall.Listener listener; + private final Context.CancellableContext context; + + public ServerStreamListenerImpl( + ServerCallImpl call, ServerCall.Listener listener, + Context.CancellableContext context) { + this.call = checkNotNull(call, "call"); + this.listener = checkNotNull(listener, "listener must not be null"); + this.context = checkNotNull(context, "context"); + // Wire ourselves up so that if the context is cancelled, our flag call.cancelled also + // reflects the new state. Use a DirectExecutor so that it happens in the same thread + // as the caller of {@link Context#cancel}. + this.context.addListener( + new Context.CancellationListener() { + @Override + public void cancelled(Context context) { + // If the context has a cancellation cause then something exceptional happened + // and we should also mark the call as cancelled. + if (context.cancellationCause() != null) { + ServerStreamListenerImpl.this.call.cancelled = true; + } + } + }, + MoreExecutors.directExecutor()); + } + + @Override + public void messagesAvailable(MessageProducer producer) { + PerfMark.startTask("ServerStreamListener.messagesAvailable", call.tag); + try { + messagesAvailableInternal(producer); + } finally { + PerfMark.stopTask("ServerStreamListener.messagesAvailable", call.tag); + } + } + + @SuppressWarnings("Finally") // The code avoids suppressing the exception thrown from try + private void messagesAvailableInternal(final MessageProducer producer) { + if (call.cancelled) { + GrpcUtil.closeQuietly(producer); + return; + } + + InputStream message; + try { + while ((message = producer.next()) != null) { + try { + listener.onMessage(call.method.parseRequest(message)); + } catch (Throwable t) { + GrpcUtil.closeQuietly(message); + throw t; + } + message.close(); + } + } catch (Throwable t) { + GrpcUtil.closeQuietly(producer); + Throwables.throwIfUnchecked(t); + throw new RuntimeException(t); + } + } + + @Override + public void halfClosed() { + PerfMark.startTask("ServerStreamListener.halfClosed", call.tag); + try { + if (call.cancelled) { + return; + } + + listener.onHalfClose(); + } finally { + PerfMark.stopTask("ServerStreamListener.halfClosed", call.tag); + } + } + + @Override + public void closed(Status status) { + PerfMark.startTask("ServerStreamListener.closed", call.tag); + try { + closedInternal(status); + } finally { + PerfMark.stopTask("ServerStreamListener.closed", call.tag); + } + } + + private void closedInternal(Status status) { + try { + if (status.isOk()) { + listener.onComplete(); + } else { + call.cancelled = true; + listener.onCancel(); + } + } finally { + // Cancel context after delivering RPC closure notification to allow the application to + // clean up and update any state based on whether onComplete or onCancel was called. + // Note that in failure situations JumpToApplicationThreadServerStreamListener has already + // closed the context. In these situations this cancel() call will be a no-op. + context.cancel(null); + } + } + + @Override + public void onReady() { + PerfMark.startTask("ServerStreamListener.onReady", call.tag); + try { + if (call.cancelled) { + return; + } + listener.onReady(); + } finally { + PerfMark.stopTask("ServerCall.closed", call.tag); + } + } + } +} diff --git a/src/federation-common/src/main/java/tak/server/federation/hub/FederationHubCache.java b/src/federation-common/src/main/java/tak/server/federation/hub/FederationHubCache.java new file mode 100644 index 00000000..1dab43b5 --- /dev/null +++ b/src/federation-common/src/main/java/tak/server/federation/hub/FederationHubCache.java @@ -0,0 +1,26 @@ +package tak.server.federation.hub; + +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.configuration.CacheConfiguration; + +import tak.server.federation.FederationPolicyGraph; + +public class FederationHubCache { + + public static final String POLICY_GRAPH_CACHE_KEY = "policyGraph"; + private static IgniteCache configurationCache; + public static IgniteCache getFederationHubPolicyStoreCache(Ignite ignite) { + if (configurationCache == null) { + CacheConfiguration cfg = new CacheConfiguration(); + + cfg.setName("FederationHubPolicyStore"); + cfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); + configurationCache = ignite.getOrCreateCache(cfg); + } + + return configurationCache; + } + +} diff --git a/src/federation-common/src/main/java/tak/server/federation/hub/broker/HubConnectionStore.java b/src/federation-common/src/main/java/tak/server/federation/hub/broker/HubConnectionStore.java index eb4ba5a2..1291e3b5 100644 --- a/src/federation-common/src/main/java/tak/server/federation/hub/broker/HubConnectionStore.java +++ b/src/federation-common/src/main/java/tak/server/federation/hub/broker/HubConnectionStore.java @@ -26,6 +26,7 @@ public class HubConnectionStore { private final Map sessionMap = new ConcurrentHashMap<>(); private final Map connectionInfos = new ConcurrentHashMap<>(); + public Collection getConnectionInfos() { return connectionInfos.values(); } diff --git a/src/federation-common/src/main/java/tak/server/federation/hub/policy/FederationHubPolicyManager.java b/src/federation-common/src/main/java/tak/server/federation/hub/policy/FederationHubPolicyManager.java index acbf24aa..92599b94 100644 --- a/src/federation-common/src/main/java/tak/server/federation/hub/policy/FederationHubPolicyManager.java +++ b/src/federation-common/src/main/java/tak/server/federation/hub/policy/FederationHubPolicyManager.java @@ -1,20 +1,19 @@ package tak.server.federation.hub.policy; +import java.util.Collection; +import java.util.List; + import tak.server.federation.Federate; import tak.server.federation.FederateGroup; import tak.server.federation.FederationException; import tak.server.federation.FederationPolicyGraph; - import tak.server.federation.hub.ui.graph.FederationPolicyModel; import tak.server.federation.hub.ui.graph.PolicyObjectCell; -import java.util.Collection; -import java.util.List; - public interface FederationHubPolicyManager { void addCaGroup(FederateGroup federateGroup); Collection getCaGroups(); - void addCaFederate(Federate federate, List federateCaNames); + FederationPolicyGraph addCaFederate(Federate federate, List federateCaNames); FederationPolicyGraph getPolicyGraph(); void setPolicyGraph(FederationPolicyModel newPolicyModel, Object updateFile) throws FederationException; Collection getPolicyCells(); diff --git a/src/federation-common/src/main/java/tak/server/federation/hub/policy/FederationHubPolicyManagerImpl.java b/src/federation-common/src/main/java/tak/server/federation/hub/policy/FederationHubPolicyManagerImpl.java index 303816fa..1c7d50d9 100644 --- a/src/federation-common/src/main/java/tak/server/federation/hub/policy/FederationHubPolicyManagerImpl.java +++ b/src/federation-common/src/main/java/tak/server/federation/hub/policy/FederationHubPolicyManagerImpl.java @@ -10,6 +10,7 @@ import java.util.Map; import java.util.Set; +import org.apache.ignite.Ignite; import org.apache.ignite.services.Service; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,15 +28,17 @@ import tak.server.federation.FederationException; import tak.server.federation.FederationNode; import tak.server.federation.FederationPolicyGraph; +import tak.server.federation.hub.FederationHubCache; import tak.server.federation.hub.ui.graph.FederationPolicyModel; import tak.server.federation.hub.ui.graph.PolicyObjectCell; public class FederationHubPolicyManagerImpl implements FederationHubPolicyManager, Service { private static final long serialVersionUID = 1012094988435086891L; + + private final Ignite ignite; - private static final String DEFAULT_POLICY_PATH = "/opt/tak/federation-hub/"; - private static final String DEFAULT_POLICY_FILENAME = "ui_generated_policy.json"; + private static String DEFAULT_POLICY_FILE = "/opt/tak/federation-hub/ui_generated_policy.json"; /* JSON file constants. */ private static final String POLICY_NAME = "name"; @@ -58,39 +61,59 @@ public class FederationHubPolicyManagerImpl implements FederationHubPolicyManage private Set groupCas; private Set dynamicallyAddedFederates; private Collection cells = new ArrayList<>(); + + private FederationPolicyGraph federationPolicyGraph; + + public FederationHubPolicyManagerImpl(Ignite ignite, String DEFAULT_POLICY_FILE) { + this.ignite = ignite; + + if (!Strings.isNullOrEmpty(DEFAULT_POLICY_FILE)) + FederationHubPolicyManagerImpl.DEFAULT_POLICY_FILE = DEFAULT_POLICY_FILE; + } @Override public FederationPolicyGraph getPolicyGraph() { - return FederationHubPolicyStore.getInstance().getPolicyGraph(); - } + return federationPolicyGraph; + } + + public void cachePolicyGraph() { + // cache events will alert other services of the change so they don't have to constantly ask for the + // most up to date policy + FederationHubCache.getFederationHubPolicyStoreCache(ignite).put(FederationHubCache.POLICY_GRAPH_CACHE_KEY, federationPolicyGraph); + } @Override - public void addCaFederate(Federate federate, List federateCaNames) { + public FederationPolicyGraph addCaFederate(Federate federate, List federateCaNames) { for (String caName : federateCaNames) { - if (getPolicyGraph().getGroup(caName) != null) { + if (federationPolicyGraph.getGroup(caName) != null) { federate.addGroupIdentity(new FederateIdentity(caName)); - getPolicyGraph().getGroup(caName).addFederateToGroup(federate); + federationPolicyGraph.getGroup(caName).addFederateToGroup(federate); } } - if (getPolicyGraph().getNode(federate.getFederateIdentity().getFedId()) == null) { + if (federationPolicyGraph.getNode(federate.getFederateIdentity().getFedId()) == null) { // Add iff federate had certs that matched CA groups in the policy graph if (!federate.getGroupIdentities().isEmpty()) { - getPolicyGraph().addFederate(federate); + federationPolicyGraph.addFederate(federate); dynamicallyAddedFederates.add(federate); } } else { for (String caName : federateCaNames) { - if (getPolicyGraph().getGroup(caName) != null) { - getPolicyGraph().getFederate(federate.getFederateIdentity().getFedId()).addGroupIdentity(new FederateIdentity(caName)); + if (federationPolicyGraph.getGroup(caName) != null) { + federationPolicyGraph.getFederate(federate.getFederateIdentity().getFedId()).addGroupIdentity(new FederateIdentity(caName)); } } } + + cachePolicyGraph(); + + return federationPolicyGraph; } @Override public void addCaGroup(FederateGroup federateGroup) { groupCas.add(federateGroup); + cachePolicyGraph(); } @Override @@ -99,7 +122,7 @@ public Collection getCaGroups() { } private void updateFederate(Federate node) { - Federate currentFederate = getPolicyGraph().getFederate(node.getFederateIdentity().getFedId()); + Federate currentFederate = federationPolicyGraph.getFederate(node.getFederateIdentity().getFedId()); // Add objects to current federate first, then add combined list to new federate // This lets new objects supersede old ones. We don't just update the current federate @@ -108,11 +131,11 @@ private void updateFederate(Federate node) { currentFederate.getAttributes().putAll(node.getAttributes()); node.getGroupIdentities().addAll(currentFederate.getGroupIdentities()); node.getAttributes().putAll(currentFederate.getAttributes()); - getPolicyGraph().addNode(node); + federationPolicyGraph.addNode(node); } private void updateGroup(FederateGroup node) { - FederateGroup currentGroup = getPolicyGraph().getGroup(node.getFederateIdentity()); + FederateGroup currentGroup = federationPolicyGraph.getGroup(node.getFederateIdentity()); // Add objects to current group first, then add combined list to new group // This lets new objects supersede old ones. We don't just update the current federate @@ -121,7 +144,7 @@ private void updateGroup(FederateGroup node) { currentGroup.getAttributes().putAll(node.getAttributes()); node.getFederatesInGroup().addAll(currentGroup.getFederatesInGroup()); node.getAttributes().putAll(currentGroup.getAttributes()); - getPolicyGraph().addNode(node); + federationPolicyGraph.addNode(node); } private void updateNode(FederationNode node) { @@ -134,22 +157,22 @@ private void updateNode(FederationNode node) { private void updateNodes(Collection federationNodes) { for (FederationNode node : federationNodes) { - if (getPolicyGraph().getNode(node.getFederateIdentity().getFedId()) != null) { + if (federationPolicyGraph.getNode(node.getFederateIdentity().getFedId()) != null) { updateNode(node); } else { - getPolicyGraph().addNode(node); + federationPolicyGraph.addNode(node); } } } private void updateEdges(FederationPolicyGraph federationPolicyGraph) throws FederationException { - getPolicyGraph().getEdgeSet().clear(); - getPolicyGraph().getEdgeSet().addAll(federationPolicyGraph.getEdgeSet()); + federationPolicyGraph.getEdgeSet().clear(); + federationPolicyGraph.getEdgeSet().addAll(federationPolicyGraph.getEdgeSet()); } private void updateFile(Object updateFile) { /* TODO allow policy file to be specified via configuration. */ - String policyFilePath = DEFAULT_POLICY_PATH + DEFAULT_POLICY_FILENAME; + String policyFilePath = DEFAULT_POLICY_FILE; if (policyFilePath.contains(".yml")) policyFilePath = policyFilePath.replace(".yml", ".json"); ObjectMapper mapper = new ObjectMapper(); @@ -163,8 +186,8 @@ private void updateFile(Object updateFile) { @Override public void updatePolicyGraph(FederationPolicyModel federationPolicyModel, Object updateFile) throws FederationException { - getPolicyGraph().getNodes().clear(); - getPolicyGraph().getEdgeSet().clear(); + federationPolicyGraph.getNodes().clear(); + federationPolicyGraph.getEdgeSet().clear(); FederationPolicyGraph federationPolicyGraph = federationPolicyModel.getPolicyGraphFromModel(); updateNodes(federationPolicyGraph.getNodes()); @@ -175,13 +198,15 @@ public void updatePolicyGraph(FederationPolicyModel federationPolicyModel, if (updateFile != null) { updateFile(updateFile); } + cachePolicyGraph(); } @Override public void setPolicyGraph(FederationPolicyModel newPolicyModel, Object updateFile) throws FederationException { FederationPolicyGraph newPolicyGraph = newPolicyModel.getPolicyGraphFromModel(); - FederationHubPolicyStore.getInstance().setPolicyGraph(newPolicyGraph); + this.federationPolicyGraph = newPolicyGraph; + cachePolicyGraph(); if (newPolicyModel.getCells() != null) cells = newPolicyModel.getCells(); @@ -366,13 +391,12 @@ private boolean initializePolicyGraphFromFile(String policyFile) { } catch (Exception e) { logger.error("err",e); } - - FederationHubPolicyStore.getInstance().setPolicyGraph(newPolicyGraph); + this.federationPolicyGraph = newPolicyGraph; + cachePolicyGraph(); } catch (IOException | RuntimeException e) { logger.error("Could not load policy graph from file: " + e); return false; } - return true; } @@ -388,10 +412,12 @@ public void init() throws Exception { } /* TODO allow policy file to be specified via configuration. */ - String policyFilename = DEFAULT_POLICY_PATH + DEFAULT_POLICY_FILENAME; + String policyFilename = DEFAULT_POLICY_FILE; File policyFile = new File(policyFilename); - if (!policyFile.exists() || !initializePolicyGraphFromFile(policyFilename)) - FederationHubPolicyStore.getInstance().setPolicyGraph(getEmptyPolicy()); + if (!policyFile.exists() || !initializePolicyGraphFromFile(policyFilename)) { + this.federationPolicyGraph = getEmptyPolicy(); + cachePolicyGraph(); + } groupCas = new HashSet<>(); dynamicallyAddedFederates = new HashSet<>(); diff --git a/src/federation-common/src/main/java/tak/server/federation/hub/policy/FederationHubPolicyStore.java b/src/federation-common/src/main/java/tak/server/federation/hub/policy/FederationHubPolicyStore.java deleted file mode 100644 index 81546721..00000000 --- a/src/federation-common/src/main/java/tak/server/federation/hub/policy/FederationHubPolicyStore.java +++ /dev/null @@ -1,25 +0,0 @@ -package tak.server.federation.hub.policy; - -import tak.server.federation.FederationPolicyGraph; - -public class FederationHubPolicyStore { - - private static FederationHubPolicyStore instance; - public static synchronized FederationHubPolicyStore getInstance() { - if (instance == null) - instance = new FederationHubPolicyStore(); - - return instance; - } - - private FederationPolicyGraph policyGraph; - - public FederationPolicyGraph getPolicyGraph() { - return policyGraph; - } - - public void setPolicyGraph(FederationPolicyGraph policyGraph) { - this.policyGraph = policyGraph; - } - -} diff --git a/src/federation-common/src/main/java/tak/server/federation/hub/policy/FederationPolicyGraphImpl.java b/src/federation-common/src/main/java/tak/server/federation/hub/policy/FederationPolicyGraphImpl.java index 2770318c..41aac493 100644 --- a/src/federation-common/src/main/java/tak/server/federation/hub/policy/FederationPolicyGraphImpl.java +++ b/src/federation-common/src/main/java/tak/server/federation/hub/policy/FederationPolicyGraphImpl.java @@ -57,11 +57,11 @@ public Set allReachableFederates(FederationNode source) throws Federat @Override public Set allReachableFederates(String sourceUid) throws FederationException { - FederationNode FederationNode = getNode(sourceUid); - if (FederationNode == null) { + FederationNode federationNode = getNode(sourceUid); + if (federationNode == null) { throw new FederationException("The passed sourceID " + sourceUid + " was not found in the policy graph."); } - return allReachableFederates(FederationNode); + return allReachableFederates(federationNode); } @Override @@ -224,7 +224,7 @@ public void addEdge(FederateEdge federateEdge) throws FederationException { } @Override - public List getFiltersAlongPath(FederationNode FederationNode, FederationNode FederationNode1) { + public List getFiltersAlongPath(FederationNode federationNode, FederationNode federationNode1) { return null; } @@ -328,10 +328,6 @@ private Set findReachableFederates(FederationNode source, Set, which will only work for in-process plugins + * Currently expected to be a Map{@literal <}String,Object{@literal >}, which will only work for in-process plugins */ public static final String ParentMetadata = "parentMetadata"; public static final String NumChildrenCreated = "numChildrenCreated"; @@ -48,7 +48,7 @@ public class MetadataConstants { public static final String NumSendAttempts = "numSendAttempts"; public static final String HttpRequestURI = "http request uri"; - + /** Expected to be a String[] */ public static final String HttpHeaders = "http_headers"; public static final String SingleShotMessage = "single_shot_message"; @@ -58,4 +58,4 @@ public class MetadataConstants { public static final String Callsign = "callsign"; -} \ No newline at end of file +} diff --git a/src/federation-common/src/main/java/tak/server/federation/message/MetadataUtils.java b/src/federation-common/src/main/java/tak/server/federation/message/MetadataUtils.java index c7073c6a..dba517dc 100644 --- a/src/federation-common/src/main/java/tak/server/federation/message/MetadataUtils.java +++ b/src/federation-common/src/main/java/tak/server/federation/message/MetadataUtils.java @@ -16,10 +16,10 @@ /** - * + * * Currently, this class provides a mapping from file extensions to valid Mime Type Strings.
- * We may want to extend it do offer more functionality. - * + * We may want to extend it do offer more functionality. + * * TODO Discuss putting in the PluginContext for use by all plugins * */ @@ -29,17 +29,17 @@ public final class MetadataUtils { private static MetadataUtils instance = null; private static final String DEFAULT_FILE_NAME = "mimetypes.csv"; //private static final Logger LOGGER = LoggerFactory.getLogger(MetadataUtils.class); - - + + private MetadataUtils() throws IOException{ // file only contains "image/jpeg" mimeTypeToFileExtension.put("image/jpg", ".jpg"); - - // Compatibility for ATAK. Since this is an old MIME Type, + + // Compatibility for ATAK. Since this is an old MIME Type, // we only want to recognize it, not use it based on the file extension mimeTypeToFileExtension.put("application/x-zip-compressed", ".zip"); - - InputStream inputStream = MetadataUtils.class.getClassLoader().getResourceAsStream(DEFAULT_FILE_NAME); + + InputStream inputStream = MetadataUtils.class.getClassLoader().getResourceAsStream(DEFAULT_FILE_NAME); BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8")); String fileLine = reader.readLine(); while (fileLine != null){ @@ -47,7 +47,7 @@ private MetadataUtils() throws IOException{ // index 0 is description, don't need for now fileExtensionToMimeTypeString.put(mimeTypeInfo[2], mimeTypeInfo[1]); mimeTypeToFileExtension.put(mimeTypeInfo[1], mimeTypeInfo[2]); - + fileLine = reader.readLine(); } reader.close(); @@ -60,48 +60,48 @@ public static synchronized MetadataUtils getInstance() throws IOException{ } return instance; } - + public String getMimeTypeFromFileName(String fileName){ - + int lastPeriod = fileName.lastIndexOf('.'); - + if (lastPeriod == -1){ return null; } String extension = fileName.substring(lastPeriod); return getMimeType(extension); } - + /** - * + * * @param fileExtension, including period * @return MIME Type in String format */ public String getMimeType(String fileExtension){ return fileExtensionToMimeTypeString.get(fileExtension.toLowerCase(Locale.ENGLISH)); } - + /** - * + * * @param mimeType Type in String format * @return fileExtension, including period */ public String getFileExtension(String mimeType){ return mimeTypeToFileExtension.get(mimeType.toLowerCase(Locale.ENGLISH)); } - + /** - * + * * @param possibleMime * @return true if possibleMime is the string representation of a recognized MIME Type according to ROGER */ public boolean isValidMimeType(String possibleMime){ return mimeTypeToFileExtension.containsKey(possibleMime); } - + /** - * Replaces file extension, where file extension is the characters after the last period in fileName, according to the mimeType provided. - * Appends file extension if none is found. + * Replaces file extension, where file extension is the characters after the last period in fileName, according to the mimeType provided. + * Appends file extension if none is found. * @param fileName * @return */ @@ -109,9 +109,9 @@ public String changeFileExtension(String fileName, String mimeType){ String coreName = removeFileExtension(fileName); return coreName + getFileExtension(mimeType); } - + /** - * Remove file extension, where file extension is the characters after the last period in fileName + * Remove file extension, where file extension is the characters after the last period in fileName * @param fileName * @return a new string with no extension */ @@ -123,7 +123,7 @@ public String removeFileExtension(String fileName){ } return coreName; } - + private int lastIndexOf(String string, char character){ //if index = -1 right away, return -1 because the character isn't there int lastIndex = -1; @@ -134,13 +134,13 @@ private int lastIndexOf(String string, char character){ } // now index is -1 return lastIndex; } - - + + /** - * This method makes a "deep clone" of any Java object it is given. + * This method makes a "deep clone" of any Java object it is given. * Returns null for null object - * @throws IOException - * @throws ClassNotFoundException + * @throws IOException + * @throws ClassNotFoundException */ public static Object deepClone(Object object) throws IOException, ClassNotFoundException { if (object == null) { @@ -156,12 +156,11 @@ public static Object deepClone(Object object) throws IOException, ClassNotFoundE ObjectInputStream ois = new ObjectInputStream(bais); return ois.readObject(); } - - + + /** * Creates parent metadata by combining metadata from child messages (does not use parentMetadata from metadataConstants) * @param messages - * @param metadataUtils * @return */ public Map createParentMetadata(Message... messages){ diff --git a/src/federation-common/src/main/java/tak/server/federation/message/Payload.java b/src/federation-common/src/main/java/tak/server/federation/message/Payload.java index fed83564..b4aedd01 100644 --- a/src/federation-common/src/main/java/tak/server/federation/message/Payload.java +++ b/src/federation-common/src/main/java/tak/server/federation/message/Payload.java @@ -2,40 +2,39 @@ /** * This interface exports the methods which are common to kinds of message payloads. - * - * The payload can be accessed either accessed via its binary representation (byte array), or as a generic object (parameterized type X). - * + * + * The payload can be accessed either accessed via its binary representation (byte array), or as a generic object (parameterized type X). + * * Important note: Concrete implementations of this interface are responsible for managing the relationship between the binary form and the generic form. - * + * *

Serialization

- * In order for a payload class that implements this interface to be serializable by the default PayloadSerializationPlugin, - * the payload must have a zero-argument constructor and be able to set its payload through setBytes. - * @see PayloadSerializationPlugin - * + * In order for a payload class that implements this interface to be serializable by the default PayloadSerializationPlugin, + * the payload must have a zero-argument constructor and be able to set its payload through setBytes. + * */ public interface Payload { - + /* * Access the payload in binary form */ byte[] getBytes(); - + /* * Set the payload in binary form - * + * * @param bytes The binary form of the payload */ @SuppressWarnings("PMD.ArrayIsStoredDirectly") void setBytes(byte[] bytes); - + /* * Access the payload as a parameterized type */ X getContent(); - + /* * Mutate the paramaterized type form of the payload - * + * * @param X The parameterized type form of the payload */ void setContent(X content); diff --git a/src/federation-hub-broker/build.gradle b/src/federation-hub-broker/build.gradle index 003db12c..4c5e1b6c 100644 --- a/src/federation-hub-broker/build.gradle +++ b/src/federation-hub-broker/build.gradle @@ -43,15 +43,14 @@ dependencies { compile group: 'org.apache.ignite', name: 'ignite-slf4j', version: ignite_version compile group: 'io.netty', name: 'netty-handler', version: netty_handler_version - compile group: 'io.netty', name: 'netty-tcnative-boringssl-static', version: netty_tcnative_version compile group: 'io.netty', name: 'netty-tcnative', version: netty_tcnative_version + compile group: 'io.netty', name: 'netty-tcnative-classes', version: netty_tcnative_version compile group: 'io.netty', name: 'netty-transport-native-epoll', version: netty_version, classifier: 'linux-x86_64' - - implementation('io.netty:netty-tcnative-classes') { - version { - strictly '2.0.53.Final' - } - } + compile "io.netty:netty-tcnative-boringssl-static:$netty_tcnative_version:linux-x86_64" + compile "io.netty:netty-tcnative-boringssl-static:$netty_tcnative_version:linux-aarch_64" + compile "io.netty:netty-tcnative-boringssl-static:$netty_tcnative_version:osx-x86_64" + compile "io.netty:netty-tcnative-boringssl-static:$netty_tcnative_version:osx-aarch_64" + compile "io.netty:netty-tcnative-boringssl-static:$netty_tcnative_version:windows-x86_64" compile group: 'org.springframework.boot', name: 'spring-boot-starter-actuator' compile group: 'org.springframework.boot', name: 'spring-boot-loader', version: spring_boot_version diff --git a/src/federation-hub-broker/scripts/federation-hub b/src/federation-hub-broker/scripts/federation-hub index a88c5ab9..4ced581c 100755 --- a/src/federation-hub-broker/scripts/federation-hub +++ b/src/federation-hub-broker/scripts/federation-hub @@ -1,4 +1,9 @@ #!/bin/bash +### BEGIN INIT INFO +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: federation-hub-init init script +### END INIT INFO # # /etc/rc.d/init.d/federation-hub-init # @@ -17,7 +22,9 @@ # description: Federation Hub for Team Awareness Kit (TAK) # Source function library -. /etc/rc.d/init.d/functions +if [ -f /etc/rc.d/init.d/functions ]; then + . /etc/rc.d/init.d/functions +fi SERVICE="Federation Hub" FEDERATION_HUB_HOME=/opt/tak/federation-hub diff --git a/src/federation-hub-broker/scripts/federation-hub-broker b/src/federation-hub-broker/scripts/federation-hub-broker index d4acb7e1..d167f371 100755 --- a/src/federation-hub-broker/scripts/federation-hub-broker +++ b/src/federation-hub-broker/scripts/federation-hub-broker @@ -1,4 +1,9 @@ #!/bin/bash +### BEGIN INIT INFO +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: federation-hub-broker init script +### END INIT INFO # # /etc/rc.d/init.d/federation-hub-broker # @@ -16,7 +21,9 @@ # description: Federation Hub broker service. # Source function library -. /etc/rc.d/init.d/functions +if [ -f /etc/rc.d/init.d/functions ]; then + . /etc/rc.d/init.d/functions +fi SERVICE="Federation Hub Broker" FEDERATION_HUB_HOME=/opt/tak/federation-hub diff --git a/src/federation-hub-broker/src/main/java/tak/server/federation/hub/broker/FederationHubBrokerService.java b/src/federation-hub-broker/src/main/java/tak/server/federation/hub/broker/FederationHubBrokerService.java index 48eaadf5..868c2bfd 100644 --- a/src/federation-hub-broker/src/main/java/tak/server/federation/hub/broker/FederationHubBrokerService.java +++ b/src/federation-hub-broker/src/main/java/tak/server/federation/hub/broker/FederationHubBrokerService.java @@ -38,6 +38,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import javax.cache.event.CacheEntryEvent; import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLException; import javax.net.ssl.SSLPeerUnverifiedException; @@ -48,6 +49,9 @@ import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.tree.ParseTree; +import org.apache.ignite.Ignite; +import org.apache.ignite.binary.BinaryObjectException; +import org.apache.ignite.cache.query.ContinuousQuery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationListener; @@ -118,6 +122,7 @@ import tak.server.federation.FederationException; import tak.server.federation.FederationPolicyGraph; import tak.server.federation.GuardedStreamHolder; +import tak.server.federation.hub.FederationHubCache; import tak.server.federation.hub.broker.events.BrokerServerEvent; import tak.server.federation.hub.broker.events.HubClientDisconnectEvent; import tak.server.federation.hub.broker.events.RestartServerEvent; @@ -127,6 +132,8 @@ import tak.server.federation.hub.ui.graph.PolicyObjectCell; public class FederationHubBrokerService implements ApplicationListener { + + private final Ignite ignite; private static final String SSL_SESSION_ID = "sslSessionId"; private static final String FEDERATED_ID_KEY = "federatedIdentity"; @@ -171,14 +178,37 @@ public class FederationHubBrokerService implements ApplicationListener continuousConfigurationQuery = new ContinuousQuery<>(); - public FederationHubBrokerService(SSLConfig sslConfig, FederationHubServerConfig fedHubConfig, FederationHubPolicyManager fedHubPolicyManager, HubConnectionStore hubConnectionStore) { + public FederationHubBrokerService(Ignite ignite, SSLConfig sslConfig, FederationHubServerConfig fedHubConfig, FederationHubPolicyManager fedHubPolicyManager, HubConnectionStore hubConnectionStore) { instance = this; + this.ignite = ignite; this.sslConfig = sslConfig; this.fedHubConfig = fedHubConfig; this.fedHubPolicyManager = fedHubPolicyManager; this.hubConnectionStore = hubConnectionStore; setupFederationServers(); + + // rather than hitting ignite every time we need the policy graph, + // use event driven approach to always have the updated graph + // available here for instant access + continuousConfigurationQuery.setLocalListener((evts) -> { + for (CacheEntryEvent e : evts) { + federationPolicyGraph = e.getValue(); + } + }); + + FederationHubCache.getFederationHubPolicyStoreCache(ignite).query(continuousConfigurationQuery); + } + + private FederationPolicyGraph federationPolicyGraph; + + public FederationPolicyGraph getFederationPolicyGraph() { + if (federationPolicyGraph == null) + federationPolicyGraph = fedHubPolicyManager.getPolicyGraph(); + + return federationPolicyGraph; } private void removeInactiveClientStreams() { @@ -270,7 +300,7 @@ private SslContext buildServerSslContext(FederationHubServerConfig fedHubConfig) } public void sendContactMessagesV1(NioNettyFederationHubServerHandler handler) { - FederationPolicyGraph fpg = fedHubPolicyManager.getPolicyGraph(); + FederationPolicyGraph fpg = getFederationPolicyGraph(); if (fpg == null) { logger.error("Cannot send contact messages; policy manager is null"); return; @@ -497,7 +527,7 @@ public void onApplicationEvent(BrokerServerEvent event) { addFederateToGroupPolicyIfMissingV2(hubConnectionStore.getSessionMap().get(entry.getKey()) ,entry.getValue()); // check if the currently connected spoke is still allowed to be connected after the policy change - FederationPolicyGraph fpg = fedHubPolicyManager.getPolicyGraph(); + FederationPolicyGraph fpg = getFederationPolicyGraph(); Federate clientNode = checkFederateExistsInPolicy(entry.getValue(), session, fpg); if (clientNode == null) { logger.info("Permission Denied. Federate/CA Group not found in the policy graph: " + entry.getValue().getFederateIdentity()); @@ -514,7 +544,6 @@ public void onApplicationEvent(BrokerServerEvent event) { hubConnectionStore.removeSession(entry.getKey()); } }); - updateOutgoingConnections(((UpdatePolicy) event).getOutgoings()); } } @@ -693,13 +722,15 @@ public void addCaFederateToPolicyGraph(FederateIdentity federateIdentity, Certif } Federate federate = new Federate(federateIdentity); - fedHubPolicyManager.addCaFederate(federate, caCertNames); + synchronized (federationPolicyGraph) { + federationPolicyGraph = fedHubPolicyManager.addCaFederate(federate, caCertNames); + } } public void addFederateToGroupPolicyIfMissingV1(Certificate[] certArray, FederateIdentity federateIdentity) { String fedId = federateIdentity.getFedId(); - if (fedHubPolicyManager.getPolicyGraph().getNode(fedId) == null) { + if (getFederationPolicyGraph().getNode(fedId) == null) { addCaFederateToPolicyGraph(federateIdentity, certArray); } } @@ -707,7 +738,10 @@ public void addFederateToGroupPolicyIfMissingV1(Certificate[] certArray, public void addFederateToGroupPolicyIfMissingV2(SSLSession session, GuardedStreamHolder holder) { hubConnectionStore.addSession(new BigInteger(session.getId()).toString(), session); String fedId = holder.getFederateIdentity().getFedId(); - if (fedHubPolicyManager.getPolicyGraph().getNode(fedId) == null) { + + FederationPolicyGraph fpg = getFederationPolicyGraph(); + + if (fpg.getNode(fedId) == null) { try { Certificate[] certArray = session.getPeerCertificates(); addCaFederateToPolicyGraph(holder.getFederateIdentity(), certArray); @@ -731,6 +765,8 @@ private class FederatedChannelService extends FederatedChannelGrpc.FederatedChan private final FederationHubBrokerService broker = FederationHubBrokerService.this; AtomicReference start = new AtomicReference<>(); + + @Override public void serverFederateGroupsStream(Subscription request, StreamObserver responseObserver) { @@ -744,7 +780,6 @@ public void serverFederateGroupsStream(Subscription request, StreamObserver streamHolder = new GuardedStreamHolder( responseObserver, request.getIdentity().getName(), FederationUtils.getBytesSHA256(clientCertArray[0].getEncoded()), session, request, @@ -754,13 +789,12 @@ public int compare(FederateGroups a, FederateGroups b) { return ComparisonChain.start().compare(a.hashCode(), b.hashCode()).result(); } }, true); - - if (fedHubConfig.isUseCaGroups()) { - addFederateToGroupPolicyIfMissingV2(session, streamHolder); - } - - FederationPolicyGraph fpg = fedHubPolicyManager.getPolicyGraph(); - requireNonNull(fpg, "federation policy graph object"); + + addFederateToGroupPolicyIfMissingV2(session, streamHolder); + + FederationPolicyGraph fpg = getFederationPolicyGraph(); + + requireNonNull(fpg, "federation policy graph object"); Federate clientNode = checkFederateExistsInPolicy(streamHolder, session, fpg); if (clientNode == null) { @@ -794,16 +828,17 @@ public StreamObserver clientFederateGroupsStream(StreamObserver< public void onNext(FederateGroups fedGroups) { SSLSession session = (SSLSession)sslSessionKey.get(Context.current()); String id = new BigInteger(session.getId()).toString(); - if (fedHubConfig.isUseCaGroups()) { - GuardedStreamHolder holder = hubConnectionStore.getClientStreamMap().get(id); - if (holder != null) { - addFederateToGroupPolicyIfMissingV2(session, hubConnectionStore.getClientStreamMap().get(id)); - } - GuardedStreamHolder groupHolder = hubConnectionStore.getClientGroupStreamMap().get(id); - if (groupHolder != null) { - addFederateToGroupPolicyIfMissingV2(session, hubConnectionStore.getClientGroupStreamMap().get(id)); - } - } + + GuardedStreamHolder holder = hubConnectionStore.getClientStreamMap().get(id); + if (holder != null) { + addFederateToGroupPolicyIfMissingV2(session, hubConnectionStore.getClientStreamMap().get(id)); + } + + GuardedStreamHolder groupHolder = hubConnectionStore.getClientGroupStreamMap().get(id); + if (groupHolder != null) { + addFederateToGroupPolicyIfMissingV2(session, hubConnectionStore.getClientGroupStreamMap().get(id)); + } + addFederateGroups(id, fedGroups); } @@ -844,9 +879,7 @@ public void onNext(BinaryBlob value) { value.getSerializedSize() + " bytes (serialized) latency: " + latency + " ms"); - if (fedHubConfig.isUseCaGroups()) { - addFederateToGroupPolicyIfMissingV2(session, hubConnectionStore.getClientStreamMap().get(new BigInteger(session.getId()).toString())); - } + addFederateToGroupPolicyIfMissingV2(session, hubConnectionStore.getClientStreamMap().get(new BigInteger(session.getId()).toString())); FederationHubBrokerService.this.handleRead(value, new BigInteger(session.getId()).toString()); } @@ -946,11 +979,9 @@ public int compare(FederatedEvent a, FederatedEvent b) { }, true ); - if (fedHubConfig.isUseCaGroups()) { - addFederateToGroupPolicyIfMissingV2(session, streamHolder); - } + addFederateToGroupPolicyIfMissingV2(session, streamHolder); - FederationPolicyGraph fpg = fedHubPolicyManager.getPolicyGraph(); + FederationPolicyGraph fpg = getFederationPolicyGraph(); requireNonNull(fpg, "federation policy graph object"); Federate clientNode = checkFederateExistsInPolicy(streamHolder, session, fpg); @@ -1061,11 +1092,9 @@ public int compare(ROL a, ROL b) { }, true ); - if (fedHubConfig.isUseCaGroups()) { - addFederateToGroupPolicyIfMissingV2(session, rolStreamHolder); - } + addFederateToGroupPolicyIfMissingV2(session, rolStreamHolder); - FederationPolicyGraph fpg = fedHubPolicyManager.getPolicyGraph(); + FederationPolicyGraph fpg = getFederationPolicyGraph(); requireNonNull(fpg, "federation policy graph object"); Federate clientNode = checkFederateExistsInPolicy(rolStreamHolder, session, fpg); @@ -1102,9 +1131,7 @@ public void onNext(ROL clientROL) { SSLSession session = (SSLSession) sslSessionKey.get(Context.current()); String sessionId = new BigInteger(session.getId()).toString(); - if (fedHubConfig.isUseCaGroups()) { - addFederateToGroupPolicyIfMissingV2(session, hubConnectionStore.getClientStreamMap().get(sessionId)); - } + addFederateToGroupPolicyIfMissingV2(session, hubConnectionStore.getClientStreamMap().get(sessionId)); parseRol(clientROL, sessionId); } catch (Exception e) { @@ -1141,7 +1168,7 @@ public StreamObserver serverEventStream(StreamObserver() { @Override @@ -1150,12 +1177,10 @@ public void onNext(FederatedEvent fe) { clientByteAccumulator.addAndGet(fe.getSerializedSize()); // Add federate to group in case policy was updated during connection - if (fedHubConfig.isUseCaGroups()) { - GuardedStreamHolder holder = hubConnectionStore.getClientStreamMap().get(id); - if (holder != null) { - addFederateToGroupPolicyIfMissingV2(session, hubConnectionStore.getClientStreamMap().get(id)); - } - } + GuardedStreamHolder holder = hubConnectionStore.getClientStreamMap().get(id); + if (holder != null) { + addFederateToGroupPolicyIfMissingV2(session, hubConnectionStore.getClientStreamMap().get(id)); + } // submit to orchestrator FederationHubBrokerService.this.handleRead(fe, new BigInteger(session.getId()).toString()); } @@ -1258,8 +1283,7 @@ public ServerCall.Listener interceptCall( public void assignMessageSourceAndDestinationsFromPolicy(Message message, FederateIdentity federateIdentity) throws FederationException { - assignMessageSourceAndDestinationsFromPolicy(message, federateIdentity, - fedHubPolicyManager.getPolicyGraph()); + assignMessageSourceAndDestinationsFromPolicy(message, federateIdentity, getFederationPolicyGraph()); } private void assignMessageSourceAndDestinationsFromPolicy(Message message, @@ -1303,7 +1327,7 @@ private void sendRolMessage(Message message) { FederateIdentity src = (FederateIdentity)message.getSource().getEntity(); FederateIdentity dest = (FederateIdentity)entity.getEntity(); - FederationPolicyGraph policyGraph = fedHubPolicyManager.getPolicyGraph(); + FederationPolicyGraph policyGraph = getFederationPolicyGraph(); Federate srcNode = policyGraph.getFederate(src.getFedId()); Federate destNode = policyGraph.getFederate(dest.getFedId()); @@ -1407,7 +1431,7 @@ public void sendFederatedEventV1(Message message) { FederateIdentity src = (FederateIdentity)message.getSource().getEntity(); FederateIdentity dest = (FederateIdentity)entity.getEntity(); - FederationPolicyGraph policyGraph = fedHubPolicyManager.getPolicyGraph(); + FederationPolicyGraph policyGraph = getFederationPolicyGraph(); Federate srcNode = policyGraph.getFederate(src.getFedId()); Federate destNode = policyGraph.getFederate(dest.getFedId()); @@ -1442,7 +1466,7 @@ private void sendFederatedEvent(Message message) { FederateIdentity src = (FederateIdentity)message.getSource().getEntity(); FederateIdentity dest = (FederateIdentity)entity.getEntity(); - FederationPolicyGraph policyGraph = fedHubPolicyManager.getPolicyGraph(); + FederationPolicyGraph policyGraph = getFederationPolicyGraph(); Federate srcNode = policyGraph.getFederate(src.getFedId()); Federate destNode = policyGraph.getFederate(dest.getFedId()); @@ -1483,7 +1507,7 @@ private void sendFederatedGroup(Message message) { FederateIdentity dest = (FederateIdentity)entity.getEntity(); FederateGroups groups = (FederateGroups)message.getPayload().getContent(); - FederationPolicyGraph policyGraph = fedHubPolicyManager.getPolicyGraph(); + FederationPolicyGraph policyGraph = getFederationPolicyGraph(); Federate srcNode = policyGraph.getFederate(src.getFedId()); Federate destNode = policyGraph.getFederate(dest.getFedId()); @@ -1546,7 +1570,7 @@ public void handleRead(BinaryBlob event, String streamKey) { try { assignMessageSourceAndDestinationsFromPolicy(federatedMessage, streamHolder.getFederateIdentity(), - fedHubPolicyManager.getPolicyGraph()); + getFederationPolicyGraph()); sendMessage(federatedMessage); } catch (FederationException e) { @@ -1574,7 +1598,7 @@ public void handleRead(ROL event, String streamKey) { try { assignMessageSourceAndDestinationsFromPolicy(federatedMessage, streamHolder.getFederateIdentity(), - fedHubPolicyManager.getPolicyGraph()); + getFederationPolicyGraph()); sendMessage(federatedMessage); } catch (FederationException e) { logger.error("Could not get destinations from policy graph", e); @@ -1604,7 +1628,7 @@ public void handleRead(FederatedEvent event, String streamKey) { try { assignMessageSourceAndDestinationsFromPolicy(federatedMessage, streamHolder.getFederateIdentity(), - fedHubPolicyManager.getPolicyGraph()); + getFederationPolicyGraph()); sendMessage(federatedMessage); if (event != null && event.hasContact()) { @@ -1680,7 +1704,7 @@ public void addFederateGroups(String sourceId, FederateGroups groups) { try { assignMessageSourceAndDestinationsFromPolicy(federatedMessage, ident, - fedHubPolicyManager.getPolicyGraph()); + getFederationPolicyGraph()); sendMessage(federatedMessage); } catch (FederationException e) { logger.error("Could not get destinations from policy graph", e); diff --git a/src/federation-hub-broker/src/main/java/tak/server/federation/hub/broker/FederationHubServer.java b/src/federation-hub-broker/src/main/java/tak/server/federation/hub/broker/FederationHubServer.java index 9046bc7c..ed57f880 100644 --- a/src/federation-hub-broker/src/main/java/tak/server/federation/hub/broker/FederationHubServer.java +++ b/src/federation-hub-broker/src/main/java/tak/server/federation/hub/broker/FederationHubServer.java @@ -43,11 +43,14 @@ public class FederationHubServer implements CommandLineRunner { private static String configFile; public static void main(String[] args) { + if (args.length > 1) { System.err.println("Usage: java -jar federation-hub-broker.jar [CONFIG_FILE_PATH]"); return; } else if (args.length == 1) { configFile = args[0]; + } else if (!Strings.isNullOrEmpty(System.getProperty("FEDERATION_HUB_BROKER_CONFIG"))) { + configFile = System.getProperties().getProperty("FEDERATION_HUB_BROKER_CONFIG"); } else { configFile = DEFAULT_CONFIG_FILE; } @@ -118,8 +121,8 @@ public FederationHubServerConfig getFedHubConfig() @Bean @Order(Ordered.LOWEST_PRECEDENCE) - public FederationHubBrokerService FederationHubBrokerService(SSLConfig getSslConfig, FederationHubServerConfig fedHubConfig, FederationHubPolicyManager fedHubPolicyManager, HubConnectionStore hubConnectionStore) { - return new FederationHubBrokerService(getSslConfig, fedHubConfig, fedHubPolicyManager, hubConnectionStore); + public FederationHubBrokerService FederationHubBrokerService(Ignite ignite, SSLConfig getSslConfig, FederationHubServerConfig fedHubConfig, FederationHubPolicyManager fedHubPolicyManager, HubConnectionStore hubConnectionStore) { + return new FederationHubBrokerService(ignite, getSslConfig, fedHubConfig, fedHubPolicyManager, hubConnectionStore); } private FederationHubServerConfig loadConfig(String configFile) diff --git a/src/federation-hub-broker/src/main/java/tak/server/federation/hub/broker/HubFigClient.java b/src/federation-hub-broker/src/main/java/tak/server/federation/hub/broker/HubFigClient.java index 5f0feb16..eabc22da 100644 --- a/src/federation-hub-broker/src/main/java/tak/server/federation/hub/broker/HubFigClient.java +++ b/src/federation-hub-broker/src/main/java/tak/server/federation/hub/broker/HubFigClient.java @@ -65,7 +65,6 @@ import tak.server.federation.GuardedStreamHolder; import tak.server.federation.hub.FederationHubDependencyInjectionProxy; import tak.server.federation.hub.broker.events.HubClientDisconnectEvent; -import tak.server.federation.hub.broker.events.UpdatePolicy; import tak.server.federation.hub.ui.graph.FederationOutgoingCell; /* @@ -185,26 +184,6 @@ public void onError(Throwable t) { @Override public void onCompleted() {} }); - - asyncFederatedChannel.clientROLStream( - Subscription.newBuilder().setFilter("") - .setIdentity(Identity.newBuilder().setName(fedName).setUid(clientUid).build()).build(), - new StreamObserver() { - - @Override - public void onNext(ROL value) { - FederationHubBrokerService.getInstance().parseRol(value, fedName); - } - - @Override - public void onError(Throwable t) { - logger.error("ROL Stream Error: ", t); - processDisconnect(); - } - - @Override - public void onCompleted() {} - }); asyncFederatedChannel.serverFederateGroupsStream( Subscription.newBuilder().setFilter("") @@ -243,6 +222,8 @@ public void onError(Throwable t) { @Override public void onCompleted() {} }); + + final AtomicBoolean initROLStream = new AtomicBoolean(false); healthScheduler = scheduler.scheduleWithFixedDelay(() -> { ClientHealth clientHealth = ClientHealth.newBuilder().setStatus(ClientHealth.ServingStatus.SERVING).build(); @@ -263,6 +244,31 @@ public void onNext(ServerHealth value) { processDisconnect(); throw new RuntimeException("Not Healthy"); } + + if (initROLStream.compareAndSet(false, true)) { + // open the client ROL stream only after getting a health check back. This will trigger transmission of federated mission changes. + // Subscription / Stream to receive ROL messages from server + asyncFederatedChannel.clientROLStream( + Subscription.newBuilder().setFilter("") + .setIdentity(Identity.newBuilder().setName(fedName).setUid(clientUid).build()).build(), + new StreamObserver() { + + @Override + public void onNext(ROL value) { + FederationHubBrokerService.getInstance().parseRol(value, fedName); + } + + @Override + public void onError(Throwable t) { + logger.error("ROL Stream Error: ", t); + processDisconnect(); + } + + @Override + public void onCompleted() {} + }); + } + } @Override @@ -311,12 +317,11 @@ public X509Certificate[] propogate(X509Certificate[] certs) { } } - FederationPolicyGraph fpg = FederationHubDependencyInjectionProxy.getInstance().fedHubPolicyManager().getPolicyGraph(); + FederationPolicyGraph fpg = FederationHubBrokerService.getInstance().getFederationPolicyGraph(); requireNonNull(fpg, "federation policy graph object"); - Federate clientNode = FederationHubDependencyInjectionProxy.getInstance() - .fedHubPolicyManager() - .getPolicyGraph() + Federate clientNode = FederationHubBrokerService.getInstance() + .getFederationPolicyGraph() .getFederate(new FederateIdentity(fedName)); requireNonNull(clientNode, "federation policy node for newly connected client"); @@ -335,13 +340,14 @@ public X509Certificate[] propogate(X509Certificate[] certs) { } private void setupGroupStreamSender() { - groupsCall = channel.newCall( - io.grpc.MethodDescriptor.create(MethodDescriptor.MethodType.CLIENT_STREAMING, - generateFullMethodName("com.atakmap.FederatedChannel", "ClientFederateGroupsStream"), - io.grpc.protobuf.ProtoUtils.marshaller(com.atakmap.Tak.FederateGroups.getDefaultInstance()), - io.grpc.protobuf.ProtoUtils.marshaller(com.atakmap.Tak.Subscription.getDefaultInstance())), - asyncFederatedChannel.getCallOptions()); - + MethodDescriptor methodDescripton = MethodDescriptor.newBuilder() + .setType(MethodDescriptor.MethodType.CLIENT_STREAMING) + .setFullMethodName(generateFullMethodName("com.atakmap.FederatedChannel", "ClientFederateGroupsStream")) + .setRequestMarshaller(io.grpc.protobuf.ProtoUtils.marshaller(com.atakmap.Tak.FederateGroups.getDefaultInstance())) + .setResponseMarshaller(io.grpc.protobuf.ProtoUtils.marshaller(com.atakmap.Tak.Subscription.getDefaultInstance())) + .build(); + + groupsCall = channel.newCall(methodDescripton, asyncFederatedChannel.getCallOptions()); // use listener to respect flow control, and send messages to the server when it // is ready groupsCall.start(new ClientCall.Listener() { @@ -350,6 +356,7 @@ private void setupGroupStreamSender() { public void onMessage(Subscription response) { // Notify gRPC to receive one additional response. groupsCall.request(1); + } @Override @@ -371,14 +378,38 @@ public int compare(FederateGroups a, FederateGroups b) { ); FederationHubDependencyInjectionProxy.getInstance().hubConnectionStore().addGroupStream(fedName, groupStreamHolder); } + + private void setServerSubscriptionForConnection() { + if (eventStreamHolder != null && serverSubscription != null) { + eventStreamHolder.setSubscription(serverSubscription); + + List existingConnectionsFromRemoteServer = FederationHubDependencyInjectionProxy.getInstance() + .hubConnectionStore() + .getConnectionInfos() + .stream() + .filter(i -> i.getRemoteServerId().equals(serverSubscription.getIdentity().getServerId())) + .collect(Collectors.toList()); + + // if we already have a connection to/from this server, don't allow another. force close without reconnect. + if (existingConnectionsFromRemoteServer.size() > 0) { + logger.info("Error: Connection to/from " + fedName + " already exists. Disallowing duplicate"); + processDisconnectWithoutRetry(); + } else { + FederationHubDependencyInjectionProxy.getInstance().hubConnectionStore().addConnectionInfo(fedName, info); + } + logger.info("Outgoing connection for {} established ", fedName); + } + } public void setupEventStreamSender() { - clientCall = channel.newCall( - io.grpc.MethodDescriptor.create(io.grpc.MethodDescriptor.MethodType.CLIENT_STREAMING, - generateFullMethodName("com.atakmap.FederatedChannel", "ServerEventStream"), - io.grpc.protobuf.ProtoUtils.marshaller(com.atakmap.Tak.FederatedEvent.getDefaultInstance()), - io.grpc.protobuf.ProtoUtils.marshaller(com.atakmap.Tak.Subscription.getDefaultInstance())), - asyncFederatedChannel.getCallOptions()); + MethodDescriptor methodDescripton = MethodDescriptor.newBuilder() + .setType(MethodDescriptor.MethodType.CLIENT_STREAMING) + .setFullMethodName(generateFullMethodName("com.atakmap.FederatedChannel", "ServerEventStream")) + .setRequestMarshaller(io.grpc.protobuf.ProtoUtils.marshaller(com.atakmap.Tak.FederatedEvent.getDefaultInstance())) + .setResponseMarshaller(io.grpc.protobuf.ProtoUtils.marshaller(com.atakmap.Tak.Subscription.getDefaultInstance())) + .build(); + + clientCall = channel.newCall(methodDescripton, asyncFederatedChannel.getCallOptions()); // use listener to respect flow control, and send messages to the server when it // is ready @@ -387,25 +418,7 @@ public void setupEventStreamSender() { @Override public void onMessage(Subscription response) { serverSubscription = response; - if (eventStreamHolder != null) { - eventStreamHolder.setSubscription(serverSubscription); - - List existingConnectionsFromRemoteServer = FederationHubDependencyInjectionProxy.getInstance() - .hubConnectionStore() - .getConnectionInfos() - .stream() - .filter(i -> i.getRemoteServerId().equals(serverSubscription.getIdentity().getServerId())) - .collect(Collectors.toList()); - - // if we already have a connection to/from this server, don't allow another. force close without reconnect. - if (existingConnectionsFromRemoteServer.size() > 0) { - logger.info("Error: Connection to/from " + fedName + " already exists. Disallowing duplicate"); - processDisconnectWithoutRetry(); - } else { - FederationHubDependencyInjectionProxy.getInstance().hubConnectionStore().addConnectionInfo(fedName, info); - } - logger.info("Outgoing connection for {} established ", fedName); - } + setServerSubscriptionForConnection(); // Notify gRPC to receive one additional response. clientCall.request(1); } @@ -428,14 +441,12 @@ public int compare(FederatedEvent a, FederatedEvent b) { }, true ); - if (serverSubscription != null) { - eventStreamHolder.setSubscription(serverSubscription); - } + setServerSubscriptionForConnection(); eventStreamHolder.send(FederatedEvent.newBuilder().build()); FederationHubDependencyInjectionProxy.getInstance().hubConnectionStore().addClientStreamHolder(fedName, eventStreamHolder); - FederationPolicyGraph fpg = FederationHubDependencyInjectionProxy.getInstance().fedHubPolicyManager().getPolicyGraph(); + FederationPolicyGraph fpg = FederationHubBrokerService.getInstance().getFederationPolicyGraph(); String fedId = eventStreamHolder.getFederateIdentity().getFedId(); Federate clientNode = fpg.getFederate(fedId); @@ -466,14 +477,15 @@ public int compare(FederatedEvent a, FederatedEvent b) { } public void setupRolStreamSender() { - // open a channel to the FIG server for the purpose of sending ROL messages - rolCall = channel.newCall(io.grpc.MethodDescriptor.create(io.grpc.MethodDescriptor.MethodType.CLIENT_STREAMING, - generateFullMethodName("com.atakmap.FederatedChannel", "ServerROLStream"), - io.grpc.protobuf.ProtoUtils.marshaller(com.atakmap.Tak.ROL.getDefaultInstance()), - io.grpc.protobuf.ProtoUtils - .marshaller(com.atakmap.Tak.Subscription.getDefaultInstance())), - asyncFederatedChannel.getCallOptions()); - + MethodDescriptor methodDescripton = MethodDescriptor.newBuilder() + .setType(MethodDescriptor.MethodType.CLIENT_STREAMING) + .setFullMethodName(generateFullMethodName("com.atakmap.FederatedChannel", "ServerROLStream")) + .setRequestMarshaller(io.grpc.protobuf.ProtoUtils.marshaller(com.atakmap.Tak.ROL.getDefaultInstance())) + .setResponseMarshaller(io.grpc.protobuf.ProtoUtils.marshaller(com.atakmap.Tak.Subscription.getDefaultInstance())) + .build(); + + rolCall = channel.newCall(methodDescripton, asyncFederatedChannel.getCallOptions()); + rolCall.start(new ClientCall.Listener() { @Override diff --git a/src/federation-hub-broker/src/main/resources/logback-broker.xml b/src/federation-hub-broker/src/main/resources/logback-broker.xml index 84237958..6db26fca 100644 --- a/src/federation-hub-broker/src/main/resources/logback-broker.xml +++ b/src/federation-hub-broker/src/main/resources/logback-broker.xml @@ -1,4 +1,9 @@ + + + 1 + + /opt/tak/federation-hub/logs/federation-hub-broker.log diff --git a/src/federation-hub-policy/scripts/federation-hub-policy b/src/federation-hub-policy/scripts/federation-hub-policy index e229fdc0..de0d692e 100755 --- a/src/federation-hub-policy/scripts/federation-hub-policy +++ b/src/federation-hub-policy/scripts/federation-hub-policy @@ -1,4 +1,9 @@ #!/bin/bash +### BEGIN INIT INFO +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: federation-hub-policy init script +### END INIT INFO # # /etc/rc.d/init.d/federation-hub-policy # @@ -16,7 +21,9 @@ # description: Federation Hub policy manager service. # Source function library -. /etc/rc.d/init.d/functions +if [ -f /etc/rc.d/init.d/functions ]; then + . /etc/rc.d/init.d/functions +fi SERVICE="Federation Hub Policy Manager" FEDERATION_HUB_HOME=/opt/tak/federation-hub diff --git a/src/federation-hub-policy/src/main/java/tak/server/federation/hub/policy/FederationHubPolicyManagerService.java b/src/federation-hub-policy/src/main/java/tak/server/federation/hub/policy/FederationHubPolicyManagerService.java index bbbbd092..d7fed286 100644 --- a/src/federation-hub-policy/src/main/java/tak/server/federation/hub/policy/FederationHubPolicyManagerService.java +++ b/src/federation-hub-policy/src/main/java/tak/server/federation/hub/policy/FederationHubPolicyManagerService.java @@ -1,17 +1,17 @@ package tak.server.federation.hub.policy; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import org.apache.ignite.Ignite; import org.apache.ignite.Ignition; import org.apache.ignite.cluster.ClusterGroup; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; -import org.springframework.context.annotation.Bean; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.ApplicationContext; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.CommandLineRunner; +import org.springframework.context.annotation.Bean; + +import com.google.common.base.Strings; import tak.server.federation.hub.FederationHubConstants; import tak.server.federation.hub.FederationHubUtils; @@ -24,7 +24,7 @@ public class FederationHubPolicyManagerService implements CommandLineRunner { private static final Logger logger = LoggerFactory.getLogger(FederationHubPolicyManagerService.class); private static Ignite ignite = null; - + public static void main(String[] args) { SpringApplication application = new SpringApplication(FederationHubPolicyManagerService.class); @@ -45,7 +45,12 @@ public Ignite getIgnite() { @Override public void run(String... args) throws Exception { - FederationHubPolicyManagerImpl hpm = new FederationHubPolicyManagerImpl(); + String defaultUiPolicyFile = null; + if (!Strings.isNullOrEmpty(System.getProperty("FEDERATION_HUB_POLICY_CONFIG"))) { + defaultUiPolicyFile = System.getProperties().getProperty("FEDERATION_HUB_POLICY_CONFIG"); + } + + FederationHubPolicyManagerImpl hpm = new FederationHubPolicyManagerImpl(ignite, defaultUiPolicyFile); ClusterGroup cg = ignite.cluster().forAttribute( FederationHubConstants.FEDERATION_HUB_IGNITE_PROFILE_KEY, FederationHubConstants.FEDERATION_HUB_POLICY_IGNITE_PROFILE); diff --git a/src/federation-hub-policy/src/main/resources/logback-policy.xml b/src/federation-hub-policy/src/main/resources/logback-policy.xml index dbcf7833..da3c4a1c 100644 --- a/src/federation-hub-policy/src/main/resources/logback-policy.xml +++ b/src/federation-hub-policy/src/main/resources/logback-policy.xml @@ -1,4 +1,9 @@ + + + 1 + + /opt/tak/federation-hub/logs/federation-hub-policy.log diff --git a/src/federation-hub-ui/scripts/federation-hub-ui b/src/federation-hub-ui/scripts/federation-hub-ui index f6a6d9d8..3728fb29 100755 --- a/src/federation-hub-ui/scripts/federation-hub-ui +++ b/src/federation-hub-ui/scripts/federation-hub-ui @@ -1,4 +1,9 @@ #!/bin/bash +### BEGIN INIT INFO +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: federation-hub-ui init script +### END INIT INFO # # /etc/rc.d/init.d/federation-hub-ui # @@ -16,7 +21,9 @@ # description: Federation Hub UI service. # Source function library -. /etc/rc.d/init.d/functions +if [ -f /etc/rc.d/init.d/functions ]; then + . /etc/rc.d/init.d/functions +fi SERVICE="Federation Hub UI" FEDERATION_HUB_HOME=/opt/tak/federation-hub diff --git a/src/federation-hub-ui/src/main/java/tak/server/federation/hub/ui/FederationHubUIServer.java b/src/federation-hub-ui/src/main/java/tak/server/federation/hub/ui/FederationHubUIServer.java index 7b6a16e8..c405edfb 100644 --- a/src/federation-hub-ui/src/main/java/tak/server/federation/hub/ui/FederationHubUIServer.java +++ b/src/federation-hub-ui/src/main/java/tak/server/federation/hub/ui/FederationHubUIServer.java @@ -131,7 +131,7 @@ public AuthorizationFileWatcher authFileWatcher(FederationHubUIConfig fedHubConf @Bean @Order(Ordered.LOWEST_PRECEDENCE) - public FederationHubUIService FederationHubBrokerService() { + public FederationHubUIService federationHubUIService() { return new FederationHubUIService(); } } diff --git a/src/federation-hub-ui/src/main/resources/logback-ui.xml b/src/federation-hub-ui/src/main/resources/logback-ui.xml index 89267383..967e7b0d 100644 --- a/src/federation-hub-ui/src/main/resources/logback-ui.xml +++ b/src/federation-hub-ui/src/main/resources/logback-ui.xml @@ -1,4 +1,9 @@ + + + 1 + + /opt/tak/federation-hub/logs/federation-hub-ui.log diff --git a/src/gradle.properties b/src/gradle.properties index 5dc18afc..bd5bf6a3 100644 --- a/src/gradle.properties +++ b/src/gradle.properties @@ -10,7 +10,9 @@ httpcomponents_version = 4.5.13 slf4j_version = 1.7.32 logback_version = 1.2.11 log4j_api_version = 2.17.1 -postgres_version = 42.2.5 +# Switched versions of of postgres to address xray reported vulnerability +#postgres_version = 42.2.5 +postgres_version = 42.3.3 # Current 5.3.3 released 01/2021 # Candidate 5.3.19 released 04/2022 @@ -20,7 +22,9 @@ spring_version = 5.3.21 # Candidate 5.3.13.RELEASE released 12/2021 # Candidate 5.5.6 released 04/2022 # Candidate 5.6.3 released 04/2022 -spring_security_version = 5.7.2 +#spring_security_version = 5.7.2 +# Version recommended by xray +spring_security_version = 5.7.5 # Current 2.5.0.RELEASE released 05/2020 # Candidate 2.5.1.RELEASE released 04/2021 @@ -83,6 +87,7 @@ commons_collections_version = 4.2 commons_fileupload_version = 1.4 commons_io_version = 2.11.0 commons_pool_version = 2.6.0 +commons_validator_version = 1.7 concurrent_hashmap_version = 1.0 esapi_version = 2.2.0.0 @@ -140,14 +145,15 @@ xerces_version = 2.12.2 xpp3_version = 1.1.4c hamcrest_version = 1.3 junit_version = 4.12 -flyway_version = 6.4.0 +flyway_version = 9.8.3 # Current 2.1.4 released 10/2008 with no known vulnberabilities as of 06/2022 # Candidate 2.1.7 released 04/2009 with no known vulnberabilities as of 06/2022 # Candidate 2.2.10 released 01/2015 with no known vulnberabilities as of 06/2022 # Candidate 2.3.5 released 08/2021 with no known vulnberabilities as of 06/2022 # Candidate 3.0.2 released 08/2021 with no known vulnberabilities as of 06/2022 -jaxwsrt_version = 2.1.4 +#jaxwsrt_version = 2.1.4 # Attempting version update to 2.3.5 to accommodate Raspberry Pi OS LITE installations +jaxwsrt_version = 2.3.5 # Current 2.9.0 released 02/2022 with no known vulnberabilities as of 06/2022 gson_version = 2.9.0 @@ -162,18 +168,16 @@ json_org_version = 20180813 opencsv_version = 4.4 # Netty and GRPC must be valid with one another -#Go to github.com/grpc-java in SECURITY.md file look for the table of known working versions - -# Current 4.1.76.Final released 04/2022 -netty_version = 4.1.76.Final - -# try this: -netty_handler_version = 4.1.58.Final -netty_tcnative_version = 2.0.36.Final - - -grpc_version = 1.35.0 +# These four are the lastest in sync versions according to +# https://github.com/grpc/grpc-java/blob/master/SECURITY.md +# Other than netty_version being slightly newer +netty_version = 4.1.77.Final +netty_handler_version = 4.1.77.Final +netty_tcnative_version = 2.0.53.Final +grpc_version = 1.49.1 +# keep this up to date with the version gRPC is expecting +perfmark_api_version = 0.25.0 docker_plugin_version = 1.2 gradle_ospackage_version = 8.3.0 @@ -202,3 +206,4 @@ annotation_api_version = 1.3.2 snake_yaml_version = 1.30 protobuf_java_version = 3.16.1 springdoc_version = 1.6.9 +okhttp3_version = 4.10.0 diff --git a/src/takserver-cluster/deployments/helm/Chart.yaml b/src/takserver-cluster/deployments/helm/Chart.yaml index 826f6541..6eefa475 100644 --- a/src/takserver-cluster/deployments/helm/Chart.yaml +++ b/src/takserver-cluster/deployments/helm/Chart.yaml @@ -6,7 +6,7 @@ type: application dependencies: - name: postgresql - version: "10.16.2" + version: "12.1.6" repository: "https://charts.bitnami.com/bitnami" condition: postgresql.enabled tags: diff --git a/src/takserver-cluster/docker-files/Dockerfile.ca b/src/takserver-cluster/docker-files/Dockerfile.ca index f04d288e..f9a47794 100644 --- a/src/takserver-cluster/docker-files/Dockerfile.ca +++ b/src/takserver-cluster/docker-files/Dockerfile.ca @@ -1,5 +1,5 @@ # need java for keytool -FROM openjdk:11-jdk-stretch +FROM openjdk:11-jdk-bullseye COPY takserver-core/certs/* / ARG ARG_CA_NAME ENV CA_NAME=$ARG_CA_NAME @@ -14,4 +14,4 @@ RUN apt update && \ apt install -y openssl && \ apt install -y vim && \ ./generateClusterCerts.sh -CMD ["bash"] \ No newline at end of file +CMD ["bash"] diff --git a/src/takserver-cluster/docker-files/Dockerfile.database-setup b/src/takserver-cluster/docker-files/Dockerfile.database-setup index f3fedae0..6b7a6485 100644 --- a/src/takserver-cluster/docker-files/Dockerfile.database-setup +++ b/src/takserver-cluster/docker-files/Dockerfile.database-setup @@ -1,4 +1,4 @@ -FROM openjdk:11-jdk-stretch +FROM openjdk:11-jdk-bullseye COPY takserver-schemamanager/SchemaManager.jar ./ COPY takserver-schemamanager/generic-cluster-database-configuration.sh ./ COPY CoreConfig.xml opt/tak/CoreConfig.xml @@ -6,4 +6,4 @@ COPY takserver-schemamanager/db-connection-configuration.sh . RUN chmod +x generic-cluster-database-configuration.sh RUN chmod +x db-connection-configuration.sh RUN ./db-connection-configuration.sh -CMD ["./generic-cluster-database-configuration.sh"] \ No newline at end of file +CMD ["./generic-cluster-database-configuration.sh"] diff --git a/src/takserver-cluster/docker-files/Dockerfile.rds b/src/takserver-cluster/docker-files/Dockerfile.rds index 5f3c144c..c5263bee 100644 --- a/src/takserver-cluster/docker-files/Dockerfile.rds +++ b/src/takserver-cluster/docker-files/Dockerfile.rds @@ -1,4 +1,4 @@ -FROM openjdk:11-stretch +FROM openjdk:11-jdk-bullseye RUN apt-get update && \ apt-get install -y \ python \ diff --git a/src/takserver-cluster/docker-files/Dockerfile.takserver-base b/src/takserver-cluster/docker-files/Dockerfile.takserver-base index 44b83cb7..333c9899 100644 --- a/src/takserver-cluster/docker-files/Dockerfile.takserver-base +++ b/src/takserver-cluster/docker-files/Dockerfile.takserver-base @@ -1,4 +1,4 @@ -FROM openjdk:11-jdk-stretch +FROM openjdk:11-jdk-bullseye COPY takserver-core/takserver-core*.war takserver.war COPY takserver-usermanager/UserManager.jar . COPY CoreConfig.xml . diff --git a/src/takserver-cluster/scripts/build-eks.py b/src/takserver-cluster/scripts/build-eks.py index b32d2988..b9a7a2b1 100644 --- a/src/takserver-cluster/scripts/build-eks.py +++ b/src/takserver-cluster/scripts/build-eks.py @@ -196,8 +196,8 @@ def setupDBParameterGroups(): except botocore.exceptions.ClientError as e: create_db_param_group_res = boto3.client('rds', region_name=TAK_CLUSTER_REGION).create_db_parameter_group( DBParameterGroupName='takserver-rds-pg', - DBParameterGroupFamily='postgres10', - Description='Takserver RDS parameter group for postgres10' + DBParameterGroupFamily='postgres15', + Description='Takserver RDS parameter group for postgres15' ) print('\nCreating RDS Parameter Group') diff --git a/src/takserver-cluster/scripts/build-kops-gossip.py b/src/takserver-cluster/scripts/build-kops-gossip.py index 8196b72a..82c17069 100755 --- a/src/takserver-cluster/scripts/build-kops-gossip.py +++ b/src/takserver-cluster/scripts/build-kops-gossip.py @@ -185,8 +185,8 @@ def setupDBParameterGroups(): except botocore.exceptions.ClientError as e: create_db_param_group_res = boto3.client('rds', region_name=TAK_CLUSTER_REGION).create_db_parameter_group( DBParameterGroupName='takserver-rds-pg', - DBParameterGroupFamily='postgres10', - Description='Takserver RDS parameter group for postgres10' + DBParameterGroupFamily='postgres15', + Description='Takserver RDS parameter group for postgres15' ) print('\nCreating RDS Parameter Group') diff --git a/src/takserver-cluster/scripts/build-kops.py b/src/takserver-cluster/scripts/build-kops.py index 004b63ba..ea248296 100755 --- a/src/takserver-cluster/scripts/build-kops.py +++ b/src/takserver-cluster/scripts/build-kops.py @@ -185,8 +185,8 @@ def setupDBParameterGroups(): except botocore.exceptions.ClientError as e: create_db_param_group_res = boto3.client('rds', region_name=TAK_CLUSTER_REGION).create_db_parameter_group( DBParameterGroupName='takserver-rds-pg', - DBParameterGroupFamily='postgres10', - Description='Takserver RDS parameter group for postgres10' + DBParameterGroupFamily='postgres15', + Description='Takserver RDS parameter group for postgres15' ) print('\nCreating RDS Parameter Group') diff --git a/src/takserver-common/build.gradle b/src/takserver-common/build.gradle index 11f3a11b..e092f90d 100644 --- a/src/takserver-common/build.gradle +++ b/src/takserver-common/build.gradle @@ -27,7 +27,11 @@ dependencies { exclude group: 'xpp' } */ - compile group: 'io.netty', name: 'netty-tcnative-boringssl-static', version: netty_tcnative_version + compile "io.netty:netty-tcnative-boringssl-static:$netty_tcnative_version:linux-x86_64" + compile "io.netty:netty-tcnative-boringssl-static:$netty_tcnative_version:linux-aarch_64" + compile "io.netty:netty-tcnative-boringssl-static:$netty_tcnative_version:osx-x86_64" + compile "io.netty:netty-tcnative-boringssl-static:$netty_tcnative_version:osx-aarch_64" + compile "io.netty:netty-tcnative-boringssl-static:$netty_tcnative_version:windows-x86_64" // compile group: 'io.netty', name: 'netty-tcnative', version: netty_tcnative_version compile group: 'io.netty', name: 'netty-transport-native-epoll', version: netty_version, classifier: 'linux-x86_64' diff --git a/src/takserver-common/src/main/java/com/bbn/marti/remote/FederationManager.java b/src/takserver-common/src/main/java/com/bbn/marti/remote/FederationManager.java index 44ad44ba..a2104e9a 100644 --- a/src/takserver-common/src/main/java/com/bbn/marti/remote/FederationManager.java +++ b/src/takserver-common/src/main/java/com/bbn/marti/remote/FederationManager.java @@ -41,6 +41,7 @@ public interface FederationManager { void removeFederateInboundGroupsMap(@NotNull String federateUID, @NotNull String remoteGroup, @NotNull String localGroup); void removeInboundGroupFromCA(@NotNull String caID, @NotNull Set localGroupNames); void removeOutboundGroupFromCA(@NotNull String caID, @NotNull Set localGroupNames); + void updateFederateMissionSettings(@NotNull String federateUID, @NotNull boolean missionFederateDefault, @NotNull List federateMissions); // Get outgoing config object by address and port List getOutgoingConnections(@NotNull String address, int port); @@ -74,10 +75,12 @@ public interface FederationManager { // Send ROL to messaging process, to be federated subject to group filtering. // Attach outbound groups to the ROL for federates using group mapping void submitFederateROL(ROL rol, NavigableSet groups); + void submitMissionFederateROL(ROL rol, NavigableSet groups, String missionName); // Send ROL to messaging process, to be federated subject to group filtering. - // Attach outbound groups to the ROL for federates using group mapping + // Attach outbound groups to the ROL for federates using group mapping void submitFederateROL(ROL rol, NavigableSet groups, String fileHash); + void submitMissionFederateROL(ROL rol, NavigableSet groups, String fileHash, String missionName); void reconfigureFederation(); diff --git a/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/ConnectionModifyResult.java b/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/ConnectionModifyResult.java index 8d37c9cc..459383ad 100644 --- a/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/ConnectionModifyResult.java +++ b/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/ConnectionModifyResult.java @@ -5,9 +5,9 @@ /** * An enum used to contain all the possible results from manipulating an input or static subscription. - *

+ *

* If the provided change matches the current state, a SUCCESS is expected to be returned. - *

+ *

* Created on 2/26/16. */ diff --git a/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/FileUserManagementInterface.java b/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/FileUserManagementInterface.java index 76581932..2f2d7a48 100644 --- a/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/FileUserManagementInterface.java +++ b/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/FileUserManagementInterface.java @@ -22,15 +22,15 @@ /** * Abstract interface used to define what can be done for user management - *

+ *

* Created on 9/21/15. */ public interface FileUserManagementInterface { - + UserAuthenticationFile getUserAuthenticationFile(); boolean userExists(@NotNull String userIdentifier); - + FileAuthenticatorControl addOrUpdateUser(@NotNull String userIdentifier, @Nullable String userPassword, boolean wasPasswordAlreadyHashed); FileAuthenticatorControl addOrUpdateUserFromCertificate(@NotNull X509Certificate certificate); @@ -73,9 +73,9 @@ public interface FileUserManagementInterface { String getUserFingerprint(@NotNull String userIdentifier); User getFirstUser(String userIdentifier); - + SimpleGroupWithUsersModel getUsersInGroup(String groupName); - + @NotNull Set getGroupNames(); } diff --git a/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/GroupManager.java b/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/GroupManager.java index 0af6f4d5..a754d451 100644 --- a/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/GroupManager.java +++ b/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/GroupManager.java @@ -17,8 +17,8 @@ /* * Operations for keeping track of users and groups. - * - * + * + * */ public interface GroupManager { @@ -26,37 +26,37 @@ public interface GroupManager { * Get all groups of which a user is a member. */ NavigableSet getGroups(User user); - + /* * Get all users. */ Collection getAllUsers(); - + /* * Get all connection ids. */ Set getAllConnectionIds(); - + /* * Get all groups. */ Collection getAllGroups(); - + /* * Add user to group. Create user and group if they don't exist. */ void addUserToGroup(User user, Group group); - + /* * Add user, creating if it doesn't exist. */ void addUser(User user); - + /* * Get group by id and direction */ Group getGroup(String id, Direction direction); - + /* * Find a User object by connectionId */ @@ -66,7 +66,7 @@ public interface GroupManager { * Get all groups by connection id */ NavigableSet getGroupsByConnectionId(String connectionId); - + /* * Get cached group vector by connection id */ @@ -84,31 +84,31 @@ public interface GroupManager { * handler is an Object, not a ConnectionInfo, to keep this interface decoupled from the NIO code. */ Reachability getReachability(Object connectionInfo); - + /* * Remove a user from a group. - * + * */ void removeUserFromGroup(User user, Group group); - + /* * Remove a user and associated group memberships - * + * */ - void removeUser(User user); - + void removeUser(User user); + /* - * Update group membership for a user, comparing the current group membership set with the provided group set. - * + * Update group membership for a user, comparing the current group membership set with the provided group set. + * */ void updateGroups(User user, Set groups); - + /* * Get the set of groups that set a but not in set b - * + * */ Set getGroupDiff(Set a, Set b); - + /* * Generate a Set from a list of group names. */ @@ -118,12 +118,12 @@ public interface GroupManager { * Finds a Set from a list of existing groups. */ Set findGroups(List groupNames); - + /* * Explicity track user by connectionId */ void putUserByConnectionId(User user, String connectionId); - + /* * Set subscription for a user */ @@ -133,7 +133,7 @@ public interface GroupManager { * Get subscription for a user */ RemoteSubscription getSubscriptionForUser(User user); - + /* * Register an authenticator */ @@ -151,10 +151,9 @@ public interface GroupManager { /** * Searches ldap groups (e.g., to help user configure items that require a group distinguished name reference) - * + * * @param groupNameFilter String optional filter applied to the LDAP cn attribute - * @return List instances - * @throws RemoteException + * @return List of @See LdapGroup instances */ List searchGroups(String groupNameFilter, boolean exactMatch); @@ -165,7 +164,7 @@ public interface GroupManager { LdapUser searchUser(String username); String getGroupPrefix(); - + /* * Make a copy of this user and its current group membership, so that it be independently managed. The copied user will be assigned a randomly generated connection id. */ @@ -173,13 +172,13 @@ public interface GroupManager { /* * Fill in Group details using the configured cache / DB - * + * */ Group hydrateGroup(Group group); - + /* * make an LDAP connection to the configured server - * + * */ DirContext connectLdap(); @@ -194,10 +193,10 @@ public interface GroupManager { * */ boolean testLdap(); - + /* - * Get the representation of a group vector, as a NavigableSet data structure - * + * Get the representation of a group vector, as a NavigableSet data structure + * */ NavigableSet groupVectorToGroupSet(String groupVector); diff --git a/src/takserver-common/src/main/java/com/bbn/marti/remote/service/RetentionQueryService.java b/src/takserver-common/src/main/java/com/bbn/marti/remote/service/RetentionQueryService.java index bca0dafe..7c865f7c 100644 --- a/src/takserver-common/src/main/java/com/bbn/marti/remote/service/RetentionQueryService.java +++ b/src/takserver-common/src/main/java/com/bbn/marti/remote/service/RetentionQueryService.java @@ -5,6 +5,8 @@ import org.dom4j.Element; +import com.bbn.marti.sync.model.Mission; + public interface RetentionQueryService { void deleteMissionByExpiration(Long ttl); @@ -17,4 +19,6 @@ public interface RetentionQueryService { boolean restoreMission(Map files, Map properties, List groups, String defaultRole, List defaultPermissions); void restoreCoT(String missionName, List files, List groups); void restoreContent(String missionName, byte[] file, Element missionContent, List groups) throws Exception; + public List getAllMissions(boolean passwordProtected, boolean defaultRole, String tool) throws Exception; + } diff --git a/src/takserver-common/src/main/java/tak/server/cache/TakIgniteSpringCacheManager.java b/src/takserver-common/src/main/java/tak/server/cache/TakIgniteSpringCacheManager.java index 6b034d1f..5a32a499 100644 --- a/src/takserver-common/src/main/java/tak/server/cache/TakIgniteSpringCacheManager.java +++ b/src/takserver-common/src/main/java/tak/server/cache/TakIgniteSpringCacheManager.java @@ -5,6 +5,9 @@ import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import javax.cache.expiry.Duration; +import javax.cache.expiry.TouchedExpiryPolicy; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; @@ -80,7 +83,11 @@ public TakIgniteSpringCacheManager(Ignite ignite) { if (cache == null) { CacheConfiguration cacheConfig = new CacheConfiguration<>(name); - + + if (config.getRemoteConfiguration().getBuffer().getQueue().isEnableCacheGroup()) { + cacheConfig.setGroupName("takserver-cache-group"); + } + if (config.getRemoteConfiguration().getNetwork().isCloudwatchEnable()) { cacheConfig.setStatisticsEnabled(true); } @@ -116,6 +123,11 @@ public TakIgniteSpringCacheManager(Ignite ignite) { } } + int cacheLastTouchedExpiryMinutes = config.getRemoteConfiguration().getBuffer().getQueue().getCacheLastTouchedExpiryMinutes(); + if (cacheLastTouchedExpiryMinutes != -1) { + cacheConfig.setExpiryPolicyFactory( + TouchedExpiryPolicy.factoryOf(new Duration(TimeUnit.MINUTES, cacheLastTouchedExpiryMinutes))); + } // near cache defaults to off but can be configured if (!messagingProfileActive && config.getRemoteConfiguration().getBuffer().getQueue().getNearCacheMaxSize() > 0) { diff --git a/src/takserver-common/src/main/java/tak/server/feeds/DataFeed.java b/src/takserver-common/src/main/java/tak/server/feeds/DataFeed.java index c8e73f3b..ba50f821 100644 --- a/src/takserver-common/src/main/java/tak/server/feeds/DataFeed.java +++ b/src/takserver-common/src/main/java/tak/server/feeds/DataFeed.java @@ -57,6 +57,8 @@ public enum DataFeedType { private int syncCacheRetentionSeconds = 3600; + private boolean federated; + private DataFeed() {} public DataFeed(String uuid, String name, DataFeedType type, List tags) { @@ -82,7 +84,8 @@ public DataFeed(com.bbn.marti.config.DataFeed datafeed) { this.coreVersion = datafeed.getCoreVersion(); this.coreVersion2TlsVersions = datafeed.getCoreVersion2TlsVersions(); this.filterGroups = datafeed.getFiltergroup(); - this.syncCacheRetentionSeconds = datafeed.getSyncCacheRetentionSeconds(); + this.syncCacheRetentionSeconds = datafeed.getSyncCacheRetentionSeconds(); + this.federated = datafeed.isFederated(); if (datafeed.getPort() == 0) { this.port = null; @@ -239,14 +242,22 @@ public void setSyncCacheRetentionSeconds(int syncCacheRetentionSeconds) { this.syncCacheRetentionSeconds = syncCacheRetentionSeconds; } + public boolean isFederated() { + return federated; + } + + public void setFederated(boolean federated) { + this.federated = federated; + } + @Override public String toString() { return "DataFeed [uuid=" + uuid + ", name=" + name + ", type=" + type + ", tags=" + tags + ", auth=" + auth + ", port=" + port + ", authRequired=" + authRequired + ", protocol=" + protocol + ", group=" + group + ", iface=" + iface + ", archive=" + archive + ", anongroup=" + anongroup + ", archiveOnly=" + archiveOnly + ", sync=" + sync + ", coreVersion=" + coreVersion + ", coreVersion2TlsVersions=" - + coreVersion2TlsVersions + ", filterGroups=" + filterGroups + ", latestSACacheSeconds=" - + syncCacheRetentionSeconds + "]"; + + coreVersion2TlsVersions + ", filterGroups=" + filterGroups + ", syncCacheRetentionSeconds=" + + syncCacheRetentionSeconds + ", federated=" + federated + "]"; } @Override diff --git a/src/takserver-common/src/main/java/tak/server/feeds/PluginDataFeed.java b/src/takserver-common/src/main/java/tak/server/feeds/PluginDataFeed.java deleted file mode 100644 index 85541e6d..00000000 --- a/src/takserver-common/src/main/java/tak/server/feeds/PluginDataFeed.java +++ /dev/null @@ -1,15 +0,0 @@ -package tak.server.feeds; - -import java.util.List; - -public class PluginDataFeed extends DataFeed { - - public PluginDataFeed(String uuid, String name, DataFeedType type, List tags) { - super(uuid, name, type, tags); - } - - public PluginDataFeed(com.bbn.marti.config.DataFeed datafeed) { - super(datafeed); - } - -} diff --git a/src/takserver-common/src/main/java/tak/server/feeds/StreamingDataFeed.java b/src/takserver-common/src/main/java/tak/server/feeds/StreamingDataFeed.java deleted file mode 100644 index 1254c4ed..00000000 --- a/src/takserver-common/src/main/java/tak/server/feeds/StreamingDataFeed.java +++ /dev/null @@ -1,19 +0,0 @@ -package tak.server.feeds; - -import java.util.List; - -/* - * - * Model class representing a generic TAK Server data feed - * - */ -public class StreamingDataFeed extends DataFeed { - - public StreamingDataFeed(String uuid, String name, DataFeedType type, List tags) { - super(uuid, name, type, tags); - } - - public StreamingDataFeed(com.bbn.marti.config.DataFeed datafeed) { - super(datafeed); - } -} diff --git a/src/takserver-common/src/main/java/tak/server/ignite/IgniteConfigurationHolder.java b/src/takserver-common/src/main/java/tak/server/ignite/IgniteConfigurationHolder.java index 05e87d7e..6ab140ba 100644 --- a/src/takserver-common/src/main/java/tak/server/ignite/IgniteConfigurationHolder.java +++ b/src/takserver-common/src/main/java/tak/server/ignite/IgniteConfigurationHolder.java @@ -60,11 +60,11 @@ public void setConfiguration(IgniteConfiguration configuration) { public IgniteConfiguration getIgniteConfiguration(String igniteProfile, String igniteHost, boolean isCluster, boolean isKubernetes, boolean isEmbedded, boolean isMulticastDiscovery, @Nullable Integer nonMulticastDiscoveryPort, @Nullable Integer nonMulticastDiscoveryPortCount, Integer communicationPort, Integer communicationPortCount, int maxQueue, long workerTimeoutMilliseconds, long dataRegionInitialSize, long dataRegionMaxSize) { - return getIgniteConfiguration(igniteProfile, igniteHost, isCluster, isKubernetes, isEmbedded, isMulticastDiscovery, nonMulticastDiscoveryPort, nonMulticastDiscoveryPortCount, communicationPort, communicationPortCount, maxQueue, workerTimeoutMilliseconds, dataRegionInitialSize, dataRegionMaxSize, -1, false); + return getIgniteConfiguration(igniteProfile, igniteHost, isCluster, isKubernetes, isEmbedded, isMulticastDiscovery, nonMulticastDiscoveryPort, nonMulticastDiscoveryPortCount, communicationPort, communicationPortCount, maxQueue, workerTimeoutMilliseconds, dataRegionInitialSize, dataRegionMaxSize, -1, false, -1.f); } public IgniteConfiguration getIgniteConfiguration(String igniteProfile, String igniteHost, boolean isCluster, boolean isKubernetes, boolean isEmbedded, boolean isMulticastDiscovery, @Nullable Integer nonMulticastDiscoveryPort, - @Nullable Integer nonMulticastDiscoveryPortCount, Integer communicationPort, Integer communicationPortCount, int maxQueue, long workerTimeoutMilliseconds, long dataRegionInitialSize, long dataRegionMaxSize, int poolSize, boolean enablePersistence) { + @Nullable Integer nonMulticastDiscoveryPortCount, Integer communicationPort, Integer communicationPortCount, int maxQueue, long workerTimeoutMilliseconds, long dataRegionInitialSize, long dataRegionMaxSize, int poolSize, boolean enablePersistence, double evictionThreashold) { if (isCluster) { IgniteConfiguration clusterConf = new IgniteConfiguration(); @@ -167,6 +167,10 @@ public IgniteConfiguration getIgniteConfiguration(String igniteProfile, String i DataRegionConfiguration takserverStorageRegion = new DataRegionConfiguration(); takserverStorageRegion.setName("takserver-cache-region"); takserverStorageRegion.setPageEvictionMode(DataPageEvictionMode.RANDOM_2_LRU); // cache eviction policy + + if (evictionThreashold != -1.f) { + takserverStorageRegion.setEvictionThreshold(evictionThreashold); + } // try to allocate off-heap memory as soon as possible, to head off any memory issues takserverStorageRegion.setInitialSize(dataRegionInitialSize); diff --git a/src/takserver-common/src/main/xsd/CoreConfig.xsd b/src/takserver-common/src/main/xsd/CoreConfig.xsd index 7d97241a..e67eb51a 100644 --- a/src/takserver-common/src/main/xsd/CoreConfig.xsd +++ b/src/takserver-common/src/main/xsd/CoreConfig.xsd @@ -264,6 +264,9 @@ + + + @@ -497,9 +500,10 @@ + - - + + @@ -527,12 +531,21 @@ - + - Authorization server public key used to validated tokens + OAuth authorization server - + + + + + + + + + + @@ -897,6 +910,20 @@ Name of a local group whose traffic will be shared with the specified federate. + + + This value will be applied for new missions as well as missions whose federation value are not specifically set in the mission lists + + + + + Enable/Disable federation on a per-mission basis for this federate. + + + + + + @@ -932,7 +959,7 @@ If true, remote groups are automatically mapped to local groups for this federate. - + @@ -1161,10 +1188,13 @@ use="optional" /> - + + + + @@ -1176,7 +1206,7 @@ - + diff --git a/src/takserver-core/build.gradle b/src/takserver-core/build.gradle index df61daf7..25a7bee2 100644 --- a/src/takserver-core/build.gradle +++ b/src/takserver-core/build.gradle @@ -109,10 +109,10 @@ writeVerFile.finalizedBy copyVerFile bootWar { enabled = true - + manifest { attributes 'Main-Class': 'org.springframework.boot.loader.PropertiesLauncher' - } + } } jar { @@ -136,9 +136,13 @@ dependencies { // Google's OpenSSL variant // Important Note: be cautious with this dependency version, in combination with the gprc-* above. - compile 'io.netty:netty-tcnative-boringssl-static:' + netty_tcnative_version + compile "io.netty:netty-tcnative-boringssl-static:$netty_tcnative_version:linux-x86_64" + compile "io.netty:netty-tcnative-boringssl-static:$netty_tcnative_version:linux-aarch_64" + compile "io.netty:netty-tcnative-boringssl-static:$netty_tcnative_version:osx-x86_64" + compile "io.netty:netty-tcnative-boringssl-static:$netty_tcnative_version:osx-aarch_64" + compile "io.netty:netty-tcnative-boringssl-static:$netty_tcnative_version:windows-x86_64" - compile group: 'io.netty', name: 'netty-tcnative-classes', version: '2.0.53.Final' + compile group: 'io.netty', name: 'netty-tcnative-classes', version: netty_tcnative_version // compile group: 'io.netty', name: 'netty-tcnative-boringssl-static', version: netty_tcnative_version // compile group: 'io.netty', name: 'netty-tcnative', version: netty_tcnative_version @@ -157,10 +161,10 @@ dependencies { compile group: 'com.h2database', name: 'h2', version: h2_version compile group: 'javax.cache', name: 'cache-api', version: '1.1.1' - + compile project(':takserver-common') compile project(':federation-common') - + compile group: 'org.locationtech.spatial4j', name: 'spatial4j', version: '0.6' // implementation group: 'org.locationtech.spatial4j', name: 'spatial4j', version: '0.6' @@ -174,25 +178,27 @@ dependencies { exclude group: 'org.springframework', module: 'spring-context' } - compile group: 'org.springframework.boot', name: 'spring-boot-starter-actuator', version: spring_boot_version + compile group: 'org.springframework.boot', name: 'spring-boot-starter-actuator', version: spring_boot_version compile group: 'org.springframework.boot', name: 'spring-boot-starter-web', version: spring_boot_version compile group: 'org.springframework.boot', name: 'spring-boot-starter-cache', version: spring_boot_version - + compile group: 'io.micrometer', name: 'micrometer-core' - - - + + + // cloudwatch metrics compile group: 'io.micrometer', name: 'micrometer-registry-cloudwatch', version: micrometer_cloudwatch_version - + compile(project(':takserver-core:takserver-war')) { exclude group: 'ch.qos.logback', module: 'logback-classic' exclude group: 'org.slf4j' exclude group: 'log4j', module: 'log4j' + // Added to exclude dom4j 1.6.1, a transitive dependency + exclude group: 'dom4j', module: 'dom4j' } compile group: 'org.slf4j', name: 'log4j-over-slf4j', version: slf4j_version compile group: 'org.slf4j', name: 'jul-to-slf4j', version: slf4j_version - + // tomcat compile group: 'org.apache.tomcat.embed', name: 'tomcat-embed-jasper', version: tomcat_version compile group: 'org.apache.tomcat.embed', name: 'tomcat-embed-el', version: tomcat_version @@ -200,8 +206,8 @@ dependencies { compile group: 'org.apache.tomcat', name: 'tomcat-jsp-api', version: tomcat_version compile group: 'org.apache.tomcat', name: 'tomcat-el-api', version: tomcat_version compile group: 'org.apache.tomcat', name: 'tomcat-annotations-api', version: tomcat_version - - + + compile group: 'javax.servlet', name: 'jstl' compile group: 'javax.mail', name: 'javax.mail-api' compile group: 'org.apache.tomcat.embed', name: 'tomcat-embed-core', version: tomcat_version @@ -216,49 +222,49 @@ dependencies { compile group: 'org.springframework.security', name: 'spring-security-web', version: spring_security_version compile group: 'org.hibernate', name: 'hibernate-core', version: hibernate_version compile group: 'org.hibernate', name: 'hibernate-entitymanager', version: hibernate_version - - - + + + // core deps - + compile group: 'commons-codec', name: 'commons-codec', version: commons_codec_version compile group: 'org.apache.httpcomponents', name: 'fluent-hc', version: httpcomponents_version compile group: 'org.apache.httpcomponents', name: 'httpclient', version: httpcomponents_version compile group: 'org.apache.httpcomponents', name: 'httpmime', version: httpcomponents_version - + // nats message queue compile group: 'io.nats', name: 'jnats', version: nats_version - + // apache ignite (cache and distributed service grid) testCompile group: 'org.apache.ignite', name: 'ignite-indexing', version: ignite_version testCompile group: 'org.apache.ignite', name: 'ignite-kubernetes', version: ignite_version testCompile group: 'org.apache.ignite', name: 'ignite-log4j', version: ignite_version testCompile group: 'org.apache.ignite', name: 'ignite-slf4j', version: ignite_version - + testCompile group: 'junit', name: 'junit', version: junit_version testCompile group: 'org.mockito', name: 'mockito-core', version: mockito_version testCompile("org.springframework.boot:spring-boot-starter-test") { exclude group: "com.vaadin.external.google", module:"android-json" } - + testCompile group: 'ch.qos.logback', name: 'logback-classic', version: logback_version integrationTestCompile group: 'org.apache.logging.log4j', name: 'log4j-api', version: log4j_api_version integrationTestCompile group: 'org.apache.logging.log4j', name: 'log4j-to-slf4j', version: log4j_api_version integrationTestCompile group: 'org.slf4j', name: 'slf4j-api', version: slf4j_version - + integrationTestCompile project(':takserver-takcl-core') // required to fix version conflict for h2 between ignite and spring boot integrationTestCompile group: 'com.h2database', name: 'h2', version: h2_version integrationTestCompile group: 'xerces', name: 'xercesImpl', version: xerces_version - + } clean { diff --git a/src/takserver-core/docker/Dockerfile.takserver b/src/takserver-core/docker/Dockerfile.takserver index ebd15a09..f60ceac5 100644 --- a/src/takserver-core/docker/Dockerfile.takserver +++ b/src/takserver-core/docker/Dockerfile.takserver @@ -1,5 +1,5 @@ -FROM openjdk:11-jdk-stretch +FROM openjdk:11-jdk-bullseye RUN apt update && \ apt-get install -y emacs-nox net-tools netcat vim -ENTRYPOINT ["/bin/bash", "-c", "/opt/tak/configureInDocker.sh init &>> /opt/tak/logs/takserver.log"] \ No newline at end of file +ENTRYPOINT ["/bin/bash", "-c", "/opt/tak/configureInDocker.sh init &>> /opt/tak/logs/takserver.log"] diff --git a/src/takserver-core/docker/buildDocker.sh b/src/takserver-core/docker/buildDocker.sh new file mode 100755 index 00000000..67bebb07 --- /dev/null +++ b/src/takserver-core/docker/buildDocker.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +DIRNAME=`basename $PWD` +if [ "$DIRNAME"=docker ]; then + echo "In docker directory. Changing to base directory." + cd .. +fi + +echo "Building and setting up takserver-db container" +docker build -t takserver-db:"$(cat tak/version.txt)" -f docker/Dockerfile.takserver-db . + +echo "Building and setting up takserver container" +docker build -t takserver:"$(cat tak/version.txt)" -f docker/Dockerfile.takserver . diff --git a/src/takserver-core/docker/configureInDocker.sh b/src/takserver-core/docker/configureInDocker.sh index 67d777e1..fc23741d 100755 --- a/src/takserver-core/docker/configureInDocker.sh +++ b/src/takserver-core/docker/configureInDocker.sh @@ -8,7 +8,8 @@ cd /opt/tak . ./setenv.sh java -jar -Xmx${MESSAGING_MAX_HEAP}m -Dspring.profiles.active=messaging takserver.war & java -jar -Xmx${API_MAX_HEAP}m -Dspring.profiles.active=api -Dkeystore.pkcs12.legacy takserver.war & -java -jar -Xmx${PLUGIN_MANAGER_MAX_HEAP}m takserver-pm.jar & +java -jar -Xmx${RETENTION_MAX_HEAP}m takserver-retention.jar & +java -jar -Xmx${PLUGIN_MANAGER_MAX_HEAP}m -Dloader.path=WEB-INF/lib-provided,WEB-INF/lib,WEB-INF/classes,file:lib/ takserver-pm.jar & if ! [ $# -eq 0 ] then diff --git a/src/takserver-core/docker/CoreConfig.xml b/src/takserver-core/docker/full/CoreConfig.docker-full.xml similarity index 100% rename from src/takserver-core/docker/CoreConfig.xml rename to src/takserver-core/docker/full/CoreConfig.docker-full.xml diff --git a/src/takserver-core/docker/full/Dockerfile.takserver b/src/takserver-core/docker/full/Dockerfile.takserver index a1380cdb..13cf79f1 100644 --- a/src/takserver-core/docker/full/Dockerfile.takserver +++ b/src/takserver-core/docker/full/Dockerfile.takserver @@ -1,4 +1,4 @@ -FROM openjdk:11-jdk-stretch +FROM openjdk:11-jdk-bullseye RUN apt update && apt-get install -y emacs-nox net-tools netcat vim nmon python-lxml COPY tak /opt/tak diff --git a/src/takserver-core/docker/full/docker-compose.yml b/src/takserver-core/docker/full/docker-compose.yml index d8bb46d2..c1a55810 100644 --- a/src/takserver-core/docker/full/docker-compose.yml +++ b/src/takserver-core/docker/full/docker-compose.yml @@ -24,7 +24,7 @@ services: - '9001:9001' takdb: - image: postgis/postgis:10-3.1 + image: postgis/postgis:15-3.3 networks: - taknet env_file: diff --git a/src/takserver-core/docker/full/docker_entrypoint.sh b/src/takserver-core/docker/full/docker_entrypoint.sh index 56a7a35c..d5a90f43 100644 --- a/src/takserver-core/docker/full/docker_entrypoint.sh +++ b/src/takserver-core/docker/full/docker_entrypoint.sh @@ -120,7 +120,7 @@ java -jar -Xmx${MESSAGING_MAX_HEAP}m -Dspring.profiles.active=messaging takserve MESSAGING_PID=$! java -jar -Xmx${API_MAX_HEAP}m -Dspring.profiles.active=api -Dkeystore.pkcs12.legacy takserver.war & API_PID=$! -java -jar -Xmx${PLUGIN_MANAGER_MAX_HEAP}m takserver-pm.jar & +java -jar -Xmx${PLUGIN_MANAGER_MAX_HEAP}m -Dloader.path=WEB-INF/lib-provided,WEB-INF/lib,WEB-INF/classes,file:lib/ takserver-pm.jar & PM_PID=$! sleep 16 diff --git a/src/takserver-core/docker/hardened/Dockerfile.ca b/src/takserver-core/docker/hardened/Dockerfile.ca index 71adc88d..ae7f9d58 100644 --- a/src/takserver-core/docker/hardened/Dockerfile.ca +++ b/src/takserver-core/docker/hardened/Dockerfile.ca @@ -1,5 +1,5 @@ # need java for keytool -FROM openjdk:11-jdk-stretch +FROM openjdk:11-jdk-bullseye RUN apt update && \ apt install -y apt-utils && \ apt install -y openssl && \ @@ -22,4 +22,4 @@ RUN chmod u=rwx /tak/certs \ && chmod u=rw /tak/certs/*.* \ && chmod u=rx /tak/certs/*.sh WORKDIR /tak/certs/ -ENTRYPOINT ["/tak/certs/generateClusterCertsIfNoneExist.sh"] \ No newline at end of file +ENTRYPOINT ["/tak/certs/generateClusterCertsIfNoneExist.sh"] diff --git a/src/takserver-core/docker/hardened/health/takserver-db/check_file_integrity.sh b/src/takserver-core/docker/hardened/tak/health/takserver-db/check_file_integrity.sh old mode 100755 new mode 100644 similarity index 100% rename from src/takserver-core/docker/hardened/health/takserver-db/check_file_integrity.sh rename to src/takserver-core/docker/hardened/tak/health/takserver-db/check_file_integrity.sh diff --git a/src/takserver-core/docker/hardened/health/takserver-db/critical_file_list.txt b/src/takserver-core/docker/hardened/tak/health/takserver-db/critical_file_list.txt similarity index 100% rename from src/takserver-core/docker/hardened/health/takserver-db/critical_file_list.txt rename to src/takserver-core/docker/hardened/tak/health/takserver-db/critical_file_list.txt diff --git a/src/takserver-core/docker/hardened/health/takserver-db/health_check.sh b/src/takserver-core/docker/hardened/tak/health/takserver-db/health_check.sh old mode 100755 new mode 100644 similarity index 100% rename from src/takserver-core/docker/hardened/health/takserver-db/health_check.sh rename to src/takserver-core/docker/hardened/tak/health/takserver-db/health_check.sh diff --git a/src/takserver-core/docker/hardened/health/takserver/check_file_integrity.sh b/src/takserver-core/docker/hardened/tak/health/takserver/check_file_integrity.sh old mode 100755 new mode 100644 similarity index 100% rename from src/takserver-core/docker/hardened/health/takserver/check_file_integrity.sh rename to src/takserver-core/docker/hardened/tak/health/takserver/check_file_integrity.sh diff --git a/src/takserver-core/docker/hardened/health/takserver/critical_file_list.txt b/src/takserver-core/docker/hardened/tak/health/takserver/critical_file_list.txt similarity index 100% rename from src/takserver-core/docker/hardened/health/takserver/critical_file_list.txt rename to src/takserver-core/docker/hardened/tak/health/takserver/critical_file_list.txt diff --git a/src/takserver-core/docker/hardened/health/takserver/health_check.sh b/src/takserver-core/docker/hardened/tak/health/takserver/health_check.sh old mode 100755 new mode 100644 similarity index 100% rename from src/takserver-core/docker/hardened/health/takserver/health_check.sh rename to src/takserver-core/docker/hardened/tak/health/takserver/health_check.sh diff --git a/src/takserver-core/docker/hardened/epel-release-8-10.el8.noarch.rpm b/src/takserver-core/docker/hardened/tak/security/epel-release-8-10.el8.noarch.rpm similarity index 100% rename from src/takserver-core/docker/hardened/epel-release-8-10.el8.noarch.rpm rename to src/takserver-core/docker/hardened/tak/security/epel-release-8-10.el8.noarch.rpm diff --git a/src/takserver-core/docker/hardened/rpms/repos/Rocky-Linux-AppStream.repo b/src/takserver-core/docker/hardened/tak/security/rpms/repos/Rocky-Linux-AppStream.repo similarity index 100% rename from src/takserver-core/docker/hardened/rpms/repos/Rocky-Linux-AppStream.repo rename to src/takserver-core/docker/hardened/tak/security/rpms/repos/Rocky-Linux-AppStream.repo diff --git a/src/takserver-core/docker/hardened/rpms/repos/Rocky-Linux-BaseOS.repo b/src/takserver-core/docker/hardened/tak/security/rpms/repos/Rocky-Linux-BaseOS.repo similarity index 100% rename from src/takserver-core/docker/hardened/rpms/repos/Rocky-Linux-BaseOS.repo rename to src/takserver-core/docker/hardened/tak/security/rpms/repos/Rocky-Linux-BaseOS.repo diff --git a/src/takserver-core/docker/hardened/rpms/repos/Rocky-Linux-PowerTools.repo b/src/takserver-core/docker/hardened/tak/security/rpms/repos/Rocky-Linux-PowerTools.repo similarity index 100% rename from src/takserver-core/docker/hardened/rpms/repos/Rocky-Linux-PowerTools.repo rename to src/takserver-core/docker/hardened/tak/security/rpms/repos/Rocky-Linux-PowerTools.repo diff --git a/src/takserver-core/docker/hardened/rpms/signatures/RPM-GPG-KEY-EPEL-8 b/src/takserver-core/docker/hardened/tak/security/rpms/signatures/RPM-GPG-KEY-EPEL-8 similarity index 100% rename from src/takserver-core/docker/hardened/rpms/signatures/RPM-GPG-KEY-EPEL-8 rename to src/takserver-core/docker/hardened/tak/security/rpms/signatures/RPM-GPG-KEY-EPEL-8 diff --git a/src/takserver-core/docker/hardened/rpms/signatures/RPM-GPG-KEY-rockyofficial b/src/takserver-core/docker/hardened/tak/security/rpms/signatures/RPM-GPG-KEY-rockyofficial similarity index 100% rename from src/takserver-core/docker/hardened/rpms/signatures/RPM-GPG-KEY-rockyofficial rename to src/takserver-core/docker/hardened/tak/security/rpms/signatures/RPM-GPG-KEY-rockyofficial diff --git a/src/takserver-core/docker/hardened/CoreConfig.xml b/src/takserver-core/example/CoreConfig.example.docker-hardened.xml similarity index 99% rename from src/takserver-core/docker/hardened/CoreConfig.xml rename to src/takserver-core/example/CoreConfig.example.docker-hardened.xml index 04b865ee..17878217 100644 --- a/src/takserver-core/docker/hardened/CoreConfig.xml +++ b/src/takserver-core/example/CoreConfig.example.docker-hardened.xml @@ -172,4 +172,4 @@ --> - + \ No newline at end of file diff --git a/src/takserver-core/docker/full/CoreConfig.xml b/src/takserver-core/example/CoreConfig.example.docker.xml similarity index 100% rename from src/takserver-core/docker/full/CoreConfig.xml rename to src/takserver-core/example/CoreConfig.example.docker.xml diff --git a/src/takserver-core/scripts/API/takserver-api b/src/takserver-core/scripts/API/takserver-api index b8a84fab..940bd676 100644 --- a/src/takserver-core/scripts/API/takserver-api +++ b/src/takserver-core/scripts/API/takserver-api @@ -1,4 +1,9 @@ #!/bin/bash +### BEGIN INIT INFO +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: takserver-api init script +### END INIT INFO # # /etc/rc.d/init.d/takserver-api # @@ -18,7 +23,9 @@ # description: Team Awareness Kit (TAK) API service. # Source function library -. /etc/rc.d/init.d/functions +if [ -f /etc/rc.d/init.d/functions ]; then + . /etc/rc.d/init.d/functions +fi SERVICE="TAK server API" TAK_HOME=/opt/tak diff --git a/src/takserver-core/scripts/API/takserver-api-cluster.sh b/src/takserver-core/scripts/API/takserver-api-cluster.sh index 7bf71ae5..0c26e1ea 100644 --- a/src/takserver-core/scripts/API/takserver-api-cluster.sh +++ b/src/takserver-core/scripts/API/takserver-api-cluster.sh @@ -1,4 +1,4 @@ - #!/bin/sh +#!/bin/sh . ./setenv.sh diff --git a/src/takserver-core/scripts/API/takserver-api.sh b/src/takserver-core/scripts/API/takserver-api.sh index 74d1f114..6d83f9a0 100644 --- a/src/takserver-core/scripts/API/takserver-api.sh +++ b/src/takserver-core/scripts/API/takserver-api.sh @@ -1,4 +1,4 @@ - #!/bin/sh +#!/bin/sh . ./setenv.sh diff --git a/src/takserver-core/scripts/certs/makeCert.sh b/src/takserver-core/scripts/certs/makeCert.sh index 3df99af6..3ff6e5b2 100755 --- a/src/takserver-core/scripts/certs/makeCert.sh +++ b/src/takserver-core/scripts/certs/makeCert.sh @@ -64,6 +64,12 @@ else CONFIG=../config.cfg fi +openssl list -providers 2>&1 | grep "\(invalid command\|unknown option\)" >/dev/null +if [ $? -ne 0 ] ; then + echo "Using legacy provider" + LEGACY_PROVIDER="-legacy" +fi + SUBJ=$SUBJBASE"CN=$SNAME" echo "Making a $1 cert for " $SUBJ openssl req -new -newkey rsa:2048 -sha256 -keyout "${SNAME}".key -passout pass:$PASS -out "${SNAME}".csr -subj "$SUBJ" @@ -79,14 +85,14 @@ cat ca-trusted.pem >> "${SNAME}"-trusted.pem # now make pkcs12 and jks keystore files if [[ "$1" == "server" || "$1" == "client" ]]; then - openssl pkcs12 -export -in "${SNAME}".pem -inkey "${SNAME}".key -out "${SNAME}".p12 -name "${SNAME}" -CAfile ca.pem -passin pass:${PASS} -passout pass:${PASS} + openssl pkcs12 ${LEGACY_PROVIDER} -export -in "${SNAME}".pem -inkey "${SNAME}".key -out "${SNAME}".p12 -name "${SNAME}" -CAfile ca.pem -passin pass:${PASS} -passout pass:${PASS} keytool -importkeystore -deststorepass "${PASS}" -destkeypass "${PASS}" -destkeystore "${SNAME}".jks -srckeystore "${SNAME}".p12 -srcstoretype PKCS12 -srcstorepass "${PASS}" -alias "${SNAME}" else # a CA - openssl pkcs12 -export -in "${SNAME}"-trusted.pem -out truststore-"${SNAME}".p12 -nokeys -passout pass:${CAPASS} + openssl pkcs12 ${LEGACY_PROVIDER} -export -in "${SNAME}"-trusted.pem -out truststore-"${SNAME}".p12 -nokeys -passout pass:${CAPASS} keytool -import -trustcacerts -file "${SNAME}".pem -keystore truststore-"${SNAME}".jks -storepass "${CAPASS}" -noprompt # include a CA signing keystore; NOT FOR DISTRIBUTION TO CLIENTS - openssl pkcs12 -export -in "${SNAME}".pem -inkey "${SNAME}".key -out "${SNAME}"-signing.p12 -name "${SNAME}" -passin pass:${PASS} -passout pass:${PASS} + openssl pkcs12 ${LEGACY_PROVIDER} -export -in "${SNAME}".pem -inkey "${SNAME}".key -out "${SNAME}"-signing.p12 -name "${SNAME}" -passin pass:${PASS} -passout pass:${PASS} keytool -importkeystore -deststorepass "${PASS}" -destkeypass "${PASS}" -destkeystore "${SNAME}"-signing.jks -srckeystore "${SNAME}"-signing.p12 -srcstoretype PKCS12 -srcstorepass "${PASS}" -alias "${SNAME}" ## create empty crl diff --git a/src/takserver-core/scripts/certs/makeRootCa.sh b/src/takserver-core/scripts/certs/makeRootCa.sh index 45bef29d..b0daa3aa 100755 --- a/src/takserver-core/scripts/certs/makeRootCa.sh +++ b/src/takserver-core/scripts/certs/makeRootCa.sh @@ -25,12 +25,18 @@ if [[ "$canamelen" -lt 5 ]]; then CA_NAME=`date +%N` fi +openssl list -providers 2>&1 | grep "\(invalid command\|unknown option\)" >/dev/null +if [ $? -ne 0 ] ; then + echo "Using legacy provider" + LEGACY_PROVIDER="-legacy" +fi + SUBJ=$SUBJBASE"CN=$CA_NAME" echo "Making a CA for " $SUBJ openssl req -new -sha256 -x509 -days 3652 -extensions v3_ca -keyout ca-do-not-share.key -out ca.pem -passout pass:${CAPASS} -config ../config.cfg -subj "$SUBJ" openssl x509 -in ca.pem -addtrust clientAuth -addtrust serverAuth -setalias "${CA_NAME}" -out ca-trusted.pem -openssl pkcs12 -export -in ca-trusted.pem -out truststore-root.p12 -nokeys -caname "${CA_NAME}" -passout pass:${CAPASS} +openssl pkcs12 ${LEGACY_PROVIDER} -export -in ca-trusted.pem -out truststore-root.p12 -nokeys -caname "${CA_NAME}" -passout pass:${CAPASS} keytool -import -trustcacerts -file ca.pem -keystore truststore-root.jks -alias "${CA_NAME}" -storepass "${CAPASS}" -noprompt cp truststore-root.jks fed-truststore.jks diff --git a/src/takserver-core/scripts/launcher/takserver b/src/takserver-core/scripts/launcher/takserver index dc4a51a3..ff6096f4 100644 --- a/src/takserver-core/scripts/launcher/takserver +++ b/src/takserver-core/scripts/launcher/takserver @@ -1,4 +1,9 @@ #!/bin/bash +### BEGIN INIT INFO +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: takserver init script +### END INIT INFO # # /etc/rc.d/init.d/takserver # @@ -18,7 +23,9 @@ # description: Team Awareness Kit (TAK) information brokering and management service. # Source function library -. /etc/rc.d/init.d/functions +if [ -f /etc/rc.d/init.d/functions ]; then + . /etc/rc.d/init.d/functions +fi SERVICE="TAK server" TAK_HOME=/opt/tak diff --git a/src/takserver-core/scripts/launcher/takserver-noplugins b/src/takserver-core/scripts/launcher/takserver-noplugins index b24aaad9..edb68896 100644 --- a/src/takserver-core/scripts/launcher/takserver-noplugins +++ b/src/takserver-core/scripts/launcher/takserver-noplugins @@ -1,4 +1,9 @@ #!/bin/bash +### BEGIN INIT INFO +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: takserver-noplugins init script +### END INIT INFO # # /etc/rc.d/init.d/takserver-noplugins # @@ -18,7 +23,9 @@ # description: Team Awareness Kit (TAK) information brokering and management service. # Source function library -. /etc/rc.d/init.d/functions +if [ -f /etc/rc.d/init.d/functions ]; then + . /etc/rc.d/init.d/functions +fi SERVICE="TAK server noplugins" TAK_HOME=/opt/tak diff --git a/src/takserver-core/scripts/messaging/takserver-messaging b/src/takserver-core/scripts/messaging/takserver-messaging index 27683145..4a02d1ce 100644 --- a/src/takserver-core/scripts/messaging/takserver-messaging +++ b/src/takserver-core/scripts/messaging/takserver-messaging @@ -1,4 +1,9 @@ #!/bin/bash +### BEGIN INIT INFO +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: takserver-messaging init script +### END INIT INFO # # /etc/rc.d/init.d/takserver-messaging # @@ -20,7 +25,9 @@ # description: Team Awareness Kit (TAK) messaging service. # Source function library -. /etc/rc.d/init.d/functions +if [ -f /etc/rc.d/init.d/functions ]; then + . /etc/rc.d/init.d/functions +fi SERVICE="TAK server Messaging" TAK_HOME=/opt/tak diff --git a/src/takserver-core/scripts/messaging/takserver-messaging-cluster.sh b/src/takserver-core/scripts/messaging/takserver-messaging-cluster.sh index 8f13a5a6..5840ed6c 100644 --- a/src/takserver-core/scripts/messaging/takserver-messaging-cluster.sh +++ b/src/takserver-core/scripts/messaging/takserver-messaging-cluster.sh @@ -1,4 +1,4 @@ - #!/bin/sh +#!/bin/sh . ./setenv.sh diff --git a/src/takserver-core/scripts/messaging/takserver-messaging.sh b/src/takserver-core/scripts/messaging/takserver-messaging.sh index 1329223e..c32abb99 100644 --- a/src/takserver-core/scripts/messaging/takserver-messaging.sh +++ b/src/takserver-core/scripts/messaging/takserver-messaging.sh @@ -1,4 +1,4 @@ - #!/bin/sh +#!/bin/sh rm -rf ./tmp/ . ./setenv.sh diff --git a/src/takserver-core/scripts/plugins/takserver-plugins b/src/takserver-core/scripts/plugins/takserver-plugins index 5c121348..48301e77 100644 --- a/src/takserver-core/scripts/plugins/takserver-plugins +++ b/src/takserver-core/scripts/plugins/takserver-plugins @@ -1,4 +1,9 @@ #!/bin/bash +### BEGIN INIT INFO +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: takserver-plugins init script +### END INIT INFO # # /etc/rc.d/init.d/takserver-plugins # @@ -18,7 +23,9 @@ # description: Team Awareness Kit (TAK) plugins service. # Source function library -. /etc/rc.d/init.d/functions +if [ -f /etc/rc.d/init.d/functions ]; then + . /etc/rc.d/init.d/functions +fi SERVICE="TAK server plugins" TAK_HOME=/opt/tak diff --git a/src/takserver-core/scripts/plugins/takserver-plugins.sh b/src/takserver-core/scripts/plugins/takserver-plugins.sh index 43a09207..922900fb 100644 --- a/src/takserver-core/scripts/plugins/takserver-plugins.sh +++ b/src/takserver-core/scripts/plugins/takserver-plugins.sh @@ -2,4 +2,7 @@ . ./setenv.sh +# add the lib/ directory to the loader.path for the plugin manager +export JDK_JAVA_OPTIONS="-Dloader.path=WEB-INF/lib-provided,WEB-INF/lib,WEB-INF/classes,file:lib/ -Dio.netty.tmpdir=/opt/tak -Djava.io.tmpdir=/opt/tak -Dio.netty.native.workdir=/opt/tak -Djava.net.preferIPv4Stack=true -Djava.security.egd=file:/dev/./urandom -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_QUIET=true -Djdk.tls.client.protocols=TLSv1.2" + java -Xms128m -Xmx${PLUGIN_MANAGER_MAX_HEAP}m -jar takserver-pm.jar $@ diff --git a/src/takserver-core/scripts/retention/takserver-retention b/src/takserver-core/scripts/retention/takserver-retention index a8e38b7f..a4fb11b2 100644 --- a/src/takserver-core/scripts/retention/takserver-retention +++ b/src/takserver-core/scripts/retention/takserver-retention @@ -1,4 +1,9 @@ #!/bin/bash +### BEGIN INIT INFO +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: takserver-retention init script +### END INIT INFO # # /etc/rc.d/init.d/takserver-retention # @@ -18,7 +23,9 @@ # description: Team Awareness Kit (TAK) retention service. # Source function library -. /etc/rc.d/init.d/functions +if [ -f /etc/rc.d/init.d/functions ]; then + . /etc/rc.d/init.d/functions +fi SERVICE="TAK server retention" TAK_HOME=/opt/tak diff --git a/src/takserver-core/scripts/retention/takserver-retention.sh b/src/takserver-core/scripts/retention/takserver-retention.sh index 39aef858..9be28ee7 100644 --- a/src/takserver-core/scripts/retention/takserver-retention.sh +++ b/src/takserver-core/scripts/retention/takserver-retention.sh @@ -2,6 +2,4 @@ . ./setenv.sh -sleep 32 - -java -Xms16m -Xmx${RETENTION_MAX_HEAP}m -Dspring.main.banner-mode=off -DIGNITE_NO_ASCII=true -DIGNITE_PERFORMANCE_SUGGESTIONS_DISABLED=true -DIGNITE_UPDATE_NOTIFIER=false -jar takserver-retention.jar $@ +java -Xms16m -Xmx${RETENTION_MAX_HEAP}m -Dspring.main.banner-mode=off -DIGNITE_NO_ASCII=true -DIGNITE_PERFORMANCE_SUGGESTIONS_DISABLED=true -DIGNITE_UPDATE_NOTIFIER=false -jar takserver-retention.jar $@ diff --git a/src/takserver-core/scripts/setenv-cluster.sh b/src/takserver-core/scripts/setenv-cluster.sh index 2b0a2219..2194199c 100644 --- a/src/takserver-core/scripts/setenv-cluster.sh +++ b/src/takserver-core/scripts/setenv-cluster.sh @@ -1,4 +1,4 @@ - #!/bin/sh +#!/bin/sh # set up execution environment export JDK_JAVA_OPTIONS="-Dloader.path=WEB-INF/lib-provided,WEB-INF/lib,WEB-INF/classes,file:lib/ -Dio.netty.tmpdir=/opt/tak -Djava.io.tmpdir=/opt/tak -Dio.netty.native.workdir=/opt/tak -Djava.net.preferIPv4Stack=true -Djava.security.egd=file:/dev/./urandom -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_QUIET=true -Djdk.tls.client.protocols=TLSv1.2" diff --git a/src/takserver-core/scripts/setenv.sh b/src/takserver-core/scripts/setenv.sh index 4d7d7b15..d8ff7b45 100644 --- a/src/takserver-core/scripts/setenv.sh +++ b/src/takserver-core/scripts/setenv.sh @@ -1,7 +1,7 @@ - #!/bin/sh +#!/bin/sh # set up execution environment -export JDK_JAVA_OPTIONS="-Dloader.path=WEB-INF/lib-provided,WEB-INF/lib,WEB-INF/classes,file:lib/ -Dio.netty.tmpdir=/opt/tak -Djava.io.tmpdir=/opt/tak -Dio.netty.native.workdir=/opt/tak -Djava.net.preferIPv4Stack=true -Djava.security.egd=file:/dev/./urandom -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_QUIET=true -Djdk.tls.client.protocols=TLSv1.2" +export JDK_JAVA_OPTIONS="-Dloader.path=WEB-INF/lib-provided,WEB-INF/lib,WEB-INF/classes -Dio.netty.tmpdir=/opt/tak -Djava.io.tmpdir=/opt/tak -Dio.netty.native.workdir=/opt/tak -Djava.net.preferIPv4Stack=true -Djava.security.egd=file:/dev/./urandom -DIGNITE_UPDATE_NOTIFIER=false -DIGNITE_QUIET=true -Djdk.tls.client.protocols=TLSv1.2" export IGNITE_HOME="/opt/tak" # pull in defaults diff --git a/src/takserver-core/scripts/setup-for-extracted-war.sh b/src/takserver-core/scripts/setup-for-extracted-war.sh new file mode 100755 index 00000000..93c1646e --- /dev/null +++ b/src/takserver-core/scripts/setup-for-extracted-war.sh @@ -0,0 +1,75 @@ +#!/bin/sh + +if [ ! -f "/opt/tak/takserver-api.sh" ] && \ + [ ! -f "/opt/tak/takserver-messaging.sh" ]; then + echo "Skipping war file extraction" + exit 0 +fi + +cd /opt/tak +if [ -d /opt/tak/extract ]; then + echo "Deleting current /opt/tak/extract" + rm -fR /opt/tak/extract +fi +mkdir /opt/tak/extract + +echo "Extracting /opt/tak/takserver.war into /opt/tak/extract directory" +(cd /opt/tak/extract && exec jar -xf /opt/tak/takserver.war) +chown -R tak:tak /opt/tak/extract + +TAKSERVER_COMMON_JAR=$(ls extract/WEB-INF/lib/takserver-common-*.jar) +VERSION=$(echo $(basename $TAKSERVER_COMMON_JAR) | sed -En "s/takserver-common-(.+)\.jar/\1/p") + +if [ -f "/opt/tak/takserver-api.sh" ]; then + if [ ! -f "/opt/tak/takserver-api.sh.bak" ]; then + mv /opt/tak/takserver-api.sh /opt/tak/takserver-api.sh.bak + fi + + cat << EOF > /opt/tak/takserver-api.sh +#!/bin/sh +. ./setenv.sh + +java -verbose:class -Xmx\${API_MAX_HEAP}m \\ + -XX:+CompactStrings \\ + -Dspring.profiles.active=api \\ + -Dkeystore.pkcs12.legacy \\ + -Dorg.xml.sax.parser="com.sun.org.apache.xerces.internal.parsers.SAXParser" \\ + -Djavax.xml.parsers.DocumentBuilderFactory="com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl" \\ + -Djavax.xml.parsers.SAXParserFactory="com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl" \\ + -classpath "extract/WEB-INF/lib/takserver-common-${VERSION}.jar:extract/WEB-INF/lib/takserver-war-${VERSION}.jar:extract/WEB-INF/lib/takserver-fig-core-${VERSION}.jar:extract/WEB-INF/lib/takserver-plugins-${VERSION}.jar:extract/WEB-INF/lib/*:extract/WEB-INF/classes/*:extract/WEB-INF/classes/." \\ + tak.server.ServerConfiguration $@ + +# To run with plugin support enable in CoreConfig and place plugin jars in /opt/tak/lib. +EOF + echo "New takserver-api.sh written" + chown tak:tak /opt/tak/takserver-api.sh + chmod u+x /opt/tak/takserver-api.sh +fi + +if [ -f "/opt/tak/takserver-messaging.sh" ]; then + if [ ! -f "/opt/tak/takserver-messaging.sh.bak" ]; then + mv /opt/tak/takserver-messaging.sh /opt/tak/takserver-messaging.sh.bak + fi + + cat << EOF > /opt/tak/takserver-messaging.sh +#!/bin/sh +rm -rf ./tmp/ +. ./setenv.sh + +java -Xmx\${MESSAGING_MAX_HEAP}m \\ + -XX:+CompactStrings \\ + -Dspring.profiles.active=messaging \\ + -Dorg.xml.sax.parser="com.sun.org.apache.xerces.internal.parsers.SAXParser" \\ + -Djavax.xml.parsers.DocumentBuilderFactory="com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl" \\ + -Djavax.xml.parsers.SAXParserFactory="com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl" \\ + -classpath "extract/WEB-INF/lib/takserver-common-${VERSION}.jar:extract/WEB-INF/lib/takserver-war-${VERSION}.jar:extract/WEB-INF/lib/takserver-fig-core-${VERSION}.jar:extract/WEB-INF/lib/takserver-plugins-${VERSION}.jar:extract/WEB-INF/lib/*:extract/WEB-INF/classes/*:extract/WEB-INF/classes/." \\ + tak.server.ServerConfiguration $@ + +# To run with plugin support enable in CoreConfig and place plugin jars in /opt/tak/lib. +EOF + echo "New takserver-messaging.sh written" + chown tak:tak /opt/tak/takserver-messaging.sh + chmod u+x /opt/tak/takserver-messaging.sh +fi +echo +echo \ No newline at end of file diff --git a/src/takserver-core/scripts/takserver.sh b/src/takserver-core/scripts/takserver.sh index 154549e1..eea8ed81 100644 --- a/src/takserver-core/scripts/takserver.sh +++ b/src/takserver-core/scripts/takserver.sh @@ -1,4 +1,4 @@ - #!/bin/sh +#!/bin/sh . ./setenv.sh diff --git a/src/takserver-core/scripts/utils/takserver-combined b/src/takserver-core/scripts/utils/takserver-combined index a115c3a2..74b852eb 100644 --- a/src/takserver-core/scripts/utils/takserver-combined +++ b/src/takserver-core/scripts/utils/takserver-combined @@ -1,4 +1,9 @@ #!/bin/bash +### BEGIN INIT INFO +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: takserver-combined init script +### END INIT INFO # # /etc/rc.d/init.d/takserver-combined # @@ -20,7 +25,9 @@ # description: Team Awareness Kit (TAK) monolith service. # Source function library -. /etc/rc.d/init.d/functions +if [ -f /etc/rc.d/init.d/functions ]; then + . /etc/rc.d/init.d/functions +fi SERVICE="TAK server combined" TAK_HOME=/opt/tak diff --git a/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/AbstractFederationTests.java b/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/AbstractFederationTests.java index 84035610..d7b73baa 100644 --- a/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/AbstractFederationTests.java +++ b/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/AbstractFederationTests.java @@ -34,8 +34,8 @@ public static void setup() { throw new RuntimeException(e); } // Federate things tend to take a little longer to propagate... - engine.setSleepMultiplier(3.0); - engine.setSendValidationDelayMultiplier(10); + engine.setSleepMultiplier(4.0); + engine.setSendValidationDelayMultiplier(20); } public void executeBasicFederationTest(boolean useV1Federation, boolean useV2Federation, String sessionIdentifier) { @@ -70,7 +70,12 @@ public void executeBasicFederationTest(boolean useV1Federation, boolean useV2Fed e.printStackTrace(System.err); Assert.fail(e.getMessage()); } finally { - engine.stopServers(testServers); + try { + engine.stopServers(testServers); + } catch (Exception e) { + System.err.println(e.getMessage()); + e.printStackTrace(System.err); + } } } @@ -126,7 +131,12 @@ public void executeBasicMultiInputFederationTest(boolean useV1Federation, boolea e.printStackTrace(System.err); Assert.fail(e.getMessage()); } finally { - engine.stopServers(testServers); + try { + engine.stopServers(testServers); + } catch (Exception e) { + System.err.println(e.getMessage()); + e.printStackTrace(System.err); + } } } @@ -205,7 +215,12 @@ public void executeAdvancedFederationTest(boolean useV1Federation, boolean useV2 e.printStackTrace(System.err); Assert.fail(e.getMessage()); } finally { - engine.stopServers(testServers); + try { + engine.stopServers(testServers); + } catch (Exception e) { + System.err.println(e.getMessage()); + e.printStackTrace(System.err); + } } } @@ -267,7 +282,12 @@ public void executeFederateConnectionInitiatorWaitTest(boolean useV1Federation, e.printStackTrace(System.err); Assert.fail(e.getMessage()); } finally { - engine.stopServers(testServers); + try { + engine.stopServers(testServers); + } catch (Exception e) { + System.err.println(e.getMessage()); + e.printStackTrace(System.err); + } } } diff --git a/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/FedHubTests.java b/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/FedHubTests.java new file mode 100644 index 00000000..2d46bee5 --- /dev/null +++ b/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/FedHubTests.java @@ -0,0 +1,417 @@ +package com.bbn.marti.tests; + +import java.io.File; +import java.io.FileInputStream; +import java.security.KeyStore; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Enumeration; +import java.util.Map; +import java.util.UUID; + +import javax.net.ssl.TrustManagerFactory; + +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.bbn.marti.takcl.SSLHelper; +import com.bbn.marti.takcl.connectivity.server.AbstractRunnableServer; +import com.bbn.marti.takcl.connectivity.server.ServerProcessDefinition; +import com.bbn.marti.test.shared.AbstractTestClass; +import com.bbn.marti.test.shared.data.generated.ImmutableUsers; +import com.bbn.marti.test.shared.data.servers.ImmutableServerProfiles; +import com.bbn.marti.test.shared.engines.ActionEngine; +import com.bbn.marti.test.shared.engines.TestEngine; +import com.bbn.roger.fig.FederationUtils; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +import tak.server.federation.hub.broker.FederationHubServerConfig; +import tak.server.federation.hub.ui.graph.EdgeCell; +import tak.server.federation.hub.ui.graph.EdgeProperties; +import tak.server.federation.hub.ui.graph.FederateOutgoingProperties; +import tak.server.federation.hub.ui.graph.FederationOutgoingCell; +import tak.server.federation.hub.ui.graph.FederationPolicyModel; +import tak.server.federation.hub.ui.graph.FilterNode; +import tak.server.federation.hub.ui.graph.GroupCell; +import tak.server.federation.hub.ui.graph.GroupProperties; +import tak.server.federation.hub.ui.graph.PolicyObjectCell; + +public class FedHubTests extends AbstractTestClass { + private static final ImmutableServerProfiles[] testServers = new ImmutableServerProfiles[]{ImmutableServerProfiles.SERVER_0, ImmutableServerProfiles.SERVER_1, ImmutableServerProfiles.SERVER_2, ImmutableServerProfiles.FEDHUB_0, ImmutableServerProfiles.FEDHUB_1}; + + + @BeforeClass + public static void setup() { + try { + SSLHelper.genCertsIfNecessary(); + if (engine != null) { + engine.engineFactoryReset(); + } + AbstractRunnableServer.setLogDirectory(TEST_ARTIFACT_DIRECTORY); + com.bbn.marti.takcl.TestLogger.setFileLogging(TEST_ARTIFACT_DIRECTORY); + engine = new TestEngine(defaultServerProfile); + } catch (Exception e) { + System.err.println(e.getMessage()); + e.printStackTrace(System.err); + throw new RuntimeException(e); + } + // Federate things tend to take a little longer to propagate... + engine.setSleepMultiplier(4.0); + engine.setSendValidationDelayMultiplier(20); + } + + private FederationHubServerConfig getFederationHubConfig() { + FederationHubServerConfig fedBrokerConfig = null; + try { + String DEFAULT_FEDERATION_BROKER_CONFIG_FILE = ImmutableServerProfiles.FEDHUB_0.getServerPath() + "/federation-hub/configs/federation-hub-broker.yml"; + + fedBrokerConfig = new ObjectMapper(new YAMLFactory()).readValue(new FileInputStream(DEFAULT_FEDERATION_BROKER_CONFIG_FILE), FederationHubServerConfig.class); + + fedBrokerConfig.setKeystoreFile("/opt/tak/TEST_RESULTS/TEST_CERTS/" + ImmutableServerProfiles.FEDHUB_0.getConsistentUniqueReadableIdentifier() + ".jks"); + fedBrokerConfig.setTruststoreFile("/opt/tak/TEST_RESULTS/TEST_CERTS/fed-truststore.jks"); + fedBrokerConfig.setV1Port(ImmutableServerProfiles.FEDHUB_0.getFederationV1ServerPort()); + fedBrokerConfig.setV2Port(ImmutableServerProfiles.FEDHUB_0.getFederationV2ServerPort()); + fedBrokerConfig.setId(ImmutableServerProfiles.FEDHUB_0.getConsistentUniqueReadableIdentifier()); + + ObjectMapper om = new ObjectMapper(new YAMLFactory()); + om.writeValue(new File(DEFAULT_FEDERATION_BROKER_CONFIG_FILE), fedBrokerConfig); + + } catch (Exception e) { + e.printStackTrace(System.err); + } + return fedBrokerConfig; + } + + private String getHubCAGroupName(FederationHubServerConfig fedBrokerConfig) { + KeyStore trust = null; + try { + TrustManagerFactory trustMgrFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trust = KeyStore.getInstance(fedBrokerConfig.getTruststoreType()); + trust.load(new FileInputStream(fedBrokerConfig.getTruststoreFile()), fedBrokerConfig.getTruststorePassword().toCharArray()); + trustMgrFactory.init(trust); + + for (Enumeration e = trust.aliases(); e.hasMoreElements();) { + String alias = e.nextElement(); + X509Certificate cert = (X509Certificate)trust.getCertificate(alias); + String issuerName = cert.getIssuerX500Principal().getName(); + String groupName = issuerName + "-" + FederationUtils.getBytesSHA256(cert.getEncoded()); + } + + CertificateFactory factory = CertificateFactory.getInstance("X.509"); + X509Certificate ca = (X509Certificate)factory.generateCertificate(new FileInputStream(new File("/opt/tak/TEST_RESULTS/TEST_CERTS/ca.pem"))); + String issuerName = ca.getIssuerX500Principal().getName(); + return issuerName + "-" + FederationUtils.getBytesSHA256(ca.getEncoded()); + } catch (Exception e) { + e.printStackTrace(System.err); + } + return null; + } + + private void createPolicyForHubCA(FederationHubServerConfig fedBrokerConfig) { + // get the ca name from our hub cert + String caGroupName = getHubCAGroupName(fedBrokerConfig); + + FederationPolicyModel policyModel = new FederationPolicyModel(); + GroupCell group1Cell = createGroupCell(caGroupName, true); + + Collection cells = new ArrayList<>(); + cells.add(group1Cell); + + policyModel.setCells(cells); + policyModel.setName(this.getClass().getName()); + + String DEFAULT_FEDERATION_POLICY_CONFIG_FILE = ImmutableServerProfiles.FEDHUB_0.getServerPath() + "/federation-hub/ui_generated_policy.json"; + ObjectMapper mapper = new ObjectMapper(); + try { + policyModel.toString(); + policyModel.getFederationPolicyObjectFromModel(); + mapper.writerWithDefaultPrettyPrinter().writeValue(new File(DEFAULT_FEDERATION_POLICY_CONFIG_FILE), policyModel.getFederationPolicyObjectFromModel()); + } catch (Exception e) { + e.printStackTrace(); + System.out.println("Could not write policy to file " + e.getStackTrace()); + } + } + + @Test(timeout = 920000) + public void basicFedHubTest() { + try { + String sessionIdentifier = initTestMethod(); + + // initial setup of the hub without actually starting it + enableFedHub(); + ActionEngine.getRunnableInstanceAndBuildIfnecessary(ImmutableServerProfiles.FEDHUB_0); + // get fedhub broker config + FederationHubServerConfig fedBrokerConfig = getFederationHubConfig(); + // create policy for allowing connections from the hub's ca (all servers will have same CA, aka interconnected) + createPolicyForHubCA(fedBrokerConfig); + + // start fedhub + engine.startServer(ImmutableServerProfiles.FEDHUB_0, sessionIdentifier); + + // disable all the fedhub flags so we can go back to starting TAK Servers + enableTakServer(); + + engine.offlineFederateServers(false, true, ImmutableServerProfiles.SERVER_0, ImmutableServerProfiles.SERVER_1); + engine.offlineAddOutboundFederateConnection(true, ImmutableServerProfiles.SERVER_0, ImmutableServerProfiles.FEDHUB_0); + engine.offlineAddOutboundFederateConnection(true, ImmutableServerProfiles.SERVER_1, ImmutableServerProfiles.FEDHUB_0); + engine.offlineAddFederate(ImmutableServerProfiles.SERVER_0, ImmutableServerProfiles.FEDHUB_0); + engine.offlineAddFederate(ImmutableServerProfiles.SERVER_1, ImmutableServerProfiles.FEDHUB_0); + engine.offlineAddOutboundFederateGroup(ImmutableServerProfiles.SERVER_0, ImmutableServerProfiles.FEDHUB_0, "group0"); + engine.offlineAddOutboundFederateGroup(ImmutableServerProfiles.SERVER_1, ImmutableServerProfiles.FEDHUB_0, "group0"); + engine.offlineAddInboundFederateGroup(ImmutableServerProfiles.SERVER_0, ImmutableServerProfiles.FEDHUB_0, "group0"); + engine.offlineAddInboundFederateGroup(ImmutableServerProfiles.SERVER_1, ImmutableServerProfiles.FEDHUB_0, "group0"); + engine.offlineAddUsersAndConnectionsIfNecessary(ImmutableUsers.s0_stcp0_anonuser_0f); + engine.offlineAddUsersAndConnectionsIfNecessary(ImmutableUsers.s1_stcp0_anonuser_0f); + + // start both TAK Servers + engine.startServer(ImmutableServerProfiles.SERVER_0, sessionIdentifier); + engine.startServer(ImmutableServerProfiles.SERVER_1, sessionIdentifier); + +// // Inserting sleep since the servers need some time to federate + Thread.sleep(30000); + + engine.connectClientAndVerify(true, ImmutableUsers.s0_stcp0_anonuser_0f); + engine.connectClientAndVerify(true, ImmutableUsers.s1_stcp0_anonuser_0f); + + engine.attemptSendFromUserAndVerify(ImmutableUsers.s0_stcp0_anonuser_0f); + engine.attemptSendFromUserAndVerify(ImmutableUsers.s1_stcp0_anonuser_0f); + } catch (Exception e) { + System.err.println(e.getMessage()); + e.printStackTrace(System.err); + Assert.fail(e.getMessage()); + } finally { + try { + Thread.sleep(10000); + engine.stopServers(testServers); + } catch (Exception e) { + System.err.println(e.getMessage()); + e.printStackTrace(System.err); + } + } + } + + @Test(timeout = 960000) + public void basicMultiInputFedHubTest() { + try { + String sessionIdentifier = initTestMethod(); + + // initial setup of the hub without actually starting it + enableFedHub(); + ActionEngine.getRunnableInstanceAndBuildIfnecessary(ImmutableServerProfiles.FEDHUB_0); + // get fedhub broker config + FederationHubServerConfig fedBrokerConfig = getFederationHubConfig(); + // create policy for allowing connections from the hub's ca (all servers will have same CA, aka interconnected) + createPolicyForHubCA(fedBrokerConfig); + + // start fedhub + engine.startServer(ImmutableServerProfiles.FEDHUB_0, sessionIdentifier); + + // disable all the fedhub flags so we can go back to starting TAK Servers + enableTakServer(); + + engine.offlineFederateServers(false, true, ImmutableServerProfiles.SERVER_0, ImmutableServerProfiles.SERVER_2); + engine.offlineAddOutboundFederateConnection(true, ImmutableServerProfiles.SERVER_0, ImmutableServerProfiles.FEDHUB_0); + engine.offlineAddOutboundFederateConnection(true, ImmutableServerProfiles.SERVER_2, ImmutableServerProfiles.FEDHUB_0); + engine.offlineAddFederate(ImmutableServerProfiles.SERVER_0, ImmutableServerProfiles.FEDHUB_0); + engine.offlineAddFederate(ImmutableServerProfiles.SERVER_2, ImmutableServerProfiles.FEDHUB_0); + engine.offlineAddInboundFederateGroup(ImmutableServerProfiles.SERVER_0, ImmutableServerProfiles.FEDHUB_0, "group0"); + engine.offlineAddInboundFederateGroup(ImmutableServerProfiles.SERVER_2, ImmutableServerProfiles.FEDHUB_0, "group0"); + engine.offlineAddOutboundFederateGroup(ImmutableServerProfiles.SERVER_0, ImmutableServerProfiles.FEDHUB_0, "group0"); + engine.offlineAddOutboundFederateGroup(ImmutableServerProfiles.SERVER_2, ImmutableServerProfiles.FEDHUB_0, "group0"); + + ImmutableUsers[] users = new ImmutableUsers[]{ + ImmutableUsers.s0_authstcp_authuser01_01f, + ImmutableUsers.s2_authstcp_authuser01_01f, + ImmutableUsers.s0_ssl_anonuser_t, + ImmutableUsers.s2_ssl_anonuser_t, + ImmutableUsers.s0_stcp0_anonuser_0f, + ImmutableUsers.s2_stcp0_anonuser_0f, + ImmutableUsers.s0_stcp12_anonuser_12f, + ImmutableUsers.s2_stcp12_anonuser_12f, + ImmutableUsers.s0_udp12t_anonuser_12t, + ImmutableUsers.s2_udp12t_anonuser_12t + }; + + for (ImmutableUsers user : users) { + engine.offlineAddUsersAndConnectionsIfNecessary(user); + } + + engine.startServer(ImmutableServerProfiles.SERVER_0, sessionIdentifier); + engine.startServer(ImmutableServerProfiles.SERVER_2, sessionIdentifier); + + // Inserting sleep since the servers need some time to federate + Thread.sleep(30000); + + for (ImmutableUsers user : users) { + if (user.getConnection().getProtocol().canConnect()) { + engine.connectClientAndVerify(true, user); + } + } + + for (ImmutableUsers user : users) { + engine.attemptSendFromUserAndVerify(user); + } + + + } catch (InterruptedException e) { + System.err.println(e.getMessage()); + e.printStackTrace(System.err); + Assert.fail(e.getMessage()); + } finally { + try { + engine.stopServers(testServers); + } catch (Exception e) { + System.err.println(e.getMessage()); + e.printStackTrace(System.err); + } + } + } + + @Test(timeout = 8400000) + public void advancedFedHubTest() { + try { + String sessionIdentifier = initTestMethod(); + + // initial setup of the hub without actually starting it + enableFedHub(); + ActionEngine.getRunnableInstanceAndBuildIfnecessary(ImmutableServerProfiles.FEDHUB_0); + // get fedhub broker config + FederationHubServerConfig fedBrokerConfig = getFederationHubConfig(); + // create policy for allowing connections from the hub's ca (all servers will have same CA, aka interconnected) + createPolicyForHubCA(fedBrokerConfig); + + // start fedhub + engine.startServer(ImmutableServerProfiles.FEDHUB_0, sessionIdentifier); + + // disable all the fedhub flags so we can go back to starting TAK Servers + enableTakServer(); + + engine.offlineFederateServers(false, true, ImmutableServerProfiles.SERVER_0, ImmutableServerProfiles.SERVER_1); + + engine.offlineAddOutboundFederateConnection(true, ImmutableServerProfiles.SERVER_0, ImmutableServerProfiles.FEDHUB_0); + engine.offlineAddOutboundFederateConnection(true, ImmutableServerProfiles.SERVER_1, ImmutableServerProfiles.FEDHUB_0); + + engine.offlineAddFederate(ImmutableServerProfiles.SERVER_0, ImmutableServerProfiles.FEDHUB_0); + engine.offlineAddFederate(ImmutableServerProfiles.SERVER_1, ImmutableServerProfiles.FEDHUB_0); + + engine.offlineAddOutboundFederateGroup(ImmutableServerProfiles.SERVER_0, ImmutableServerProfiles.FEDHUB_0, "group0"); + engine.offlineAddOutboundFederateGroup(ImmutableServerProfiles.SERVER_0, ImmutableServerProfiles.FEDHUB_0, "group2"); + engine.offlineAddOutboundFederateGroup(ImmutableServerProfiles.SERVER_1, ImmutableServerProfiles.FEDHUB_0, "group0"); + engine.offlineAddOutboundFederateGroup(ImmutableServerProfiles.SERVER_1, ImmutableServerProfiles.FEDHUB_0, "group1"); + + engine.offlineAddInboundFederateGroup(ImmutableServerProfiles.SERVER_0, ImmutableServerProfiles.FEDHUB_0, "group1"); + engine.offlineAddInboundFederateGroup(ImmutableServerProfiles.SERVER_1, ImmutableServerProfiles.FEDHUB_0, "group2"); + + ImmutableUsers[] users = new ImmutableUsers[]{ + ImmutableUsers.s0_authstcp_authuser01_01f, + ImmutableUsers.s1_authstcp_authuser01_01f, + ImmutableUsers.s0_ssl_anonuser_t, + ImmutableUsers.s1_ssl_anonuser_t, + ImmutableUsers.s0_stcp0_anonuser_0f, + ImmutableUsers.s1_stcp0_anonuser_0f, + ImmutableUsers.s0_stcp12_anonuser_12f, + ImmutableUsers.s1_stcp12_anonuser_12f, + ImmutableUsers.s0_udp12t_anonuser_12t, + ImmutableUsers.s1_udp12t_anonuser_12t, + }; + + for (ImmutableUsers user : users) { + engine.offlineAddUsersAndConnectionsIfNecessary(user); + } + + engine.startServer(ImmutableServerProfiles.SERVER_0, sessionIdentifier); + engine.startServer(ImmutableServerProfiles.SERVER_1, sessionIdentifier); + + // Inserting sleep since the servers need some time to federate + Thread.sleep(30000); + + for (ImmutableUsers user : users) { + if (user.getConnection().getProtocol().canConnect()) { + engine.connectClientAndVerify(true, user); + } + } + + for (ImmutableUsers user : users) { + engine.attemptSendFromUserAndVerify(user); + } + + } catch (InterruptedException e) { + System.err.println(e.getMessage()); + e.printStackTrace(System.err); + Assert.fail(e.getMessage()); + } finally { + try { + engine.stopServers(testServers); + } catch (Exception e) { + System.err.println(e.getMessage()); + e.printStackTrace(System.err); + } + } + } + + private void enableFedHub() { + ServerProcessDefinition.FederationHubPolicy.setEnabled(true); + ServerProcessDefinition.FederationHubBroker.setEnabled(true); + ServerProcessDefinition.MessagingService.setEnabled(false); + ServerProcessDefinition.ApiService.setEnabled(false); + ServerProcessDefinition.RetentionService.setEnabled(false); + ServerProcessDefinition.PluginManager.setEnabled(false); + } + + private void enableTakServer() { + ServerProcessDefinition.MessagingService.setEnabled(true); + ServerProcessDefinition.ApiService.setEnabled(true); + ServerProcessDefinition.FederationHubPolicy.setEnabled(false); + ServerProcessDefinition.FederationHubBroker.setEnabled(false); + ServerProcessDefinition.RetentionService.setEnabled(false); + ServerProcessDefinition.PluginManager.setEnabled(false); + } + + private GroupCell createGroupCell(String caName, boolean interconnected) { + GroupProperties groupProps = new GroupProperties(); + groupProps.setFilters(new ArrayList()); + groupProps.setAttributes(new ArrayList()); + groupProps.setName(caName); + groupProps.setId(caName); + groupProps.setInterconnected(interconnected); + + GroupCell groupCell = new GroupCell(); + groupCell.setProperties(groupProps); + groupCell.setId(UUID.randomUUID().toString()); + + return groupCell; + } + + private EdgeCell createEdgeCell(GroupCell source, GroupCell dest) { + EdgeProperties edgeProperties = new EdgeProperties(); + edgeProperties.setFilters(new ArrayList()); + + EdgeCell edgeCell = new EdgeCell(); + edgeCell.setProperties(edgeProperties); + edgeCell.setId(UUID.randomUUID().toString()); + edgeCell.addOther("source", Map.of("id", source.getId())); + edgeCell.addOther("target", Map.of("id", dest.getId())); + + return edgeCell; + } + + private FederationOutgoingCell createOutgoingCell(String host, int port, String name, boolean enabled) { + FederateOutgoingProperties outgoingProps = new FederateOutgoingProperties(); + outgoingProps.setHost(host); + outgoingProps.setPort(port); + outgoingProps.setName(name); + outgoingProps.setOutgoingName(name); + outgoingProps.setId(name); + outgoingProps.setOutgoingEnabled(enabled); + + FederationOutgoingCell outgoing = new FederationOutgoingCell(); + outgoing.setProperties(outgoingProps); + outgoing.setId(UUID.randomUUID().toString()); + + return outgoing; + } +} diff --git a/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/PluginStartupTests.java b/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/PluginStartupTests.java new file mode 100644 index 00000000..9dbfd69e --- /dev/null +++ b/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/PluginStartupTests.java @@ -0,0 +1,62 @@ +package com.bbn.marti.tests; + +import com.bbn.marti.test.shared.AbstractTestClass; +import com.bbn.marti.takcl.connectivity.server.ServerProcessDefinition; +import com.bbn.marti.test.shared.data.generated.ImmutableConnections; +import com.bbn.marti.test.shared.data.generated.ImmutableUsers; +import com.bbn.marti.test.shared.data.servers.ImmutableServerProfiles; +import com.bbn.marti.test.shared.engines.ActionEngine; + +import javax.annotation.concurrent.Immutable; + +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class PluginStartupTests extends AbstractTestClass { + + private static final String className = "PluginStartupTests"; + + @BeforeClass + public static void setup() { + ServerProcessDefinition.PluginManager.setEnabled(true); + ServerProcessDefinition.RetentionService.setEnabled(false); + AbstractTestClass.setup(); + } + + @Test(timeout = 420000) + public void pluginStartupValiationTest() { + try { + String sessionIdentifier = initTestMethod(); + + System.out.println("--- Starting offlineAddUsersAndConnectionsIfNecessary for s0_stcp_anonuser_t_A ..."); + engine.offlineAddUsersAndConnectionsIfNecessary(ImmutableUsers.s0_stcp_anonuser_t_A); + System.out.println("--- Done with offlineAddUsersAndConnectionsIfNecessary for s0_stcp_anonuser_t_A"); + + System.out.println("--- Starting offlineEnableLatestSA for SERVER_0 ..."); + engine.offlineEnableLatestSA(true, ImmutableServerProfiles.SERVER_0); + System.out.println("--- Done with offlineEnableLatestSA for SERVER_0"); + + System.out.println("--- Starting startServerWithStartupValidation ..."); + engine.startServerWithStartupValidation(ImmutableServerProfiles.SERVER_0, sessionIdentifier, true, false); + System.out.println("--- Done with startServerWithStartupValidation"); + + System.out.println("--- Starting connectClientsAndVerify ..."); + engine.connectClientsAndVerify(true, ImmutableUsers.s0_stcp_anonuser_t_A); + System.out.println("--- Done with connectClientsAndVerify"); + + System.out.println("Sleep for 10s..."); + Thread.sleep(10000); + + System.out.println("--- Starting verifyReceivedMessageSentFromPlugin ..."); + engine.verifyReceivedMessageSentFromPlugin(ImmutableUsers.s0_stcp_anonuser_t_plugin1, ImmutableUsers.s0_stcp_anonuser_t_A); + System.out.println("--- Done with verifyReceivedMessageSentFromPlugin"); + + } catch (Exception e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } finally { + engine.stopServers(ImmutableServerProfiles.SERVER_0); + } + } +} diff --git a/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/StartupTests.java b/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/StartupTests.java index 3864a2f7..1ac17a5f 100644 --- a/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/StartupTests.java +++ b/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/StartupTests.java @@ -1,11 +1,16 @@ package com.bbn.marti.tests; +import com.bbn.marti.takcl.SSLHelper; +import com.bbn.marti.takcl.connectivity.server.AbstractRunnableServer; +import com.bbn.marti.takcl.connectivity.server.ServerProcessDefinition; import com.bbn.marti.test.shared.AbstractTestClass; import com.bbn.marti.test.shared.data.generated.ImmutableConnections; import com.bbn.marti.test.shared.data.generated.ImmutableUsers; import com.bbn.marti.test.shared.data.servers.ImmutableServerProfiles; import com.bbn.marti.test.shared.engines.ActionEngine; +import com.bbn.marti.test.shared.engines.TestEngine; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; /** @@ -15,6 +20,13 @@ public class StartupTests extends AbstractTestClass { private static final String className = "StartupTests"; + @BeforeClass + public static void setup() { + ServerProcessDefinition.PluginManager.setEnabled(false); + ServerProcessDefinition.RetentionService.setEnabled(true); + AbstractTestClass.setup(); + } + @Test(timeout = 420000) public void jarStartupValiationTest() { try { @@ -23,7 +35,7 @@ public void jarStartupValiationTest() { engine.offlineAddUsersAndConnectionsIfNecessary(ImmutableUsers.s0_stcp_anonuser_t_A); engine.offlineAddUsersAndConnectionsIfNecessary(ImmutableUsers.s0_stcp_anonuser_t_B); engine.offlineEnableLatestSA(true, ImmutableServerProfiles.SERVER_0); - engine.startServerWithStartupValidation(ImmutableServerProfiles.SERVER_0, sessionIdentifier, true, true); + engine.startServerWithStartupValidation(ImmutableServerProfiles.SERVER_0, sessionIdentifier, false, true); engine.connectClientsAndVerify(true, ImmutableUsers.s0_stcp_anonuser_t_A, ImmutableUsers.s0_stcp_anonuser_t_B); engine.attemptSendFromUserAndVerify(ImmutableUsers.s0_stcp_anonuser_t_A); diff --git a/src/takserver-core/src/main/java/com/bbn/cot/filter/DataFeedFilter.java b/src/takserver-core/src/main/java/com/bbn/cot/filter/DataFeedFilter.java index 1f8d1d43..8e4bcdb4 100644 --- a/src/takserver-core/src/main/java/com/bbn/cot/filter/DataFeedFilter.java +++ b/src/takserver-core/src/main/java/com/bbn/cot/filter/DataFeedFilter.java @@ -2,6 +2,8 @@ import java.util.ArrayList; import java.util.List; +import java.util.NavigableSet; +import java.util.concurrent.ConcurrentSkipListSet; import java.util.stream.Collectors; import org.dom4j.DocumentHelper; @@ -15,6 +17,9 @@ import com.bbn.marti.config.GeospatialFilter; import com.bbn.marti.config.GeospatialFilter.BoundingBox; import com.bbn.marti.feeds.DataFeedService; +import com.bbn.marti.remote.groups.Direction; +import com.bbn.marti.remote.groups.Group; +import com.bbn.marti.remote.groups.GroupManager; import com.bbn.marti.remote.InputMetric; import com.bbn.marti.remote.exception.TakException; import com.bbn.marti.service.DistributedConfiguration; @@ -22,7 +27,6 @@ import com.bbn.marti.service.SubscriptionStore; import com.bbn.marti.sync.model.DataFeedDao; import com.bbn.marti.sync.model.MinimalMission; -import com.bbn.marti.sync.model.Mission; import com.bbn.marti.sync.service.DistributedDataFeedCotService; import com.bbn.marti.util.GeomUtils; import com.bbn.marti.util.MessagingDependencyInjectionProxy; @@ -46,10 +50,13 @@ public class DataFeedFilter { private static DataFeedFilter instance = null; @Autowired - DataFeedService dataFeedService; - + private DataFeedService dataFeedService; + + @Autowired + private GroupManager groupManager; + @Autowired - ObjectMapper mapper; + private ObjectMapper mapper; public static DataFeedFilter getInstance() { if (instance == null) { @@ -64,7 +71,12 @@ public static DataFeedFilter getInstance() { public void filter(CotEventContainer cot, DataFeed dataFeed) { + if (logger.isDebugEnabled()) { + logger.debug("Calling filter for dataFeed: {}, {}", dataFeed.getName(), dataFeed.getUuid()); + } + if (cot != null && dataFeed != null) { + cot.setContext(Constants.DATA_FEED_KEY, dataFeed); cot.setContext(Constants.DATA_FEED_UUID_KEY, dataFeed.getUuid()); cot.setContext(Constants.ARCHIVE_EVENT_KEY, dataFeed.isArchive()); @@ -84,13 +96,26 @@ public void filter(CotEventContainer cot, DataFeed dataFeed) { tagElement.addText(tag); sourceElement.add(tagElement); }); - + + if (dataFeed.getFiltergroup() != null && !dataFeed.getFiltergroup().isEmpty()) { + NavigableSet groups = new ConcurrentSkipListSet<>(); + dataFeed.getFiltergroup().forEach(groupName -> + groups.add(groupManager.hydrateGroup(new Group(groupName, Direction.IN)))); + + if (logger.isDebugEnabled()) { + logger.debug("Setting groups on datafeed cot: group count {} groups: {}", groups.size(), groups); + } + + cot.setContext(Constants.GROUPS_KEY, groups); + } + // submit data feed message for in memory caching DistributedDataFeedCotService.getInstance().cacheDataFeedEvent(dataFeed, cot); // if vbm is enabled, only broker messages to clients subscribed to a mission that is linked to this data feed // this is achieved by adding an explicit endpoint for the cot, meaning it won't hit implicit brokering if (DistributedConfiguration.getInstance().getRemoteConfiguration().getVbm().isEnabled()) { + String dataFeedUuid = (String) cot.getContextValue(Constants.DATA_FEED_UUID_KEY); if (logger.isTraceEnabled()) { @@ -103,35 +128,59 @@ public void filter(CotEventContainer cot, DataFeed dataFeed) { // deserialize the mission from JSON to address pokey binary marshaller try { for (String missionJson : MessagingDependencyInjectionProxy.getInstance().missionService().getMinimalMissionsJsonForDataFeed(dataFeedUuid)) { + if (logger.isDebugEnabled()) { + logger.debug("missionJson: {}", missionJson); + + } MinimalMission m = mapper.readValue(missionJson, MinimalMission.class); feedMissions.add(m); } } catch (JsonProcessingException e) { throw new TakException(e); } - + if (logger.isDebugEnabled()) { logger.debug("deserialized " + feedMissions.size() + " feed missions from JSON"); + logger.debug("DistributedConfiguration.getInstance().getRemoteConfiguration().getNetwork().getMissionCopTool(): {}", DistributedConfiguration.getInstance().getRemoteConfiguration().getNetwork().getMissionCopTool()); } // if there was a vbm match and we're mission federating, pass the data feed message to each fig client boolean vbmMatch = feedMissions.stream().anyMatch(m -> DistributedConfiguration.getInstance().getRemoteConfiguration().getNetwork().getMissionCopTool().equals(m.getTool().toLowerCase())); if (vbmMatch && isMissionDataFeedFederation()) { - // submit data feed message to each fig federate. - SubscriptionStore.getInstanceFederatedSubscriptionManager().getFederateSubscriptions().forEach(fedSub -> { - if (fedSub instanceof FigFederateSubscription) { - FigFederateSubscription figSub = (FigFederateSubscription) fedSub; - try { - figSub.submit(cot); - } catch (Exception e) { - logger.error("Could not submit data feed Cot to Fig Sub", e); + + DataFeedDao dataFeedDao = dataFeedService.getDataFeedByUid(dataFeed.getUuid()); + + if (dataFeedDao.getFederated()) { // whether or not to federate per datafeed + + if (logger.isDebugEnabled()) { + logger.debug("Sending datafeed {} to federation", dataFeed.getUuid()); + } + + // submit data feed message to each fig federate. + SubscriptionStore.getInstanceFederatedSubscriptionManager().getFederateSubscriptions().forEach(fedSub -> { + if (fedSub instanceof FigFederateSubscription) { + FigFederateSubscription figSub = (FigFederateSubscription) fedSub; + try { + figSub.submit(cot); + } catch (Exception e) { + logger.error("Could not submit data feed Cot to Fig Sub", e); + } } + }); + }else { + if (logger.isDebugEnabled()) { + logger.debug("Not sending datafeed {} to federation", dataFeed.getUuid()); } - }); + } + + } else { + if (logger.isDebugEnabled()) { + logger.debug("Not sending datafeed {} to federation because vbmMatch: {}, isMissionDataFeedFederation: {}", dataFeed.getUuid(), vbmMatch, isMissionDataFeedFederation()); + } } handleVbmMissions(cot, feedMissions); - } + } } } @@ -145,7 +194,7 @@ public void filterFederatedDataFeed(CotEventContainer cot) { // if vbm is enabled and we're mission federating, only broker messages to clients subscribed to a mission that is linked to this data feed // otherwise, this message will continue through federation brokering if (isVbm() && isMissionDataFeedFederation()) { - if (dataFeed != null) { + if (dataFeed != null && dataFeed.getFederated()) { // update fed feed counter InputMetric metric = SubmissionService.getInstance().getInputMetric(dataFeed.getName()); metric.getReadsReceived().getAndIncrement(); diff --git a/src/takserver-core/src/main/java/com/bbn/cot/filter/FlowTagFilter.java b/src/takserver-core/src/main/java/com/bbn/cot/filter/FlowTagFilter.java index 8729a787..cebc976e 100644 --- a/src/takserver-core/src/main/java/com/bbn/cot/filter/FlowTagFilter.java +++ b/src/takserver-core/src/main/java/com/bbn/cot/filter/FlowTagFilter.java @@ -62,7 +62,7 @@ public CotEventContainer filter(CotEventContainer c) { * * Intended to modify this message so that this server's broker will OK the message. * - * @note This method mutates the container's contained xml. + * This method mutates the container's contained xml. * @return Flag indicating whether the message was modified. */ public boolean unfilter(CotEventContainer c) { @@ -74,7 +74,7 @@ public boolean unfilter(CotEventContainer c) { * * Intended to modify the message so that any federation route will be successful. * - * @throw ClassCastException if the message's flow tag Nodes are not all of type element + * @throws ClassCastException if the message's flow tag Nodes are not all of type element * @return The list of server Ids that were removed from the document. Empty if nothing was removed. */ public List unfilterAll(CotEventContainer c) { @@ -97,7 +97,7 @@ public List unfilterAll(Document doc) { /** * Removes the flow tag filter for the given server id. * - * @note In the case that the given xpath is malformed due to a flawed serverId, the error is caught, and false is returned. + * In the case that the given xpath is malformed due to a flawed serverId, the error is caught, and false is returned. * * @return whether an attribute with the given serverId name was actually removed ie, whether the xpath /event/detail/_flow-tags_/@serverId existed */ diff --git a/src/takserver-core/src/main/java/com/bbn/cot/filter/Images.java b/src/takserver-core/src/main/java/com/bbn/cot/filter/Images.java index e616bce3..c5b3bf61 100644 --- a/src/takserver-core/src/main/java/com/bbn/cot/filter/Images.java +++ b/src/takserver-core/src/main/java/com/bbn/cot/filter/Images.java @@ -24,15 +24,15 @@ import com.bbn.marti.util.Tuple; /** - * This has been changed innto a static class of + * This has been changed innto a static class of * non-mutating processing utilies for images. See the ImageProcessingFilter - * and ImageFormattingFilter for detachment of the xml-encoded image from the + * and ImageFormattingFilter for detachment of the xml-encoded image from the * cot message, and then reattachment for different formats. * - * The main change here is that ImageData is now tightly coupled with + * The main change here is that ImageData is now tightly coupled with * the element that generates it. Upon construction, the ImageData object holds - * canonical versions of the byte arrays (full and thumb), as well as the XML - * element that generated those. The XML element is copy constructed from the + * canonical versions of the byte arrays (full and thumb), as well as the XML + * element that generated those. The XML element is copy constructed from the * provided one. For presentation, the processing methods will generate a new * Image Element from the ImageData object, according to the given format. For * URL annotations, the ImageData object is partially copied, with the byte @@ -48,13 +48,13 @@ */ public class Images { public static final ImagePref DEFAULT_IMAGE_PREF = ImagePref.THUMBNAIL; - + // XPath values public static final String DETAIL_PATH = "/event/detail"; public static final String IMAGE_PATH = "/event/detail/image"; public static final String IMAGE_ELEMENT_NAME = "image"; public static final String IMAGE_FORMAT = "jpg"; - + private static final Logger log = Logger.getLogger(Images.class); /** @@ -69,13 +69,13 @@ public static class ImageData { private byte[] thumbnail = null; private Element imageElem = null; private Integer primaryKey = null; - + // Fluent setter public ImageData encodedImage(String text) { this.encodedImage = text; return this; } - + public String encodedImage() { return this.encodedImage; } @@ -96,7 +96,7 @@ public ImageData thumbBytes(byte[] thumbnail) { this.thumbnail = thumbnail; return this; } - + // get thumb bytes public byte[] thumbBytes() { return this.thumbnail; @@ -107,13 +107,13 @@ public ImageData element(Element imageElement) { this.imageElem = imageElement; return this; } - + // returns a reference to the mutable element held by this data wrapper // users should copy before modifying, as changes will be reflected internally public Element element() { return this.imageElem; } - + // fluent setter for the db primary key public ImageData primaryKey(Integer primaryKey) { this.primaryKey = primaryKey; @@ -138,13 +138,13 @@ public ImageData copy() { // preexisting code public byte[] getFullResImage() { return fullResImg; } public byte[] getThumbnail() { return thumbnail; } - public Element getElement() { + public Element getElement() { Assertion.notNull(imageElem); - return imageElem; + return imageElem; } - public Integer getPrimaryKey() { + public Integer getPrimaryKey() { Assertion.notNull(primaryKey); - return primaryKey; + return primaryKey; } } @@ -172,12 +172,12 @@ public static ImageData imageDataFromElement(Element imageElem) { ByteArrayOutputStream thumbStream = new ByteArrayOutputStream(); ImageIO.write(thumbImg, IMAGE_FORMAT, thumbStream); - // create copy (obtuse logic with temporarily stripping out the text, we want to avoid potentially allocating + // create copy (obtuse logic with temporarily stripping out the text, we want to avoid potentially allocating // another text node with the encoded image data copied, though this might not be the case) imageElem.setText(""); Element clonedElem = imageElem.createCopy(); imageElem.setText(rawDataString); - + // create ImageData wrapper imageData = new ImageData() .encodedImage(rawDataString) @@ -188,7 +188,7 @@ public static ImageData imageDataFromElement(Element imageElem) { } catch (Exception e) { log.warn(" Image Conversion error: " + e.getMessage(), e); } - + return imageData; } @@ -222,7 +222,7 @@ public static Element generateElementFromData(ImageData imageData, ImagePref pre // DEBUG line Assertion.fail(); } - + return newImageElem; } @@ -273,18 +273,18 @@ public static Integer parseTextToPrimaryKey(Element imageElem) { log.error("Error reading image element text: couldn't parse to an integer"); } } - + return primaryKey; } /** * Tries to parse a group of images for a specific cot message from the database. - * - * The imageMap is an *ordered map* (insertion ordering) primary key (image database) -> full/thumb byte arrays, + * + * The imageMap is an *ordered map* (insertion ordering) primary key (image database) -{@literal >} full/thumb byte arrays, * and the imageElems list is the list of image elements in the order as retrieved by the database, where the image * element text should contain the matching image data primary key. - * - * This method attempts to pair image elements with the image data by parsing the image element text and looking + * + * This method attempts to pair image elements with the image data by parsing the image element text and looking * in the image map. If no element exists, we default to the zipped ordering */ public static List parseDatabaseFormat(Map imageMap, List imageElems) { @@ -311,7 +311,7 @@ public static List parseDatabaseFormat(Map imageMa dataList.add(toAssociate .element(imageElem.createCopy()) ); - + // remove the element from the xml tree -- will not affect list membership w/dom4j imageElem.detach(); } diff --git a/src/takserver-core/src/main/java/com/bbn/cot/filter/StreamingEndpointRewriteFilter.java b/src/takserver-core/src/main/java/com/bbn/cot/filter/StreamingEndpointRewriteFilter.java index 8ecd9a12..9a3eb085 100644 --- a/src/takserver-core/src/main/java/com/bbn/cot/filter/StreamingEndpointRewriteFilter.java +++ b/src/takserver-core/src/main/java/com/bbn/cot/filter/StreamingEndpointRewriteFilter.java @@ -9,6 +9,7 @@ import java.util.List; import java.util.NavigableSet; import java.util.Set; +import javax.naming.ldap.LdapName; import org.dom4j.Element; import org.dom4j.Node; @@ -152,7 +153,7 @@ public CotEventContainer filter(final CotEventContainer cot) { } else if(detached.attribute(UID_ATTR) != null) { uids.add(detached.attributeValue(UID_ATTR)); } else if(detached.attribute(MISSION_ATTR) != null) { - missionNames.add(detached.attributeValue(MISSION_ATTR).toLowerCase()); + missionNames.add(detached.attributeValue(MISSION_ATTR)); logger.debug("mission destination specified in message: " + detached.attributeValue(MISSION_ATTR)); } } @@ -185,8 +186,21 @@ public CotEventContainer filter(final CotEventContainer cot) { MissionSubscription missionSubscription = null; User user = (User) cot.getContextValue(Constants.USER_KEY); if (user != null) { - missionSubscription = missionSubscriptionRepository.findByMissionNameAndClientUidAndUsernameNoMission( + missionSubscription = missionSubscriptionRepository + .findByMissionNameAndClientUidAndUsernameNoMission( missionName, clientUid, user.getName()); + + if (missionSubscription == null + && user.getCert() != null && user.getCert().getSubjectX500Principal() != null) { + // lookup the mission subscription based on CN, needed when input auth=ldap or auth=file + String cn = new LdapName(user.getCert().getSubjectX500Principal().getName()) + .getRdns().stream().filter(i -> i.getType().equalsIgnoreCase("CN")) + .findFirst().get().getValue().toString(); + missionSubscription = missionSubscriptionRepository + .findByMissionNameAndClientUidAndUsernameNoMission( + missionName, clientUid, cn); + } + } else { missionSubscription = missionSubscriptionRepository.findByMissionNameAndClientUidNoMission( missionName, clientUid); @@ -308,8 +322,7 @@ public CotEventContainer filter(final CotEventContainer cot) { logger.debug("rol to federate for mission change " + rol + " to groups " + groups); } - federationManager.submitFederateROL(rol, groups); - + federationManager.submitMissionFederateROL(rol, groups, missionName); } else { logger.warn("unable to federate mission uid add - cot message specified no groups"); } diff --git a/src/takserver-core/src/main/java/com/bbn/marti/config/ConfigHelper.java b/src/takserver-core/src/main/java/com/bbn/marti/config/ConfigHelper.java index 7e76b0b3..6ceeab81 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/config/ConfigHelper.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/config/ConfigHelper.java @@ -9,7 +9,7 @@ /** * Class used to do miscellaneous tasks to the generated Configuration file such as load default objects or check * content values - *

+ *

* Created on 8/31/15. */ public class ConfigHelper { diff --git a/src/takserver-core/src/main/java/com/bbn/marti/groups/LdapSSLSocketFactory.java b/src/takserver-core/src/main/java/com/bbn/marti/groups/LdapSSLSocketFactory.java index 399df5d5..9f8cb462 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/groups/LdapSSLSocketFactory.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/groups/LdapSSLSocketFactory.java @@ -1,5 +1,6 @@ package com.bbn.marti.groups; +import com.bbn.marti.config.Tls; import com.bbn.marti.service.DistributedConfiguration; import javax.net.SocketFactory; import javax.net.ssl.SSLContext; @@ -61,7 +62,8 @@ public LdapSSLSocketFactory() { KeyStore keyStore = KeyStore.getInstance(truststore); keyStore.load(new FileInputStream(truststoreFile), truststorePass.toCharArray()); trustManagerFactory.init(keyStore); - SSLContext sslContext = SSLContext.getInstance("TLS"); + Tls tlsConfig = config.getRemoteConfiguration().getSecurity().getTls(); + SSLContext sslContext = SSLContext.getInstance(tlsConfig.getContext()); sslContext.init(null, trustManagerFactory.getTrustManagers(), new java.security.SecureRandom()); // save the socket factory from the context diff --git a/src/takserver-core/src/main/java/com/bbn/marti/groups/OAuthAuthenticator.java b/src/takserver-core/src/main/java/com/bbn/marti/groups/OAuthAuthenticator.java index 0e0da697..e2ba56e3 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/groups/OAuthAuthenticator.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/groups/OAuthAuthenticator.java @@ -113,9 +113,12 @@ private AuthResult auth(User user) { if (oAuth2AccessToken == null || oAuth2AccessToken.isExpired()) { throw new OAuth2Exception("defaultTokenServices.readAccessToken failed!"); } - } else { + } else if (claims.get("email") != null) { // For jwt's from keycloak, get the username from the email claim username = (String) claims.get("email"); + } else { + // For other trusted tokens, assign a random username if we don't have an attribute mapping + username = UUID.randomUUID().toString(); } if (user instanceof AuthenticatedUser) { diff --git a/src/takserver-core/src/main/java/com/bbn/marti/groups/X509Authenticator.java b/src/takserver-core/src/main/java/com/bbn/marti/groups/X509Authenticator.java index 5d5a10c7..31eba5d8 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/groups/X509Authenticator.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/groups/X509Authenticator.java @@ -443,7 +443,13 @@ public void assignGroupsCheckCache(Set groups, User user, String username activeGroupCacheHelper.setActiveGroupsForUser(username, activeGroups); // notify the user that their cache has been updated - DistributedSubscriptionManager.getInstance().sendGroupsUpdatedMessage(username, null); + try { + DistributedSubscriptionManager.getInstance().sendGroupsUpdatedMessage(username, null); + } catch (Exception e) { + if (logger.isDebugEnabled()) { + logger.debug("exception calling sendGroupsUpdatedMessage!", e); + } + } } // remove any inactive cache entries prior to push the groups to the user diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/binder/impls/MulticastServerBinder.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/binder/impls/MulticastServerBinder.java index af1c9a73..f0cf3c6c 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/binder/impls/MulticastServerBinder.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/binder/impls/MulticastServerBinder.java @@ -23,7 +23,7 @@ /** * A server binder for opening and binding a multicast udp server datagram to a given port/group ip-addr/interface * -* TODO: implement logic for rejoining the group and checking on the membership key (ie, does it expire or fail validation when +* TODO: implement logic for rejoining the group and checking on the membership key (ie, does it expire or fail validation when * we leave the multicast group) */ public class MulticastServerBinder extends UdpServerBinder { @@ -33,15 +33,15 @@ public class MulticastServerBinder extends UdpServerBinder { * Returns a Multicast UDP data listener. Will bind to the given port, and push any accepted data into a channel handler * that has ... (see above for UDP server binder) * - * @note the network interface and the inet address group are *not* ignored + * The network interface and the inet address group are *not* ignored */ public final static BinderFactory mudpBinderFactory = new BinderFactory() { public ServerBinder instance( int port, OrderedExecutor boundExecutor, - StreamInstantiator streamInstantiator, + StreamInstantiator streamInstantiator, List interfs, - InetAddress group) + InetAddress group) { return new MulticastServerBinder() .withInterfaces(interfs) @@ -52,7 +52,7 @@ public ServerBinder instance( } }; - + private InetAddress group; // the address of the multicast group we're joining private final List interfs = new LinkedList(); // the list of network interfaces that we're going to try and join the group on @@ -63,7 +63,7 @@ public final MulticastServerBinder withGroup(InetAddress group) { this.group = group; return this; } - + public final MulticastServerBinder withInterface(NetworkInterface interf) { Assertion.notNull(interf); @@ -76,13 +76,13 @@ public final MulticastServerBinder withInterface(NetworkInterface interf) { } catch (Exception e) { log.error("Error encountered trying to determine multicast properties of network interface -- excluding from multicast list: " + interf, e); } - + return this; } - + public MulticastServerBinder withInterfaces(List interfs) { Assertion.notNull(interfs); - + for (NetworkInterface interf : interfs) { withInterface(interf); } @@ -92,7 +92,7 @@ public MulticastServerBinder withInterfaces(List interfs) { /** * Either returns the list of user-specified network interfaces for - * multicast, if nonempty, or the list of all network interfaces that support + * multicast, if nonempty, or the list of all network interfaces that support * multicast, if empty. */ private List interfsOrAllMulticastInterfs() { @@ -106,19 +106,19 @@ private List interfsOrAllMulticastInterfs() { @Override protected DatagramChannel doBind() throws IOException { DatagramChannel serverChannel = super.doBind(); - + if (serverChannel != null) { // TODO: do something with the membership keys List joinedKeys = joinInterfaces(serverChannel); } - + return serverChannel; } private List joinInterfaces(DatagramChannel serverChannel) throws IOException { List toJoin = interfsOrAllMulticastInterfs(); List joinedKeys = new LinkedList(); - + if (!toJoin.isEmpty()) { for (NetworkInterface interf : toJoin) { try { @@ -128,14 +128,14 @@ private List joinInterfaces(DatagramChannel serverChannel) throws log.warn(String.format("%s encountered io exception encountered trying to join %s", this, interf)); } } - + if (joinedKeys.isEmpty()) { throw new IOException(this + " could not join any network interfaces that support multicast"); } } else { log.warn(this + " has no available network interfaces that support multicast"); } - + return joinedKeys; } @@ -147,4 +147,4 @@ public String toString() { port() ); } -} \ No newline at end of file +} diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/binder/impls/UdpServerBinder.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/binder/impls/UdpServerBinder.java index 4ed060d1..d1718d11 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/binder/impls/UdpServerBinder.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/binder/impls/UdpServerBinder.java @@ -34,19 +34,19 @@ public class UdpServerBinder extends AbstractServerBinder { private final static Logger log = Logger.getLogger(UdpServerBinder.class); /** - * Returns a UDP data listener. Will bind to the given port, and push any accepted data into a channel handler that + * Returns a UDP data listener. Will bind to the given port, and push any accepted data into a channel handler that * has no active connection, but contains a record of where the traffic came from (the source ip/port that were - * given when the packet was unloaded). + * given when the packet was unloaded). * - * @note the network interface and the inet address group are ignored - */ + * The network interface and the inet address group are ignored + */ public final static BinderFactory udpBinderFactory = new BinderFactory() { public ServerBinder instance( int port, OrderedExecutor boundExecutor, - StreamInstantiator streamInstantiator, + StreamInstantiator streamInstantiator, List interfs, - InetAddress group) + InetAddress group) { return new UdpServerBinder() .withPort(port) @@ -58,7 +58,7 @@ public ServerBinder instance( protected DatagramChannel doBind() throws IOException { // instantiate datagram server channel DatagramChannel serverChannel = null; - + try { // Force IPV4 (INET) since it is left up to the platform otherwise serverChannel = DatagramChannel.open(StandardProtocolFamily.INET); @@ -69,7 +69,7 @@ protected DatagramChannel doBind() throws IOException { } catch (IOException e) { // drop resources on error NetUtils.guardedClose(serverChannel); - + // throw error at the binder throw e; } @@ -82,17 +82,17 @@ public ChannelWrapper handleBind(Server server) throws IOException { Assertion.notNull(server); DatagramChannel serverChannel = doBind(); - + if (serverChannel != null) { // build handler for server datagram channel ChannelHandler serverHandler = new UdpServerChannelHandler() .withChannel(serverChannel) .withExecutor(boundExecutor()) .withServer(server); // one COULD put an ordered view here, that would limit inputs from a specific udp input - + // build chain around handler instantiator().instantiate(serverHandler); - + // wrap and register with the server return new ChannelWrapper() .withChannel(serverChannel) @@ -102,7 +102,7 @@ public ChannelWrapper handleBind(Server server) throws IOException { return null; } } - + @Override public String toString() { return new String("UDP local port: " + port()); diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/codec/ByteCodec.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/codec/ByteCodec.java index ea6b3e4d..eb3f06f3 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/codec/ByteCodec.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/codec/ByteCodec.java @@ -8,7 +8,7 @@ import com.bbn.marti.util.concurrent.future.AsyncFuture; /** -* A codec that handles network -> application, and application -> network traffic conversion. +* A codec that handles network -{@literal >} application, and application -{@literal >} network traffic conversion. * * Codecs are guaranteed that: * @@ -21,14 +21,14 @@ * passed to decode when more data arrives * * -- the order of any data emitted from encode/decode will be kept in the same -* order w.r.t. a happens-before relation of encode/decode calls. (within an encode, or a +* order w.r.t. a happens-before relation of encode/decode calls. (within an encode, or a * decode, scope. encode and decode are themselves disjoint.) * * Codecs must guarantee that: * -* -- within the scope of a single decode call, all progress that can be made on the -* given data *is* made. ie, if a codec does not process all of the data passed to -* a decode call, then an arbitrary number of decode calls with that remaining data +* -- within the scope of a single decode call, all progress that can be made on the +* given data *is* made. ie, if a codec does not process all of the data passed to +* a decode call, then an arbitrary number of decode calls with that remaining data * will always emit the empty buffer, and will always the passed in buffer marked * as completely unread * @@ -47,24 +47,24 @@ public interface ByteCodec { * * * The Pipeline expects that, until the returned future is triggered: - * + * * -- no read traffic is propagated beyond the given codec, ie. a decode * call does not produce a nonempty ByteBuffer (Pipelines can handle * a future being triggered *during* a read call) */ - AsyncFuture onConnect(); + AsyncFuture onConnect(); /** * Receives incoming, network side traffic and decodes it into another byte buffer * - * A synchronous call from the callee pipeline, ie. no side effects are expected to take place outside + * A synchronous call from the callee pipeline, ie. no side effects are expected to take place outside * the scope of this call * * Pipeline implementors guarantee that this call will not be made until the onConnect call has * returned */ ByteBuffer decode(ByteBuffer buffer); - + /** * Receives outgoing, application side traffic and encodes it into another byte buffer * @@ -81,7 +81,7 @@ public interface ByteCodec { * After this call, encode calls may be placed, but can be discarded. */ void onInboundClose(); - + /** * Outbound close is propagated in an application-to-network order * @@ -90,8 +90,8 @@ public interface ByteCodec { * After this calls, decode calls may be placed, and should be serviced. */ void onOutboundClose(); - + void setConnectionInfo(ConnectionInfo connectionInfo); - + ConnectionInfo getConnectionInfo(); -} \ No newline at end of file +} diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/codec/ByteCodecFactory.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/codec/ByteCodecFactory.java index 0a6816c8..158c12ee 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/codec/ByteCodecFactory.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/codec/ByteCodecFactory.java @@ -7,11 +7,11 @@ /* * An interface for a byte codec constructor * -* @note the returned ByteCodec cannot be null--a null ByteCodec will generate a RuntimeException +* The returned ByteCodec cannot be null--a null ByteCodec will generate a RuntimeException * during BytePipeline construction */ public interface ByteCodecFactory { - + /** * Constructs a codec that uses the given pipeline context * as a view for communicating with the surrounding pipeline @@ -19,12 +19,12 @@ public interface ByteCodecFactory { ByteCodec buildCodec(PipelineContext ctx); /** - * May be called by the pipeline when construction of + * May be called by the pipeline when construction of * a new codec occurs, allowing a codec to specify - * an underlying executor that can/should be used + * an underlying executor that can/should be used * to call into the codec * - * Allows for specific threads to be dedicated to + * Allows for specific threads to be dedicated to * specific tasks (such as ssl) * * If the codec does not have such an execution @@ -32,4 +32,4 @@ public interface ByteCodecFactory { * will default to using its own executor. */ OrderedExecutor codecExecutor(); -} \ No newline at end of file +} diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/grpc/NioGrpcChannelHandler.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/grpc/NioGrpcChannelHandler.java index 4e71a129..ab4a3e52 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/grpc/NioGrpcChannelHandler.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/grpc/NioGrpcChannelHandler.java @@ -35,7 +35,7 @@ public NioGrpcChannelHandler(Input input, StreamObserver stream, X50 } public void submitTakMessage(TakMessage message) { - CotEventContainer cotEventContainer = StreamingProtoBufHelper.getInstance().proto2cot(message); + CotEventContainer cotEventContainer = StreamingProtoBufHelper.proto2cot(message); if (isNotDOSLimited(cotEventContainer) && isNotReadLimited(cotEventContainer)) { if (isDataFeedInput()) { @@ -94,7 +94,7 @@ private void setReader() { private void setWriter() { writer = (data) -> { - stream.onNext(StreamingProtoBufHelper.getInstance().cot2protoBuf(data)); + stream.onNext(StreamingProtoBufHelper.cot2protoBuf(data)); }; } diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/listener/ProtocolListener.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/listener/ProtocolListener.java index 634ef57b..29cbdc92 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/listener/ProtocolListener.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/listener/ProtocolListener.java @@ -16,22 +16,22 @@ public interface ProtocolListener { * @param protocol The source datatype parser/deparser */ public void onConnect(ChannelHandler handler, Protocol protocol); - + /** * Called when data is received by transport * @param data The data received - * @param transport The transport over which data was received + * @param handler The handler over which data was received * @param protocol The protocol over which data was received */ public void onDataReceived(T data, ChannelHandler handler, Protocol protocol); - + /** * Called with the handler is closed * * onInboundClose marks the end of the read stream -- once this is called, - * we can expect no more data from the given handler. + * we can expect no more data from the given handler. * - * onOutboundClose signals that the protocol/handler will no longer + * onOutboundClose signals that the protocol/handler will no longer * accept outbound data. Writing to the protocol after this call has been * received may produce a runtime exception * diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/listener/ProtocolListenerInstantiator.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/listener/ProtocolListenerInstantiator.java index 25194d76..77bb05be 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/listener/ProtocolListenerInstantiator.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/listener/ProtocolListenerInstantiator.java @@ -6,13 +6,13 @@ import com.bbn.marti.nio.protocol.Protocol; /** -* An interface for a factory that produces a protocol +* An interface for a factory that produces a protocol * listener, in response to newInstance being called -* with a ChannelHandler/Protocol pair +* with a ChannelHandler/Protocol{@literal <}T{@literal >} pair * -* @note If the factory does not wish to register a ProtocolListener -* for the given Handler/Protocol, it may return null +* If the factory does not wish to register a ProtocolListener{@literal <}T{@literal >} +* for the given Handler/Protocol{@literal <}T{@literal >}, it may return null */ public interface ProtocolListenerInstantiator { public ProtocolListener newInstance(ChannelHandler handler, Protocol protocol); -} \ No newline at end of file +} diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/NioNettyBuilder.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/NioNettyBuilder.java index 9b1c568b..70f51ab4 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/NioNettyBuilder.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/NioNettyBuilder.java @@ -38,6 +38,7 @@ import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelOption; import io.netty.channel.EventLoopGroup; +import io.netty.channel.FixedRecvByteBufAllocator; import io.netty.channel.WriteBufferWaterMark; import io.netty.channel.epoll.Epoll; import io.netty.channel.epoll.EpollEventLoopGroup; @@ -114,7 +115,10 @@ public NioDatagramChannel newChannel() { } }) .handler(new NioNettyUdpHandler(input)) + .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) .option(ChannelOption.AUTO_CLOSE, true) + .option(ChannelOption.SO_RCVBUF, input.getMaxMessageReadSizeBytes()) + .option(ChannelOption.RCVBUF_ALLOCATOR, new FixedRecvByteBufAllocator(input.getMaxMessageReadSizeBytes())) .option(ChannelOption.SO_BROADCAST, true); portToNettyServer.put(input.getPort(), bootstrap.bind(input.getPort()).sync().channel().closeFuture()); @@ -161,6 +165,9 @@ public NioDatagramChannel newChannel() { } }) .handler(new NioNettyUdpHandler(input)) + .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) + .option(ChannelOption.SO_RCVBUF, input.getMaxMessageReadSizeBytes()) + .option(ChannelOption.RCVBUF_ALLOCATOR, new FixedRecvByteBufAllocator(input.getMaxMessageReadSizeBytes())) .option(ChannelOption.AUTO_CLOSE, true); InetSocketAddress groupAddress = SocketUtils.socketAddress(group.getHostAddress(), input.getPort()); @@ -222,6 +229,8 @@ private void startServer(NioNettyInitializer initializer, int port) { bootstrap.group(bossGroup, workerGroup) .channel(isEpoll ? EpollServerSocketChannel.class : NioServerSocketChannel.class) .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) + .option(ChannelOption.SO_RCVBUF, initializer.getInput().getMaxMessageReadSizeBytes()) + .option(ChannelOption.RCVBUF_ALLOCATOR, new FixedRecvByteBufAllocator(initializer.getInput().getMaxMessageReadSizeBytes())) .childHandler(initializer) .childOption(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark(lowMark, highMark)) .childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT); diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyHandlerBase.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyHandlerBase.java index 12865aee..ffcec3ec 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyHandlerBase.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyHandlerBase.java @@ -509,11 +509,11 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { Exception bottomOfExceptionChain = spelunkToBottomOfExceptionChain(cause); if (bottomOfExceptionChain != null && bottomOfExceptionChain instanceof CertificateExpiredException) { - log.error(String.format("Connection rejected for expired client certificate: %s, %s, %s.", - getCertificateName(cause), bottomOfExceptionChain.getMessage())); + log.error(String.format("Connection rejected for expired client certificate: %s, %s, %s", + getCertificateName(cause), bottomOfExceptionChain.getMessage(), additionalInfo.toString())); } else if (bottomOfExceptionChain != null && bottomOfExceptionChain instanceof CertificateRevokedException) { - log.error(String.format("Connection rejected for revoked client certificate: %s, %s.", - getCertificateName(cause), bottomOfExceptionChain.getMessage())); + log.error(String.format("Connection rejected for revoked client certificate: %s, %s, %s", + getCertificateName(cause), bottomOfExceptionChain.getMessage(), additionalInfo.toString())); } else { // get certificate info if available SslHandler sslhandler = (SslHandler) ctx.channel().pipeline().get("ssl"); diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyTlsServerHandler.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyTlsServerHandler.java index 8436944f..14d122a0 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyTlsServerHandler.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyTlsServerHandler.java @@ -369,7 +369,7 @@ protected void convertAndSubmitProtoBufBytesAsCot(ByteBuffer msg) { // parse and broadcast the message TakMessage takMessage = TakMessage.parseFrom(eventBytes); - CotEventContainer cotEventContainer = StreamingProtoBufHelper.getInstance().proto2cot(takMessage); + CotEventContainer cotEventContainer = StreamingProtoBufHelper.proto2cot(takMessage); if (isNotDOSLimited(cotEventContainer) && isNotReadLimited(cotEventContainer)) { diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/initializers/NioNettyInitializer.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/initializers/NioNettyInitializer.java index 319cc2b4..0b74b204 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/initializers/NioNettyInitializer.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/initializers/NioNettyInitializer.java @@ -50,7 +50,11 @@ public abstract class NioNettyInitializer extends ChannelInitializer application data +* network -{@literal >} application data * * The onOutboundClose call signals that the channel handler can no longer receive -* application -> network traffic. +* application -{@literal >} network traffic. * * A channel listener can expect that all onDataReceived calls, if any, come after * an onConnect call, and that no further onDataReceived calls will be received -* after an onInboundClose call. +* after an onInboundClose call. * * There are no ordering constraints on onInbound/OutboundClose calls. * @@ -52,13 +52,13 @@ public interface ChannelListener { public void onDataReceived(ByteBuffer buffer, ChannelHandler handler); /** - * Called to mark the EOS for network -> application traffic + * Called to mark the EOS for network -{@literal >} application traffic */ - public void onInboundClose(ChannelHandler handler); - + public void onInboundClose(ChannelHandler handler); + /** - * Called to mark the end of application -> network traffic. Further + * Called to mark the end of application -{@literal >} network traffic. Further * attempts to write to the handler may throw an exception. */ public void onOutboundClose(ChannelHandler handler); -} \ No newline at end of file +} diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/DecoratedStreamInstantiator.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/DecoratedStreamInstantiator.java index c6d437e8..1a5e034b 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/DecoratedStreamInstantiator.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/DecoratedStreamInstantiator.java @@ -20,17 +20,17 @@ * * Specifically, given a channel handler, this partially-instantiated factory will (*synchronously*): * -* - construct a ChannelListener/Protocol instance using the given ProtocolInstantiator, and link it +* - construct a ChannelListener/Protocol{@literal <}T{@literal >} instance using the given ProtocolInstantiator{@literal <}T{@literal >}, and link it * to the ChannelHandler (see ChannelHandler.listener). * -* - construct a list of ProtocolListener instances using the given ChannelHandler, Protocol pair, and -* store them into the Protocol. -* @note the ordering of ProtocolListenerInstantiator defines the (possibly flatMapped) ordering of ProtocolListener +* - construct a list of ProtocolListener{@literal <}T{@literal >} instances using the given ChannelHandler, Protocol{@literal <}T{@literal >} pair, and +* store them into the Protocol{@literal <}T{@literal >}. +* The ordering of ProtocolListenerInstantiator{@literal <}T{@literal >} defines the (possibly flatMapped) ordering of ProtocolListener{@literal <}T{@literal >} * * - return * -* @note if a Protocol is null or throws an exception, the callee is responsible for handling a runtime exception, -* while if a ProtocolListener is null or throws an exception, it is not included in the stream processing chain (TODO: justify this asymmetry) +* If a Protocol{@literal <}T{@literal >} is null or throws an exception, the callee is responsible for handling a runtime exception, +* while if a ProtocolListener{@literal <}T{@literal >} is null or throws an exception, it is not included in the stream processing chain (TODO: justify this asymmetry) */ public class DecoratedStreamInstantiator implements StreamInstantiator { private final static Logger log = Logger.getLogger(DecoratedStreamInstantiator.class); @@ -38,38 +38,38 @@ public class DecoratedStreamInstantiator implements StreamInstantiator { private ProtocolInstantiator protocolInstantiator; // factory used to convert ChannelHandler -> Protocol private LinkedBlockingQueue> listenerInstantiators; // factories used for instantiating ProtocolListeners from the ChannelHandler/Protocol pair private String listenerInstantiatorsStr = ""; // cached string representation of the protocol listeners, for fast toString conversion - + public DecoratedStreamInstantiator withProtocolInstantiator(ProtocolInstantiator protocolInstantiator) { Assertion.notNull(protocolInstantiator); - + this.protocolInstantiator = protocolInstantiator; return this; } - + public DecoratedStreamInstantiator withProtocolListenerInstantiators(LinkedBlockingQueue> listenerInstantiators) { Assertion.notNull(listenerInstantiators); Assertion.areNotNull(listenerInstantiators); - + this.listenerInstantiators = listenerInstantiators; - + updateListenerStr(); - + return this; } - + /** * Stores the current, joined list of ListenerInstantiators.toString */ private void updateListenerStr() { this.listenerInstantiatorsStr = Joiner.on(", ").join(this.listenerInstantiators); } - + /** * Build all the parts with the instantiators, link together */ public void instantiate(ChannelHandler handler) { log.trace(this + " building stream for " + handler); - + // build channel listener / protocol and set value in the channel handler Protocol protocol = instantiateProtocol(handler, protocolInstantiator); @@ -81,19 +81,17 @@ public void instantiate(ChannelHandler handler) { log.trace(this + " built stream for " + handler); } - + /** - * Instantiates the given Protocol using the given handler, and attaches the ChannelListener + * Instantiates the given Protocol{@literal <}T{@literal >} using the given handler, and attaches the ChannelListener * to the handler * - * Returns the Protocol - * - * @throws an AssertionException if the protocol is null + * Returns the Protocol{@literal <}T{@literal >} * */ public static Protocol instantiateProtocol(ChannelHandler handler, ProtocolInstantiator protocolInstantiator) { Protocol protocol = null; - + try { protocol = protocolInstantiator.newInstance(handler); } catch (Exception e) { @@ -101,24 +99,24 @@ public static Protocol instantiateProtocol(ChannelHandler handler, Protoc } Assertion.notNull(protocol); - + handler.listener(protocol); - + return protocol; } - + public static void attachListeners(Protocol target, List> listeners) { for (ProtocolListener listener : listeners) { target.addProtocolListener(listener); } } - + /** * For each protocol listener instantiator, call new instance, guardedly, and aggregate the results into a list */ public static List> instantiateListeners(ChannelHandler handler, Protocol protocol, LinkedBlockingQueue> instantiators) { List> listeners = new ArrayList>(instantiators.size()); - + for (ProtocolListenerInstantiator instantiator : instantiators) { try { ProtocolListener listener = instantiator.newInstance(handler, protocol); @@ -130,10 +128,10 @@ public static List> instantiateListeners(ChannelHandler log.warn(String.format("Error instantiating protocol listener for handler/protocol pair -- handler: %s, protocol: %s", handler, protocol), e); } } - + return listeners; } - + public String toString() { return String.format("%s --> {%s}", this.protocolInstantiator, this.listenerInstantiatorsStr); } diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/Protocol.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/Protocol.java index 8479acf8..b14a5e9c 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/Protocol.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/Protocol.java @@ -13,11 +13,11 @@ */ public interface Protocol extends ChannelListener { /** - * Writes the given data to the transport, returning an - * asynchronous integer that will be set with the number of + * Writes the given data to the transport, returning an + * asynchronous integer that will be set with the number of * bytes written out to the wire on account of the given data - * - * If the data could not be sent, the async integer may be + * + * If the data could not be sent, the async integer may be * triggered with an exceptional status (see AsyncCallback onFailure) */ public AsyncFuture write(T data, ChannelHandler handler); @@ -25,7 +25,7 @@ public interface Protocol extends ChannelListener { /** * add/removes the given protocol listener from this protocol * - * @note Should not be called from the same thread context as a + * Should not be called from the same thread context as a * protocol listener on_ call originating from this protocol. * May deadlock due to lock read/write non-reentrancy in some implementations. */ diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/SingleProtobufOrCotProtocol.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/SingleProtobufOrCotProtocol.java index 846475ba..d58233bc 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/SingleProtobufOrCotProtocol.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/SingleProtobufOrCotProtocol.java @@ -107,7 +107,7 @@ public static CotEventContainer byteBufToCot(ByteBuffer buffer, ChannelHandler h if (firstByte == StreamingProtoBufHelper.MAGIC) { - int version = StreamingProtoBufHelper.getInstance().readVarint(buffer); + int version = StreamingProtoBufHelper.readVarint(buffer); if (Integer.toString(version).compareTo(StreamingProtoBufHelper.TAK_PROTO_VERSION) != 0) { log.error("Failed to find supported protocol version ! : " + version); return null; @@ -123,7 +123,7 @@ public static CotEventContainer byteBufToCot(ByteBuffer buffer, ChannelHandler h // parse the protobuf Takmessage.TakMessage takMessage = Takmessage.TakMessage.parseFrom(eventBytes); - cotEventContainer = StreamingProtoBufHelper.getInstance().proto2cot(takMessage); + cotEventContainer = StreamingProtoBufHelper.proto2cot(takMessage); } else if (firstByte == 0x3C) { diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/StreamingCotProtocol.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/StreamingCotProtocol.java index 06b39d9f..5f18a240 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/StreamingCotProtocol.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/StreamingCotProtocol.java @@ -24,11 +24,11 @@ /** * Cursor-on-Target (CoT) protocol that re-uses a socket instead of opening and closing it between each message. * - * In the network -> application direction, we decode a series of CoT messages laid head-to-tail, without any + * In the network -{@literal >} application direction, we decode a series of CoT messages laid head-to-tail, without any * headers or delimiters that might indicate message boundaries and potentially make our parsing methodology * far more efficient. Such is life. * - * In the application -> network direction, we encode a single message to its byte representation and pass + * In the application -{@literal >} network direction, we encode a single message to its byte representation and pass * it onward, to the channel handler. * */ @@ -160,7 +160,7 @@ public void onInboundClose(ChannelHandler handler) { // void out parser and builder this.messageStringBuffer = null; this.parser = null; - + if (log.isTraceEnabled()) { log.trace(String.format( "%s received network close signal -- handler: %s", @@ -193,7 +193,7 @@ public void onOutboundClose(ChannelHandler handler) { // notify listeners super.broadcastOutboundClose(handler); } - + /** * A static function that appends the given data to an existing * buffer, and checks to see if any new messages can be parsed with @@ -268,11 +268,11 @@ public static List add(StringBuffer messageStringBuffer, Char } /** - * @note DO NOT put the channel handler in the string -- typically prints out its listener as + * DO NOT put the channel handler in the string -- typically prints out its listener as * part of its toString method */ @Override public String toString() { return "server_streaming_CoT"; } -} \ No newline at end of file +} diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/StreamingProtoBufOrCoTProtocol.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/StreamingProtoBufOrCoTProtocol.java index 3e811ed5..25f47fb6 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/StreamingProtoBufOrCoTProtocol.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/StreamingProtoBufOrCoTProtocol.java @@ -123,7 +123,7 @@ public void onOutboundClose(ChannelHandler handler) { } /** - * @note DO NOT put the channel handler in the string -- typically prints out its listener as + * DO NOT put the channel handler in the string -- typically prints out its listener as * part of its toString method */ @Override @@ -154,7 +154,7 @@ private static String getVersion() { if (version == null) { synchronized (StreamingProtoBufOrCoTProtocol.class) { try { - + return MessagingDependencyInjectionProxy.getInstance().versionBean().getVer(); } catch (Exception e) { @@ -275,7 +275,7 @@ private void setupNegotiationListener() { public final ProtocolListenerInstantiator negotiationCallback = new AbstractAutoProtocolListener() { - + private static final long serialVersionUID = 9879877823L; @Override @@ -291,4 +291,4 @@ public String toString() { return "negotiation_listener"; } }; -} \ No newline at end of file +} diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/StreamingProtoBufProtocol.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/StreamingProtoBufProtocol.java index 36d4deb1..15dc52b4 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/StreamingProtoBufProtocol.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/StreamingProtoBufProtocol.java @@ -138,7 +138,7 @@ public void onDataReceived(ByteBuffer buffer, ChannelHandler handler) { // parse and broadcast the message TakMessage takMessage = TakMessage.parseFrom(eventBytes); - CotEventContainer cotEventContainer = StreamingProtoBufHelper.getInstance().proto2cot(takMessage); + CotEventContainer cotEventContainer = StreamingProtoBufHelper.proto2cot(takMessage); super.broadcastDataReceived(cotEventContainer, handler); // reset parser state @@ -185,7 +185,7 @@ private static TakMessage createFileTransferRequest(CotEventContainer data) { SAXReader reader = new SAXReader(); Document doc = reader.read(new ByteArrayInputStream(fileTransferXml.getBytes())); CotEventContainer container = new CotEventContainer(doc); - TakMessage takMessage = StreamingProtoBufHelper.getInstance().cot2protoBuf(container); + TakMessage takMessage = StreamingProtoBufHelper.cot2protoBuf(container); return takMessage; } catch (Exception e) { @@ -204,15 +204,15 @@ public AsyncFuture write(CotEventContainer data, ChannelHandler handler try { Assertion.condition(!outboundClosed, "!outboundClosed"); Assertion.areNotNull(data, handler); - + ByteBuffer buffer = data.getProtoBufBytes(); - + if (buffer == null) { buffer = convertCotToProtoBufBytes(data); } else { Metrics.counter(Constants.METRIC_MESSAGE_PRECONVERT_COUNT, "takserver", "messaging").increment(); } - + if(buffer == null) { return null; }; @@ -231,7 +231,7 @@ public static ByteBuffer convertCotToProtoBufBytes(CotEventContainer data) { // // Convert CotEventContainer to protobuf // - TakMessage takMessage = StreamingProtoBufHelper.getInstance().cot2protoBuf(data); + TakMessage takMessage = StreamingProtoBufHelper.cot2protoBuf(data); if (takMessage == null) { log.error("cot2protoBuf failed to parse message!"); return null; @@ -246,13 +246,13 @@ public static ByteBuffer convertCotToProtoBufBytes(CotEventContainer data) { if (xml.length() > MAX_LOG_SIZE) { xml = xml.substring(0, MAX_LOG_SIZE) + "..."; } - + if (log.isDebugEnabled()) { log.debug("Attempt to write message greater than max size : " + takMessageSize + ", " + xml); } else { log.error("Attempt to write message greater than max size : " + takMessageSize); } - + // overwrite the failed message with a file transfer request, and write it out instead takMessage = createFileTransferRequest(data); if (takMessage == null) { @@ -306,17 +306,17 @@ public void onOutboundClose(ChannelHandler handler) { Assertion.notNull(handler); this.outboundClosed = true; - + // notify listeners super.broadcastOutboundClose(handler); } /** - * @note DO NOT put the channel handler in the string -- typically prints out its listener as + * DO NOT put the channel handler in the string -- typically prints out its listener as * part of its toString method */ @Override public String toString() { return "server_streaming_protobuf"; } -} \ No newline at end of file +} diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/server/NioServer.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/server/NioServer.java index d63007b7..57a161b3 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/server/NioServer.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/server/NioServer.java @@ -39,40 +39,40 @@ * The nio server is a single thread that can be started and stopped. On start, the registered binders (an interface that allows for users * to specify initial instances of server/client connects) are called, and their channels are registered with the selector. The main thread * then listens on the selector, which is triggered whenever a registered channel receives a signal, or whenever wakeup is called by another -* thread. This wakeup mechanism is used here to signal the arrival of a selector change request, which can be processed in the absence of +* thread. This wakeup mechanism is used here to signal the arrival of a selector change request, which can be processed in the absence of * any waiting IO events. * * Application-side users interact with the server (see Server interface) by submitting registration or interestOp modifications, and by implementing a ChannelHandler. * These registration/interestOp modifications either register/deregister a channel handler from the server's calls, or change * the set of calls that a handler can receivecalls. A handler receives a call for each IO event that the server receives from the selector * for the corresponding channel. -* +* * Any client can submit a selector change request, which returns an asynchronous future containing either the submitted channel handler, upon * success, or an exception, upon failure. * * When an IOEvent is received for a specific channel, the server calls into the handler to notify: the handler should (quickly) process * the IO event such that the selector will not be retriggered for that particular event, and spin off any tasks necessary to fully process the data -* received. This handler call type returns a flag indicating whether or not the handler should be resubscribed to the IOEvent category it just received. +* received. This handler call type returns a flag indicating whether or not the handler should be resubscribed to the IOEvent category it just received. * -* The bit-flag format and logic of the InterestOp integer is hidden by means of the typed IOEvent enumeration. Each +* The bit-flag format and logic of the InterestOp integer is hidden by means of the typed IOEvent enumeration. Each * IOEvent enumeration (READ, WRITE, ACCEPT, CONNECT) corresponds to a single high bit in some integer bit vector, which is tightly coupled * in the enumeration definition. A subscription to multiple IOEvent is encapsulated with an EnumSet. Conversion to and from the bit vector * is done on the client thread (in the channel wrapper) to avoid computation on the selector thread while avoiding cumbersome and easily mangled * bitwise operations. * -* Client handler and binder writers are responsible for ensuring a few details. All handler implementations should be resilient to spurious calls for IOEvent +* Client handler and binder writers are responsible for ensuring a few details. All handler implementations should be resilient to spurious calls for IOEvent * handling: it cannot be guaranteed that asynchronous calls to unsubscribe or close will be honored before the server calls in. * -* All channels that are passed to the server, either through a binder or through the channel registration method, should have their channels +* All channels that are passed to the server, either through a binder or through the channel registration method, should have their channels * configured to nonblocking upon instantiation. * */ public class NioServer implements Server, Runnable, Serializable { - + private static final long serialVersionUID = -6983711481588072641L; - + private static NioServer instance; - + public static NioServer getInstance() { if (instance == null) { synchronized (NioServer.class) { @@ -81,12 +81,12 @@ public static NioServer getInstance() { } } } - + return instance; } private final static Logger log = Logger.getLogger(NioServer.class); - + private Selector selector = null; // selector for listening for nio events private Thread serverThread = null; // thread pointer for a current, running server private Queue selectorChanges; // queue for storing application-side requests for IO subscription changes until the main thread can process them @@ -102,7 +102,7 @@ public NioServer() throws IOException { } // /** -// * Selector change submission guard: if the nio server is dead (and not processing any +// * Selector change submission guard: if the nio server is dead (and not processing any // * connections, any change request will immediately throw an exception on submission. // */ // private boolean serverIsDead() { @@ -113,15 +113,15 @@ public NioServer() throws IOException { * Enqueues the change if the server is awake, or sets an exception if the server is not * * This method is not enitrely race free with respect to the server being alive/dead, ie, at the moment - * the server is shut down, it is possible for a submitting thread to enqueue a change that is not + * the server is shut down, it is possible for a submitting thread to enqueue a change that is not * set to except by the server shutdown method (which excepts all pending changes) */ private boolean wakeupOrExceptChange(AbstractSelectorChange change) { if (!selectorChanges.offer(change)) { - if(log.isWarnEnabled()) { + if(log.isWarnEnabled()) { log.warn("Server received change submitted to dead server " + change.toString()); } - + // server is dead or the change was not accepted into the queue change.future.setException(new IllegalStateException("Server is not running, or server queue did not accept change")); return false; @@ -208,7 +208,7 @@ public AsyncFuture removeIterestOp(SelectableChannel channel, IO return change.future; } - + public AsyncFuture removeIterestOps(SelectableChannel channel, EnumSet events) { Assertion.areNotNull(channel, events, "None of the arguments passed to the remove interest call can be null"); @@ -218,9 +218,9 @@ public AsyncFuture removeIterestOps(SelectableChannel channel, E return change.future; } - + /** - * Method for attempting to open a new selector for this server. + * Method for attempting to open a new selector for this server. */ private void openSelector() throws IOException { if (this.selector != null && this.selector.isOpen()) @@ -232,7 +232,7 @@ private void openSelector() throws IOException { throw new IOException( "Error opening selector", e); - } + } } /** @@ -270,12 +270,11 @@ public void bind(@NotNull ServerBinder binder, @NotNull Input input) throws IOEx } } } - + /** * * @param binder * @param name - * @param connectionId Iff set, this will be propagated to client connections as their connectionId * @throws IOException */ public void bind(@NotNull ServerBinder binder, @NotNull String name) throws IOException { @@ -356,7 +355,7 @@ private void registerWrapper(ServerBinder source, ChannelWrapper wrapper) throws * Starts listening and processing incoming traffic. Asserts that no thread * is running already, and that the selector is open (ie, bind has been called) * - * @note not thread-safe + * Not thread-safe */ public void listen() { Assertion.condition(this.serverThread == null || !this.serverThread.isAlive(), "Server is already listening"); @@ -364,28 +363,28 @@ public void listen() { try { long mem = Runtime.getRuntime().maxMemory(); - + log.info("max memory (bytes): " + mem); } catch (Throwable t) { log.error("exeception getting memory available", t); } - + log.info("Server started"); - + this.serverThread = new Thread(this, "NIO Server"); this.serverThread.start(); } - + /** * Interrupts the current thread if it is running, and then (synchronously) joins it. */ public void stop() { Assertion.condition(this.serverThread != null && this.serverThread.isAlive(), "Server is not running"); - + // interrupt thread (which interrupts the selector). Thread itself detects shutdown, checking once per loop cycle serverThread.interrupt(); - + try { serverThread.join(); } catch (InterruptedException e) { @@ -394,8 +393,8 @@ public void stop() { // void out thread pointer serverThread = null; } - } - + } + /** * Called by the server thread when an interrupt is detected. Iterates over all channels * in the selector and closes them, and then closes the selector. @@ -403,12 +402,12 @@ public void stop() { private void shutdown() { log.info("Server shutting down"); - // close all of the channels registered with the selector + // close all of the channels registered with the selector closeAllChannels(); // shut down the actual selector closeSelector(); - + // remove and except all pending selector changes closePendingChanges(); } @@ -440,7 +439,7 @@ private void closeSelector() { // drop the reference to it selector = null; } - } + } private void closePendingChanges() { AbstractSelectorChange change; @@ -486,11 +485,11 @@ private void listenForever() { } catch (Exception e) { log.error("Server encountered exception shutting down", e); } - + // jump out return; } - + try { // process key set processKeySet(); @@ -514,7 +513,7 @@ private void listenForever() { */ private void processSelectorChanges() { // log.trace("processing selector modifications"); - + AbstractSelectorChange change; while ((change = selectorChanges.poll()) != null) { try { @@ -531,29 +530,29 @@ private void processSelectorChanges() { */ private void processKeySet() { // log.trace("processing key set"); - + Iterator iter = selector.selectedKeys().iterator(); while (iter.hasNext()) { SelectionKey key = iter.next(); // remove key from the select key set -- otherwise, it stays in the triggered set iter.remove(); - + if (!key.isValid()) { - // don't have anything to handle + // don't have anything to handle if(log.isTraceEnabled()) { log.trace("Server skipping over invalid key"); } continue; } - - + + final SelectableChannel channel = key.channel(); final ChannelHandler handler = (ChannelHandler) key.attachment(); IOEvent event = null; boolean staySubscribed = false; // intial assumption of false assures desubscription if an exception is thrown (they get closed anyhow) - + try { if (key.isReadable()) { event = IOEvent.READ; @@ -569,7 +568,7 @@ private void processKeySet() { staySubscribed = handler.handleConnect(channel, this); } else { // jump out, have nothing to do - if(log.isWarnEnabled()) { + if(log.isWarnEnabled()) { log.warn("Encountered active key with nothing to do"); } continue; @@ -597,7 +596,7 @@ public String toString() { return "Nio Server instance (synchronous io strategy)"; } - + } diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/server/Server.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/server/Server.java index 10eafeee..cadeaed5 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/server/Server.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/server/Server.java @@ -13,17 +13,17 @@ import com.bbn.marti.util.concurrent.future.AsyncFuture; /** -* Interface for a nio server that is controlled through a bind/listen/stop sequence as a persistent service, +* Interface for a nio server that is controlled through a bind/listen/stop sequence as a persistent service, * and through register/deregister/modify change operations as an event publisher on the channels it regulates. * -*

The server is responsible for handling scheduling and guaranteeing ordering of calls. See {@link SchedulingStrategy}, -* where the server makes different guarantees about different scheduling strategies in terms of the number of threads servicing +*

The server is responsible for handling scheduling and guaranteeing ordering of calls. +* A server makes different guarantees about different scheduling strategies in terms of the number of threads servicing * a type of request at a given time. * *

Channel registration/deregistration and modification of interest is done asynchronously, with the listenable future triggered -* when the operation is completed, or when an exception is generated. ChannelHandlers should be made resilient to unwanted calls that -* occur between the time when the change request is submitted, and the time when the server processes it. -* The Server does guarantee that InterestOp modifications are applied in the order they were received (subject to a happens before +* when the operation is completed, or when an exception is generated. ChannelHandlers should be made resilient to unwanted calls that +* occur between the time when the change request is submitted, and the time when the server processes it. +* The Server does guarantee that InterestOp modifications are applied in the order they were received (subject to a happens before * relation with the entry and exit of the server's submit method call. * */ @@ -32,10 +32,10 @@ public interface Server { // called to start the server listening public void listen(); - + // called to stop the server listening and release all resources public void stop(); - + // application side interface for adding/removing channels from the selection set public AsyncFuture registerChannel(SelectableChannel channel, ChannelHandler handler, EnumSet events); @@ -43,21 +43,21 @@ public interface Server { public AsyncFuture deregisterChannel(SelectableChannel channel, ChannelHandler handler); - // application side interface for changing the selection key registration for a specific channel (ie, what we want the channel handler to receive + // application side interface for changing the selection key registration for a specific channel (ie, what we want the channel handler to receive // notifications about) // absolute set -- blows away current interest sets public AsyncFuture setInterestOps(SelectableChannel channel, EnumSet events); - public AsyncFuture setInterestOp(SelectableChannel channel, IOEvent event); + public AsyncFuture setInterestOp(SelectableChannel channel, IOEvent event); // relative add -- ORs in given event. Idempotent if already registered for that IO flag public AsyncFuture addInterestOp(SelectableChannel channel, IOEvent event); - + public AsyncFuture addInterestOps(SelectableChannel channel, EnumSet events); public AsyncFuture removeIterestOp(SelectableChannel channel, IOEvent event); - + public AsyncFuture removeIterestOps(SelectableChannel channel, EnumSet events); - // relative subtract -- ANDs out given event. Idempotent if already not registered for that IO flag -} \ No newline at end of file + // relative subtract -- ANDs out given event. Idempotent if already not registered for that IO flag +} diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/util/DatagramState.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/util/DatagramState.java index 2769f34c..524dba20 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/util/DatagramState.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/util/DatagramState.java @@ -9,18 +9,18 @@ * Set of states for a connectionless protocol, with no session layer protocols (that the handler is aware of) * * INIT is used to guard against errors during construction/initialization of the handler/listener/stream processing chain -* +* * OPEN/CLOSED represent the network state of a datagram socket. CLOSING is used to differentiate between the time when the application * calls close, but data has yet to be written out; the application is no longer aware of the handler during closing. * -* At CLOSING -> CLOSED, network resources are released. +* At CLOSING -{@literal >} CLOSED, network resources are released. * -* If a forceClose is issued from OPEN -> CLOSED, all pending write data is dropped, in addition to any actions taken in CLOSING -> CLOSED +* If a forceClose is issued from OPEN -{@literal >} CLOSED, all pending write data is dropped, in addition to any actions taken in CLOSING -{@literal >} CLOSED */ public enum DatagramState { INIT, // handler is built, registration/instantiation in progress OPEN, // fully connected--can receive application writes, deliver read traffic to the application - CLOSING, // in process of closing, initiated from the application side + CLOSING, // in process of closing, initiated from the application side CLOSED; // fully closed -- no more network or application traffic. // states in which a (client) handler can receive a connect call (ChannelHandler.connect) from the application @@ -34,10 +34,10 @@ public enum DatagramState { // states in which a handler can receive a close call (ChannelHandler.close) from the application public final static Set closeReceiveStates = EnumSet.of(OPEN, CLOSING); - + // states in which a handler should expect that the network connection is active public final static Set netConnectStates = EnumSet.of(OPEN, CLOSING); // states in which a handler should not apply a force close ie. it's already done public final static Set forceClosedStates = EnumSet.of(CLOSED); -} \ No newline at end of file +} diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/websockets/NioWebSocketHandler.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/websockets/NioWebSocketHandler.java index f1da8eb8..b60d174b 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/websockets/NioWebSocketHandler.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/websockets/NioWebSocketHandler.java @@ -163,7 +163,7 @@ private void setReader() { // parse and broadcast the message TakMessage takMessage = TakMessage.parseFrom(eventBytes); - CotEventContainer cotEventContainer = StreamingProtoBufHelper.getInstance().proto2cot(takMessage); + CotEventContainer cotEventContainer = StreamingProtoBufHelper.proto2cot(takMessage); if (isNotDOSLimited(cotEventContainer) && isNotReadLimited(cotEventContainer)) protocolListeners.forEach(listener -> listener.onDataReceived(cotEventContainer, channelHandler, protocol)); diff --git a/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedConfiguration.java b/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedConfiguration.java index 15e92826..5160c31c 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedConfiguration.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedConfiguration.java @@ -219,6 +219,26 @@ public ConnectionModifyResult setArchiveOnlyFlagNoSave(String inputName, boolean input.setArchiveOnly(desiredState); return ConnectionModifyResult.SUCCESS; } + + public ConnectionModifyResult setFederatedFlagNoSave(String inputName, boolean desiredState) { + Input input = getInputByName(inputName); + if (input == null) { + return ConnectionModifyResult.FAIL_NONEXISTENT; + } + input.setFederated(desiredState); + return ConnectionModifyResult.SUCCESS; + } + + public ConnectionModifyResult setSyncCacheRetentionSeconds(String inputName, int desiredState) { + Input input = getInputByName(inputName); + if (input == null) { + return ConnectionModifyResult.FAIL_NONEXISTENT; + } + input.setSyncCacheRetentionSeconds(desiredState); + return ConnectionModifyResult.SUCCESS; + } + + @SuppressWarnings("rawtypes") public synchronized ConnectionModifyResult updateTagsNoSave(String inputName, List newTagList) throws RemoteException { diff --git a/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedRetentionQueryManager.java b/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedRetentionQueryManager.java index 37c75b94..ce9a2037 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedRetentionQueryManager.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedRetentionQueryManager.java @@ -3,6 +3,7 @@ import java.util.Date; import java.util.List; import java.util.Map; +import java.util.NavigableSet; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -22,6 +23,7 @@ import com.bbn.marti.remote.sync.MissionContent; import com.bbn.marti.remote.util.RemoteUtil; import com.bbn.marti.sync.Metadata; +import com.bbn.marti.sync.model.Mission; import com.bbn.marti.sync.model.MissionRole; import com.bbn.marti.sync.model.MissionRole.Role; import com.bbn.marti.sync.service.MissionService; @@ -56,8 +58,9 @@ public void deleteMissionByExpiration(Long expiration) { } if (expiration == null || expiration.longValue() < 0) { - logger.error(" bad argument, delete MissionByExpiration is ignored expiration: " + expiration); - throw new IllegalArgumentException("invalid expiration: " + expiration); + if (logger.isWarnEnabled()) { + logger.warn(" bad argument, delete MissionByExpiration is ignored expiration: " + expiration); + } } if (logger.isInfoEnabled()) { @@ -87,8 +90,10 @@ public void deleteMissionByTtl(Integer ttl) { } if (ttl.intValue() < 0) { - logger.error(" bad argument, delete MissionByTtl is ignored ttl: " + ttl); - throw new IllegalArgumentException("invalid expiration: " + ttl); + if (logger.isWarnEnabled()) { + logger.warn(" bad argument, delete MissionByTtl is ignored ttl: " + ttl); + } + return; } if (logger.isInfoEnabled()) { @@ -180,7 +185,7 @@ public boolean restoreMission(Map files, Map pro missionService().createMission(properties.get("mission_name"), properties.get("creatorUid"), groupVector, properties.get("description"), properties.get("chatroom"), properties.get("baseLayer"), - properties.get("bbox"), properties.get("path"), properties.get("classification"), properties.get("tool"), properties.get("password_hash"), missionRole, expiration, properties.get("bounding_polygon")); + properties.get("bbox"), properties.get("path"), properties.get("classification"), properties.get("tool"), properties.get("password_hash"), missionRole, expiration, properties.get("bounding_polygon"), Boolean.valueOf(properties.get("invite_only"))); return true; } @@ -264,4 +269,20 @@ public void restoreContent(String missionName, byte[] file, Element missionConte missionService().addMissionContent(missionName, mc, missionContent.attributeValue("creatorUid"), groupVector); } + + @Override + public List getAllMissions(boolean passwordProtected, boolean defaultRole, String tool) + throws Exception { + + if (logger.isDebugEnabled()) { + logger.debug(" getAllMissions service called"); + } + + NavigableSet allGroups = (NavigableSet) groupManager.getAllGroups(); + + MissionService missionService = missionService(); + List missions = missionService.getAllMissions(passwordProtected, defaultRole, tool, allGroups); + return missions; + } + } diff --git a/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedSubscriptionManager.java b/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedSubscriptionManager.java index f3b73a4c..ed7fa7c7 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedSubscriptionManager.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedSubscriptionManager.java @@ -40,13 +40,11 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; -import java.util.logging.Handler; import org.apache.ignite.events.DiscoveryEvent; import org.apache.ignite.events.EventType; import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.services.ServiceContext; -import org.apache.naming.HandlerRef; import org.dom4j.Document; import org.dom4j.DocumentException; import org.dom4j.DocumentHelper; @@ -80,7 +78,6 @@ import com.bbn.marti.nio.protocol.base.AbstractBroadcastingProtocol; import com.bbn.marti.nio.server.NioServer; import com.bbn.marti.nio.websockets.NioWebSocketHandler; -import com.bbn.marti.remote.InputMetric; import com.bbn.marti.remote.RemoteSubscription; import com.bbn.marti.remote.RemoteSubscriptionMetrics; import com.bbn.marti.remote.SubscriptionManagerLite; @@ -107,14 +104,15 @@ import com.bbn.marti.util.concurrent.future.AsyncFuture; import com.bbn.marti.util.spring.SpringContextBeanForApi; import com.bbn.metrics.dto.MetricSubscription; -import com.bbn.security.web.MartiValidator; import com.bbn.security.web.MartiValidatorConstants; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.collect.Sets.SetView; import io.micrometer.core.instrument.Metrics; +import tak.server.CommonConstants; import tak.server.Constants; import tak.server.cluster.ClusterManager; import tak.server.cot.CotEventContainer; @@ -122,6 +120,7 @@ import tak.server.federation.FigFederateSubscription; import tak.server.ignite.IgniteHolder; import tak.server.ignite.cache.IgniteCacheHolder; +import tak.server.messaging.MessageConverter; public class DistributedSubscriptionManager implements SubscriptionManager, org.apache.ignite.services.Service { @@ -1918,6 +1917,8 @@ public void submitAnnounceMissionChangeCot(String missionName, CotEventContainer changeMessage.setContext(Constants.TOPICS_KEY, explicitTopics); } + + sendToPlugins(changeMessage); } @Override @@ -2002,13 +2003,15 @@ public void submitBroadcastMissionAnnouncementCot(String creatorUid, String grou } } } - + + sendToPlugins(message); + if (!websocketHits.isEmpty()) { WebsocketMessagingBroker.brokerWebSocketMessage(websocketHits, message); } } - public void sendMissionInvite(String missionName, String[] uids, String authorUid, String tool, String token, String roleXml) { + public void sendMissionInvite(String missionName, String[] uids, String authorUid, String tool, String token, String roleXml) { if (logger.isDebugEnabled()) { logger.debug("send mission invites for mission " + missionName); } @@ -2052,6 +2055,8 @@ public void submitSendMissionInviteCot(String[] uids, CotEventContainer inviteMe if (!websocketHits.isEmpty()) { WebsocketMessagingBroker.brokerWebSocketMessage(websocketHits, inviteMessage); } + + sendToPlugins(inviteMessage); } @Override @@ -2086,11 +2091,27 @@ public void submitSendMissionRoleChangeCot(String uid, CotEventContainer roleCha } else { sub.submit(roleChangeMessage); } + + sendToPlugins(roleChangeMessage); + } catch (Exception e) { logger.warn("exception sending mission role change message " + e.getMessage(), e); } } + private void sendToPlugins(CotEventContainer message) { + try { + byte[] rawMessage = MessageConverter.cotToDataMessage(new CotEventContainer(message, true, + ImmutableSet.of(Constants.SOURCE_TRANSPORT_KEY, Constants.SOURCE_PROTOCOL_KEY, Constants.USER_KEY)), + true, MessagingDependencyInjectionProxy.getInstance().serverInfo().getServerId()); + IgniteHolder.getInstance().getIgnite().message().send(CommonConstants.PLUGIN_SUBSCRIBE_TOPIC, rawMessage); + } catch (Exception e) { + if (logger.isDebugEnabled()) { + logger.debug("exception in sendToPlugins", e); + } + } + } + @Override public void putMissionContentUid(String missionName, String contentUid) { if (Strings.isNullOrEmpty(missionName) || Strings.isNullOrEmpty(contentUid)) { diff --git a/src/takserver-core/src/main/java/com/bbn/marti/service/LocalConfiguration.java b/src/takserver-core/src/main/java/com/bbn/marti/service/LocalConfiguration.java index 71b78c9d..e32bd671 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/service/LocalConfiguration.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/service/LocalConfiguration.java @@ -306,7 +306,8 @@ else if (messagingProfileActive) (offheapSize == null ? configuration.getBuffer().getQueue().getCacheOffHeapInitialSizeBytes() : offheapSize), (offheapSize == null ? configuration.getBuffer().getQueue().getCacheOffHeapMaxSizeBytes() : offheapSize), ignitePoolSize, - configuration.getBuffer().getQueue().isEnableCachePersistence())); + configuration.getBuffer().getQueue().isEnableCachePersistence(), + configuration.getBuffer().getQueue().getCacheOffHeapEvictionThreshold())); } diff --git a/src/takserver-core/src/main/java/com/bbn/marti/service/RepeaterService.java b/src/takserver-core/src/main/java/com/bbn/marti/service/RepeaterService.java index ad8861d5..cb0f4ce6 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/service/RepeaterService.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/service/RepeaterService.java @@ -32,18 +32,18 @@ * This service acts as a repeater for certain messages. Any messages added to this service * will be repeated out to all interested parties at regular intervals. This service will * adapt itself to the configuration specified in the config file. Those parameters are: - * + * * KEY | TYPE | DEFAULT | DESCRIPTION * -------------------------------------------------------------------------------------------------------------------- * enable | Boolean | false | determines if this service will be started or not * maxAllowedRepeatables | Integer | int max | the maximum number of messages that this service will manage * periodMillis | Integer | 10000 | the period in milliseconds in which this service will initiate dissemination - * + * * In addition the repeater node can contain 0 or more sub-nodes of the form: - * + * {@literal <}repeatableType initiate-test="/event/detail/initiate-repeat" cancel-test="/event/detail/cancel" _name="TestType1"/{@literal >} * Each such sub-node will specify the test that must be satisfied by an incoming message in order to initiate repeating * and the test that must be satisfied in order to cancel an already repeating message. - * + * * */ @@ -60,7 +60,7 @@ public class RepeaterService extends BaseService { private BrokerService brokerService; private CoreConfig coreConfig; private GroupManager groupManager; - + // ** configuration driven private Boolean active = Boolean.TRUE; private Integer maxAllowedRepeatables; @@ -80,7 +80,7 @@ public RepeaterService(CoreConfig coreConfig, BrokerService brokerService, Group loadRepeatableTypesFromConfig(); loadBuildItRepeatableTypes(); initiatePeriodicExecution(); - + LOGGER.debug("Repeater Service created [maxAllowedRepeatables=" + maxAllowedRepeatables + ", periodMillis=" + repeaterManager.getPeriodMillis() + "]"); } @@ -107,7 +107,7 @@ private void loadBuildItRepeatableTypes() { */ private void initiatePeriodicExecution() { final Runnable periodicTask = new Runnable() { - + @Override public void run() { try { @@ -119,7 +119,7 @@ public void run() { } } }; - + //scheduler.scheduleAtFixedRate(periodicTask, 0, repeaterManager.getPeriodMillis(), TimeUnit.MILLISECONDS); Resources.repeaterPool.schedule(periodicTask, repeaterManager.getPeriodMillis(), TimeUnit.MILLISECONDS); } @@ -141,33 +141,33 @@ public boolean addToInputQueue(CotEventContainer cotMsg) { // ** All other messages are ignored by this service. Tuple isInitiateMsg = isRepeatInitiatorMessage(cotMsg); if(isInitiateMsg.left()) { - - - + + + // User user = (User) cotMsg.getContextValue(SubmissionService.USER_KEY); -// +// // if (user == null) { // throw new TakException("repeatable message " + cotMsg + " does not contain a user - skipping"); // } - + LOGGER.debug("New repeatable message request found. "); - + // Get a copy of the user try { - - + + User user = (User) cotMsg.getContextValue(Constants.USER_KEY); if (user != null) { User rptUser = groupManager.replicateUserAndGroupMembership(user); cotMsg.setContext(Constants.USER_KEY, rptUser); } - + cotMsg.setContext(Constants.REPEATER_KEY, true); } catch (Exception e) { throw new TakException(e); } - + if(hasRoomInQueueFor(cotMsg)) { repeaterManager.addMessage(cotMsg, isInitiateMsg.right()); return true; @@ -186,7 +186,7 @@ public boolean addToInputQueue(CotEventContainer cotMsg) { return true; } else { LOGGER.warn("Received request to cancel repeat treatment for " + cotMsg.getUid() + ", however messages for this UID were not being repeated."); - return false; + return false; } } else { return false; @@ -207,7 +207,7 @@ private XPath getXPath(String xpath) { } /** - * Determine if the given message is a cancellation message or not. + * Determine if the given message is a cancellation message or not. * @param cotMsg The message to analyze * @return True if it represents a cancellation order for a repeated message */ @@ -218,7 +218,7 @@ private boolean isRepeatCancellationMessage(CotEventContainer cotMsg) { return true; } } - + return false; } @@ -243,11 +243,11 @@ private Tuple isRepeatInitiatorMessage(CotEventContainer cotMsg protected void processNextEvent() { // ** No processing occurs on a per message basis, instead it occurs periodically. See executePeriodicTask(void). } - + private static String getConfigurationKey(String propertyName) { return CONFIG_KEY_BASE + propertyName; } - + /** * Allows for pausing and resuming of this service. Set to true to allow service to execute, false to pause. * @param active @@ -255,17 +255,17 @@ private static String getConfigurationKey(String propertyName) { public void setActive(boolean active) { this.active = active; } - + @Override public void startService() { setActive(true); } - + @Override public void stopService(boolean wait) { setActive(false); } - + /** * This is the core logic that is executed on a periodic basis. It results in all repeatable messages being disseminated. */ @@ -275,7 +275,7 @@ private void executePeriodicTask() { long now = System.currentTimeMillis(); cotMsg.setTime(DateUtil.toCotTime(now)); cotMsg.setStale(DateUtil.toCotTime(now + coreConfig.getRemoteConfiguration().getRepeater().getStaleDelayMillis())); - + if (LOGGER.isDebugEnabled()) { LOGGER.debug("submitted repeated message to broker service: " + cotMsg); @@ -291,7 +291,7 @@ private void executePeriodicTask() { } } } - + brokerService.addToInputQueue(cotMsg); } @@ -305,16 +305,16 @@ private void executePeriodicTask() { cotMsg.setType("b-a-o-can"); String callsign = cotMsg.getCallsign(); - + Node detailNode = cotMsg.getDocument().selectSingleNode("/event/detail"); detailNode.detach(); - + Element eventElement = (Element)cotMsg.getDocument().nodeIterator().next(); Element newDetail = eventElement.addElement("detail"); Element newEmergency = newDetail.addElement("emergency"); newEmergency.addAttribute("cancel", "true"); newEmergency.setText(callsign); - + brokerService.addToInputQueue(cotMsg); removedEntries.add(cotMsg.getUid()); diff --git a/src/takserver-core/src/main/java/com/bbn/marti/service/RepositoryService.java b/src/takserver-core/src/main/java/com/bbn/marti/service/RepositoryService.java index c2f7db19..ab14c17c 100755 --- a/src/takserver-core/src/main/java/com/bbn/marti/service/RepositoryService.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/service/RepositoryService.java @@ -41,6 +41,7 @@ import com.bbn.cot.filter.ImageFormattingFilter; import com.bbn.marti.config.DataFeed; import com.bbn.marti.config.Repository; +import com.bbn.marti.groups.GroupFederationUtil; import com.bbn.marti.remote.ConnectionEventTypeValue; import com.bbn.marti.remote.ImagePref; import com.bbn.marti.remote.QueueMetric; @@ -143,9 +144,9 @@ public boolean addToInputQueue(CotEventContainer c) { if (c.getContextValue(Constants.ARCHIVE_EVENT_KEY) != null && !((Boolean) c.getContextValue(Constants.ARCHIVE_EVENT_KEY)).booleanValue()) { return false; } - - // This is the case where c is a datafeed message, and the datafeed archive is disabled - if (c.getContextValue(Constants.DATA_FEED_KEY) != null && ((DataFeed) c.getContextValue(Constants.DATA_FEED_KEY)).isArchive() == false) { + + // This is the case where c is a datafeed message, and the datafeed archive is disabled + if (c.getContextValue(Constants.DATA_FEED_KEY) != null && ((DataFeed) c.getContextValue(Constants.DATA_FEED_KEY)).isArchive() == false) { return false; } } catch (Exception e) { @@ -234,12 +235,12 @@ protected void processNextEvent() { } } }); - + } catch (RejectedExecutionException ree) { // count how often full queue has blocked message send Metrics.counter(Constants.METRIC_REPOSITORY_QUEUE_FULL_SKIP).increment(); } - } + } } private static final String cotRouterTableName = "cot_router"; @@ -259,7 +260,7 @@ public void insertBatchCotData(List events) { + "how, point_hae, point_ce, point_le, groups, " + "id, servertime) VALUES " + "(?,ST_GeometryFromText(?, 4326),?,?,?,?,?,?,?,?,?,?,?,?,(?)::bit(" + RemoteUtil.GROUPS_BIT_VECTOR_LEN + "), nextval('cot_router_seq'),?) ")) { - + // formats each cot message as we iterate over it Iterable imageFormattedEvents = Iterables.filter(events, imageFilter); LinkedList toRemove = new LinkedList<>(); @@ -304,15 +305,15 @@ public void insertBatchCotData(List events) { } continue; } - + event.setContext(Constants.GROUPS_BIT_VECTOR_KEY, groupsBitVector); - - // CoT is valid, but came from a data feed. add it to the list and let {#archiveBatchDataFeedCot()} handle it + + // CoT is valid, but came from a data feed. add it to the list and let {#archiveBatchDataFeedCot()} handle it if (event.getContextValue(Constants.DATA_FEED_KEY) != null) { dataFeedEvents.add(event); continue; } - + setCotQueryParams(cotRouterInsert, event); cotRouterInsert.addBatch(); @@ -333,7 +334,7 @@ public void insertBatchCotData(List events) { log.debug("exception executing CoT insert batch ", e); } } - + archiveBatchDataFeedCot(dataFeedEvents, connection); } catch (SQLException eee) { @@ -342,7 +343,7 @@ public void insertBatchCotData(List events) { } } } - + private void setCotQueryParams(PreparedStatement dataFeedInsert, CotEventContainer event) throws SQLException { dataFeedInsert.setString(1, event.getUid()); dataFeedInsert.setString(2, "POINT(" + event.getLon() + " " @@ -366,7 +367,7 @@ private void setCotQueryParams(PreparedStatement dataFeedInsert, CotEventContain dataFeedInsert.setDouble(14, event.getLe()); dataFeedInsert.setString(15, RemoteUtil.getInstance().bitVectorToString((boolean[]) event.getContext(Constants.GROUPS_BIT_VECTOR_KEY))); - + // // check to see if this event has serverTime (set by SubmissionService.processNextEvent) // @@ -379,8 +380,8 @@ private void setCotQueryParams(PreparedStatement dataFeedInsert, CotEventContain dataFeedInsert.setTimestamp(16, new Timestamp(DatatypeConverter .parseDateTime(serverTime).getTimeInMillis()), utcCalendar); } - - // link the Cot UID to the data feed it came from + + // link the Cot UID to the data feed it came from private void archiveBatchDataFeedCot(List events, Connection connection) { if (events.size() != 0) { @@ -392,14 +393,14 @@ private void archiveBatchDataFeedCot(List events, Connection + "access, qos, opex, " + "how, point_hae, point_ce, point_le, groups, " + "id, servertime) VALUES " - + "(?,ST_GeometryFromText(?, 4326),?,?,?,?,?,?,?,?,?,?,?,?,(?)::bit(" - + RemoteUtil.GROUPS_BIT_VECTOR_LEN + "), nextval('cot_router_seq'),?) returning id)" + + "(?,ST_GeometryFromText(?, 4326),?,?,?,?,?,?,?,?,?,?,?,?,(?)::bit(" + + RemoteUtil.GROUPS_BIT_VECTOR_LEN + "), nextval('cot_router_seq'),?) returning id)" + " INSERT INTO data_feed_cot (cot_router_id, data_feed_id) VALUES ((SELECT id FROM inserted_row), (SELECT id FROM data_feed WHERE uuid = ?))")) { - + for (CotEventContainer event : events) { try { setCotQueryParams(dataFeedInsert, event); - + dataFeedInsert.setString(17, ((DataFeed) event.getContext(Constants.DATA_FEED_KEY)).getUuid()); dataFeedInsert.execute(); @@ -761,7 +762,6 @@ public boolean hasRoomInQueueFor(CotEventContainer c) { * @param callsign String (required) * @param uid String (required) * @param eventType ConnectionEventTypeValue (required) - * @return boolean indicating success */ public void auditCallsignUIDEventAsync(String callsign, String uid, String username, ConnectionEventTypeValue eventType, String groupVector) { @@ -773,6 +773,13 @@ public void auditCallsignUIDEventAsync(String callsign, String uid, String usern return; } + // don't include the generated uuid for anonymous users in the audit + if (username.startsWith(GroupFederationUtil.ANONYMOUS_USERNAME_BASE)) { + username = GroupFederationUtil.ANONYMOUS_USERNAME_BASE; + } + + final String fusername = username; + Resources.callsignAuditExecutor.execute(() -> { try { @@ -788,7 +795,7 @@ public void auditCallsignUIDEventAsync(String callsign, String uid, String usern "select ce.id, cet.id, current_timestamp, (?)::bit(" + RemoteUtil.GROUPS_BIT_VECTOR_LEN + ") " + "from client_endpoint ce join connection_event_type cet on cet.event_name = ? " + "where ce.callsign = ? and ce.uid = ? and username = ?"); - + if (log.isDebugEnabled()) { log.debug("Insert client endpoint callsign: " + callsign + " uid: " + uid); } @@ -798,10 +805,10 @@ public void auditCallsignUIDEventAsync(String callsign, String uid, String usern try (PreparedStatement ps_ep = conn.prepareStatement(ep_sql)) { ps_ep.setString(1, callsign); ps_ep.setString(2, uid); - ps_ep.setString(3, username); + ps_ep.setString(3, fusername); ps_ep.setString(4, callsign); ps_ep.setString(5, uid); - ps_ep.setString(6, username); + ps_ep.setString(6, fusername); ps_ep.executeUpdate(); // Insert a client endpoint row @@ -810,7 +817,7 @@ public void auditCallsignUIDEventAsync(String callsign, String uid, String usern ps_epe.setString(2, eventType.value()); ps_epe.setString(3, callsign); ps_epe.setString(4, uid); - ps_epe.setString(5, username); + ps_epe.setString(5, fusername); ps_epe.executeUpdate(); } } @@ -832,8 +839,6 @@ public void auditCallsignUIDEventAsync(String callsign, String uid, String usern /** * Insert Disconnect events for any client endpoints that may have not been properly * disconnected due to a sudden CORE broker shutdown. Called from MartiMain during startup. - * - * @return boolean indicating success */ // @CacheEvict(value = Constants.CONTACTS_CACHE, allEntries = true) diff --git a/src/takserver-core/src/main/java/com/bbn/marti/service/SubmissionService.java b/src/takserver-core/src/main/java/com/bbn/marti/service/SubmissionService.java index 104e81ed..c9c7dd99 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/service/SubmissionService.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/service/SubmissionService.java @@ -74,6 +74,7 @@ import com.bbn.marti.config.Dropfilter; import com.bbn.marti.config.Federation.FederationServer; import com.bbn.marti.config.Filter; +import com.bbn.marti.config.GeospatialFilter; import com.bbn.marti.config.Input; import com.bbn.marti.config.MicrosoftCAConfig; import com.bbn.marti.config.NameEntries; @@ -245,7 +246,8 @@ public static SubmissionService getInstance() { "t-b-a", "t-b-c", "t-b-q", - "t-x-c-t", + "t-x-c-f", + "t-x-c-t", "t-x-c-t-r", "t-x-takp-q", "t-x-c-m", @@ -416,7 +418,7 @@ public void init() throws IOException, DocumentException { feed.getAuth().toString(), feed.getPort(), feed.isAuthRequired(), feed.getProtocol(), feed.getGroup(), feed.getIface(), feed.isArchive(), feed.isAnongroup(), feed.isArchiveOnly(), feed.getCoreVersion(), feed.getCoreVersion2TlsVersions(), - feed.isSync(), feed.getSyncCacheRetentionSeconds()); + feed.isSync(), feed.getSyncCacheRetentionSeconds(), feed.isFederated()); if (feed.getTag().size() > 0) { dataFeedRepository.removeAllDataFeedTagsById(dataFeedId); @@ -432,7 +434,7 @@ public void init() throws IOException, DocumentException { feed.getAuth().toString(), feed.getPort(), feed.isAuthRequired(), feed.getProtocol(), feed.getGroup(), feed.getIface(), feed.isArchive(), feed.isAnongroup(), feed.isArchiveOnly(), feed.getCoreVersion(), feed.getCoreVersion2TlsVersions(), - feed.isSync(), feed.getSyncCacheRetentionSeconds(), groupVector); + feed.isSync(), feed.getSyncCacheRetentionSeconds(), groupVector, feed.isFederated()); if (feed.getTag().size() > 0) { dataFeedRepository.removeAllDataFeedTagsById(dataFeedId); @@ -626,10 +628,16 @@ public void init() throws IOException, DocumentException { } else { List tags = dataFeedRepository.getDataFeedTagsById(dataFeedInfo.get(0).getId()); - + + Set groupNames = RemoteUtil.getInstance().getGroupNamesForBitVectorString( + dataFeedInfo.get(0).getGroupVector(), groupManager.getAllGroups()); + // update cache List pluginDatafeeds = new ArrayList<>(); - tak.server.plugins.PluginDataFeed pluginDataFeed = new tak.server.plugins.PluginDataFeed(m.getFeedUuid(), dataFeedInfo.get(0).getName(), tags, dataFeedInfo.get(0).getArchive(), dataFeedInfo.get(0).isSync()); + tak.server.plugins.PluginDataFeed pluginDataFeed = new tak.server.plugins.PluginDataFeed( + m.getFeedUuid(), dataFeedInfo.get(0).getName(), tags, dataFeedInfo.get(0).getArchive(), + dataFeedInfo.get(0).isSync(), new ArrayList(groupNames), dataFeedInfo.get(0).getFederated()); + pluginDatafeeds.add(pluginDataFeed); pluginDatafeedCacheHelper.cachePluginDatafeed(m.getFeedUuid(), pluginDatafeeds); @@ -639,13 +647,16 @@ public void init() throws IOException, DocumentException { dataFeed.getTag().addAll(tags); dataFeed.setArchive(dataFeedInfo.get(0).getArchive()); dataFeed.setSync(dataFeedInfo.get(0).isSync()); - if (logger.isDebugEnabled()) { - logger.debug("Retrieve Datafeed info from dataFeedRepository: uuid: {}, name: {}, tags: {}, archive: {}, sync: {}", dataFeed.getUuid(), dataFeed.getName(), dataFeed.getTag(), dataFeed.isArchive(), dataFeed.isSync()); + dataFeed.getFiltergroup().addAll(groupNames); + dataFeed.setFederated(dataFeedInfo.get(0).getFederated()); + + if (logger.isDebugEnabled()) { + logger.debug("Retrieve Datafeed info from dataFeedRepository: uuid: {}, name: {}, tags: {}, archive: {}, sync: {}, federated: {}, filtergroup: {}", dataFeed.getUuid(), dataFeed.getName(), dataFeed.getTag(), dataFeed.isArchive(), dataFeed.isSync(), dataFeed.isFederated(), dataFeed.getFiltergroup()); } - + DataFeedFilter.getInstance().filter(pluginCotEvent, dataFeed); - MessagingDependencyInjectionProxy.getInstance().cotMessenger().send(pluginCotEvent); + MessagingDependencyInjectionProxy.getInstance().cotMessenger().send(pluginCotEvent); InputMetric inputMetric = getInputMetric(dataFeed.getName()); if (inputMetric != null) { inputMetric.getMessagesReceived().incrementAndGet(); @@ -668,14 +679,16 @@ public void init() throws IOException, DocumentException { dataFeed.getTag().addAll(cacheResult.get(0).getTags()); dataFeed.setArchive(cacheResult.get(0).isArchive()); dataFeed.setSync(cacheResult.get(0).isSync()); - - if (logger.isDebugEnabled()) { - logger.debug("Retrieve Datafeed info from cache: uuid: {}, name: {}, tags: {}, archive: {}, sync: {}", dataFeed.getUuid(), dataFeed.getName(), dataFeed.getTag(), dataFeed.isArchive(), dataFeed.isSync()); + dataFeed.getFiltergroup().addAll(cacheResult.get(0).getFilterGroups()); + dataFeed.setFederated(cacheResult.get(0).isFederated()); + + if (logger.isDebugEnabled()) { + logger.debug("Retrieve Datafeed info from cache: uuid: {}, name: {}, tags: {}, archive: {}, sync: {}, federated: {}, filtergroup: {}", dataFeed.getUuid(), dataFeed.getName(), dataFeed.getTag(), dataFeed.isArchive(), dataFeed.isSync(), dataFeed.isFederated(), dataFeed.getFiltergroup()); } DataFeedFilter.getInstance().filter(pluginCotEvent, dataFeed); - MessagingDependencyInjectionProxy.getInstance().cotMessenger().send(pluginCotEvent); + MessagingDependencyInjectionProxy.getInstance().cotMessenger().send(pluginCotEvent); InputMetric inputMetric = getInputMetric(dataFeed.getName()); if (inputMetric != null) { inputMetric.getMessagesReceived().incrementAndGet(); @@ -1642,7 +1655,10 @@ public void processControlMessage(CotEventContainer c) throws UnknownHostExcepti case "t-b": processSubscriptionMessage(c); break; - case "t-b-q": + case "t-x-c-f": + processFilterMessage(c); + break; + case "t-b-q": logger.info("ignoring durable messaging (t-b-q) control message"); break; case "t-x-c-t": @@ -1686,22 +1702,6 @@ private void processSubscriptionMessage(CotEventContainer msg) throws UnknownHos return; } - GeospatialEventFilter geospatialEventFilter = null; - Node filterNode = subNode.selectSingleNode("*[local-name() = 'filter']"); - if (filterNode != null) { - try { - JAXBContext jaxbContext = JAXBContext.newInstance(Filter.class); - Unmarshaller unmarshaller = jaxbContext.createUnmarshaller(); - StringReader sr = new StringReader(filterNode.asXML()); - Filter filter = (Filter) unmarshaller.unmarshal(sr); - if (filter != null) { - geospatialEventFilter = new GeospatialEventFilter(filter.getGeospatialFilter()); - } - } catch (JAXBException e) { - logger.error("Exception reading filter from subscription! " + e.getMessage()); - } - } - TransportCotEvent transportType = TransportCotEvent.findByID(tokens[0]); Tuple> handlerAndProtocol = null; Subscription subscription = null; @@ -1718,7 +1718,6 @@ private void processSubscriptionMessage(CotEventContainer msg) throws UnknownHos logger.debug("updating subscription: " + subscription); } subscription.xpath = xpath; - subscription.geospatialEventFilter = geospatialEventFilter; } else { logger.warn("can't update a subscription that doesn't exist"); } @@ -1764,6 +1763,34 @@ private void processSubscriptionMessage(CotEventContainer msg) throws UnknownHos } } + private void processFilterMessage(CotEventContainer msg) { + try { + GeospatialEventFilter filter = null; + + List bboxNodes = msg.getDocument().selectNodes( + "/event/detail/subscription/geospatialFilter/boundingBox"); + if (bboxNodes != null && !bboxNodes.isEmpty()) { + GeospatialFilter geospatialFilter = new GeospatialFilter(); + for (Node bboxNode : bboxNodes) { + GeospatialFilter.BoundingBox boundingBox = new GeospatialFilter.BoundingBox(); + boundingBox.setMinLongitude(Double.valueOf(bboxNode.valueOf("@minLongitude"))); + boundingBox.setMinLatitude(Double.valueOf(bboxNode.valueOf("@minLatitude"))); + boundingBox.setMaxLongitude(Double.valueOf(bboxNode.valueOf("@maxLongitude"))); + boundingBox.setMaxLatitude(Double.valueOf(bboxNode.valueOf("@maxLatitude"))); + geospatialFilter.getBoundingBox().add(boundingBox); + } + filter = new GeospatialEventFilter(geospatialFilter); + } + + ChannelHandler handler = msg.getContext(Constants.SOURCE_TRANSPORT_KEY, ChannelHandler.class); + Subscription subscription = subscriptionStore.getByHandler(handler); + subscription.geospatialEventFilter = filter; + + } catch (Exception e) { + logger.error("Exception in processFilterMessage!", e); + } + } + private void processMetricsMessage(CotEventContainer msg) { try { Subscription subscription = subscriptionStore.getByHandler((ChannelHandler) msg.getContextValue(Constants.SOURCE_TRANSPORT_KEY)); @@ -2099,7 +2126,6 @@ public ConnectionModifyResult modifyInputAndSave(String inputName, Input modifie } synchronized (configLock) { - try { Input currentState = config.getInputByName(inputName); @@ -2146,8 +2172,12 @@ public ConnectionModifyResult modifyInputAndSave(String inputName, Input modifie } // Modify the archive flags - if (currentState.isArchive() != modifiedInput.isArchive() || currentState.isArchiveOnly() != modifiedInput.isArchiveOnly()) { + if (currentState.isArchive() != modifiedInput.isArchive() || currentState.isArchiveOnly() != modifiedInput.isArchiveOnly() || currentState.isFederated() != modifiedInput.isFederated() || currentState.getSyncCacheRetentionSeconds() != modifiedInput.getSyncCacheRetentionSeconds()) { // Modify the archive flag + logger.info("current arch, arch_only, federated: " + currentState.isArchive() + "," + currentState.isArchiveOnly() + + "," + currentState.isFederated()); + logger.info("modified arch, arch_only, federated: " + modifiedInput.isArchive() + "," + modifiedInput.isArchiveOnly() + + "," + modifiedInput.isFederated()); if (currentState.isArchive() != modifiedInput.isArchive()) { result = config.setArchiveFlagNoSave(inputName, modifiedInput.isArchive()); if (result != ConnectionModifyResult.SUCCESS) { @@ -2162,13 +2192,33 @@ public ConnectionModifyResult modifyInputAndSave(String inputName, Input modifie return result; } } + + // Modify federated flag + if (currentState.isFederated() != modifiedInput.isFederated()) { + result = config.setFederatedFlagNoSave(inputName, modifiedInput.isFederated()); + if (result != ConnectionModifyResult.SUCCESS) { + return result; + } + } + // Modify syncCacheRetentionSeconds value + + if (currentState.getSyncCacheRetentionSeconds() != + modifiedInput.getSyncCacheRetentionSeconds()) + { + result = config.setSyncCacheRetentionSeconds(inputName, modifiedInput.getSyncCacheRetentionSeconds()); + if(result != ConnectionModifyResult.SUCCESS) + { + return result; + } + } + // Save the flag changes; updateProtocolListeners(config.getInputByName(inputName)); nettyBuilder.modifyServerInput(modifiedInput); config.saveChangesAndUpdateCache(); } - + // Modify data feed attributes if (modifiedInput instanceof DataFeed && currentState instanceof DataFeed) { DataFeed modifiedDataFeed = (DataFeed) modifiedInput; diff --git a/src/takserver-core/src/main/java/com/bbn/marti/util/CircularlyLinkedQueue.java b/src/takserver-core/src/main/java/com/bbn/marti/util/CircularlyLinkedQueue.java index 2140642b..c370e188 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/util/CircularlyLinkedQueue.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/util/CircularlyLinkedQueue.java @@ -9,30 +9,30 @@ import java.util.NoSuchElementException; /** -* A CircularlyLinkedQueue implementation for supporting fast and potentially constant-time, zero-copy appends to other +* A CircularlyLinkedQueue implementation for supporting fast and potentially constant-time, zero-copy appends to other * CircularlyLinkedQueues. * * A circularly linked queue is essentially a singly linked list, except that a pointer to the tail element is -* kept instead of a head reference. The last element is circularly linked to the head of the list, allowing for constant +* kept instead of a head reference. The last element is circularly linked to the head of the list, allowing for constant * time reads from the head, and constant time writes to the tail. * -* The last element is called the tail, and refers to the most recently inserted element, and can be +* The last element is called the tail, and refers to the most recently inserted element, and can be * addressed by logical index (size() - 1). If the list is empty, the tail field is null, and the size is 0. This -* creates a separate case which must be addressed when an element is added to an empty queue, and when the last element of +* creates a separate case which must be addressed when an element is added to an empty queue, and when the last element of * the queue is removed. * * Internally, this queue relies on the Node structure to store the parameterized type and a pointer to the next element. * The NodeIterator internal class allows for contiguous iteration over the chain of nodes - it returns a Node with each next, -* and advances an internal Node pointer to the next Node. It starts pointing at the head. +* and advances an internal Node pointer to the next Node. It starts pointing at the head. * * The NodeValueIterator is a shallow wrapper around the NodeIterator class, and simply returns the value in each returned Node. * * Neither Iterator supports concurrent modification, but are not written to explicitly fail if this occurs. * -* This queue was written (in particular) to support a fast drainTo operation: this method takes another CircularlyLinkedQueue, and an optional +* This queue was written (in particular) to support a fast drainTo operation: this method takes another CircularlyLinkedQueue, and an optional * size limit, and splices at most limit nodes into the given queue. This is done without instantiating a new Node structure for -* each element (ie, zero-copy). If the size limit is larger than the size of this queue, the entire list can be spliced in constant time. -* Otherwise, limit nodes are selected in time proportional to limit (specifically, the time it takes to iterate across limit nodes), and spliced in. +* each element (ie, zero-copy). If the size limit is larger than the size of this queue, the entire list can be spliced in constant time. +* Otherwise, limit nodes are selected in time proportional to limit (specifically, the time it takes to iterate across limit nodes), and spliced in. * The number of elements actually spliced is returned. * */ @@ -157,20 +157,20 @@ public int size() { * This method hooks the tails of two circularly linked queues together, such that the * tail of the second queue becomes the tail of the new queue. * - * Given - * Queue 1: 0 -> 1 -> .... -> n (first tail) + * Given + * Queue 1: 0 -{@literal >} 1 -{@literal >} .... -{@literal >} n (first tail) * ^ | - * | <- <- <- V + * | {@literal <}- {@literal <}- {@literal <}- V * - * Queue 2: 0 -> 1 -> .... -> n (second tail) + * Queue 2: 0 -{@literal >} 1 -{@literal >} .... -{@literal >} (second tail) * ^ | - * | <- <- <- V - * + * | {@literal <}- {@literal <}- {@literal <}- V + * * * The new queue is - * 0 -> 1 -> .... -> n (first tail) -> 0 -> 1 -> .... -> n (second tail) + * 0 -{@literal >} 1 -{@literal >} .... -{@literal >} n (first tail) -{@literal >} 0 -{@literal >} 1 -{@literal >} .... -{@literal >} n (second tail) * ^ | - * | <- <- <- <- <- <- <- <- <- <- <- <- V + * | {@literal <}- {@literal <}- {@literal <}- {@literal <}- {@literal <}- {@literal <}- {@literal <}- {@literal <}- {@literal <}- {@literal <}- {@literal <}- {@literal <}- V * */ public static void hookTails(Node firstTail, Node secondTail) { @@ -212,8 +212,8 @@ public static Node buildTail(Collection coll) { /** * A constant-time, append all operation that hooks this entire queue onto the end of the given one. - * - * If this queue is empty, nothing is done. If their queue is empty, our tail node is copied + * + * If this queue is empty, nothing is done. If their queue is empty, our tail node is copied * into their tail field. Otherwise (both nonempty), both loops are linked together such that * their tail points to our head, and our tail points to their head. * @@ -242,7 +242,7 @@ public int drainTo(CircularlyLinkedQueue in) { /** * A batched append operation that hooks at most maxAdd nodes onto the end of the given queue. - * + * * If maxAdd is at least as large as this queue, we append the whole queue with the other drainTo method. * Otherwise, we select maxAdd Nodes, unhook them from our queue, and link them circularly. If the target * queue is empty, this loop becomes their new queue. Otherwise, we hook our queue fragment onto the end of theirs. @@ -254,10 +254,10 @@ public int drainTo(CircularlyLinkedQueue in, int maxAdd) { // POST: maxAdd is positive if (maxAdd < size()) { // POST: size is at least 2 : we can safely batch a remove from this queue without moving the tail - // select maxAdd nodes, link them into the given structure + // select maxAdd nodes, link them into the given structure Node finger = tail; Node head = tail.next; - + for (int i = 0; i < maxAdd; i++) { finger = finger.next; } @@ -292,7 +292,7 @@ public Iterator iterator() { @Override public boolean isEmpty() { - return size() == 0; + return size() == 0; } @Override @@ -359,7 +359,7 @@ public boolean addAll(int index, Collection coll) { else if (index == size()) return addAll(coll); // defer to simple tail append // build new circular queue, returns tail - Node tempTail = buildTail(coll); + Node tempTail = buildTail(coll); if (tempTail == null) return false; // splice segment in @@ -379,9 +379,9 @@ public boolean addAll(int index, Collection coll) { */ public Node getNodeAt(int index) { // primary case, make fastest - if (index >= 0 && index < size() - 1) + if (index >= 0 && index < size() - 1) return getNodeBefore(index).next; - else if (index < 0 || index >= size()) + else if (index < 0 || index >= size()) throw new IndexOutOfBoundsException(); else // index == size - 1 return tail; @@ -393,13 +393,13 @@ else if (index < 0 || index >= size()) * index == 0 is special cased to return the tail. */ public Node getNodeBefore(int index) { - if (index < 0 || index >= size()) throw new IndexOutOfBoundsException(); + if (index < 0 || index >= size()) throw new IndexOutOfBoundsException(); else if (index == 0) return tail; // POST: index is at least 1 NodeIterator iter = new NodeIterator(this); - // advance the iterator index - 2 times + // advance the iterator index - 2 times while(--index > 0) { iter.next(); } @@ -493,9 +493,9 @@ public void add(int index, E element) { */ @Override public E remove(int index) { - Node before = getNodeBefore(index); + Node before = getNodeBefore(index); Node target = before.next; - // getNodeBefore takes care of bounds check internally + // getNodeBefore takes care of bounds check internally // POST: size > 0 if (size() > 1) { @@ -530,28 +530,28 @@ public int indexOf(Object o) { /** * Advances the iterator until the given element is found, or the iterator is depleted - * - * When depleted, 0 is returned. Otherwise, the count of how many iterator positions were clicked before the - * element was found. Note that this means that the + * + * When depleted, 0 is returned. Otherwise, the count of how many iterator positions were clicked before the + * element was found. Note that this means that the */ public int findNextDistanceTo(E elem, Iterator iter) { int idx = 0; - + while (iter.hasNext()) { idx++; if (elem.equals(iter.next())) { return idx; } } - + return 0; } /** * This function uses a single iterator to seek (in a forwards direction) over all the instances of the given object in the list. - * If the findNextDistanceTo function never reports an instance, then -1 is returned. + * If the findNextDistanceTo function never reports an instance, then -1 is returned. * - * Otherwise, the findNextDistanceTo function is called on the same iterator: each call returns the number of elements + * Otherwise, the findNextDistanceTo function is called on the same iterator: each call returns the number of elements * it passed over before it found the given element. This count is summed into the travelled variable, which always holds the index * of the last instance found. Once no more instances can be found, we stop searching. */ @@ -587,4 +587,4 @@ public List subList(int fromIndex, int toIndex) { // TODO Auto-generated method stub return null; } -} \ No newline at end of file +} diff --git a/src/takserver-core/src/main/java/com/bbn/marti/util/concurrent/Transitions.java b/src/takserver-core/src/main/java/com/bbn/marti/util/concurrent/Transitions.java index c84a9a4e..f3a0ca71 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/util/concurrent/Transitions.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/util/concurrent/Transitions.java @@ -9,20 +9,20 @@ public class Transitions { /** - * Repeatedly tries to transition the state of the given atomic reference to dest, + * Repeatedly tries to transition the state of the given atomic reference to dest, * until the transition is successfully completed, or until the given untilSet * contains the atomic reference's state. * * Returns the state observed prior to either a successful transition or a member * of untilSet - * - * @note the caller should take care to all but prove convergence + * + * The caller should take care to all but prove convergence */ public static > E doUntilSetTransition(AtomicReference state, E dest, Set untilSet) { Assertion.areNotNull(state, dest, untilSet); - + E current; - + do { current = state.get(); } while (!untilSet.contains(current) @@ -38,15 +38,15 @@ public static > E doUntilSetTransition(AtomicReference stat * * Returns the state observed prior to either a successful transition or a member * of the complement of whileSet - * - * @note the caller should take care to all but prove convergence, as the opportunities for infinite + * + * The caller should take care to all but prove convergence, as the opportunities for infinite * looping abound */ public static > E doWhileSetTransition(AtomicReference state, E dest, Set whileSet) { Assertion.areNotNull(state, dest, whileSet); E current; - + do { current = state.get(); } while (whileSet.contains(current) diff --git a/src/takserver-core/src/main/java/com/bbn/marti/util/concurrent/executor/OrderedExecutorView.java b/src/takserver-core/src/main/java/com/bbn/marti/util/concurrent/executor/OrderedExecutorView.java index 45c0f575..195a2834 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/util/concurrent/executor/OrderedExecutorView.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/util/concurrent/executor/OrderedExecutorView.java @@ -5,28 +5,28 @@ import com.bbn.marti.util.concurrent.future.AsyncFuture; /** -* A view into an asynchronous executor that has the property that +* A view into an asynchronous executor that has the property that * all jobs are executed in the order they are received, while sharing -* the underlying thread pool with other views, and optionally imposing limits +* the underlying thread pool with other views, and optionally imposing limits * on the number of pending jobs allowed in the view. */ public interface OrderedExecutorView extends AsyncExecutor { /** * Returns whether there are any pending jobs in the view * - * Intended to be a low-cost facility for determining whether + * Intended to be a low-cost facility for determining whether * there are any jobs in the queue, to bypass more costly - * techniques of inserting a runnable unto a queue and waiting + * techniques of inserting a runnable unto a queue and waiting * for it to be called. This use is obviously only helpful if one - * can guarantee atomicity of the query -> skip/insert self + * can guarantee atomicity of the query -{@literal >} skip/insert self * logic. */ public boolean isEmpty(); /** * When the given AsyncFuture reaches the head of the queue, - * the executor view halts execution of client jobs until the given + * the executor view halts execution of client jobs until the given * future (used here as a barrier) is triggered. */ public void barrier(AsyncFuture barrier); -} \ No newline at end of file +} diff --git a/src/takserver-core/src/main/java/tak/server/ServerConfiguration.java b/src/takserver-core/src/main/java/tak/server/ServerConfiguration.java index e88f8ec8..a95c4ff2 100644 --- a/src/takserver-core/src/main/java/tak/server/ServerConfiguration.java +++ b/src/takserver-core/src/main/java/tak/server/ServerConfiguration.java @@ -91,6 +91,7 @@ import com.bbn.marti.service.kml.KmlIconStrategyJaxb; import com.bbn.marti.sync.cache.AllCopMissionsCacheKeyGenerator; import com.bbn.marti.sync.cache.AllMissionsCacheKeyGenerator; +import com.bbn.marti.sync.cache.InviteOnlyMissionCacheKeyGenerator; import com.bbn.marti.sync.cache.MethodNameMultiStringArgCacheKeyGenerator; import com.bbn.marti.sync.model.MissionChange; import com.bbn.marti.sync.model.MissionFeed; @@ -515,6 +516,11 @@ public KeyGenerator allCopsMissionsCacheKeyGenerator() { return new AllCopMissionsCacheKeyGenerator(); } + @Bean + public KeyGenerator inviteOnlyMissionsCacheKeyGenerator() { + return new InviteOnlyMissionCacheKeyGenerator(); + } + @Bean public KeyGenerator methodNameMultiStringArgCacheKeyGenerator() { return new MethodNameMultiStringArgCacheKeyGenerator(); diff --git a/src/takserver-core/src/main/java/tak/server/api/DistributedPluginMissionApi.java b/src/takserver-core/src/main/java/tak/server/api/DistributedPluginMissionApi.java index 1d6bcac8..1f6b3f12 100644 --- a/src/takserver-core/src/main/java/tak/server/api/DistributedPluginMissionApi.java +++ b/src/takserver-core/src/main/java/tak/server/api/DistributedPluginMissionApi.java @@ -146,12 +146,24 @@ public Mission readMission(String name, boolean changes, boolean logs, Long seca return mission; } - + + @Override + public Mission createMission(String name, String creatorUid, String[] groupNames, + String description, String chatRoom, String baseLayer, String bbox, + List boundingPolygonParam, String path, String classification, String tool, + String password, String roleParam, Long expiration, byte[] missionPackage) throws Exception { + + return createMission(name, creatorUid, groupNames, + description, chatRoom, baseLayer, bbox, + boundingPolygonParam, path, classification, tool, + password, roleParam, expiration, false, missionPackage); + } + @Override public Mission createMission(String name, String creatorUid, String[] groupNames, String description, String chatRoom, String baseLayer, String bbox, List boundingPolygonParam, String path, String classification, String tool, - String password, String roleParam, Long expiration, byte[] missionPackage) throws Exception { + String password, String roleParam, Long expiration, Boolean inviteOnly, byte[] missionPackage) throws Exception { if (Strings.isNullOrEmpty(name)) { throw new IllegalArgumentException("Mission name cannot be empty or null"); @@ -287,10 +299,10 @@ public Mission createMission(String name, String creatorUid, String[] groupNames if (expiration != null) { mission = missionService.createMission(name, creatorUid, groupVectorMission, description, chatRoom, baseLayer, bbox, path, classification, tool, passwordHash, defaultRole, expiration, - boundingPolygon); + boundingPolygon, inviteOnly); } else { mission = missionService.createMission(name, creatorUid, groupVectorMission, description, chatRoom, - baseLayer, bbox, path, classification, tool, passwordHash, defaultRole, -1L, boundingPolygon); + baseLayer, bbox, path, classification, tool, passwordHash, defaultRole, -1L, boundingPolygon, inviteOnly); } MissionRole ownerRole = missionRoleRepository.findFirstByRole(MissionRole.Role.MISSION_OWNER); diff --git a/src/takserver-core/src/main/java/tak/server/config/ApiConfiguration.java b/src/takserver-core/src/main/java/tak/server/config/ApiConfiguration.java index a2dcaa68..b2062ea4 100644 --- a/src/takserver-core/src/main/java/tak/server/config/ApiConfiguration.java +++ b/src/takserver-core/src/main/java/tak/server/config/ApiConfiguration.java @@ -44,6 +44,7 @@ import com.bbn.marti.KmlMasterSaServlet; import com.bbn.marti.LatestKMLServlet; import com.bbn.marti.MissionKMLServlet; +import com.bbn.marti.oauth.OAuthApi; import com.bbn.marti.ResubscribeServlet; import com.bbn.marti.TracksKMLServlet; import com.bbn.marti.citrap.CITrapReportAPI; @@ -80,7 +81,7 @@ import com.bbn.marti.network.SecurityAuthenticationApi; import com.bbn.marti.network.SubmissionApi; import com.bbn.marti.network.UIDSearchApi; -import com.bbn.marti.oauth.admin.TokenApi; +import com.bbn.marti.oauth.TokenApi; import com.bbn.marti.remote.CoreConfig; import com.bbn.marti.remote.FederationConfigInterface; import com.bbn.marti.remote.groups.FileUserManagementInterface; @@ -874,6 +875,11 @@ public QoSApi qosApi() { public LoginAccessController loginAcccessController() { return new LoginAccessController(); } + + @Bean + public OAuthApi OAuthApi() { + return new OAuthApi(); + } @Bean public FileUserAccountManagementApi fileUserAccountManagementApi() { diff --git a/src/takserver-core/src/main/java/tak/server/federation/DistributedFederationManager.java b/src/takserver-core/src/main/java/tak/server/federation/DistributedFederationManager.java index 092bb5b0..9ff27497 100644 --- a/src/takserver-core/src/main/java/tak/server/federation/DistributedFederationManager.java +++ b/src/takserver-core/src/main/java/tak/server/federation/DistributedFederationManager.java @@ -62,6 +62,7 @@ import com.bbn.cot.filter.GeospatialEventFilter; import com.bbn.marti.config.Federation; import com.bbn.marti.config.Federation.Federate; +import com.bbn.marti.config.Federation.Federate.Mission; import com.bbn.marti.config.Federation.FederationOutgoing; import com.bbn.marti.config.Federation.FederationServer.FederationPort; import com.bbn.marti.config.Filter; @@ -2064,6 +2065,12 @@ public void submitFederateROL(final ROL rol, final NavigableSet groups) { submitFederateROL(rol, groups, null); } + + @Override + public void submitMissionFederateROL(final ROL rol, final NavigableSet groups, String missionName) { + + submitMissionFederateROL(rol, groups, null, missionName); + } @Override public void submitFederateROL(ROL rol, final NavigableSet groups, String fileHash) { @@ -2194,6 +2201,152 @@ public void run() { Metrics.counter(Constants.METRIC_FEDERATE_ROL_SKIP).increment(); } } + + @Override + public void submitMissionFederateROL(ROL rol, final NavigableSet groups, String fileHash, String missionName) { + try { + // Federate this ROL message if there is a reachability relationship + Resources.federationROLExecutor.execute(new Runnable() { + @Override + public void run() { + if (logger.isDebugEnabled()) { + logger.debug("Federated ROL: " + rol.getProgram() + " groups: " + groups); + } + + ROL finalRol = rol; + + // Populate the file contents here in messaging if no content provided. Avoids serializing file over ignite + try { + if (rol.getPayloadList().isEmpty() && !Strings.isNullOrEmpty(fileHash)) { + + String groupVector = RemoteUtil.getInstance().bitVectorToString(RemoteUtil.getInstance().getBitVectorForGroups(groups)); + + // use the file hash to load the file from db / cache + byte[] fileBytes = MessagingDependencyInjectionProxy.getInstance().esyncService().getContentByHash(fileHash, groupVector); + + if (logger.isDebugEnabled()) { + if (fileBytes == null) { + logger.debug("null bytes for file " + fileHash); + } else { + + logger.debug("fetched " + fileBytes.length + " for hash " + fileHash); + + } + } + + BinaryBlob filePayload = BinaryBlob.newBuilder().setData(ByteString.readFrom(new ByteArrayInputStream(fileBytes))).build(); + + ROL.Builder rolBuilder = rol.toBuilder(); + + rolBuilder.addPayload(filePayload); + + if (logger.isDebugEnabled()) { + logger.debug("Added file payload size " + fileBytes.length + " bytes to rol"); + } + + finalRol = rolBuilder.build(); + + } + } catch (Exception e) { + if (logger.isWarnEnabled()) { + logger.warn("exception fetching file for federation from data layer", e); + } + } + + try { + + for (FederateSubscription destFed : SubscriptionStore.getInstanceFederatedSubscriptionManager().getFederateSubscriptions()) { + + if (CommonGroupDirectedReachability.getInstance().isReachable(groups, destFed.getUser())) { + + Federate federate = null; + if (destFed instanceof FigServerFederateSubscription) { + federate = getFederate(((FigFederateSubscription) destFed).getFederate().getId()); + } else if (destFed instanceof FigFederateSubscription) { + federate = ((FigFederateSubscription) destFed).getFigClient().getFederate(); + } + + if (federate == null) { + if (logger.isDebugEnabled()) { + logger.debug("unable to add federate groups to ROL, federate is null "); + } + continue; + } + + // Decide whether to send this mission to the federate + boolean isFederateThisMission = (federate.isMissionFederateDefault() != null)?federate.isMissionFederateDefault(): true; + + for (Mission missionFederateConfig: federate.getMission()) { + if (missionFederateConfig.getName().equals(missionName)) { + isFederateThisMission = missionFederateConfig.isEnabled(); + break; + } + } + if (!isFederateThisMission) { + if (logger.isDebugEnabled()) { + logger.debug("Not sending mission {} to federate {}", missionName, federate.getName()); + } + continue; + } + + ROL.Builder builder = finalRol.toBuilder(); + + if (federate.isFederatedGroupMapping()) { + Set outGroups = GroupFederationUtil.getInstance().filterFedOutboundGroups( + federate.getOutboundGroup(), groups, federate.getId()); + builder.addAllFederateGroups(outGroups); + } + + ROL rolWithGroups = builder.build(); + + if (destFed instanceof FigServerFederateSubscription) { + if (logger.isDebugEnabled()) { + logger.debug("for FigServerFederateSubscription - sending federated ROL " + + rolWithGroups.getProgram() + " from groups " + groups + " to " + destFed.getUser()); + } + ((FigServerFederateSubscription) destFed).lazyGetROLClientStream().send(rolWithGroups); + + trackSendChangesEventForFederate(((FigServerFederateSubscription) destFed).getFederate().getId(), ((FigServerFederateSubscription) destFed).getFederate().getName(), true); + + try { + Metrics.counter(Constants.METRIC_FED_ROL_MESSAGE_READ_COUNT, "takserver", "messaging").increment(); + } catch (Exception ex) { + if (logger.isDebugEnabled()) { + logger.debug("error recording fed message read metric", ex); + } + } + + } else if (destFed instanceof FigFederateSubscription) { + if (logger.isDebugEnabled()) { + logger.debug("for FigFederateSubscription - sending federated ROL " + rolWithGroups.getProgram() + + " from groups " + groups + " to " + destFed.getUser()); + } + ((FigFederateSubscription) destFed).getFigClient().getRolCall().sendMessage(rolWithGroups); + + trackSendChangesEventForFederate(((FigFederateSubscription) destFed).getFederate().getId(), ((FigFederateSubscription) destFed).getFederate().getName(), false); + + try { + Metrics.counter(Constants.METRIC_FED_ROL_MESSAGE_WRITE_COUNT, "takserver", "messaging").increment(); + } catch (Exception ex) { + if (logger.isDebugEnabled()) { + logger.debug("error recording fed message write metric", ex); + } + } + } + } + } + } catch (Exception e) { + if (logger.isWarnEnabled()) { + logger.warn("exception federating mission change", e); + } + } + } + }); + } catch (RejectedExecutionException ree) { + // count how often full queue has blocked ROL send + Metrics.counter(Constants.METRIC_FEDERATE_ROL_SKIP).increment(); + } + } @Override public void reconfigureFederation() { @@ -2251,4 +2404,21 @@ public void trackDisconnectEventForFederate(String fedId, String fedName, boolea } } } + + @Override + public void updateFederateMissionSettings(String federateUID, boolean missionFederateDefault, List federateMissions) { + Federation fedConfig = coreConfig.getRemoteConfiguration().getFederation(); + + List federates = fedConfig.getFederate(); + + for (Federate f : federates) { + if (f.getId().compareTo(federateUID) == 0) { + f.setMissionFederateDefault(missionFederateDefault); + f.getMission().clear(); + f.getMission().addAll(federateMissions); + break; + } + } + coreConfig.setAndSaveFederation(fedConfig); + } } diff --git a/src/takserver-core/src/main/java/tak/server/federation/FederationServer.java b/src/takserver-core/src/main/java/tak/server/federation/FederationServer.java index aa0ae247..19669f42 100644 --- a/src/takserver-core/src/main/java/tak/server/federation/FederationServer.java +++ b/src/takserver-core/src/main/java/tak/server/federation/FederationServer.java @@ -129,6 +129,8 @@ import io.grpc.stub.StreamObserver; import io.micrometer.core.instrument.Metrics; import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.util.internal.logging.InternalLoggerFactory; +import io.netty.util.internal.logging.Slf4JLoggerFactory; import mil.af.rl.rol.FederationProcessor; import mil.af.rl.rol.MissionRolVisitor; import mil.af.rl.rol.Resource; @@ -241,6 +243,8 @@ public static void stopServer() { @EventListener({ContextRefreshedEvent.class}) private void init() { fedServer = this; + + InternalLoggerFactory.setDefaultFactory(Slf4JLoggerFactory.INSTANCE); try { // if the federation truststore is pointing to the root truststore - undo it and set to fed truststore. @@ -283,6 +287,8 @@ private void init() { serverConfig.setKeystorePassword(fedServerConfig.getTls().getKeystorePass()); serverConfig.setTruststoreFile(fedServerConfig.getTls().getTruststoreFile()); serverConfig.setTruststorePass(fedServerConfig.getTls().getTruststorePass()); + serverConfig.setContext(fedServerConfig.getTls().getContext()); + serverConfig.setCiphers(fedServerConfig.getTls().getCiphers()); serverConfig.setSkipGateway(true); // eliminate this serverConfig.setMaxMessageSizeBytes(fedConfig().getFederationServer().getMaxMessageSizeBytes()); // put in coreconfig serverConfig.setMetricsLogIntervalSeconds(60); // put in coreconfig @@ -325,6 +331,8 @@ private void refreshConfig() { serverConfig.setKeystorePassword(fedServerConfig.getTls().getKeystorePass()); serverConfig.setTruststoreFile(fedServerConfig.getTls().getTruststoreFile()); serverConfig.setTruststorePass(fedServerConfig.getTls().getTruststorePass()); + serverConfig.setContext(fedServerConfig.getTls().getContext()); + serverConfig.setCiphers(fedServerConfig.getTls().getCiphers()); serverConfig.setSkipGateway(true); // eliminate this serverConfig.setMaxMessageSizeBytes(134217728); // put in coreconfig serverConfig.setMetricsLogIntervalSeconds(60); // put in coreconfig @@ -369,7 +377,7 @@ public void run() { } NettyServerBuilder serverBuilder = NettyServerBuilder.forPort(config.getPort()) - .maxMessageSize(config.getMaxMessageSizeBytes()) // max message size. If not specified, defaults to 4MB + .maxInboundMessageSize(config.getMaxMessageSizeBytes()) // max message size. If not specified, defaults to 4MB .sslContext(sslConfig.getSslContext()) .executor(Resources.federationGrpcExecutor) .workerEventLoopGroup(Resources.federationGrpcWorkerEventLoopGroup) @@ -696,7 +704,7 @@ public void clientROLStream(Subscription subscription, StreamObserver clien try { // send out data feeds to federate if (DistributedConfiguration.getInstance().getRemoteConfiguration().getFederation().isAllowDataFeedFederation()) { - List feedMessages = mdm.getDataFeedEvents(); + List feedMessages = mdm.getDataFeedEventsForFederatedDataFeedOnly(); AtomicLong delayMs = new AtomicLong(100L); for (final ROL feedMessage : feedMessages) { @@ -1358,7 +1366,7 @@ public void onNext(FederateGroups value) { @Override public void onError(Throwable t) { - + logger.error("error in clientFederateGroupsStream", t); } @Override @@ -1853,7 +1861,10 @@ private void handleRead(FederatedEvent fedEvent, String sessionId, FederateSubsc cot.setContext(Constants.GROUPS_KEY, groups); } - // cot.setContextValue(RepositoryService.ARCHIVE_EVENT_KEY, config.isArchive()); // TODO: make archive configurable per FIG federate + if (!federationManager.getFederate(serverFederateMap.get(getCurrentSessionId())).isArchive()) { + cot.setContextValue(Constants.ARCHIVE_EVENT_KEY, Boolean.FALSE); + } + } else { if (logger.isDebugEnabled()) { logger.debug("user not found for federate subscription for FIG message"); diff --git a/src/takserver-core/src/main/java/tak/server/federation/FigFederateSubscription.java b/src/takserver-core/src/main/java/tak/server/federation/FigFederateSubscription.java index bb7028d3..4684ac99 100644 --- a/src/takserver-core/src/main/java/tak/server/federation/FigFederateSubscription.java +++ b/src/takserver-core/src/main/java/tak/server/federation/FigFederateSubscription.java @@ -15,7 +15,7 @@ import com.atakmap.Tak.FederatedEvent; import com.atakmap.Tak.Subscription; - +import com.bbn.marti.config.DataFeed; import com.bbn.marti.config.Federation.Federate; import com.bbn.marti.groups.GroupFederationUtil; import com.bbn.marti.groups.MessagingUtilImpl; @@ -231,6 +231,16 @@ public void submit(final CotEventContainer toSend, long hitTime) { } return; } + + if (toSend.getContextValue(Constants.DATA_FEED_KEY) != null) { + DataFeed datafeed = (DataFeed)toSend.getContext(Constants.DATA_FEED_KEY); + if (!datafeed.isFederated()) { + logger.info("~~~ In submit A: Not sending to federation"); + return; + }else { + logger.info("~~~ In submit A: Will send to federation"); + } + } // increment the hit count in the super class super.incHit(hitTime); diff --git a/src/takserver-core/src/main/java/tak/server/federation/FigServerFederateSubscription.java b/src/takserver-core/src/main/java/tak/server/federation/FigServerFederateSubscription.java index e0a3d878..de2c7369 100644 --- a/src/takserver-core/src/main/java/tak/server/federation/FigServerFederateSubscription.java +++ b/src/takserver-core/src/main/java/tak/server/federation/FigServerFederateSubscription.java @@ -12,6 +12,7 @@ import tak.server.Constants; import tak.server.cot.CotEventContainer; +import com.bbn.marti.config.DataFeed; import com.bbn.marti.config.Federation.Federate; import com.bbn.marti.groups.GroupFederationUtil; import com.bbn.marti.nio.channel.base.AbstractBroadcastingChannelHandler; @@ -115,6 +116,16 @@ public void submit(final CotEventContainer toSend, long hitTime) { } return; } + + if (toSend.getContextValue(Constants.DATA_FEED_KEY) != null) { + DataFeed datafeed = (DataFeed)toSend.getContext(Constants.DATA_FEED_KEY); + if (!datafeed.isFederated()) { + logger.info("~~~ In submit B: Not send to federation"); + return; + }else { + logger.info("~~~ In submit B: Will send to federation"); + } + } // increment the hit count in the super class super.incHit(hitTime); diff --git a/src/takserver-core/src/main/java/tak/server/federation/MissionDisruptionManager.java b/src/takserver-core/src/main/java/tak/server/federation/MissionDisruptionManager.java index 3047c466..bd182ddf 100644 --- a/src/takserver-core/src/main/java/tak/server/federation/MissionDisruptionManager.java +++ b/src/takserver-core/src/main/java/tak/server/federation/MissionDisruptionManager.java @@ -213,7 +213,6 @@ public List getMissionChangesAndTrackConnectEvent(Federate federate, String List fedMissions = new ArrayList<>(); if (DistributedConfiguration.getInstance().getRemoteConfiguration().getFederation().isFederateOnlyPublicMissions()) { - missionService.getAllMissions(true, true, null, outboundGroups); fedMissions.addAll(missionService.getAllMissions(true, true, "public", outboundGroups)); // only federating public, but make an exception for cops if vbm is enabled if (DistributedConfiguration.getInstance().getRemoteConfiguration().getVbm().isEnabled()) { @@ -226,7 +225,23 @@ public List getMissionChangesAndTrackConnectEvent(Federate federate, String // get mission changes as ROL for (Mission fedMission : fedMissions) { + + // Decide whether to send this mission to the federate + boolean isFederateThisMission = (federate.isMissionFederateDefault() != null)?federate.isMissionFederateDefault(): true; + for (com.bbn.marti.config.Federation.Federate.Mission missionFederateConfig: federate.getMission()) { + if (missionFederateConfig.getName().equals(fedMission.getName())) { + isFederateThisMission = missionFederateConfig.isEnabled(); + break; + } + } + if (!isFederateThisMission) { + if (logger.isDebugEnabled()) { + logger.debug("MissionDisruptionManager: Not sending mission {} to federate {}", fedMission.getName(), federate.getName()); + } + continue; + } + long missionRecencyMillis = missionRecencySeconds.getOrDefault(fedMission.getName(), recencySecs) * 1000; long maxMissionRecencyMillis; if (missionRecencyMillis < 0) { @@ -312,24 +327,28 @@ public List getMissionChangesAndTrackConnectEvent(Federate federate, String .filter(df -> df.getUuid().equals(createdMissionFeed.getDataFeedUid())) .findFirst().orElse(null); - DataFeedMetadata createFeedMeta = new DataFeedMetadata(); - - createFeedMeta.setDataFeedUid(createdMissionFeed.getDataFeedUid()); - createFeedMeta.setFilterBbox(createdMissionFeed.getFilterBbox()); - createFeedMeta.setFilterCallsign(createdMissionFeed.getFilterCallsign()); - createFeedMeta.setFilterType(createdMissionFeed.getFilterType()); - - createFeedMeta.setMissionFeedUid(change.getMissionFeedUid()); - createFeedMeta.setMissionName(change.getMissionName()); + if (dataFeed != null && dataFeed.isFederated()) { + + DataFeedMetadata createFeedMeta = new DataFeedMetadata(); + + createFeedMeta.setDataFeedUid(createdMissionFeed.getDataFeedUid()); + createFeedMeta.setFilterBbox(createdMissionFeed.getFilterBbox()); + createFeedMeta.setFilterCallsign(createdMissionFeed.getFilterCallsign()); + createFeedMeta.setFilterType(createdMissionFeed.getFilterType()); + + createFeedMeta.setMissionFeedUid(change.getMissionFeedUid()); + createFeedMeta.setMissionName(change.getMissionName()); + + createFeedMeta.setArchive(dataFeed.isArchive()); + createFeedMeta.setArchiveOnly(dataFeed.isArchiveOnly()); + createFeedMeta.setSync(dataFeed.isSync()); + createFeedMeta.setFeedName(dataFeed.getName()); + createFeedMeta.setAuthType(dataFeed.getAuth().toString()); + createFeedMeta.setTags(dataFeed.getTag()); + + changeROL = malrc.createDataFeedToROL(createFeedMeta); + } - createFeedMeta.setArchive(dataFeed.isArchive()); - createFeedMeta.setArchiveOnly(dataFeed.isArchiveOnly()); - createFeedMeta.setSync(dataFeed.isSync()); - createFeedMeta.setFeedName(dataFeed.getName()); - createFeedMeta.setAuthType(dataFeed.getAuth().toString()); - createFeedMeta.setTags(dataFeed.getTag()); - - changeROL = malrc.createDataFeedToROL(createFeedMeta); } } break; @@ -337,12 +356,25 @@ public List getMissionChangesAndTrackConnectEvent(Federate federate, String case DELETE_DATA_FEED: { if (DistributedConfiguration.getInstance().getRemoteConfiguration().getFederation().isAllowDataFeedFederation()) { MissionFeed deleteMissionFeed = missionService.getMissionFeed(change.getMissionFeedUid()); + if (deleteMissionFeed != null) { - DataFeedMetadata deleteFeedMeta = new DataFeedMetadata(); - deleteFeedMeta.setMissionFeedUid(change.getMissionFeedUid()); - deleteFeedMeta.setMissionName(change.getMissionName()); - deleteFeedMeta.setDataFeedUid(deleteMissionFeed.getDataFeedUid()); - changeROL = malrc.deleteDataFeedToROL(deleteFeedMeta); + DataFeed dataFeed = DistributedConfiguration.getInstance() + .getRemoteConfiguration() + .getNetwork() + .getDatafeed() + .stream() + .filter(df -> df.getUuid().equals(deleteMissionFeed.getDataFeedUid())) + .findFirst().orElse(null); + + if (dataFeed != null && dataFeed.isFederated()) { + DataFeedMetadata deleteFeedMeta = new DataFeedMetadata(); + deleteFeedMeta.setMissionFeedUid(change.getMissionFeedUid()); + deleteFeedMeta.setMissionName(change.getMissionName()); + deleteFeedMeta.setDataFeedUid(deleteMissionFeed.getDataFeedUid()); + + changeROL = malrc.deleteDataFeedToROL(deleteFeedMeta); + } + } } break; @@ -376,7 +408,7 @@ public List getMissionChangesAndTrackConnectEvent(Federate federate, String return rols; } - public List getDataFeedEvents() { + public List getDataFeedEventsForFederatedDataFeedOnly() { List rols = new CopyOnWriteArrayList<>(); DistributedConfiguration.getInstance() @@ -386,16 +418,20 @@ public List getDataFeedEvents() { .stream() .forEach(dataFeed -> { try { - DataFeedMetadata createFeedMeta = new DataFeedMetadata(); - createFeedMeta.setDataFeedUid(dataFeed.getUuid()); - createFeedMeta.setArchive(dataFeed.isArchive()); - createFeedMeta.setArchiveOnly(dataFeed.isArchiveOnly()); - createFeedMeta.setSync(dataFeed.isSync()); - createFeedMeta.setFeedName(dataFeed.getName()); - createFeedMeta.setAuthType(dataFeed.getAuth().toString()); - createFeedMeta.setTags(dataFeed.getTag()); - rols.add(malrc.updateDataFeedToROL(createFeedMeta)); + if (dataFeed.isFederated()) { + DataFeedMetadata createFeedMeta = new DataFeedMetadata(); + createFeedMeta.setDataFeedUid(dataFeed.getUuid()); + createFeedMeta.setArchive(dataFeed.isArchive()); + createFeedMeta.setArchiveOnly(dataFeed.isArchiveOnly()); + createFeedMeta.setSync(dataFeed.isSync()); + createFeedMeta.setFeedName(dataFeed.getName()); + createFeedMeta.setAuthType(dataFeed.getAuth().toString()); + createFeedMeta.setTags(dataFeed.getTag()); + + rols.add(malrc.updateDataFeedToROL(createFeedMeta)); + } + } catch (JsonProcessingException e) { if (logger.isDebugEnabled()) { logger.debug("exception in federate data feed event", e); diff --git a/src/takserver-core/src/main/java/tak/server/federation/SSLConfig.java b/src/takserver-core/src/main/java/tak/server/federation/SSLConfig.java index 29e8ac39..64bfd7c2 100644 --- a/src/takserver-core/src/main/java/tak/server/federation/SSLConfig.java +++ b/src/takserver-core/src/main/java/tak/server/federation/SSLConfig.java @@ -7,11 +7,13 @@ import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; +import java.util.Arrays; import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLEngine; import javax.net.ssl.TrustManagerFactory; +import com.google.common.base.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,13 +52,25 @@ public SslContext initSslContext(FigServerConfig config) { trust.load(getFileOrResource(config.getTruststoreFile()), config.getTruststorePassword().toCharArray()); trustMgrFactory.init(trust); - sslContext = GrpcSslContexts.configure(SslContextBuilder.forServer(keyMgrFactory), SslProvider.OPENSSL) // this ensures that we are using OpenSSL, not JRE SSL - .protocols("TLSv1.2","TLSv1.3") + SslContextBuilder sslContextBuilder = GrpcSslContexts.configure(SslContextBuilder.forServer(keyMgrFactory), SslProvider.OPENSSL) // this ensures that we are using OpenSSL, not JRE SSL .trustManager(trustMgrFactory) - .clientAuth(ClientAuth.REQUIRE) // client auth always required - .build(); + .clientAuth(ClientAuth.REQUIRE); // client auth always required + + String context = "TLSv1.2,TLSv1.3"; + + String ciphers = config.getCiphers(); + if (!Strings.isNullOrEmpty(ciphers)) { + sslContextBuilder = sslContextBuilder.ciphers(Arrays.asList(ciphers.split(","))); + // only set context from config if cipher is also present + if (!Strings.isNullOrEmpty(config.getContext())) { + context = config.getContext(); + } + } + + sslContext = sslContextBuilder.protocols(Arrays.asList(context.split(","))).build(); return sslContext; + } catch (Exception e) { throw new TakException(e); } diff --git a/src/takserver-core/src/main/java/tak/server/federation/TakFigClient.java b/src/takserver-core/src/main/java/tak/server/federation/TakFigClient.java index f74bfb40..40d08c2d 100644 --- a/src/takserver-core/src/main/java/tak/server/federation/TakFigClient.java +++ b/src/takserver-core/src/main/java/tak/server/federation/TakFigClient.java @@ -16,6 +16,7 @@ import java.security.cert.X509Certificate; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Date; import java.util.HashSet; @@ -325,12 +326,24 @@ public void start(FederationOutgoing outgoing, ConnectionStatus status) { logger.warn("exception initializing trust store", e); } - - channel = openFigConnection(outgoing.getAddress(), outgoing.getPort(), GrpcSslContexts.configure(SslContextBuilder.forClient(), SslProvider.OPENSSL) // this ensures that we are using OpenSSL, not JRE SSL - .protocols("TLSv1.2","TLSv1.3") + SslContextBuilder sslContextBuilder = GrpcSslContexts.configure(SslContextBuilder.forClient(), SslProvider.OPENSSL) // this ensures that we are using OpenSSL, not JRE SSL .keyManager(keyMgrFactory) - .trustManager(trustMgrFactory) - .build()); + .trustManager(trustMgrFactory); + + String context = "TLSv1.2,TLSv1.3"; + + String ciphers = figTls.getCiphers(); + if (!Strings.isNullOrEmpty(ciphers)) { + sslContextBuilder = sslContextBuilder.ciphers(Arrays.asList(ciphers.split(","))); + // only set context from config if cipher is also present + if (!Strings.isNullOrEmpty(figTls.getContext())) { + context = figTls.getContext(); + } + } + + channel = openFigConnection(outgoing.getAddress(), outgoing.getPort(), + sslContextBuilder.protocols(Arrays.asList(context.split(","))).build()); + } catch (Exception e) { logger.error("exception setting up TLS config", e); } @@ -604,7 +617,10 @@ public void onNext(FederatedEvent fedEvent) { cot.setContext(Constants.GROUPS_KEY, groups); } - // cot.setContextValue(RepositoryService.ARCHIVE_EVENT_KEY, config.isArchive()); // TODO: make archive configurable per FIG federate + if (!getFederate().isArchive()) { + cot.setContextValue(Constants.ARCHIVE_EVENT_KEY, Boolean.FALSE); + } + } else { if (logger.isDebugEnabled()) { logger.debug("user not found for federate subscription for federated message"); @@ -683,7 +699,15 @@ public void onNext(FederatedEvent fedEvent) { public void onError(Throwable t) { String rootCauseMsg = FederationUtils.getHumanReadableErrorMsg(t); if (logger.isDebugEnabled()) { - logger.debug("received error notification from server" + " cause " + rootCauseMsg); + logger.debug("received error notification from server" + " cause " + rootCauseMsg, t); + + if (t instanceof io.grpc.StatusRuntimeException) { + io.grpc.StatusRuntimeException tg = (io.grpc.StatusRuntimeException) t; + Status s = tg.getStatus(); + if (logger.isDebugEnabled()) { + logger.debug("status: " + s + " " + s.getDescription()); + } + } } status.setLastError(rootCauseMsg); SubscriptionStore.getInstanceFederatedSubscriptionManager().updateFederateOutgoingStatusCache(outgoing.getDisplayName(), status); @@ -823,7 +847,7 @@ public void onCompleted() { try { // send out data feeds to federate if (DistributedConfiguration.getInstance().getRemoteConfiguration().getFederation().isAllowDataFeedFederation()) { - List feedMessages = mdm.getDataFeedEvents(); + List feedMessages = mdm.getDataFeedEventsForFederatedDataFeedOnly(); AtomicLong delayMs = new AtomicLong(100L); for (final ROL feedMessage : feedMessages) { diff --git a/src/takserver-core/src/main/java/tak/server/messaging/DistributedPluginDataFeedApi.java b/src/takserver-core/src/main/java/tak/server/messaging/DistributedPluginDataFeedApi.java index 0b0424be..07dd3c8f 100644 --- a/src/takserver-core/src/main/java/tak/server/messaging/DistributedPluginDataFeedApi.java +++ b/src/takserver-core/src/main/java/tak/server/messaging/DistributedPluginDataFeedApi.java @@ -59,11 +59,11 @@ private RemoteUtil remoteUtil() { } @Override - public PluginDataFeed create(String uuid, String name, List tags, boolean archive, boolean sync, List groupNames) { + public PluginDataFeed create(String uuid, String name, List tags, boolean archive, boolean sync, List groupNames, boolean federated) { try { - logger.info("Calling create() method in DistributedPluginDataFeedApi, uuid: {}, name: {}, tags: {}, archive: {}, sync: {}, groupNames: {}", uuid, name, tags, archive, sync, groupNames); + logger.info("Calling create() method in DistributedPluginDataFeedApi, uuid: {}, name: {}, tags: {}, archive: {}, sync: {}, groupNames: {}, federated: {}", uuid, name, tags, archive, sync, groupNames, federated); PluginDatafeedCacheHelper pluginDatafeedCacheHelper = pluginDatafeedCacheHelper(); GroupManager groupManager = groupManager(); @@ -104,6 +104,7 @@ public PluginDataFeed create(String uuid, String name, List tags, boolea dataFeed.setCoreVersion2TlsVersions(""); dataFeed.setSync(sync); dataFeed.getFiltergroup().addAll(groupNames); + dataFeed.setFederated(federated); MessagingIgniteBroker.brokerServiceCalls(service -> ((InputManager) service) .modifyInput(name, dataFeed), Constants.DISTRIBUTED_INPUT_MANAGER, InputManager.class); @@ -114,7 +115,7 @@ public PluginDataFeed create(String uuid, String name, List tags, boolea dataFeedId = dataFeedRepository.updateDataFeedWithGroupVector(uuid, name, DataFeedType.Plugin.ordinal(), AuthType.ANONYMOUS.toString(), 0, false, "Plugin", - "", "", archive, false, false ,0, "", sync, 3600, groupVector); + "", "", archive, false, false ,0, "", sync, 3600, groupVector, federated); logger.info("Updated datafeed uuid {}, row id", uuid, dataFeedId); @@ -142,7 +143,7 @@ public PluginDataFeed create(String uuid, String name, List tags, boolea } // update cache - PluginDataFeed re = new PluginDataFeed(uuid, name, tags, archive, sync, groupNames); + PluginDataFeed re = new PluginDataFeed(uuid, name, tags, archive, sync, groupNames, federated); List pluginDataFeeds = new ArrayList(); pluginDataFeeds.add(re); pluginDatafeedCacheHelper.cachePluginDatafeed(uuid, pluginDataFeeds); @@ -177,6 +178,7 @@ public PluginDataFeed create(String uuid, String name, List tags, boolea dataFeed.setCoreVersion2TlsVersions(""); dataFeed.setSync(sync); dataFeed.getFiltergroup().addAll(groupNames); + dataFeed.setFederated(federated); MessagingIgniteBroker.brokerServiceCalls(service -> ((InputManager) service) .createDataFeed(dataFeed), Constants.DISTRIBUTED_INPUT_MANAGER, InputManager.class); @@ -187,7 +189,7 @@ public PluginDataFeed create(String uuid, String name, List tags, boolea dataFeedId = dataFeedRepository.addDataFeed(uuid, name, DataFeedType.Plugin.ordinal(), AuthType.ANONYMOUS.toString(), 0, false, "Plugin", - "", "", archive, false, false ,0, "", sync, 3600, groupVector); + "", "", archive, false, false ,0, "", sync, 3600, groupVector, federated); logger.info("Added datafeed uuid {}, row id", uuid, dataFeedId); @@ -215,7 +217,7 @@ public PluginDataFeed create(String uuid, String name, List tags, boolea } // update cache - PluginDataFeed re = new PluginDataFeed(uuid, name, tags, archive, sync, groupNames); + PluginDataFeed re = new PluginDataFeed(uuid, name, tags, archive, sync, groupNames, federated); List pluginDataFeeds = new ArrayList(); pluginDataFeeds.add(re); pluginDatafeedCacheHelper.cachePluginDatafeed(uuid, pluginDataFeeds); @@ -235,22 +237,28 @@ public PluginDataFeed create(String uuid, String name, List tags, boolea throw e; } } + + @Override + public PluginDataFeed create(String uuid, String name, List tags, boolean archive, boolean sync, List groupNames) { + + return create(uuid, name, tags, archive, sync, groupNames, true); + + } @Override public PluginDataFeed create(String uuid, String name, List tags, boolean archive, boolean sync) { - return create(uuid, name, tags, archive, sync, Arrays.asList(Constants.ANON_GROUP)); + return create(uuid, name, tags, archive, sync, Arrays.asList(Constants.ANON_GROUP), true); } - @Override public PluginDataFeed create(String uuid, String name, List tags) { return create(uuid, name, tags, true, false); - } - + } + @Override public void delete(String uuid, List groupNames) { diff --git a/src/takserver-core/src/main/java/tak/server/messaging/MessageConverter.java b/src/takserver-core/src/main/java/tak/server/messaging/MessageConverter.java index a461440c..7cd39616 100644 --- a/src/takserver-core/src/main/java/tak/server/messaging/MessageConverter.java +++ b/src/takserver-core/src/main/java/tak/server/messaging/MessageConverter.java @@ -72,8 +72,6 @@ public class MessageConverter { @Autowired private ServerInfo serverInfo; - private StreamingProtoBufHelper cotProtoConverter = new StreamingProtoBufHelper(); - private static final Logger logger = LoggerFactory.getLogger(MessageConverter.class); // Convert CotEventContainer to proto encoding @@ -81,8 +79,12 @@ public byte[] cotToDataMessage(CotEventContainer message) { return cotToDataMessage(message, false); } - // Convert CotEventContainer to proto encoding public byte[] cotToDataMessage(CotEventContainer message, boolean padEmptyGroups) { + return cotToDataMessage(message, padEmptyGroups, serverInfo.getServerId()); + } + + // Convert CotEventContainer to proto encoding + public static byte[] cotToDataMessage(CotEventContainer message, boolean padEmptyGroups, String serverId) { Message.Builder mb = Message.newBuilder(); @@ -112,9 +114,9 @@ public byte[] cotToDataMessage(CotEventContainer message, boolean padEmptyGroups groups.forEach((group) -> mb.addGroups(group.getName())); - mb.setSource(serverInfo.getServerId()); + mb.setSource(serverId); - mb.setPayload(cotProtoConverter.cot2protoBuf(message)); + mb.setPayload(StreamingProtoBufHelper.cot2protoBuf(message)); String clientId = (String)message.getContext().get(Constants.CLIENT_UID_KEY); if (clientId != null) { @@ -174,7 +176,7 @@ public ROL controlMessageToRol(byte[] controlMessageBytes) throws InvalidProtoco // Convert MissionAnnouncement announcement message to CoT public CotEventContainer getCotFromMissionAnnouncement(MissionAnnouncement missionannouncement) { - return cotProtoConverter.proto2cot(missionannouncement.getPayload()); + return StreamingProtoBufHelper.proto2cot(missionannouncement.getPayload()); } // Convert ClusterMissionAnnouncementDetail to proto encoding @@ -204,7 +206,7 @@ public byte[] missionAnnouncementToDataMessage(ClusterMissionAnnouncementDetail } mb.setMissionAnnouncementType(detail.missionAnnouncementType); - mb.setPayload(cotProtoConverter.cot2protoBuf(detail.cot)); + mb.setPayload(StreamingProtoBufHelper.cot2protoBuf(detail.cot)); return mb.build().toByteArray(); } @@ -237,7 +239,7 @@ public CotEventContainer dataMessageToCot(Message m, boolean setClusterKey) thro TakMessage takMessage = m.getPayload(); - CotEventContainer cot = cotProtoConverter.proto2cot(takMessage); + CotEventContainer cot = StreamingProtoBufHelper.proto2cot(takMessage); NavigableSet takGroups = new ConcurrentSkipListSet<>(); diff --git a/src/takserver-core/src/main/resources/application.properties b/src/takserver-core/src/main/resources/application.properties index 01d9bf11..7850a0db 100644 --- a/src/takserver-core/src/main/resources/application.properties +++ b/src/takserver-core/src/main/resources/application.properties @@ -25,4 +25,5 @@ spring.mvc.throwExceptionIfNoHandlerFound=true spring.mvc.threadContextInheritable=true - +spring.jackson.mapper.accept-case-insensitive-enums=true +spring.jackson.deserialization.accept-single-value-as-array=true diff --git a/src/takserver-core/src/main/resources/logback-spring.xml b/src/takserver-core/src/main/resources/logback-spring.xml index 19c7adce..2e708cf3 100644 --- a/src/takserver-core/src/main/resources/logback-spring.xml +++ b/src/takserver-core/src/main/resources/logback-spring.xml @@ -102,6 +102,7 @@ + diff --git a/src/takserver-core/src/main/resources/security-context.xml b/src/takserver-core/src/main/resources/security-context.xml index d3a87d3f..3e228923 100644 --- a/src/takserver-core/src/main/resources/security-context.xml +++ b/src/takserver-core/src/main/resources/security-context.xml @@ -154,6 +154,7 @@ + @@ -271,7 +272,7 @@ - + diff --git a/src/takserver-core/src/main/resources/validation.properties b/src/takserver-core/src/main/resources/validation.properties index 99644cba..6e14721c 100644 --- a/src/takserver-core/src/main/resources/validation.properties +++ b/src/takserver-core/src/main/resources/validation.properties @@ -5,7 +5,7 @@ Validator.CotType=^[a-zA-Z\\-\\.]+$ Validator.Double=\\-?[0-9]*\\.?[[0-9]]* Validator.Hexidecimal=^[a-fA-F0-9]+$ Validator.KmlGeometry=^\\-?[0-9].*\\.?[0-9].*,-?[0-9].*\\.?[0-9].*$ -Validator.MartiSafeString=^[\u0600-\u06FF\\w\\d\\s\\.\\(\\)!=@#$&^*_\\-\\+\\[\\]\\{\\}:,\\.\\/\\|\\\\]*$ +Validator.MartiSafeString=^[\\p{L}\\p{N}\\w\\d\\s\\.\\(\\)!=@#$&^*_\\-\\+\\[\\]\\{\\}:,\\.\\/\\|\\\\]*$ Validator.CertCommonName=[^!()<>$?]+$ Validator.NonNegativeInteger=^[0-9]+$ Validator.URL=^(ht|f)tp(s?)\\:\\/\\/[0-9a-zA-Z]([-.\\w]*[0-9a-zA-Z])*(:(0-9)*)*(\\/?)([a-zA-Z0-9\\-\\.\\?\\,\\:\\'\\/\\\\\\+=&%\\$#_]*)?$ diff --git a/src/takserver-core/src/main/webapp/Marti/LogManager.jsp b/src/takserver-core/src/main/webapp/Marti/LogManager.jsp index cd57af5f..f83dd8d7 100644 --- a/src/takserver-core/src/main/webapp/Marti/LogManager.jsp +++ b/src/takserver-core/src/main/webapp/Marti/LogManager.jsp @@ -132,6 +132,10 @@ window.open("ErrorLog?id=" + ids); } + function downloadAll() { + window.open("ErrorLog?id=ALL"); + } + function searchErrorLogs(query) { var separator = "?"; var location = window.location.href; @@ -188,6 +192,7 @@     +   

diff --git a/src/takserver-core/src/main/webapp/Marti/MissionInvite.jsp b/src/takserver-core/src/main/webapp/Marti/MissionInvite.jsp index 0365f1fa..2679549d 100644 --- a/src/takserver-core/src/main/webapp/Marti/MissionInvite.jsp +++ b/src/takserver-core/src/main/webapp/Marti/MissionInvite.jsp @@ -145,13 +145,18 @@ type : 'POST', data : formData, contentType : 'application/x-www-form-urlencoded', - }) - .done(function() { + success: function (response) { window.location = "Missions.jsp"; - }) - .fail(function() { - $.jnotify("Failed to send invitations", "error"); - }); + }, + error : function(stat, err) { + if (stat.responseJSON !== null && stat.responseJSON.message != ""){ + $.jnotify("Error: " + stat.responseJSON.message, "error", 3000); + } else{ + $.jnotify("Failed to send invitations", "error", 3000); + } + } + }); + }); }); diff --git a/src/takserver-core/src/main/webapp/Marti/Missions.jsp b/src/takserver-core/src/main/webapp/Marti/Missions.jsp index bb611c7b..21e1d5e5 100644 --- a/src/takserver-core/src/main/webapp/Marti/Missions.jsp +++ b/src/takserver-core/src/main/webapp/Marti/Missions.jsp @@ -41,12 +41,18 @@ cache : false, contentType : false, processData : false, + success: function (response){ + location.reload(); + }, error : function(stat, err) { - $.jnotify("Error deleting", "error"); + if (stat.responseJSON !== null && stat.responseJSON.message != ""){ + $.jnotify("Error: " + stat.responseJSON.message, "error", 3000); + } else{ + $.jnotify("Error deleting", "error", 3000); + } } }); - location.reload(); } function getSelected() { diff --git a/src/takserver-core/src/main/webapp/Marti/data_retention/js/controllers.js b/src/takserver-core/src/main/webapp/Marti/data_retention/js/controllers.js index cba59095..ffd29ecd 100644 --- a/src/takserver-core/src/main/webapp/Marti/data_retention/js/controllers.js +++ b/src/takserver-core/src/main/webapp/Marti/data_retention/js/controllers.js @@ -70,6 +70,7 @@ dataRetentionControllers.controller('ViewPoliciesCtrl', [ ttl_number = ttl / (YEAR * 24 * 3600); ttl_type = "years"; } + ttl_number = Math.round(ttl_number); $scope.policy_map[data_type] = { 'ttl': ttl, 'ttl_type': ttl_type, diff --git a/src/takserver-core/src/main/webapp/Marti/federation/js/app.js b/src/takserver-core/src/main/webapp/Marti/federation/js/app.js index 1c92fd8e..c1c42854 100644 --- a/src/takserver-core/src/main/webapp/Marti/federation/js/app.js +++ b/src/takserver-core/src/main/webapp/Marti/federation/js/app.js @@ -48,6 +48,10 @@ app.config(['$routeProvider', templateUrl: 'partials/federateGroups.html', controller: 'FederateGroupsCtrl' }). + when('/editFederateMissions/:id', { + templateUrl: 'partials/federateMissions.html', + controller: 'FederateMissionsCtrl' + }). when('/listFederateContacts/:id', { templateUrl: 'partials/federateContacts.html', controller: 'FederateContactsListCtrl' diff --git a/src/takserver-core/src/main/webapp/Marti/federation/js/controllers.js b/src/takserver-core/src/main/webapp/Marti/federation/js/controllers.js index 8ae60dc7..81c30239 100644 --- a/src/takserver-core/src/main/webapp/Marti/federation/js/controllers.js +++ b/src/takserver-core/src/main/webapp/Marti/federation/js/controllers.js @@ -301,16 +301,16 @@ federationManagerControllers.controller('FederateGroupsCtrl', ['$scope', }, function (apiResponse) { $scope.getFederateGroupsMap(); + window.location.reload(); }, function (apiResponse) { + alert('An error occurred adding the group. Please correct the errors and resubmit.'); $scope.serviceReportedMessages = true; $scope.messages = apiResponse.data.messages; - alert('An error occurred adding the group. Please correct the errors and resubmit.'); $scope.submitInProgress = false; } ); - } - window.location.reload(); + } } $scope.deleteGroupMap = function (remoteGroup, localGroup) { @@ -384,6 +384,150 @@ federationManagerControllers.controller('FederateGroupsCtrl', ['$scope', } ]); +federationManagerControllers.controller('FederateMissionsCtrl', ['$scope', + '$location', + 'FederateDetailsService', + 'MissionListService', + // 'FederateMissionsUpdateService', + '$routeParams', + '$http', + '$modal', + function ( + $scope, + $location, + FederateDetailsService, + MissionListService, + // FederateMissionsUpdateService, + $routeParams, + $http, + $modal + ) { + $scope.federateId = $routeParams.id; + $scope.federateName = $routeParams.name; + $scope.submitInProgress = false; + $scope.vbm_filter_checkbox = false; + $scope.mission_federate_default = null; + + function getFederateMissionsFromConfig() { + + }; + + function getAllVbmCopMissionsWithFederateSetting() { + + var federateMissionsFromConfig; + + FederateDetailsService.query({ + federateId: $scope.federateId + }, + function (apiResponse) { + console.log(apiResponse.data); + + if (apiResponse.data.mission == null){ + federateMissionsFromConfig = []; + }else{ + federateMissionsFromConfig = apiResponse.data.mission; + if (apiResponse.data.missionFederateDefault == null){ + $scope.mission_federate_default = true; //default value for mission_federate_default is true + }else{ + $scope.mission_federate_default = apiResponse.data.missionFederateDefault; + } + + } + $scope.showRmiError = false; + + MissionListService.query( + function(apiResponse2) { + var list_of_all_missions = apiResponse2.data; + $scope.missions = []; + var found; + for (var mission of list_of_all_missions) { + found = false; + for (var x of federateMissionsFromConfig){ + if (x.name == mission.name){ + $scope.missions.push({'name': mission.name , 'enabled': x.enabled, 'tool': mission.tool }); + found = true; + break; + } + } + if (found == false){ + $scope.missions.push({'name': mission.name , 'enabled': null, 'tool': mission.tool}); + } + } + console.log($scope.missions); + }, + function(apiResponse2) { + $scope.missions = []; + console.log('error'); + console.log(apiResponse2); + alert('Error when querying mission for federate'); + } + ); + + }, + function () { + $scope.showRmiError = true; + }); + }; + + $scope.backToFederates = function () { + console.log('back to federates'); + window.history.back(); + }; + + $scope.cancel = function () { + $scope.backToFederates(); + }; + + $scope.vbm_filter_func = function (mission) { + if ($scope.vbm_filter_checkbox == false || ($scope.vbm_filter_checkbox == true && mission.tool.toLowerCase() == 'vbm')){ + return mission; + } + }; + + $scope.saveFederateMissions = function () { + $scope.submitInProgress = true; + + console.log($scope.missions); + var missions_with_defined_value_only = []; // filter out the missions whose values are not set to either true or false + for (var mission of $scope.missions) { + if (mission.enabled == true || mission.enabled == false){ + missions_with_defined_value_only.push(mission); + } + } + console.log(missions_with_defined_value_only); + + // Note: Could not find a way to set both federateId in request path and the data in the request body using FederateMissionsUpdateService + if ($scope.mission_federate_default == null){ + alert("Need to set default value for mission federate"); + return; + } + + $http({ + url: '/Marti/api/federatemissions/'+ $scope.federateId, + method: "PUT", + data: { + missionFederateDefault: $scope.mission_federate_default, + missions: missions_with_defined_value_only + } + }) + .then(function(apiResponse) { + console.log(apiResponse.data); + $scope.showRmiError = false; + $scope.submitInProgress = false; + alert("Successfully updated mission federation"); + }, + function(apiResponse) { + $scope.showRmiError = true; + alert("Error saving the federate missions"); + $scope.submitInProgress = false; + }); + + } + + getAllVbmCopMissionsWithFederateSetting(); + } +]); + federationManagerControllers.controller('OutgoingConnectionCreationCtrl', ['$scope', '$location', 'OutgoingConnectionsService', diff --git a/src/takserver-core/src/main/webapp/Marti/federation/js/services.js b/src/takserver-core/src/main/webapp/Marti/federation/js/services.js index 1900e303..65c3dd3a 100644 --- a/src/takserver-core/src/main/webapp/Marti/federation/js/services.js +++ b/src/takserver-core/src/main/webapp/Marti/federation/js/services.js @@ -105,3 +105,10 @@ services.factory('MissionListService', function($resource) { 'query': {method: "GET", isArray: false} }) }); + +// services.factory('FederateMissionsUpdateService', function($resource) { +// return $resource('/Marti/api/federatemissions/:federateId', {}, { +// 'update': {method: "PUT", isArray: true} +// }); +// }); + diff --git a/src/takserver-core/src/main/webapp/Marti/federation/partials/federateMissions.html b/src/takserver-core/src/main/webapp/Marti/federation/partials/federateMissions.html new file mode 100644 index 00000000..1341552c --- /dev/null +++ b/src/takserver-core/src/main/webapp/Marti/federation/partials/federateMissions.html @@ -0,0 +1,94 @@ + + +

+

Federate Mission Settings

+ +

Federate name: {{federateName}}

+ +

Note that the below settings are only applicable when Allow Mission Federation is enabled globally

+ +
+
    +
  • {{message}}
    • +
+
+ +
+ + + This value will be applied for new missions as well as missions whose value are not specifically set on the table below +

+
+ + + + + + + + + + + + + +
Mission Enabled Tool
{{mission.name}} + + {{mission.tool}}
+ +
+   + + +
+
diff --git a/src/takserver-core/src/main/webapp/Marti/federation/partials/federates.html b/src/takserver-core/src/main/webapp/Marti/federation/partials/federates.html index ca55a710..125fcad3 100644 --- a/src/takserver-core/src/main/webapp/Marti/federation/partials/federates.html +++ b/src/takserver-core/src/main/webapp/Marti/federation/partials/federates.html @@ -44,6 +44,7 @@

Active Connections

Delete Edit Config Edit Groups + Edit Missions @@ -148,6 +149,7 @@

Federate Configuration

Delete Edit Config Edit Groups + Edit Missions diff --git a/src/takserver-core/src/main/webapp/Marti/inputs/js/controllers.js b/src/takserver-core/src/main/webapp/Marti/inputs/js/controllers.js index 9c53520a..7056402e 100644 --- a/src/takserver-core/src/main/webapp/Marti/inputs/js/controllers.js +++ b/src/takserver-core/src/main/webapp/Marti/inputs/js/controllers.js @@ -352,12 +352,14 @@ inputManagerControllers.controller('DataFeedCreationCtrl', $scope.dataFeed.tags = ''; $scope.dataFeed.auth = 'X_509'; $scope.dataFeed.protocol = 'tls'; + $scope.dataFeed.syncCacheRetentionSeconds = "3600"; $scope.dataFeed.archive = 'true'; $scope.dataFeed.anongroup = 'false'; $scope.dataFeed.archiveOnly = 'false'; $scope.dataFeed.coreVersion = "2"; $scope.dataFeed.protocol = 'tls'; $scope.dataFeed.sync = 'false'; + $scope.dataFeed.federated = 'true'; $scope.inputNameDuplicate = false; $scope.serviceReportedMessages = false; $scope.messages = []; @@ -426,6 +428,7 @@ inputManagerControllers.controller('DataFeedModificationCtrl', ['$scope', '$loca $scope.dataFeed = apiResponse.data; $scope.hideArchive = false; $scope.hideArchiveOnly = false; + $scope.hideFederated = false; var protocol = apiResponse.data.protocol.toLowerCase(); $scope.hideFilterGroupList = (apiResponse.data.auth.toLowerCase() != 'anonymous'); }, diff --git a/src/takserver-core/src/main/webapp/Marti/inputs/partials/list.html b/src/takserver-core/src/main/webapp/Marti/inputs/partials/list.html index 413a1c9c..ea46f918 100644 --- a/src/takserver-core/src/main/webapp/Marti/inputs/partials/list.html +++ b/src/takserver-core/src/main/webapp/Marti/inputs/partials/list.html @@ -80,10 +80,12 @@

Streaming Data Feeds

TLS Group Interface + Sync Cache Retention(s) Archive Filter Groups Anonymous Group Archive Only + Federated Reads Received Messages Received Number Connected @@ -107,10 +109,12 @@

Streaming Data Feeds

? x.input.coreVersion2TlsVersions : secConfig.tlsVersion) : ''}} + + @@ -137,6 +141,7 @@

Plugin Data Feeds

Tags Archive Filter Groups + Federated Messages Received Bytes Received   @@ -152,6 +157,7 @@

Plugin Data Feeds

+ Delete diff --git a/src/takserver-core/src/main/webapp/Marti/inputs/partials/modifyPluginDataFeed.html b/src/takserver-core/src/main/webapp/Marti/inputs/partials/modifyPluginDataFeed.html index daa97296..7b43be8c 100644 --- a/src/takserver-core/src/main/webapp/Marti/inputs/partials/modifyPluginDataFeed.html +++ b/src/takserver-core/src/main/webapp/Marti/inputs/partials/modifyPluginDataFeed.html @@ -81,6 +81,14 @@

Modify Plugin Data Feed Definition

+ + + + + + + diff --git a/src/takserver-core/src/main/webapp/Marti/inputs/partials/modifyStreamingDataFeed.html b/src/takserver-core/src/main/webapp/Marti/inputs/partials/modifyStreamingDataFeed.html index 1c3b50a0..3fcf3095 100644 --- a/src/takserver-core/src/main/webapp/Marti/inputs/partials/modifyStreamingDataFeed.html +++ b/src/takserver-core/src/main/webapp/Marti/inputs/partials/modifyStreamingDataFeed.html @@ -91,6 +91,20 @@

Modify Streaming Data Feed Definition

  + + + + +
Provide sync cache retention time in seconds. + + + + Must be a number in seconds + + +   + + @@ -98,6 +112,14 @@

Modify Streaming Data Feed Definition

+ + + + + + + diff --git a/src/takserver-core/src/main/webapp/Marti/inputs/partials/newPluginDataFeed.html b/src/takserver-core/src/main/webapp/Marti/inputs/partials/newPluginDataFeed.html index c95fd95e..b381ea7d 100644 --- a/src/takserver-core/src/main/webapp/Marti/inputs/partials/newPluginDataFeed.html +++ b/src/takserver-core/src/main/webapp/Marti/inputs/partials/newPluginDataFeed.html @@ -115,6 +115,21 @@

Create Plugin Data Feed

+ + + + + + + + + Federated is required + + + *Required diff --git a/src/takserver-core/src/main/webapp/Marti/inputs/partials/newStreamingDataFeed.html b/src/takserver-core/src/main/webapp/Marti/inputs/partials/newStreamingDataFeed.html index 2683a8f8..0fbd3b3c 100644 --- a/src/takserver-core/src/main/webapp/Marti/inputs/partials/newStreamingDataFeed.html +++ b/src/takserver-core/src/main/webapp/Marti/inputs/partials/newStreamingDataFeed.html @@ -196,6 +196,19 @@

Create Streaming Data Feed

+ + + + +
Provide sync cache retention time in seconds. + + + + Must be a number in seconds + + + + @@ -259,6 +272,21 @@

Create Streaming Data Feed

+ + + + + + + + + Federated is required + + + *Required diff --git a/src/takserver-core/src/main/webapp/Marti/login/index.html b/src/takserver-core/src/main/webapp/Marti/login/index.html new file mode 100644 index 00000000..0b66663f --- /dev/null +++ b/src/takserver-core/src/main/webapp/Marti/login/index.html @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + +
+ + + diff --git a/src/takserver-core/src/main/webapp/Marti/login/js/app.js b/src/takserver-core/src/main/webapp/Marti/login/js/app.js new file mode 100644 index 00000000..9aee7785 --- /dev/null +++ b/src/takserver-core/src/main/webapp/Marti/login/js/app.js @@ -0,0 +1,19 @@ +'use strict'; + +var app = angular.module('loginManager', ['ngRoute', 'ngResource', 'ngMessages', 'loginManagerControllers', 'loginManagerServices']); + +app.filter('encodeURIComponent', function() { + return window.encodeURIComponent; +}); + +app.config(['$routeProvider', + function($routeProvider) { + $routeProvider. + when('/', { + templateUrl: '/Marti/login/partials/login.html', + controller: 'loginController' + }). + otherwise({ + redirectTo: '/' + }); + }]); \ No newline at end of file diff --git a/src/takserver-core/src/main/webapp/Marti/login/js/controllers.js b/src/takserver-core/src/main/webapp/Marti/login/js/controllers.js new file mode 100644 index 00000000..e27e61d5 --- /dev/null +++ b/src/takserver-core/src/main/webapp/Marti/login/js/controllers.js @@ -0,0 +1,47 @@ +'use strict'; + +var loginControllers = angular.module('loginManagerControllers', []); + +loginControllers.controller('loginController', ['$scope', '$location', 'loginService', '$window', + function ($scope, $location, loginService, $window) { + + $scope.showLogin = true; + $scope.showError = false; + $scope.externalAuth = false; + $scope.externalAuthName = null; + + $scope.getConfig = async function() { + loginService.config.get( + function(apiResponse) { + console.log(apiResponse); + $scope.externalAuth = true; + $scope.externalAuthName = apiResponse.data; + }, + function() { $scope.showRmiError = true; }); + } + + $scope.loginWithAuthServer = async function() { + window.location = "/login/auth"; + } + + $scope.onSubmit = async function (){ + var xhr = new XMLHttpRequest(); + var url = "/oauth/token?grant_type=password" + + "&username=" + encodeURIComponent(username.value) + + "&password=" + encodeURIComponent(password.value); + xhr.responseType = 'json'; + xhr.open("POST", url, true); + xhr.onload = function() { + if (xhr.status == 200) { + window.location = document.referrer + } else { + $scope.showError = true; + $scope.$apply(); + } + }; + + xhr.send(); + } + + $scope.getConfig(); + }]); \ No newline at end of file diff --git a/src/takserver-core/src/main/webapp/Marti/login/js/services.js b/src/takserver-core/src/main/webapp/Marti/login/js/services.js new file mode 100644 index 00000000..22a81690 --- /dev/null +++ b/src/takserver-core/src/main/webapp/Marti/login/js/services.js @@ -0,0 +1,9 @@ +var services = angular.module('loginManagerServices', []) + + .factory('loginService', function($resource) { + return { + config: $resource('/login/authserver', {}, { + 'get': {method: "GET", isArray: false }, + }) + }; + }); diff --git a/src/takserver-core/src/main/webapp/Marti/login/login.html b/src/takserver-core/src/main/webapp/Marti/login/partials/login.html similarity index 85% rename from src/takserver-core/src/main/webapp/Marti/login/login.html rename to src/takserver-core/src/main/webapp/Marti/login/partials/login.html index afaa9f97..f9f6ebe4 100644 --- a/src/takserver-core/src/main/webapp/Marti/login/login.html +++ b/src/takserver-core/src/main/webapp/Marti/login/partials/login.html @@ -1,44 +1,12 @@ - + - - - - - Login - - @@ -49,7 +17,7 @@ width:94px; height:97px; display:inline-block; - margin-top: -4px; + margin-top: 10px; margin-left: 0px; filter: drop-shadow(0 2px 3px #cecece); } @@ -74,16 +42,16 @@ -