diff --git a/README.md b/README.md index a92b2420..ffbe6a45 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Links: * [CI Test Execution](src/takserver-takcl-core/docs/ci_testing.md) - * [Publishing](docs/publishing.md) + * [Publishing](src/docs/publishing.md) --- Clean and Build TAK Server @@ -141,7 +141,7 @@ https://localhost:8443/swagger-ui.html ### Integration Tests Integration tests are executed against master nightly. In addition to this, they can be executed on any branch as follows: -1. Navigate to the [TAKServer Dashboard](https://git.tak.gov/core/takserver). +1. Navigate to the [TAKServer Dashboard](https://git.takmaps.com/core/takserver). 2. On the sidebar, hover over 'CI/CD' and select 'Pipelines'. 3. Find your commit from the list and tap the Play button to the right, and select the test suite you would like to execute. The Main suites are what is executed nightly and execute all the tests. diff --git a/src/docs/TAK_Server_Configuration_Guide.odt b/src/docs/TAK_Server_Configuration_Guide.odt index 14abe44a..83083ccc 100644 Binary files a/src/docs/TAK_Server_Configuration_Guide.odt and b/src/docs/TAK_Server_Configuration_Guide.odt differ diff --git a/src/docs/TAK_Server_Configuration_Guide.pdf b/src/docs/TAK_Server_Configuration_Guide.pdf index bea8dca0..eb1e7ae7 100644 Binary files a/src/docs/TAK_Server_Configuration_Guide.pdf and b/src/docs/TAK_Server_Configuration_Guide.pdf differ diff --git a/src/federation-common/src/main/java/tak/server/federation/message/ByteArrayPayload.java b/src/federation-common/src/main/java/tak/server/federation/message/ByteArrayPayload.java index 77bb42a0..c7b8714a 100644 --- a/src/federation-common/src/main/java/tak/server/federation/message/ByteArrayPayload.java +++ b/src/federation-common/src/main/java/tak/server/federation/message/ByteArrayPayload.java @@ -10,7 +10,6 @@ * For this implementation, the parameterized type of this class is the same as the type of getBytes() -- this would not usually be the case, the content type * is expected to be a POJO. https://en.wikipedia.org/wiki/Plain_Old_Java_Object * - * */ public class ByteArrayPayload implements Payload { diff --git a/src/federation-common/src/main/java/tak/server/federation/message/MetadataUtils.java b/src/federation-common/src/main/java/tak/server/federation/message/MetadataUtils.java index e08eec53..c7073c6a 100644 --- a/src/federation-common/src/main/java/tak/server/federation/message/MetadataUtils.java +++ b/src/federation-common/src/main/java/tak/server/federation/message/MetadataUtils.java @@ -21,7 +21,6 @@ * We may want to extend it do offer more functionality. * * TODO Discuss putting in the PluginContext for use by all plugins - * * */ public final class MetadataUtils { diff --git a/src/federation-common/src/main/java/tak/server/federation/message/Payload.java b/src/federation-common/src/main/java/tak/server/federation/message/Payload.java index 8f4a52fa..fed83564 100644 --- a/src/federation-common/src/main/java/tak/server/federation/message/Payload.java +++ b/src/federation-common/src/main/java/tak/server/federation/message/Payload.java @@ -12,7 +12,6 @@ * the payload must have a zero-argument constructor and be able to set its payload through setBytes. * @see PayloadSerializationPlugin * - * */ public interface Payload { diff --git a/src/federation-hub-ui/src/main/webapp/modules/workflows/joint_paper_factory.js b/src/federation-hub-ui/src/main/webapp/modules/workflows/joint_paper_factory.js index dc490a09..5f1b97c0 100644 --- a/src/federation-hub-ui/src/main/webapp/modules/workflows/joint_paper_factory.js +++ b/src/federation-hub-ui/src/main/webapp/modules/workflows/joint_paper_factory.js @@ -1,4 +1,3 @@ - /* globals joint */ /* globals $ */ 'use strict'; @@ -210,6 +209,7 @@ angular.module('roger_federation.Workflows') //Custom Icons joint.shapes.bpmn.icons.dataStore = "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiA/PjwhRE9DVFlQRSBzdmcgIFBVQkxJQyAnLS8vVzNDLy9EVEQgU1ZHIDEuMS8vRU4nICAnaHR0cDovL3d3dy53My5vcmcvR3JhcGhpY3MvU1ZHLzEuMS9EVEQvc3ZnMTEuZHRkJz48c3ZnIGVuYWJsZS1iYWNrZ3JvdW5kPSJuZXcgMCAwIDI0IDI0IiBoZWlnaHQ9IjI0cHgiIGlkPSJMYXllcl8xIiB2ZXJzaW9uPSIxLjEiIHZpZXdCb3g9IjAgMCAyNCAyNCIgd2lkdGg9IjI0cHgiIHhtbDpzcGFjZT0icHJlc2VydmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGxpbmsiPjxwYXRoICBkPSJtIDAuOTA3NzQ0NTUsNC4wMTI4NTI1IDAsMTYuMjM1MDExNSBjIDAuNzQwODE4OTUsMy42MDc3ODEgMjEuNDgzNzUxNDUsMy42MDc3ODEgMjIuMjI0NTcwNDUsMCBsIDAsLTE2LjIzNTAxMTUgYyAtMC43NDA4MTksLTMuNjA3NzgwMTIgLTIxLjQ4Mzc1MTUsLTMuNjA3NzgwMTIgLTIyLjIyNDU3MDQ1LDAgMC43NDA4MTg5NSwzLjYwNzc4MDEgMjEuNDgzNzUxNDUsMy42MDc3ODAxIDIyLjIyNDU3MDQ1LDAgTSAwLjkwNzc0NDU1LDYuNTM4Mjk4OCBjIDAuNzQwODE4OTUsMy42MDc3ODAyIDIxLjQ4Mzc1MTQ1LDMuNjA3NzgwMiAyMi4yMjQ1NzA0NSwwIE0gMC45MDc3NDQ1NSw5LjA2Mzc0NDkgYyAwLjc0MDgxODk1LDMuNjA3NzgwMSAyMS40ODM3NTE0NSwzLjYwNzc4MDEgMjIuMjI0NTcwNDUsMCINCiAgICAgc3R5bGU9ImZpbGw6I2ZmZmZmZjtzdHJva2U6IzAwMDAwMDtzdHJva2Utd2lkdGg6MC44OTE5NzE0MSIgLz48L3N2Zz4="; + joint.shapes.bpmn.icons.outgoing = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAYAAAD0eNT6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAASdAAAEnQB3mYfeAAAABl0RVh0U29mdHdhcmUAd3d3Lmlua3NjYXBlLm9yZ5vuPBoAABMjSURBVHic7d07khxZcoZRD0yb0ahxB6NQ5I5a6CWwm5whuQO+2djCCLMoaqQ4ixjLUTJhBaAemekeEfdeP0dClVklXPs/VBYqtsvlEgBAL5/OPgAAOJ4AAICGBAAANCQAAKAhAQAADQkAAGhIAABAQwIAABoSAADQkAAAgIYEAAA0JAAAoCEBAAANCQAAaEgAAEBDAgAAGhIAANCQAACAhgQAADQkAACgIQEAAA0JAABoSAAAQEMCAAAaEgAA0JAAAICGBAAANCQAAKAhAQAADQkAAGhIAABAQwIAABoSAADQkAAAgIYEAAA0JAAAoCEBAAANCQAAaEgAAEBDAgAAGhIAANCQAACAhgQAADQkAACgIQEAAA0JAABoSAAAQEMCAAAaEgAA0JAAAICGBAAANCQAAKAhAQAADQkAAGhIAABAQwIAABoSAADQkAAAgIYEAAA0JAAAoCEBAAANCQAAaEgAAEBDAgAAGhIAANCQAACAhgQAADQkAACgIQEAAA0JAABoSAAAQEMCAAAaEgAA0JAAAICGBAAANCQAAKAhAQAADQkAAGhIAABAQwIAABoSAADQkAAAgIYEAAA0JAAAoCEBAAANCQAAaEgAAEBDAgAAGhIAANCQAACAhgQAADQkAACgIQEAAA0JAABoSAAAQEMCAAAaEgAA0JAAAICGBAAANCQAAKAhAQAADQkAAGhIAABAQwIAABoSAADQkAAAgIZ+OPsAoI9t2/46Iv7q7DuAiO1yuZx9w5u2bfuXiPi7iPjpcrn8+ex7gJxt2z5HxN+ffQcw8HcAtm3754j41xcfiwAAKDLkzwBcx//fXnzqx4j4w7ZtvznpJABYynAB8Mr434gAACgyVAC8M/43IgAACgwTAHeM/40IAICkIQJg27Z/ivvG/0YEAEDC6QFwHf9/f+JLRQAAPOnUAEiM/40IAIAnnBYABeN/IwIA4EGnBEDh+N+IAAB4wOEBsG3b76N2/G9EAADc6dAAuI7/f+z4V4gAALjDYQFwwPjfiAAA+MAhAXDg+N+IAAB4x+4BcML434gAAHjDrgGwbdvv4pzxvxEBAPCK3QLgOv7/udfrP0AEAMA3dgmAgcb/RgQAwAvlATDg+N+IAAC4Kg2Agcf/RgQAQBQGwLZt/xhjj/+NCACgvZIAuI7/f1W81kFEAACtpQNgwvG/EQEAtJUKgInH/0YEANDS0wGwwPjfiAAA2nkqABYa/xsRAEArDwfAtm3/EGuN/40IAKCNhwLgOv7/vdMtIxABALRwdwA0GP8bEQDA8u4KgEbjfyMCAFjahwHQcPxvRAAAy3o3ALZt+yV6jv+NCABgSW8GwHX8/+fAW0YlAgBYzqsBYPy/IwIAWMp3AWD83yQCAFjGVwFg/D8kAgBYwpcA2Lbt5zD+9xABAEzvU8SX8f/15FtmIgIAmNon4/80EQDAtD5FxG/PPmJiIgCAKX26XC6/RMTnsw+ZmAgAYDqfIiJEQJoIAGAqX/4XgAhIEwEATOOr3wMgAtJEAABT+O43AYqANBEAwPBefRaACEgTAQAM7c2nAYqANBEAwLDeDIAIEVBABAAwpHcDIEIEFBABAAznwwCIEAEFRAAAQ7krACJEQAERAMAw7g6ACBFQQAQAMISHAiBCBBQQAQCc7uEAiBABBUQAAKd6KgAiREABEQDAaX7IfPHlcvll27aIiJ9rzmnnx4iIbdt+ulwufz77GDjAnyLif88+gtP9JiL+9uwjutsul0v+Rbbt1xABGX+MCBEALO/6Xc8/xPUfQJzn6bcAXvJ2QJq3A4DlGf+xlARAhAgoIAKAZRn/8ZQFQIQIKCACgOUY/zGVBkCECCggAoBlGP9xlQdAhAgoIAKA6Rn/se0SABEioIAIAKZl/Me3WwBEiIACIgCYjvGfw64BECECCogAYBrGfx67B0CECCggAoDhGf+5HBIAESKggAgAhmX853NYAESIgAIiABiO8Z/ToQEQIQIKiABgGMZ/XocHQIQIKCACgNMZ/7mdEgARIqCACABOY/znd1oARIiAAiIAOJzxX8OpARAhAgqIAOAwxn8dpwdAhAgoIAKA3Rn/tQwRABEioIAIAHZj/NczTABEiIACIgAoZ/zXNFQARIiAAiIAKGP81zVcAESIgAIiAEgz/msbMgAiREABEQA8zfivb9gAiBABBUQA8DDj38PQARAhAgqIAOBuxr+P4QMgQgQUEAHAh4x/L1MEQIQIKCACgDcZ/36mCYAIEVBABADfMf49TRUAESKggAgAvjD+fU0XABEioIAIAIx/c1MGQIQIKCACoDHjz7QBECECCogAaMj4EzF5AESIgAIiABox/txMHwARIqCACIAGjD8vLREAESKggAiAhRl/vrVMAESIgAIiABZk/HnNUgEQIQIKiABYiPHnLcsFQIQIKCACYAHGn/csGQARIqCACICJGX8+smwARIiAAiIAJmT8ucfSARAhAgqIAJiI8S/xOSL+/+wj9rZ8AESIgAIiACZg/Et8vm7G8loEQIQIKCACYGDGv0Sb8Y9oFAARIqCACIABGf8SrcY/olkARIiAAiIABmL8S7Qb/4iGARAhAgqIABiA8S/RcvwjmgZAhAgoIALgRMa/RNvxj2gcABEioIAIgBMY/xKtxz+ieQBEiIACIgAOZPxLtB//CAEQESKggAiAAxj/Esb/SgBciYA0EQA7Mv4ljP8LAuAFEZAmAmAHxr+E8f+GAPiGCEgTAVDI+Jcw/q8QAK8QAWkiAAoY/xLG/w0C4A0iIE0EQILxL2H83yEA3iEC0kQAPMH4lzD+HxAAHxABaSIAHmD8Sxj/OwiAO4iANBEAdzD+JYz/nQTAnURAmgiAdxj/Esb/AQLgASIgTQTAK4x/CeP/IAHwIBGQJgLgBeNfwvg/QQA8QQSkiQAI41/E+D9JADxJBKSJAFoz/iWMf4IASBABaSKAlox/CeOfJACSRECaCKAV41/C+BcQAAVEQJoIoAXjX8L4FxEARURAmghgaca/hPEvJAAKiYA0EcCSjH8J419MABQTAWkigKUY/xLGfwcCYAciIE0EsATjX8L470QA7EQEpIkApmb8Sxj/HQmAHYmANBHAlIx/CeO/MwGwMxGQJgKYivEvYfwPIAAOIALSRABTMP4ljP9BBMBBRECaCGBoxr+E8T+QADiQCEgTAQzJ+Jcw/gcTAAcTAWkigKEY/xLG/wQC4AQiIE0EMATjX8L4n0QAnEQEpIkATmX8Sxj/EwmAE4mANBHAKYx/CeN/MgFwMhGQJgI4lPEvYfwHIAAGIALSRACHMP4ljP8gBMAgRECaCGBXxr+E8R+IABiICEgTAezC+Jcw/oMRAIMRAWkigFLGv4TxH5AAGJAISBMBlDD+JYz/oATAoERAmgggxfiXMP4DEwADEwFpIoCnGP8Sxn9wAmBwIiBNBPAQ41/C+E9AAExABKSJAO5i/EsY/0kIgEmIgDQRwLuMfwnjPxEBMBERkCYCeJXxL2H8JyMAJiMC0kQAXzH+JYz/hATAhERAmgggIox/EeM/KQEwKRGQJgKaM/4ljP/EBMDERECaCGjK+Jcw/pMTAJMTAWkioBnjX8L4L0AALEAEpImAJox/CeO/CAGwCBGQJgIWZ/xLGP+FCICFiIA0EbAo41/C+C9GACxGBKSJgMUY/xLGf0ECYEEiIE0ELML4lzD+ixIAixIBaSJgcsa/hPFfmABYmAhIEwGTMv4ljP/iBMDiRECaCJiM8S9h/BsQAA2IgDQRMAnjX8L4NyEAmhABaSJgcMa/hPFvRAA0IgLSRMCgjH8J49+MAGhGBKSJgMEY/xLGvyEB0JAISBMBgzD+JYx/UwKgKRGQJgJOZvxLGP/GBEBjIiBNBJzE+Jcw/s0JgOZEQJoIOJjxL2H8EQCIgAIi4CDGv4TxJyIEAFciIE0E7Mz4lzD+fCEA+EIEpImAnRj/EsafrwgAviIC0kRAMeNfwvjzHQHAd0RAmggoYvxLGH9eJQB4lQhIEwFJxr+E8edNAoA3iYA0EfAk41/C+PMuAcC7RECaCHiQ8S9h/PmQAOBDIiBNBNzJ+Jcw/txFAHAXEZAmAj5g/EsYf+4mALibCEgTAW8w/iWMPw8RADxEBKSJgG8Y/xLGn4cJAB4mAtJEwJXxL2H8eYoA4CkiIK19BBj/EsafpwkAniYC0tpGgPEvYfxJEQCkiIC0dhFg/EsYf9IEAGkiIK1NBBj/EsafEgKAEiIgbfkIMP4ljD9lBABlREDashFg/EsYf0oJAEqJgLTlIsD4lzD+lBMAlBMBactEgPEvYfzZhQBgFyIgbfoIMP4ljD+7EQDsRgSkTRsBxr+E8WdXAoBdiYC06SLA+Jcw/uxOALA7EZA2TQQY/xLGn0MIAA4hAtKGjwDjX8L4cxgBwGFEQNqwEWD8Sxh/DiUAOJQISBsuAox/CePP4QQAhxMBacNEgPEvYfw5hQDgFCIg7fQIMP4ljD+nEQCcRgSknRYBxr+E8edUAoBTiYC0wyPA+Jcw/pxOAHA6EZB2WAQY/xLGnyEIAIYgAtJ2jwDjX8L4MwwBwDBEQNpuEWD8Sxh/hiIAGIoISCuPAONfwvgzHAHAcERAWlkEGP8Sxp8hCQCGJALS0hFg/EsYf4YlABiWCEh7OgKMfwnjz9AEAEMTAWkPR4DxL2H8GZ4AYHgiIO3uCDD+JYw/UxAATEEEpH0YAca/hPFnGgKAaYiAtDcjwPiXMP5MRQAwFRGQ9l0EGP8Sxp/p/HD2AfCoy+Xyy7ZtERE/n33LpH6MiNi27afrx8Y/x/gzJQHAlERA2o9v/JnHGH+mtV0ul7NvgKdt2/ZriADOYfwXtm3b/0XEb8++Y09+BoCp+ZkATmL8mZ4AYHoigIMZf5YgAFiCCOAgxp9lCACWIQLYmfFnKQKApYgAdmL8WY4AYDkigGLGnyUJAJYkAihi/FmWAGBZIoAk48/SBABLEwE8yfizPAHA8kQADzL+tOBZALTg2QHcyfhz8zki/ubsI/bkWQC04tkBvMP404q3AGjF2wG8wfjTjgCgHRHAN4w/LQkAWhIBXBl/2hIAtCUC2jP+tCYAaE0EtGX8aU8A0J4IaMf4QwgAiAgR0IjxhysBAFciYHnGH14QAPCCCFiW8YdvCAD4hghYjvGHVwgAeIUIWIbxhzcIAHiDCJie8Yd3CAB4hwiYlvGHDwgA+IAImI7xhzsIALiDCJiG8Yc7CQC4kwgYnvGHBwgAeIAIGJbxhwcJAHiQCBiO8YcnCAB4gggYhvGHJwkAeJIIOJ3xhwQBAAki4DTGH5IEACSJgMMZfyggAKCACDiM8YciAgCKiIDdGX8oJACgkAjYjfGHYgIAiomAcsYfdiAAYAcioIzxh50IANiJCEgz/rAjAQA7EgFPM/6wMwEAOxMBDzP+cAABAAcQAXcz/nAQAQAHEQEfMv5wIAEABxIBbzL+cDABAAcTAd8x/nACAQAnEAFfGH84iQCAk4gA4w9nEgBwosYRYPzhZAIATtYwAow/DEAAwAAaRYDxh0EIABhEgwgw/jAQAQADWTgCjD8MRgDAYBaMAOMPAxIAMKCFIsD4w6AEAAxqgQgw/jAwAQADmzgCjD8MTgDA4CaMAOMPExAAMIGJIsD4wyQEAExigggw/jARAQATGTgCjD9MRgDAZAaMAOMPExIAMKGBIsD4w6QEAExqgAgw/jAxAQATOzECjD9MTgDA5E6IAOMPCxAAsIADI8D4wyIEACzigAgw/rAQAQAL2TECjD8sRgDAYnaIAOMPCxIAsKDCCDD+sCgBAIsqiADjDwsTALCwRAQYf1icAIDFPREBxh8aEADQwAMRYPyhCQEATdwRAcYfGhEA0Mg7EWD8oRkBAM28EgHGHxraLpfL2TcAJ9i27deIL0EANCMAAKAhbwEAQEMCAAAaEgAA0JAAAICGBAAANCQAAKAhAQAADQkAAGhIAABAQwIAABoSAADQkAAAgIYEAAA0JAAAoCEBAAANCQAAaEgAAEBDAgAAGhIAANCQAACAhgQAADQkAACgIQEAAA0JAABoSAAAQEMCAAAaEgAA0JAAAICGBAAANCQAAKAhAQAADQkAAGhIAABAQwIAABoSAADQkAAAgIYEAAA0JAAAoCEBAAANCQAAaEgAAEBDAgAAGhIAANCQAACAhgQAADQkAACgIQEAAA0JAABoSAAAQEMCAAAaEgAA0JAAAICGBAAANCQAAKAhAQAADQkAAGhIAABAQwIAABoSAADQkAAAgIYEAAA0JAAAoCEBAAANCQAAaEgAAEBDAgAAGhIAANCQAACAhgQAADQkAACgIQEAAA0JAABoSAAAQEMCAAAaEgAA0JAAAICGBAAANCQAAKAhAQAADQkAAGhIAABAQwIAABoSAADQkAAAgIYEAAA0JAAAoCEBAAANCQAAaEgAAEBDAgAAGhIAANCQAACAhgQAADQkAACgIQEAAA0JAABoSAAAQEMCAAAaEgAA0JAAAICGBAAANCQAAKAhAQAADQkAAGhIAABAQwIAABoSAADQ0F8AiCXRTxmjtiwAAAAASUVORK5CYII=' joint.shapes.bpmn.icons.federation = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABgAAAAWAgMAAACnE7QbAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAAAYdEVYdFNvZnR3YXJlAHBhaW50Lm5ldCA0LjAuOWwzfk4AAAAJUExURQAAAP///wAAAHPGg3EAAAACdFJOUwAAdpPNOAAAACdJREFUCNdjYGBatYqBgYkBDIinQCwHOI9xFapcA4iNkLvWtIJ0GwDTEwWTtBTeUwAAAABJRU5ErkJggg=="; joint.shapes.bpmn.icons.eventGateway = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAMAAACdt4HsAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAAAYdEVYdFNvZnR3YXJlAHBhaW50Lm5ldCA0LjAuOWwzfk4AAAB1UExURf///wAAAAEBAQsLCw0NDRERERcXFxgYGBwcHCMjIyQkJCkpKSoqKisrKy8vLzAwMDExMTMzMzU1NTk5OTs7O0JCQkNDQ0pKSktLS1FRUVJSUlRUVFlZWVpaWl5eXmFhYWNjY2lpaXNzc3x8fIeHh7q6utvb22ODD/wAAAABdFJOUwBA5thmAAAD10lEQVRYw5VXW3viOgzUbC4kbeiyZcPCNtTF9pn//xPPg3xLSKDrB39tsGXNjCVLIhuD3vuvr+vpdP368t5T/mnQXzvMRnf9vg26vw1WRvPX8VuH79OW6vVtHN9eq/Rh/9QN+l/RZ5OA0/spIvr12ARHXTbY5Tp6O+hv4wMLPOh2Tz03q6Af1MRh0wJ3ANA7itBddyWBu6t+7QFgxw3yawD4pAjNigyNoQg/AaBek4NnAGg9hbZeV6G2FPoWAM53FtT0QKELvve3rMKtD0gchUN0dDYcABwZHEG7kIHetghH8wgAboW/Y/yxsWsYbYVi0ZxJ9ioP94+U1luyp8rdc0FgR7VTEsRpOC3Xoaewm6/zAKpguMoSKZ+lZnRVcLUC4NPnAYBVaBUX5yns/LFSjFY1ywoMVCEyucrHDwDoCknCKvZ5MXsAXtiUuOgaADj8N2LOS+TLZx69wjJlnISw/GC0tOcMsFHCfApBL6yBJl29rvBcr16TuGQD1IznqgODOmDDio954Ac20/9WXeiDC05JyQ5oVqhm6nUA8MKsRM2wUWQC4MUBmLL4WaKw5xjDUcIWJ163sAN6yiX4E9y9C9ccCQH1RdgnOYywVVHU/XY1YfRQAfXPVmnz4pM7t8DPVtJTaScRkVuC7cPftmB0M+3yMxwjHoANpxqgYphV42Ez65p0dyrAhHkCXkUM8Coikshc9aALHIi8AibMIzCK/AHeU6zYDQM+UiDyDvwR+Q2MwYDOAcNlw8Ate1fsWhiQSw6JFQQt7wxcgHcJc7zo7hkCeQOmMJs5ieGWPUFQkjiXUYN3FUOJoJTRFZfqIYYSgU+XuJxvjzGUCG6LrTmYtjGEACrRaCII12t6hmGJYEoXMycU+whDadYWvsfIYpM85JCdlTIXp5zbAg0TKSEAi6RqAdx4j+AyT6rpHA3AkCnjYtTDbNTFQ1QDFQuT4WWzKZeww2qlypyYrL4WLj81TtimZGpXDZj8SLSMxyaX+/JxpTP3I6T08nH1ZX35SZ6RaNiuRWsAZ/Jjlnw9lMfjVg04ryWP6m1ZSoxaNekzet4uZc+xFtwlTgoQQ7TwuMiKy+avB9uyzKvXy7y6LPOWd5XVrNDc3Reau1mhWd0d4WIN6LQkxUtZ6oar1bpYS7oNgnae8SEGUA8/T6efQ6q9bSql14jWGvBZuf+xLD7umMSLthZTW+5uJ08Ruhes8HfX8uxjy2PMNI6TMbHl2T9peXLTtV9runR7ZR63bf6QW9Wi7UuN7OFp7xl7sxUVYj/3vPW9rLe+F/f99jm3qrmR/ecGvlRhe/f/2BV4PKllnhkAAAAASUVORK5CYII="; joint.shapes.bpmn.icons.messageThrow = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAVBAMAAADV4/HZAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAOwgAADsIBFShKgAAAABh0RVh0U29mdHdhcmUAcGFpbnQubmV0IDQuMC45bDN+TgAAABJQTFRF////AAAAEBAQICAgMDAwQEBA9ql18wAAAAF0Uk5TAEDm2GYAAABVSURBVBjT1Y25EYAwEAOXL2fNkEMHpgQ6gP6bIfAc9tABirQanQ4mG2XIDJU3wIs+eAZR6AqvsIsmuFUXOJXiDk2lSe2WS2Ittnn/jbaBOQx+9J/gAcYZFZNx+7SlAAAAAElFTkSuQmCC"; @@ -647,6 +647,23 @@ angular.module('roger_federation.Workflows') }, graphType: "GroupCell", icon: "circle" + }), + new joint.shapes.bpmn.Activity({ + position: { x: 10, y: 100 }, size: { width: 80, height: 80 }, + roger_federation: { + name: "", + type: "FederationOutgoing", + stringId: "", + description: "", + interconnected: false, + groupFilters: [], + attributes: [] + }, + graphType: "FederationOutgoingCell", + attrs: { + '.body': { fill: 'gray', stroke: 'black', opacity: '0.40' }, + }, + icon: "outgoing" }) ], 'participants'); addStencilToolip(stencil.graphs.participants); @@ -664,3 +681,4 @@ angular.module('roger_federation.Workflows') }; } ]); + diff --git a/src/federation-hub-ui/src/main/webapp/modules/workflows/workflow_services.js b/src/federation-hub-ui/src/main/webapp/modules/workflows/workflow_services.js index 53e9ed4d..e8eb1ba2 100644 --- a/src/federation-hub-ui/src/main/webapp/modules/workflows/workflow_services.js +++ b/src/federation-hub-ui/src/main/webapp/modules/workflows/workflow_services.js @@ -1,4 +1,3 @@ - 'use strict'; angular.module('roger_federation.Workflows') @@ -623,6 +622,17 @@ angular.module('roger_federation.Workflows') }); }; + workflowService.getActiveConnections = function() { + return $http.get( + ConfigService.getServerBaseUrlStrV2() + 'getActiveConnections/').then( + function(res) { + return res.data; + }, + function(reason) { + throw reason; + }); + }; + workflowService.getKnownFilters = function() { diff --git a/src/federation-hub-ui/src/main/webapp/modules/workflows/workflows_controller.js b/src/federation-hub-ui/src/main/webapp/modules/workflows/workflows_controller.js index 4a8a53c5..37e533a5 100644 --- a/src/federation-hub-ui/src/main/webapp/modules/workflows/workflows_controller.js +++ b/src/federation-hub-ui/src/main/webapp/modules/workflows/workflows_controller.js @@ -1,19 +1,65 @@ - /* globals joint */ /* globals $ */ 'use strict'; angular.module('roger_federation.Workflows') - .controller('WorkflowsController', ['$scope', '$rootScope', '$window', '$state', '$stateParams', '$interval', '$uibModal', + .controller('WorkflowsController', ['$scope', '$rootScope', '$window', '$state', '$stateParams', '$timeout', '$uibModal', '$log', '$http', 'uuid4', 'growl', 'WorkflowService', 'OntologyService', 'JointPaper', 'SemanticsService', workflowsController ]); -function workflowsController($scope, $rootScope, $window, $state, $stateParams, $interval, $uibModal, $log, $http, uuid4, growl, WorkflowService, OntologyService, JointPaper, SemanticsService) { +function workflowsController($scope, $rootScope, $window, $state, $stateParams, $timeout, $uibModal, $log, $http, uuid4, growl, WorkflowService, OntologyService, JointPaper, SemanticsService) { $scope.JointPaper = JointPaper; $scope.criticResults = []; $scope.data = "{users: 'Joe'}"; + pollActiveConnections() + + function pollActiveConnections() { + WorkflowService.getActiveConnections().then(function(activeConnections) { + setOutgoingNodesStatus(activeConnections) + }).catch(e => setOutgoingNodesStatus([])) + $timeout(pollActiveConnections, 2000); + } + + function setOutgoingNodesStatus(activeConnections) { + if (JointPaper.paper && JointPaper.paper._views) { + var outgoingCellKeys = Object.keys(JointPaper.paper._views); + for (var i = 0; i < outgoingCellKeys.length; i++) { + let cellView = JointPaper.paper._views[outgoingCellKeys[i]] + + let text = cellView.model.attributes.roger_federation.stringId +'\n\n' + + cellView.model.attributes.roger_federation.host + ':' + cellView.model.attributes.roger_federation.port + + if (cellView.model.attributes.graphType === "FederationOutgoingCell") { + let foundConnection = undefined + activeConnections.forEach(activeConnection => { + if (activeConnection.connectionId === cellView.model.attributes.roger_federation.name) + foundConnection = activeConnection + }) + if (foundConnection) { + cellView.model.attributes.attrs['.body']['fill'] = 'green' + text += '\n' + foundConnection.connectionType.toLowerCase() + } else { + if (cellView.model.attributes.roger_federation.outgoingEnabled) + cellView.model.attributes.attrs['.body']['fill'] = 'red' + else + cellView.model.attributes.attrs['.body']['fill'] = 'gray' + } + + var shapeLabel = joint.util.breakText(text, { + width: 200 + }); + + cellView.model.set('content', shapeLabel); + + cellView.update() + cellView.resize() + } + } + } + } + $scope.addSemanticSubscription = function() { $state.go('workflows.editor.addSparqlQuery', { mode: 'new_request', @@ -345,6 +391,8 @@ function workflowsController($scope, $rootScope, $window, $state, $stateParams, $state.go('workflows.editor.addBPMNFederate'); } else if (['Group'].indexOf(propertiesType) !== -1) { $state.go('workflows.editor.addFederateGroup'); + } else if (['FederationOutgoing'].indexOf(propertiesType) !== -1) { + $state.go('workflows.editor.addFederationOutgoing'); } } } diff --git a/src/gradle.properties b/src/gradle.properties index 7a9e869a..33f37f49 100644 --- a/src/gradle.properties +++ b/src/gradle.properties @@ -14,7 +14,7 @@ spring_boot_version = 2.4.2 spring_boot_spring_version = 5.3.3 spring_cloud_starter_version = 2.2.3.RELEASE micrometer_cloudwatch_version = 1.5.5 - +jts_version = 1.18.0 guava_version = 30.1-jre intellij_annotations_version = 12.0 jackson_version = 2.11.4 @@ -59,7 +59,7 @@ spring_data_version = 2.4.3 tinder_version = 1.3.0 whack_version = 2.0.1 worldwind_version = 2.0.0 -xerces_version = 2.12.0 +xerces_version = 2.12.2 xpp3_version = 1.1.4c hamcrest_version = 1.3 junit_version = 4.12 @@ -86,3 +86,4 @@ mockito_version = 2.28.2 jsonwebtoken_version = 0.9.1 caffeine_version = 3.0.2 gradle_shadow_version = 6.1.0 +whack_version = 2.0.1 diff --git a/src/lib/whack_2_0_1.jar b/src/lib/whack_2_0_1.jar deleted file mode 100644 index f53cfd79..00000000 Binary files a/src/lib/whack_2_0_1.jar and /dev/null differ diff --git a/src/lib/xml-apis-2.11.0.jar b/src/lib/xml-apis-2.11.0.jar deleted file mode 100644 index 46733464..00000000 Binary files a/src/lib/xml-apis-2.11.0.jar and /dev/null differ diff --git a/src/takserver-common/build.gradle b/src/takserver-common/build.gradle index 5276a27b..1c84c631 100644 --- a/src/takserver-common/build.gradle +++ b/src/takserver-common/build.gradle @@ -36,6 +36,8 @@ dependencies { compile group: 'com.github.ben-manes.caffeine', name: 'caffeine', version: caffeine_version + compile group: 'com.github.ben-manes.caffeine', name: 'caffeine', version: caffeine_version + compile project(':takserver-fig-core') testCompile group: 'junit', name: 'junit', version: junit_version diff --git a/src/takserver-common/src/main/java/com/bbn/marti/remote/ClientEndpoint.java b/src/takserver-common/src/main/java/com/bbn/marti/remote/ClientEndpoint.java index d2adb653..1a9e5895 100644 --- a/src/takserver-common/src/main/java/com/bbn/marti/remote/ClientEndpoint.java +++ b/src/takserver-common/src/main/java/com/bbn/marti/remote/ClientEndpoint.java @@ -12,9 +12,10 @@ public class ClientEndpoint implements Serializable { public ClientEndpoint() {} - public ClientEndpoint(String callsign, String uid, Date lastEventTime, String lastEventName, String groups) { + public ClientEndpoint(String callsign, String uid, String username, Date lastEventTime, String lastEventName, String groups) { this.callsign = callsign; this.uid = uid; + this.username = username; this.lastEventTime = lastEventTime; this.lastStatus = lastEventName; this.groups = groups; @@ -36,6 +37,14 @@ public void setUid(String uid) { this.uid = uid; } + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss.S'Z'") public Date getLastEventTime() { return lastEventTime; @@ -65,6 +74,7 @@ public void setGroups(String groups) { private String callsign; private String uid; + private String username; private Date lastEventTime; private String lastStatus; private String groups; diff --git a/src/takserver-common/src/main/java/com/bbn/marti/remote/FederationManager.java b/src/takserver-common/src/main/java/com/bbn/marti/remote/FederationManager.java index ad219769..44ad44ba 100644 --- a/src/takserver-common/src/main/java/com/bbn/marti/remote/FederationManager.java +++ b/src/takserver-common/src/main/java/com/bbn/marti/remote/FederationManager.java @@ -71,9 +71,13 @@ public interface FederationManager { List getConfiguredFederates(); - // Send ROL to core, to be federated subject to group filtering. + // Send ROL to messaging process, to be federated subject to group filtering. // Attach outbound groups to the ROL for federates using group mapping void submitFederateROL(ROL rol, NavigableSet groups); + + // Send ROL to messaging process, to be federated subject to group filtering. + // Attach outbound groups to the ROL for federates using group mapping + void submitFederateROL(ROL rol, NavigableSet groups, String fileHash); void reconfigureFederation(); diff --git a/src/takserver-common/src/main/java/com/bbn/marti/remote/InputMetric.java b/src/takserver-common/src/main/java/com/bbn/marti/remote/InputMetric.java index fa9d503c..e40aa608 100644 --- a/src/takserver-common/src/main/java/com/bbn/marti/remote/InputMetric.java +++ b/src/takserver-common/src/main/java/com/bbn/marti/remote/InputMetric.java @@ -6,7 +6,7 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicLong; -import com.bbn.marti.config.Network; +import com.bbn.marti.config.Input; import com.google.common.collect.ComparisonChain; /** @@ -18,12 +18,12 @@ public class InputMetric implements Serializable, Comparable { private final String id = UUID.randomUUID().toString().replace("-", ""); - private Network.Input input; + private Input input; private AtomicLong readsReceived = new AtomicLong(); private AtomicLong messagesReceived = new AtomicLong(); private AtomicLong numClients = new AtomicLong(); - public InputMetric(Network.Input input) { + public InputMetric(Input input) { if (input == null) { throw new IllegalArgumentException("null input"); } @@ -31,11 +31,11 @@ public InputMetric(Network.Input input) { this.input = input; } - public Network.Input getInput() { + public Input getInput() { return input; } - public void setInput(Network.Input input) { + public void setInput(Input input) { this.input = input; } diff --git a/src/takserver-common/src/main/java/com/bbn/marti/remote/MessagingConfigurator.java b/src/takserver-common/src/main/java/com/bbn/marti/remote/MessagingConfigurator.java index 7561d4b8..7fd9df66 100644 --- a/src/takserver-common/src/main/java/com/bbn/marti/remote/MessagingConfigurator.java +++ b/src/takserver-common/src/main/java/com/bbn/marti/remote/MessagingConfigurator.java @@ -4,28 +4,29 @@ import java.util.Collection; import java.util.HashMap; -import com.bbn.marti.config.Network; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.Input; import com.bbn.marti.remote.groups.ConnectionModifyResult; import com.bbn.marti.remote.groups.NetworkInputAddResult; public interface MessagingConfigurator { - NetworkInputAddResult addInputAndSave(Network.Input input); + NetworkInputAddResult addInputAndSave(Input input); void removeInputAndSave(String name); - Collection getInputMetrics(); + Collection getInputMetrics(boolean excludeDataFeeds); - ConnectionModifyResult modifyInputAndSave(String inputName, Network.Input input); + ConnectionModifyResult modifyInputAndSave(String inputName, Input input); MessagingConfigInfo getMessagingConfig(); void modifyMessagingConfig(MessagingConfigInfo info); - void addMetric(Network.Input input, InputMetric metric); + void addMetric(Input input, InputMetric metric); - InputMetric getMetric(Network.Input input); + void updateMetric(Input input); + + InputMetric getMetric(Input input); AuthenticationConfigInfo getAuthenticationConfig(); @@ -40,4 +41,6 @@ public interface MessagingConfigurator { HashMap verifyConfiguration(); void addInput(Input newInput, boolean cluster) throws IOException; + + void removeDataFeedAndSave(String name); } diff --git a/src/takserver-common/src/main/java/com/bbn/marti/remote/SubmissionInterface.java b/src/takserver-common/src/main/java/com/bbn/marti/remote/SubmissionInterface.java index 3b575a89..1321b501 100644 --- a/src/takserver-common/src/main/java/com/bbn/marti/remote/SubmissionInterface.java +++ b/src/takserver-common/src/main/java/com/bbn/marti/remote/SubmissionInterface.java @@ -22,6 +22,7 @@ public interface SubmissionInterface { public boolean submitMissionPackageCotAtTime(String cotMessage, String missionName, Date timestamp, NavigableSet groups, String clientUid); - // Submit explicitly addressed CoT message to intersection of specified groups, callsigns and uids - public boolean submitCot(String cotMessage, List uids, List callsigns, NavigableSet groups, boolean federate); + // Submit explicitly addressed CoT message to intersection of specified groups, callsigns and uids. + // resbumission indicates the event is being resent from this server, trims flow tags and turns off archiving + public boolean submitCot(String cotMessage, List uids, List callsigns, NavigableSet groups, boolean federate, boolean resubmission); } diff --git a/src/takserver-common/src/main/java/com/bbn/marti/remote/SubscriptionManagerLite.java b/src/takserver-common/src/main/java/com/bbn/marti/remote/SubscriptionManagerLite.java index efce8511..9ea7f031 100644 --- a/src/takserver-common/src/main/java/com/bbn/marti/remote/SubscriptionManagerLite.java +++ b/src/takserver-common/src/main/java/com/bbn/marti/remote/SubscriptionManagerLite.java @@ -16,7 +16,7 @@ public interface SubscriptionManagerLite { - enum ChangeType { CONTENT, LOG, KEYWORD, METADATA, EXTERNAL_DATA, UID_KEYWORD, RESOURCE_KEYWORD, MISSION_DELETE, MISSION_CREATE } + enum ChangeType { CONTENT, LOG, KEYWORD, METADATA, EXTERNAL_DATA, UID_KEYWORD, RESOURCE_KEYWORD, MISSION_DELETE, MISSION_CREATE, DATA_FEED, MAP_LAYER } RemoteSubscription getRemoteSubscriptionByClientUid(String cUid); RemoteSubscription getSubscriptionByClientUid(String cUid); @@ -39,11 +39,12 @@ enum ChangeType { CONTENT, LOG, KEYWORD, METADATA, EXTERNAL_DATA, UID_KEYWORD, R // Mission subscription management void missionSubscribe(String missionName, String uid); - void missionUnsubscribe(String missionName, String uid); + void missionUnsubscribe(String missionName, String uid, String username, boolean disconnectOnly); void missionDisconnect(String missionName, String uid); void removeAllMissionSubscriptions(String missionName); List getMissionSubscriptions(String missionName, boolean connectedOnly); + void announceMissionChange(String missionName, ChangeType changeType, String creatorUid, String tool, String changes, String xmlContentForNotification); void announceMissionChange(String missionName, ChangeType changeType, String creatorUid, String tool, String changes); void announceMissionChange(String missionName, String creatorUid, String tool, String changes); void broadcastMissionAnnouncement(String missionName, String groupVector, String creatorUid, ChangeType changeType, String tool); diff --git a/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/ConnectionInfo.java b/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/ConnectionInfo.java index 5d46dd52..a3952893 100644 --- a/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/ConnectionInfo.java +++ b/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/ConnectionInfo.java @@ -9,7 +9,7 @@ import org.jetbrains.annotations.NotNull; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.Input; import tak.server.ignite.IgniteHolder; diff --git a/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/ConnectionModifyResult.java b/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/ConnectionModifyResult.java index 06431162..8d37c9cc 100644 --- a/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/ConnectionModifyResult.java +++ b/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/ConnectionModifyResult.java @@ -58,4 +58,5 @@ public int compare(ConnectionModifyResult o1, ConnectionModifyResult o2) { public static final ConnectionModifyResult FAIL_NOMOD_PORT = new ConnectionModifyResult("ERROR! The connection port cannot be modified after initial creation!", 400); public static final ConnectionModifyResult FAIL_NOMOD_GROUP = new ConnectionModifyResult("ERROR! The connection multicast group cannot be modified after initial creation!", 400); public static final ConnectionModifyResult FAIL_NOMOD_IFACE = new ConnectionModifyResult("ERROR! The connection interface cannot be modified after initial creation!", 400); + public static final ConnectionModifyResult FAIL_NOMOD_DFEED = new ConnectionModifyResult("ERROR! The connection found was not a data feed but contained data feed attributes!", 400); } diff --git a/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/GroupManager.java b/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/GroupManager.java index 33994d8e..0af6f4d5 100644 --- a/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/GroupManager.java +++ b/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/GroupManager.java @@ -156,7 +156,7 @@ public interface GroupManager { * @return List instances * @throws RemoteException */ - List searchGroups(String groupNameFilter); + List searchGroups(String groupNameFilter, boolean exactMatch); /* * lookup an ldap user diff --git a/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/User.java b/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/User.java index 323e46ec..9c807b05 100644 --- a/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/User.java +++ b/src/takserver-common/src/main/java/com/bbn/marti/remote/groups/User.java @@ -30,7 +30,7 @@ public abstract class User implements Node, Comparable, Serializable { private static final long serialVersionUID = 8002622862665912632L; protected final String id; - protected final String name; + protected String name; protected final String connectionId; protected final ConnectionType connectionType; protected final String address; @@ -57,7 +57,7 @@ public User(@NotNull String id, @NotNull String connectionId, @NotNull Connectio this.authorities = new ConcurrentSkipListSet<>(); this.originNode = IgniteHolder.getInstance().getIgniteId(); } - + public String getId() { return id; } @@ -83,7 +83,9 @@ public String getConnectionId() { public ConnectionType getConnectionType() { return connectionType; } - + + public void setName(String name) { this.name = name; } + public String getName() { return name; } diff --git a/src/takserver-common/src/main/java/com/bbn/marti/remote/service/InputManager.java b/src/takserver-common/src/main/java/com/bbn/marti/remote/service/InputManager.java index 0ef7370d..9cb4ed5f 100644 --- a/src/takserver-common/src/main/java/com/bbn/marti/remote/service/InputManager.java +++ b/src/takserver-common/src/main/java/com/bbn/marti/remote/service/InputManager.java @@ -2,7 +2,8 @@ import java.util.Collection; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.DataFeed; +import com.bbn.marti.config.Input; import com.bbn.marti.remote.InputMetric; import com.bbn.marti.remote.MessagingConfigInfo; import com.bbn.marti.remote.groups.ConnectionModifyResult; @@ -14,11 +15,15 @@ public interface InputManager { NetworkInputAddResult createInput(Input input); + NetworkInputAddResult createDataFeed(DataFeed dataFeed); + ConnectionModifyResult modifyInput(String id, Input input); void deleteInput(String name); - Collection getInputMetrics(); + void deleteDataFeed(String name); + + Collection getInputMetrics(boolean excludeDataFeeds); MessagingConfigInfo getConfigInfo(); diff --git a/src/takserver-common/src/main/java/com/bbn/marti/remote/util/RemoteUtil.java b/src/takserver-common/src/main/java/com/bbn/marti/remote/util/RemoteUtil.java index adf5ba16..3ff45d46 100644 --- a/src/takserver-common/src/main/java/com/bbn/marti/remote/util/RemoteUtil.java +++ b/src/takserver-common/src/main/java/com/bbn/marti/remote/util/RemoteUtil.java @@ -9,12 +9,11 @@ import java.util.NavigableSet; import java.util.Set; import java.util.concurrent.ConcurrentSkipListSet; -import java.util.concurrent.atomic.AtomicBoolean; import javax.xml.bind.DatatypeConverter; +import org.apache.commons.lang3.StringUtils; import org.apache.ignite.Ignite; -import org.apache.ignite.IgniteAtomicLong; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,8 +32,6 @@ import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; -import tak.server.plugins.PluginManagerConstants; - /* * * Constants and utility functions that are shared between TAK Server microservices @@ -53,7 +50,9 @@ public class RemoteUtil { public static final String GROUP_CLAUSE = " :groupVector\\:\\:bit(" + GROUPS_BIT_VECTOR_LEN + ") & " + "lpad(groups\\:\\:character varying, " + GROUPS_BIT_VECTOR_LEN + ", '0')\\:\\:bit(" + GROUPS_BIT_VECTOR_LEN + ")\\:\\:bit varying " + "<> 0\\:\\:bit(" + GROUPS_BIT_VECTOR_LEN + ")\\:\\:bit varying "; - + + public static final String GROUP_FUNCTION_CALL = " function( 'bitwiseAndGroups', :groupVector, groups ) = TRUE "; + public static final String GROUP_VECTOR = " cast(:groupVector as bit(" + GROUPS_BIT_VECTOR_LEN + "))"; private String bitStringNoGroups = null; @@ -109,7 +108,9 @@ public boolean[] getBitVectorForGroups(Set groups, int direction) { } if (group.getBitpos() == null) { - logger.error("empty bit position in group - skipping: " + group); + if (logger.isErrorEnabled()) { + logger.error("empty bit position in group - skipping: " + StringUtils.normalizeSpace(group.toString())); + } continue; } @@ -374,30 +375,4 @@ public boolean isGroupVectorAllowed(String requestGroupVectorString, String data return false; } - - public AtomicBoolean getIsIntercept(AtomicBoolean isIntercept) { - if (ignite == null) { - throw new IllegalStateException("ignite reference in RemoteUtil is null"); - } - - if (isIntercept == null) { - IgniteAtomicLong interceptFlag = ignite.atomicLong(PluginManagerConstants.INTERCEPTOR_REGISTRATION_KEY, -1, true); - - if (interceptFlag.get() == -1) { - logger.info("Interceptor plugins registration could not be determined, interceptor plugins disabled"); - isIntercept = new AtomicBoolean(false); - } else if (interceptFlag.get() == 0) { - logger.info("No interceptor plugins registered."); - isIntercept = new AtomicBoolean(false); - } else if (interceptFlag.get() == 1) { - logger.info("One or more interceptor plugins registered."); - isIntercept = new AtomicBoolean(true); - } else { - logger.info("Invalid value " + interceptFlag.get() + " for intercept flag, interceptor plugins disabled."); - isIntercept = new AtomicBoolean(false); - } - } - - return isIntercept; - } } diff --git a/src/takserver-common/src/main/java/tak/server/PluginManager.java b/src/takserver-common/src/main/java/tak/server/PluginManager.java index aac7a67d..2482ce75 100644 --- a/src/takserver-common/src/main/java/tak/server/PluginManager.java +++ b/src/takserver-common/src/main/java/tak/server/PluginManager.java @@ -30,4 +30,20 @@ public interface PluginManager { * Stop one plugin by id. */ void stopPluginByName(String name); + + /* + * set plugin archive for one plugin + */ + void setPluginArchive(String name, boolean isArchiveEnabled); + + /* + * set plugin enabled atribute for one plugin + */ + void setPluginEnabled(String name, boolean isPluginEnabled); + + /* + * Submit data a plugin for processing. Implementation of this function is entirely delegated to the plugin. + * + */ + void submitDataToPlugin(String pluginClassName, String scope, String data, String contentType); } diff --git a/src/takserver-common/src/main/java/tak/server/feeds/APIDataFeed.java b/src/takserver-common/src/main/java/tak/server/feeds/APIDataFeed.java new file mode 100644 index 00000000..ca420d91 --- /dev/null +++ b/src/takserver-common/src/main/java/tak/server/feeds/APIDataFeed.java @@ -0,0 +1,15 @@ +package tak.server.feeds; + +import java.util.List; + +public class APIDataFeed extends DataFeed { + + public APIDataFeed(String uuid, String name, DataFeedType type, List tags) { + super(uuid, name, type, tags); + } + + public APIDataFeed(com.bbn.marti.config.DataFeed datafeed) { + super(datafeed); + } + +} diff --git a/src/takserver-common/src/main/java/tak/server/feeds/DataFeed.java b/src/takserver-common/src/main/java/tak/server/feeds/DataFeed.java new file mode 100644 index 00000000..a4c7cda3 --- /dev/null +++ b/src/takserver-common/src/main/java/tak/server/feeds/DataFeed.java @@ -0,0 +1,242 @@ +package tak.server.feeds; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.commons.lang3.EnumUtils; + +import com.bbn.marti.config.AuthType; +import com.google.common.collect.ComparisonChain; + +public class DataFeed implements Serializable, Comparable { + + private static final long serialVersionUID = -7747557200324370313L; + + public enum DataFeedType { + Streaming, + API, + Plugin + } + + private String uuid; + + private String name; + + private DataFeedType type; + + private List tags = new ArrayList<>(); + + private AuthType auth; + + private Integer port; + + private boolean authRequired; + + private String protocol; + + private String group; + + private String iface; + + private boolean archive; + + private boolean anongroup; + + private boolean archiveOnly; + + private boolean sync; + + private Integer coreVersion; + + private String coreVersion2TlsVersions; + + private List filterGroups; + + private DataFeed() {} + + public DataFeed(String uuid, String name, DataFeedType type, List tags) { + this.uuid = uuid; + this.name = name; + this.type = type; + this.tags.addAll(tags); + } + + public DataFeed(com.bbn.marti.config.DataFeed datafeed) { + this.uuid = datafeed.getUuid(); + this.name = datafeed.getName(); + this.tags = datafeed.getTag(); + this.auth = datafeed.getAuth(); + this.authRequired = datafeed.isAuthRequired(); + this.protocol = datafeed.getProtocol(); + this.group = datafeed.getGroup(); + this.iface = datafeed.getIface(); + this.archive = datafeed.isArchive(); + this.anongroup = datafeed.isAnongroup(); + this.archiveOnly = datafeed.isArchiveOnly(); + this.sync = datafeed.isSync(); + this.coreVersion = datafeed.getCoreVersion(); + this.coreVersion2TlsVersions = datafeed.getCoreVersion2TlsVersions(); + this.filterGroups = datafeed.getFiltergroup(); + + if (datafeed.getPort() == 0) { + this.port = null; + } else { + this.port = datafeed.getPort(); + } + + this.type = EnumUtils.getEnumIgnoreCase(DataFeedType.class, datafeed.getType()); + + if (type == null) this.type = DataFeedType.Streaming; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Collection getTags() { + return tags; + } + + public void setTags(List tags) { + this.tags = tags; + } + + public DataFeedType getType() { + return type; + } + + public void setType(DataFeedType type) { + this.type = type; + } + + public AuthType getAuth() { + return auth; + } + + public void setAuth(AuthType auth) { + this.auth = auth; + } + + public boolean getAuthRequired() { + return authRequired; + } + + public void setAuthRequired(boolean authRequired) { + this.authRequired = authRequired; + } + + public String getProtocol() { + return protocol; + } + + public void setProtocol(String protocol) { + this.protocol = protocol; + } + + public Integer getPort() { + return port; + } + + public void setPort(Integer port) { + this.port = port; + } + + public String getGroup() { + return group; + } + + public void setGroup(String group) { + this.group = group; + } + + public String getIface() { + return iface; + } + + public void setIface(String iface) { + this.iface = iface; + } + + public boolean getArchive() { + return archive; + } + + public void setArchive(boolean archive) { + this.archive = archive; + } + + public boolean getAnongroup() { + return anongroup; + } + + public void setAnongroup(boolean anongroup) { + this.anongroup = anongroup; + } + + public boolean getArchiveOnly() { + return archiveOnly; + } + + public void setArchiveOnly(boolean archiveOnly) { + this.archiveOnly = archiveOnly; + } + + public boolean getSync() { + return sync; + } + + public void setSync(boolean sync) { + this.sync = sync; + } + + public Integer getCoreVersion() { + return coreVersion; + } + + public void setCoreVersion(Integer coreVersion) { + this.coreVersion = coreVersion; + } + + public String getCoreVersion2TlsVersions() { + return coreVersion2TlsVersions; + } + + public void setCoreVersion2TlsVersions(String coreVersion2TlsVersions) { + this.coreVersion2TlsVersions = coreVersion2TlsVersions; + } + + public List getFilterGroups() { + return filterGroups; + } + + public void setFilterGroups(List filterGroups) { + this.filterGroups = filterGroups; + } + + @Override + public String toString() { + return "RemoteDataFeed [uuid=" + uuid + ", name=" + name + ", type=" + type + ", tags=" + tags + ", auth=" + auth + ", authRequired=" + authRequired + + ", protocol=" + protocol + ", port=" + port + ", group=" + group + ", iface=" + iface + ", archive=" + archive + ", anongroup=" + anongroup + ", sync=" + sync + + ", archiveOnly=" + archiveOnly + ", coreVersion=" + coreVersion + ", coreVersion2TlsVersions=" + coreVersion2TlsVersions + ", filterGroups=" + filterGroups + "]"; + } + + @Override + public int compareTo(DataFeed that) { + return ComparisonChain.start().compare(this.getUuid(), that.getUuid()).result(); + } + +} diff --git a/src/takserver-common/src/main/java/tak/server/feeds/PluginDataFeed.java b/src/takserver-common/src/main/java/tak/server/feeds/PluginDataFeed.java new file mode 100644 index 00000000..85541e6d --- /dev/null +++ b/src/takserver-common/src/main/java/tak/server/feeds/PluginDataFeed.java @@ -0,0 +1,15 @@ +package tak.server.feeds; + +import java.util.List; + +public class PluginDataFeed extends DataFeed { + + public PluginDataFeed(String uuid, String name, DataFeedType type, List tags) { + super(uuid, name, type, tags); + } + + public PluginDataFeed(com.bbn.marti.config.DataFeed datafeed) { + super(datafeed); + } + +} diff --git a/src/takserver-common/src/main/java/tak/server/feeds/StreamingDataFeed.java b/src/takserver-common/src/main/java/tak/server/feeds/StreamingDataFeed.java new file mode 100644 index 00000000..1254c4ed --- /dev/null +++ b/src/takserver-common/src/main/java/tak/server/feeds/StreamingDataFeed.java @@ -0,0 +1,19 @@ +package tak.server.feeds; + +import java.util.List; + +/* + * + * Model class representing a generic TAK Server data feed + * + */ +public class StreamingDataFeed extends DataFeed { + + public StreamingDataFeed(String uuid, String name, DataFeedType type, List tags) { + super(uuid, name, type, tags); + } + + public StreamingDataFeed(com.bbn.marti.config.DataFeed datafeed) { + super(datafeed); + } +} diff --git a/src/takserver-common/src/main/java/tak/server/plugins/PluginManagerConstants.java b/src/takserver-common/src/main/java/tak/server/plugins/PluginManagerConstants.java index e805dfde..0c22fe82 100644 --- a/src/takserver-common/src/main/java/tak/server/plugins/PluginManagerConstants.java +++ b/src/takserver-common/src/main/java/tak/server/plugins/PluginManagerConstants.java @@ -5,7 +5,4 @@ public class PluginManagerConstants { public static final String PLUGIN_REGISTRY_NAME = "takserver-plugin-registry"; public static final String PLUGIN_MANAGER_IGNITE_PROFILE = "takserver-plugin-manager-profile"; - - public static final String INTERCEPTOR_REGISTRATION_KEY = "takserver-plugin-starter-interceptor-reg-key"; - } diff --git a/src/takserver-common/src/main/xsd/CoreConfig.xsd b/src/takserver-common/src/main/xsd/CoreConfig.xsd index e71fe678..4b490acd 100644 --- a/src/takserver-common/src/main/xsd/CoreConfig.xsd +++ b/src/takserver-common/src/main/xsd/CoreConfig.xsd @@ -32,6 +32,8 @@ + + @@ -41,31 +43,12 @@ Networking - - - - - Network Input - - - - - - - - - - - - - - - - - - - - + + + + + + HTTP Connector @@ -122,6 +105,7 @@ + @@ -204,6 +188,7 @@ + @@ -261,8 +246,8 @@ - - + + @@ -290,6 +275,7 @@ + @@ -541,7 +527,20 @@ + + + + Authorization server public key used to validated tokens + + + + + + + + + @@ -836,6 +835,8 @@ + + @@ -1104,12 +1105,82 @@ Registration Whitelist + + + - + + + + + + + Network Input + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1140,5 +1211,19 @@ + + + + + + + + + + + + + + diff --git a/src/takserver-core/build.gradle b/src/takserver-core/build.gradle index bd99d9ec..f1260c37 100644 --- a/src/takserver-core/build.gradle +++ b/src/takserver-core/build.gradle @@ -120,6 +120,8 @@ targetCompatibility = 1.8 dependencies { + compile group: 'xerces', name: 'xercesImpl', version: xerces_version + compile group: 'org.apache.logging.log4j', name: 'log4j-api', version: log4j_api_version compile group: 'org.apache.logging.log4j', name: 'log4j-to-slf4j', version: log4j_api_version compile group: 'org.slf4j', name: 'slf4j-api', version: slf4j_version @@ -134,6 +136,9 @@ dependencies { compile project(':takserver-common') compile project(':federation-common') + + compile group: 'org.locationtech.spatial4j', name: 'spatial4j', version: '0.6' + // implementation group: 'org.locationtech.spatial4j', name: 'spatial4j', version: '0.6' compile group: 'org.dom4j', name: 'dom4j', version: dom4j_version compile group: 'com.google.code.gson', name: 'gson', version: gson_version @@ -186,7 +191,7 @@ dependencies { compile group: 'org.hibernate', name: 'hibernate-core', version: hibernate_version compile group: 'org.hibernate', name: 'hibernate-entitymanager', version: hibernate_version - compile group: 'xerces', name: 'xercesImpl', version: xerces_version + // core deps diff --git a/src/takserver-core/docker/configureInDocker.sh b/src/takserver-core/docker/configureInDocker.sh index e9700b28..67d777e1 100755 --- a/src/takserver-core/docker/configureInDocker.sh +++ b/src/takserver-core/docker/configureInDocker.sh @@ -7,7 +7,7 @@ fi cd /opt/tak . ./setenv.sh java -jar -Xmx${MESSAGING_MAX_HEAP}m -Dspring.profiles.active=messaging takserver.war & -java -jar -Xmx${API_MAX_HEAP}m -Dspring.profiles.active=api takserver.war & +java -jar -Xmx${API_MAX_HEAP}m -Dspring.profiles.active=api -Dkeystore.pkcs12.legacy takserver.war & java -jar -Xmx${PLUGIN_MANAGER_MAX_HEAP}m takserver-pm.jar & if ! [ $# -eq 0 ] diff --git a/src/takserver-core/example/CoreConfig.example.xml b/src/takserver-core/example/CoreConfig.example.xml index 32c20744..033d2d02 100644 --- a/src/takserver-core/example/CoreConfig.example.xml +++ b/src/takserver-core/example/CoreConfig.example.xml @@ -168,5 +168,6 @@ --> + diff --git a/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/InputTests.java b/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/InputTests.java index e38b846b..64cb139e 100644 --- a/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/InputTests.java +++ b/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/InputTests.java @@ -51,6 +51,12 @@ public void inputRemoveAddTest(TestManipulatorInterface manipulator) { case SUBSCRIPTION_TCP: case SUBSCRIPTION_UDP: case SUBSCRIPTION_MCAST: + case DATAFEED_TCP: + case DATAFEED_UDP: + case DATAFEED_MCAST: + case DATAFEED_STCP: + case DATAFEED_TLS: + case DATAFEED_SSL: break; case INPUT_STCP: diff --git a/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/StreamingDataFeedsTests.java b/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/StreamingDataFeedsTests.java new file mode 100644 index 00000000..197fd5a0 --- /dev/null +++ b/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/StreamingDataFeedsTests.java @@ -0,0 +1,164 @@ +package com.bbn.marti.tests; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + +import org.junit.Test; + +import com.bbn.marti.test.shared.AbstractTestClass; +import com.bbn.marti.test.shared.TestManipulatorInterface; +import com.bbn.marti.test.shared.data.connections.AbstractConnection; +import com.bbn.marti.test.shared.data.connections.ConnectionFilter; +import com.bbn.marti.test.shared.data.generated.ImmutableConnections; +import com.bbn.marti.test.shared.data.protocols.ProtocolProfiles; +import com.bbn.marti.test.shared.data.servers.ImmutableServerProfiles; +import com.bbn.marti.test.shared.data.users.AbstractUser; +import com.bbn.marti.test.shared.data.users.UserFilter; + +public class StreamingDataFeedsTests extends AbstractTestClass { + + private static final String className = "StreamingDataFeedsTests"; + + @Test(timeout = 14000000) + public void dataFeedRemoveAddTest() { + randomize = true; + randomSeed = 348057624308987L; + dataFeedRemoveAddTest(null); + } + + + public void dataFeedRemoveAddTest(TestManipulatorInterface manipulator) { + try { + String sessionIdentifier = initTestMethod(); + + // add all the inputs with their initial users + +// List dataFeedList = ServerProfiles.SERVER_0.getInputs(); + List dataFeedListDefault = new ArrayList( + ImmutableConnections.valuesFiltered(new ConnectionFilter() + .addServerProfile(ImmutableServerProfiles.SERVER_0) + .addConnectionTypes(ProtocolProfiles.ConnectionType.DATAFEED))); + + List dataFeedList = new LinkedList<>(); + for (AbstractConnection connection : dataFeedListDefault) { + switch (connection.getProtocol()) { + + case INPUT_TCP: + case INPUT_UDP: + case INPUT_MCAST: + case INPUT_STCP: + case INPUT_TLS: + case INPUT_SSL: + case SUBSCRIPTION_TCP: + case SUBSCRIPTION_UDP: + case SUBSCRIPTION_MCAST: + case DATAFEED_TCP: + case DATAFEED_UDP: + case DATAFEED_MCAST: + break; + + case SUBSCRIPTION_STCP: + case SUBSCRIPTION_TLS: + case SUBSCRIPTION_SSL: + case DATAFEED_STCP: + case DATAFEED_TLS: + case DATAFEED_SSL: + dataFeedList.add(connection); + } + } + + List userList = new ArrayList<>(2 * dataFeedList.size()); + List sendableUserList = new LinkedList<>(); + List connectableUserList = new LinkedList<>(); + + randomize(dataFeedList); + + for (AbstractConnection dataFeed : dataFeedList) { + AbstractUser[] dataFeedUsers = dataFeed.getUsers(new UserFilter().setUserActivityIsValidated(true)).toArray(new AbstractUser[0]); + AbstractUser u0 = dataFeedUsers[0]; + AbstractUser u1 = dataFeedUsers[1]; + userList.add(u0); + userList.add(u1); + engine.offlineAddUsersAndConnectionsIfNecessary(u0); + engine.offlineAddUsersAndConnectionsIfNecessary(u1); + if (dataFeed.getProtocol().canConnect()) { + connectableUserList.add(u0); + connectableUserList.add(u1); + } + if (dataFeed.getProtocol().canSend()) { + sendableUserList.add(u0); + sendableUserList.add(u1); + } + } + + if (manipulator != null) { + manipulator.preTestRunPreServerStart(engine); + } + + engine.startServer(ImmutableServerProfiles.SERVER_0, sessionIdentifier); + + if (manipulator != null) { + manipulator.preTestRunPostServerStart(engine); + } + + randomize(connectableUserList); + + engine.connectClientsAndVerify(true, connectableUserList.toArray(new AbstractUser[0])); + + randomize(sendableUserList); + + for (AbstractUser user : sendableUserList) { + engine.attemptSendFromUserAndVerify(user); + } + + randomize(dataFeedList); + + for (AbstractConnection dataFeed : dataFeedList) { + engine.onlineAddDataFeed(dataFeed); + + AbstractUser[] inputUsers = dataFeed.getUsers(new UserFilter().setUserActivityIsValidated(true)).toArray(new AbstractUser[0]); + AbstractUser u0 = inputUsers[0]; + AbstractUser u1 = inputUsers[1]; + + if (connectableUserList.contains(u0)) { + engine.connectClientAndVerify(true, u0); + } + + if (connectableUserList.contains(u1)) { + engine.connectClientAndVerify(true, u1); + } + + List looplist = engine.getUsersThatCanSend(); + randomize(looplist); + + for (AbstractUser user : looplist) { + engine.attemptSendFromUserAndVerify(user); + } + } + + randomize(dataFeedList); + + for (AbstractConnection dataFeed : dataFeedList) { + engine.onlineRemoveDataFeedAndVerify(dataFeed); + + List loopList = engine.getUsersThatCanSend(); + randomize(loopList); + + for (AbstractUser user : loopList) { + engine.attemptSendFromUserAndVerify(user); + } + } + + } finally { + if (manipulator != null) { + manipulator.postTestRunPreServerStop(engine); + } + engine.stopServers(ImmutableServerProfiles.SERVER_0); + if (manipulator != null) { + manipulator.postTestRunPostServerStop(engine); + } + } + } + +} diff --git a/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/missions/MissionDataFlowTests.java b/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/missions/MissionDataFlowTests.java index d6882063..3a07b838 100644 --- a/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/missions/MissionDataFlowTests.java +++ b/src/takserver-core/src/integrationTest/java/com/bbn/marti/tests/missions/MissionDataFlowTests.java @@ -24,7 +24,7 @@ public void b_adminAddMission() { @Test(timeout = SHORT_TIMEOUT) public void c_adminSubscribeMemberToMission() { - engine.missionSubscribe(admin, missionName, existingMember); + engine.missionSubscribe(existingMember, missionName, existingMember); } @Test(timeout = SHORT_TIMEOUT) @@ -53,7 +53,7 @@ public void h_subscribeNewMemberToMission() { engine.onlineAddUser(newMember); engine.connectClientsAndVerify(true, newMember); onfam.certmod(newMember.getCertPublicPemPath().toString(), null, null, false, null, newMember.getDefinedGroupSet().stringArray(), null, null); - engine.missionSubscribe(admin, missionName, newMember); + engine.missionSubscribe(newMember, missionName, newMember); } @Test(timeout = SHORT_TIMEOUT) diff --git a/src/takserver-core/src/main/java/com/bbn/cot/filter/DataFeedFilter.java b/src/takserver-core/src/main/java/com/bbn/cot/filter/DataFeedFilter.java new file mode 100644 index 00000000..84b2691a --- /dev/null +++ b/src/takserver-core/src/main/java/com/bbn/cot/filter/DataFeedFilter.java @@ -0,0 +1,151 @@ +package com.bbn.cot.filter; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import org.dom4j.DocumentHelper; +import org.dom4j.Element; +import org.locationtech.jts.geom.Polygon; + +import com.bbn.marti.config.DataFeed; +import com.bbn.marti.config.GeospatialFilter; +import com.bbn.marti.config.GeospatialFilter.BoundingBox; +import com.bbn.marti.util.GeomUtils; +import com.bbn.marti.service.DistributedConfiguration; +import com.bbn.marti.service.SubscriptionStore; +import com.bbn.marti.sync.model.Mission; +import com.bbn.marti.util.MessagingDependencyInjectionProxy; +import com.bbn.marti.util.spring.SpringContextBeanForApi; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.google.common.base.Strings; + +import tak.server.Constants; +import tak.server.cot.CotEventContainer; + +/* + * + */ +public class DataFeedFilter { + + private static DataFeedFilter instance = null; + + public static DataFeedFilter getInstance() { + if (instance == null) { + synchronized (DataFeedFilter.class) { + if (instance == null) { + instance = SpringContextBeanForApi.getSpringContext().getBean(DataFeedFilter.class); + } + } + } + return instance; + } + + public void filter(CotEventContainer cot, DataFeed dataFeed) { + + if (cot != null && dataFeed != null) { + Element detailElem = cot.getDocument().getRootElement().element("detail"); + if (detailElem == null) { + detailElem = DocumentHelper.makeElement(cot.getDocument().getRootElement(), "detail"); + } + + Element sourceElement = DocumentHelper.makeElement(detailElem, "source"); + sourceElement.addAttribute("type", "dataFeed"); + sourceElement.addAttribute("name", dataFeed.getName()); + sourceElement.addAttribute("uid", dataFeed.getUuid()); + + dataFeed.getTag().forEach(tag-> { + Element tagElement = DocumentHelper.createElement("tag"); + tagElement.addText(tag); + sourceElement.add(tagElement); + }); + + // if vbm is enabled, only broker messages to clients subscribed to a mission that is linked to this data feed + if (DistributedConfiguration.getInstance().getRemoteConfiguration().getVbm().isEnabled()) { + // get missions associated with this data feed + List feedMissions = MessagingDependencyInjectionProxy.getInstance().missionService().getMissionsForDataFeed(dataFeed.getUuid()); + + // figure out which missions should filter CoT based on bounding box + List feedMissionsInCotBbox = new ArrayList<>(); + for (Mission mission : feedMissions) { + // no bbox, allow mission + if (Strings.isNullOrEmpty(mission.getBbox()) && Strings.isNullOrEmpty(mission.getBoundingPolygon())) { + feedMissionsInCotBbox.add(mission); + } + // use polygon over bbox + else if (!Strings.isNullOrEmpty(mission.getBoundingPolygon())) { + Polygon polygon = GeomUtils.postgisBoundingPolygonToPolygon(mission.getBoundingPolygon()); + // valid bounding box + if (polygon != null) { + double latitude = Double.parseDouble(cot.getLat()); + double longitude = Double.parseDouble(cot.getLon()); + + // if we received back non null, cot passed the filter + if (GeomUtils.polygonContainsCoordinate(polygon, latitude, longitude)) { + feedMissionsInCotBbox.add(mission); + } + } + } + // fallback to bbox + else { + BoundingBox boundingBox = getBoundingBoxFromBboxString(mission.getBbox()); + // valid bounding box + if (boundingBox != null) { + GeospatialFilter gf = new GeospatialFilter(); + gf.getBoundingBox().add(boundingBox); + GeospatialEventFilter gef = new GeospatialEventFilter(gf, true, false); + CotEventContainer c = gef.filter(cot); + // if we received back non null, cot passed the filter + if (c != null) { + feedMissionsInCotBbox.add(mission); + } + } + } + } + + // Collect all the mission subscriber uids for valid feed missions + List feedMissionClients = feedMissionsInCotBbox + .stream() + .map(mission -> mission.getName()) + .map(missionName -> SubscriptionStore.getInstance().getLocalUidsByMission(missionName)) + .flatMap(clientUids -> clientUids.stream()) + .distinct() + .collect(Collectors.toList()); + + // by adding explicit UIDs, this CoT event will go into explicit brokering rather than implicit + cot.setContextValue(StreamingEndpointRewriteFilter.EXPLICIT_FEED_UID_KEY, feedMissionClients); + } + + cot.setContext(Constants.DATA_FEED_KEY, dataFeed); + } + } + + // compute bbox from string and cache it for instant lookup next time + BoundingBox getBoundingBoxFromBboxString(String bbox) { + + BoundingBox cachedBoundingBox = cache().getIfPresent(bbox); + if (cachedBoundingBox != null) { + return cachedBoundingBox; + } + + BoundingBox boundingBox = GeomUtils.getBoundingBoxFromBboxString(bbox); + + cache().put(bbox, boundingBox); + + return boundingBox; + } + + private Cache cache; + private Cache cache() { + if (cache == null) { + synchronized (this) { + if (cache == null) { + Caffeine builder = Caffeine.newBuilder(); + cache = builder.maximumSize(100).build(); + } + } + } + return cache; + } +} diff --git a/src/takserver-core/src/main/java/com/bbn/cot/filter/DropEventFilter.java b/src/takserver-core/src/main/java/com/bbn/cot/filter/DropEventFilter.java index 5bbcf369..e3c6bcdd 100644 --- a/src/takserver-core/src/main/java/com/bbn/cot/filter/DropEventFilter.java +++ b/src/takserver-core/src/main/java/com/bbn/cot/filter/DropEventFilter.java @@ -36,7 +36,7 @@ public CotEventContainer filter(CotEventContainer c) { if(detail != null && !c.getDetailXml().contains(detail)) { if (log.isDebugEnabled()) { - log.debug("cot event type : " + c.getDetailXml() + " didnt match filter detail: " + detail); + log.debug("cot detail : " + c.getDetailXml() + " didnt match filter detail: " + detail); } return c; } diff --git a/src/takserver-core/src/main/java/com/bbn/cot/filter/GeospatialEventFilter.java b/src/takserver-core/src/main/java/com/bbn/cot/filter/GeospatialEventFilter.java index ca82cc0b..3f7d79b5 100644 --- a/src/takserver-core/src/main/java/com/bbn/cot/filter/GeospatialEventFilter.java +++ b/src/takserver-core/src/main/java/com/bbn/cot/filter/GeospatialEventFilter.java @@ -2,6 +2,7 @@ import com.bbn.marti.config.GeospatialFilter; +import com.bbn.marti.util.GeomUtils; import tak.server.cot.CotEventContainer; import org.dom4j.Node; @@ -15,9 +16,11 @@ public class GeospatialEventFilter implements CotFilter { private GeospatialFilter filter; + private boolean noFilterCheckTypes = true; + private boolean noFilterCheckTAKClient = true; private static final List noFilterTypes = Arrays.asList( - "b-a-o-tbl", // NineOneOne + "b-a-o-tbl", // NineOneOne "b-a-o-can", // Cancel "b-a-g", // GeoFenceBreach "b-a-o-pan", // RingTheBell @@ -28,19 +31,29 @@ public GeospatialEventFilter(GeospatialFilter filter) { this.filter = filter; } + public GeospatialEventFilter(GeospatialFilter filter, boolean noFilterCheckTypes, boolean noFilterCheckTAKClient) { + this(filter); + this.noFilterCheckTypes = noFilterCheckTypes; + this.noFilterCheckTAKClient = noFilterCheckTAKClient; + } + private boolean noFilter(CotEventContainer c) { - // check the noFilterList - if (noFilterTypes.contains(c.getType())) { - return true; + if (noFilterCheckTypes) { + // check the noFilterList + if (noFilterTypes.contains(c.getType())) { + return true; + } } - // see if the cot event is for a TAK device - Node name = c.getDocument().selectSingleNode("/event/detail/__group/@name"); - Node role = c.getDocument().selectSingleNode("/event/detail/__group/@role"); - if (name != null && role != null && - name.getText() != null && role.getText() != null) { - return true; + if (noFilterCheckTAKClient) { + // see if the cot event is for a TAK device + Node name = c.getDocument().selectSingleNode("/event/detail/__group/@name"); + Node role = c.getDocument().selectSingleNode("/event/detail/__group/@role"); + if (name != null && role != null && + name.getText() != null && role.getText() != null) { + return true; + } } return false; @@ -69,16 +82,8 @@ public CotEventContainer filter(CotEventContainer c) { // iterate over the filters for (GeospatialFilter.BoundingBox bbox : filter.getBoundingBox()) { - boolean validLongitude = false; - if (bbox.getMaxLongitude() > bbox.getMinLongitude()) { - validLongitude = longitude >= bbox.getMinLongitude() && longitude <= bbox.getMaxLongitude(); - } else { - validLongitude = longitude >= bbox.getMinLongitude() || longitude <= bbox.getMaxLongitude(); - } - // return the cot event if found within one of the inputs filters - if (validLongitude && - latitude >= bbox.getMinLatitude() && latitude <= bbox.getMaxLatitude()) { + if (GeomUtils.bboxContainsCoordinate(bbox, latitude, longitude)) { return c; } } diff --git a/src/takserver-core/src/main/java/com/bbn/cot/filter/StreamingEndpointRewriteFilter.java b/src/takserver-core/src/main/java/com/bbn/cot/filter/StreamingEndpointRewriteFilter.java index 9d9cda50..ac0c71e8 100644 --- a/src/takserver-core/src/main/java/com/bbn/cot/filter/StreamingEndpointRewriteFilter.java +++ b/src/takserver-core/src/main/java/com/bbn/cot/filter/StreamingEndpointRewriteFilter.java @@ -46,6 +46,7 @@ public class StreamingEndpointRewriteFilter implements CotFilter { public static String EXPLICIT_PUBLISH_KEY = "explicitBrokeringPub"; public static String EXPLICIT_CALLSIGN_KEY = "explicitBrokeringCallsign"; public static String EXPLICIT_UID_KEY = "explicitBrokeringUid"; + public static String EXPLICIT_FEED_UID_KEY = "explicitFeedBrokeringUid"; public static String EXPLICIT_MISSION_KEY = "explicitBrokeringMission"; public static String UID_ATTR = "uid"; @@ -181,8 +182,16 @@ public CotEventContainer filter(final CotEventContainer cot) { for (final String missionName : missionNames) { - MissionSubscription missionSubscription = missionSubscriptionRepository.findByMissionNameAndClientUidNoMission( - missionName, clientUid); + MissionSubscription missionSubscription = null; + User user = (User) cot.getContextValue(Constants.USER_KEY); + if (user != null) { + missionSubscription = missionSubscriptionRepository.findByMissionNameAndClientUidAndUsernameNoMission( + missionName, clientUid, user.getName()); + } else { + missionSubscription = missionSubscriptionRepository.findByMissionNameAndClientUidNoMission( + missionName, clientUid); + } + if (missionSubscription == null) { logger.error("unable to find mission subscription for client " + missionName + ", " + clientUid); continue; diff --git a/src/takserver-core/src/main/java/com/bbn/cot/filter/VBMSASharingFilter.java b/src/takserver-core/src/main/java/com/bbn/cot/filter/VBMSASharingFilter.java new file mode 100644 index 00000000..49c329f8 --- /dev/null +++ b/src/takserver-core/src/main/java/com/bbn/cot/filter/VBMSASharingFilter.java @@ -0,0 +1,63 @@ +package com.bbn.cot.filter; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.bbn.marti.service.DistributedConfiguration; +import com.bbn.marti.service.SubmissionService; +import com.bbn.marti.util.spring.SpringContextBeanForApi; + +import tak.server.Constants; +import tak.server.cot.CotEventContainer; + +/* + * + * The purpose of this class is to filter out SA messages that didn't come come from a data feed. + * + */ +public class VBMSASharingFilter { + private static final Logger logger = LoggerFactory.getLogger(VBMSASharingFilter.class); + + private static VBMSASharingFilter instance = null; + + public static VBMSASharingFilter getInstance() { + if (instance == null) { + synchronized (VBMSASharingFilter.class) { + if (instance == null) { + instance = SpringContextBeanForApi.getSpringContext().getBean(VBMSASharingFilter.class); + } + } + } + return instance; + } + + public CotEventContainer filter(CotEventContainer cot) { + if (cot == null) return cot; + + if (SubmissionService.getInstance().isControlMessage(cot.getType())) return cot; + + // came from a data feed + if (cot.getContext(Constants.DATA_FEED_KEY) != null) { + return cot; + } + + if (DistributedConfiguration.getInstance().getVbm().isEnabled()) { + if (DistributedConfiguration.getInstance().getVbm().isDisableChatSharing()) { + if (isChatMessage(cot)) { + return null; + } + } + if (DistributedConfiguration.getInstance().getVbm().isDisableSASharing()) { + if (!isChatMessage(cot)) { + return null; + } + } + } + + return cot; + } + + private boolean isChatMessage(CotEventContainer cot) { + return "b-t-f".equals(cot.getType()); + } +} diff --git a/src/takserver-core/src/main/java/com/bbn/marti/config/ConfigHelper.java b/src/takserver-core/src/main/java/com/bbn/marti/config/ConfigHelper.java index 376ce3ed..7e76b0b3 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/config/ConfigHelper.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/config/ConfigHelper.java @@ -34,8 +34,8 @@ public static String validateConfiguration(@NotNull Configuration configuration) HashSet inputNameSet = new HashSet<>(); - List inputList = configuration.getNetwork().getInput(); - for (Network.Input input : inputList) { + List inputList = configuration.getNetwork().getInput(); + for (Input input : inputList) { // Using switch since the compiler will complain if another auth is added but not checked for here if (inputNameSet.contains(input.getName())) { diff --git a/src/takserver-core/src/main/java/com/bbn/marti/groups/DistributedPersistentGroupManager.java b/src/takserver-core/src/main/java/com/bbn/marti/groups/DistributedPersistentGroupManager.java index aea51827..73b39333 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/groups/DistributedPersistentGroupManager.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/groups/DistributedPersistentGroupManager.java @@ -761,7 +761,7 @@ public boolean testLdap() { } @Override - public List searchGroups(String groupNameFilter) { + public List searchGroups(String groupNameFilter, boolean exactMatch) { List results = new ArrayList(); @@ -782,7 +782,12 @@ public List searchGroups(String groupNameFilter) { Object[] args = null; if (groupNameFilter != null && groupNameFilter.trim().length() > 0) { - groupFilter = "(&(objectclass={0})(CN=*{1}*))"; + if (exactMatch) { + groupFilter = "(&(objectclass={0})(CN={1}))"; + } else { + groupFilter = "(&(objectclass={0})(CN=*{1}*))"; + } + args = new Object[2]; args[0] = ldapConfig.getGroupObjectClass().trim(); args[1] = groupNameFilter.trim(); diff --git a/src/takserver-core/src/main/java/com/bbn/marti/groups/FileAuthenticator.java b/src/takserver-core/src/main/java/com/bbn/marti/groups/FileAuthenticator.java index 6d35b451..66c4e181 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/groups/FileAuthenticator.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/groups/FileAuthenticator.java @@ -606,6 +606,7 @@ public synchronized void saveChanges(FileAuthenticatorControl control) throws JA userFileCache.put(control.getFileUser().getIdentifier(), control.getFileUser()); } } + IgniteHolder.getInstance().getIgnite().message().send(Constants.FILE_AUTH_TOPIC, control); } catch (Exception e) { if (logger.isDebugEnabled()) { diff --git a/src/takserver-core/src/main/java/com/bbn/marti/groups/GroupFederationUtil.java b/src/takserver-core/src/main/java/com/bbn/marti/groups/GroupFederationUtil.java index c193dc03..8666a426 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/groups/GroupFederationUtil.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/groups/GroupFederationUtil.java @@ -36,7 +36,7 @@ import com.atakmap.Tak.ROL; import com.bbn.cot.filter.StreamingEndpointRewriteFilter; import com.bbn.marti.config.Federation; -import com.bbn.marti.config.Network; +import com.bbn.marti.config.Input; import com.bbn.marti.nio.channel.base.AbstractBroadcastingChannelHandler; import com.bbn.marti.remote.groups.AuthenticatedUser; import com.bbn.marti.remote.groups.ConnectionInfo; @@ -289,7 +289,7 @@ public Set getReachableUsers(Subscription src) { return reachableUsers; } - public void updateInputGroups(@NotNull Network.Input input) { + public void updateInputGroups(@NotNull Input input) { // Get the connectionId String connectionId = MessageConversionUtil.getConnectionId(input); diff --git a/src/takserver-core/src/main/java/com/bbn/marti/groups/LdapAuthenticator.java b/src/takserver-core/src/main/java/com/bbn/marti/groups/LdapAuthenticator.java index c9a265be..ff3fa472 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/groups/LdapAuthenticator.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/groups/LdapAuthenticator.java @@ -191,6 +191,28 @@ private void logRunningTime(long rt) { } } + public static Set applyGroupPrefixFilter(Set groupNames, String prefix) { + + Set filtered = new ConcurrentSkipListSet<>(); + + // + // apply the groupPrefix filter + // + for (String groupName : groupNames) { + // move the group name on for further processing if it matches. an empty string prefix matches everything. + if (groupName.toLowerCase(Locale.ENGLISH).startsWith(prefix)) { + filtered.add(groupName); + } else { + if (logger.isDebugEnabled()) { + logger.debug("skipping non prefix-matching group " + groupName); + } + } + } + + return filtered; + } + + public Set getGroupNamesFromSearchResults(Map groupInfo) { Set groupNames = new ConcurrentSkipListSet<>(); @@ -231,21 +253,7 @@ public Set getGroupNamesFromSearchResults(Map groupInfo) groupNames.add(groupName); } - Set filtered = new ConcurrentSkipListSet<>(); - - // - // apply the groupPrefix filter - // - for (String groupName : groupNames) { - // move the group name on for further processing if it matches. an empty string prefix matches everything. - if (groupName.toLowerCase(Locale.ENGLISH).startsWith(groupPrefix)) { - filtered.add(groupName); - } else { - if (logger.isDebugEnabled()) { - logger.debug("skipping non prefix-matching group " + groupName); - } - } - } + Set filtered = applyGroupPrefixFilter(groupNames, groupPrefix); // // apply the groupNameExtractorRegex to the results of the groupPrefix filter @@ -291,14 +299,6 @@ public boolean assignGroups(Map groupInfo, User user) { logger.trace("processing group updates for " + user + " for these groups " + groupNames); } - if (getConf().getFiltergroup() != null && getConf().getFiltergroup().size() > 0) { - Set filterGroups = new ConcurrentSkipListSet<>(getConf().getFiltergroup()); - Set intersection = Sets.intersection(groupNames, filterGroups); - if (intersection.size() == 0) { - throw new UnauthorizedException(); - } - } - return processGroupUpdates(user, groupNames); } @@ -628,14 +628,19 @@ public String applyGroupNameExtractorRegex(String groupName) { } public boolean groupNamesToGroups(Set groupNames, Set groups) { + return groupNamesToGroups(groupManager, groupNames, groups, readOnlyGroup, readGroupSuffix, writeGroupSuffix); + } + + public static boolean groupNamesToGroups(GroupManager groupManager, Set groupNames, Set groups, + String readOnlyGroupName, String readSuffix, String writeSuffix) { // // check to see if the user belongs to the readOnlyGroup // boolean readOnly = false; - if (readOnlyGroup != null && readOnlyGroup.length() > 0) { + if (readOnlyGroupName != null && readOnlyGroupName.length() > 0) { for (String groupName : groupNames) { - if (groupName.compareTo(readOnlyGroup) == 0) { + if (groupName.compareTo(readOnlyGroupName) == 0) { groupNames.remove(groupName); readOnly = true; break; @@ -651,12 +656,12 @@ public boolean groupNamesToGroups(Set groupNames, Set groups) { // boolean grantReadAccess = true; boolean grantWriteAccess = true; - if (groupName.endsWith(readGroupSuffix)) { + if (groupName.endsWith(readSuffix)) { grantWriteAccess = false; - groupName = groupName.substring(0, groupName.indexOf(readGroupSuffix)); - } else if (groupName.endsWith(writeGroupSuffix)) { + groupName = groupName.substring(0, groupName.indexOf(readSuffix)); + } else if (groupName.endsWith(writeSuffix)) { grantReadAccess = false; - groupName = groupName.substring(0, groupName.indexOf(writeGroupSuffix)); + groupName = groupName.substring(0, groupName.indexOf(writeSuffix)); } if (grantWriteAccess && !readOnly) { @@ -671,12 +676,20 @@ public boolean groupNamesToGroups(Set groupNames, Set groups) { return readOnly; } - private boolean processGroupUpdates(User user, Set groupNames) { + public boolean processGroupUpdates(User user, Set groupNames) { if (user == null || groupNames == null) { throw new IllegalArgumentException("null user or groupNames list"); } + if (getConf().getFiltergroup() != null && getConf().getFiltergroup().size() > 0) { + Set filterGroups = new ConcurrentSkipListSet<>(getConf().getFiltergroup()); + Set intersection = Sets.intersection(groupNames, filterGroups); + if (intersection.size() == 0) { + throw new UnauthorizedException(); + } + } + Set groups = new ConcurrentSkipListSet<>(); boolean readOnly = groupNamesToGroups(groupNames, groups); diff --git a/src/takserver-core/src/main/java/com/bbn/marti/groups/OAuthAuthenticator.java b/src/takserver-core/src/main/java/com/bbn/marti/groups/OAuthAuthenticator.java index 43e09ac2..0e0da697 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/groups/OAuthAuthenticator.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/groups/OAuthAuthenticator.java @@ -2,31 +2,27 @@ package com.bbn.marti.groups; import java.io.Serializable; -import java.util.Map; -import java.util.Set; +import java.util.*; +import java.util.concurrent.ConcurrentSkipListSet; import javax.naming.NamingException; +import com.bbn.marti.config.Oauth; import com.google.common.base.Strings; import io.jsonwebtoken.Claims; +import io.jsonwebtoken.JwtException; import io.jsonwebtoken.SignatureAlgorithm; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.security.oauth2.common.OAuth2AccessToken; +import org.springframework.security.oauth2.common.exceptions.InvalidTokenException; import org.springframework.security.oauth2.common.exceptions.OAuth2Exception; import org.springframework.security.oauth2.provider.token.DefaultTokenServices; import com.bbn.marti.config.Auth; import com.bbn.marti.jwt.JwtUtils; import com.bbn.marti.remote.exception.TakException; -import com.bbn.marti.remote.groups.AuthCallback; -import com.bbn.marti.remote.groups.AuthResult; -import com.bbn.marti.remote.groups.AuthStatus; -import com.bbn.marti.remote.groups.AuthenticatedUser; -import com.bbn.marti.remote.groups.ConnectionType; -import com.bbn.marti.remote.groups.Group; -import com.bbn.marti.remote.groups.GroupManager; -import com.bbn.marti.remote.groups.User; +import com.bbn.marti.remote.groups.*; import com.bbn.marti.service.DistributedConfiguration; import com.bbn.marti.service.Resources; import com.bbn.marti.util.spring.SpringContextBeanForApi; @@ -42,13 +38,16 @@ public class OAuthAuthenticator extends AbstractAuthenticator implements Seriali private static final long serialVersionUID = -4317122669577006009L; - Logger logger = LoggerFactory.getLogger(OAuthAuthenticator.class); - + private Logger logger = LoggerFactory.getLogger(OAuthAuthenticator.class); + private Oauth oauthConf = DistributedConfiguration.getInstance().getAuth().getOauth(); + private Auth.Ldap ldapConf = DistributedConfiguration.getInstance().getAuth().getLdap(); + private DefaultTokenServices defaultTokenServices; private static OAuthAuthenticator instance; + private String groupPrefix; + private String readOnlyGroup; + private String readGroupSuffix; + private String writeGroupSuffix; - Auth.Ldap ldapConf = DistributedConfiguration.getInstance().getAuth().getLdap(); - - DefaultTokenServices defaultTokenServices; public static synchronized OAuthAuthenticator getInstance(GroupManager groupManager) { if (instance == null) { @@ -73,6 +72,13 @@ public OAuthAuthenticator(GroupManager groupManager) { groupManager.registerAuthenticator("oauth", this); defaultTokenServices = SpringContextBeanForApi.getSpringContext().getBean(DefaultTokenServices.class); + + if (oauthConf != null) { + groupPrefix = oauthConf.getGroupprefix().toLowerCase(); + readOnlyGroup = oauthConf.getReadOnlyGroup(); + readGroupSuffix = oauthConf.getReadGroupSuffix(); + writeGroupSuffix = oauthConf.getWriteGroupSuffix(); + } } @Override @@ -92,17 +98,25 @@ private AuthResult auth(User user) { AuthStatus authStatus = AuthStatus.SUCCESS; try { - String token = user.getName(); - OAuth2AccessToken oAuth2AccessToken = defaultTokenServices.readAccessToken(token); - if (oAuth2AccessToken == null || oAuth2AccessToken.isExpired()) { - throw new OAuth2Exception("defaultTokenServices.readAccessToken failed!"); + String token = user.getName(); + Claims claims = JwtUtils.getInstance().parseClaims(token, SignatureAlgorithm.RS256); + if (claims == null) { + throw new InvalidTokenException("Unable to parse claims from token : " + token); } - token = oAuth2AccessToken.getValue(); - - Claims claims = JwtUtils.getInstance().parseClaims(token, SignatureAlgorithm.RS256); String username = (String) claims.get("user_name"); + if (username != null) { + // jwt's from takserver will contain the user_name claim. + // Ensure the token hasn't been revoked by checking the token store + OAuth2AccessToken oAuth2AccessToken = defaultTokenServices.readAccessToken(token); + if (oAuth2AccessToken == null || oAuth2AccessToken.isExpired()) { + throw new OAuth2Exception("defaultTokenServices.readAccessToken failed!"); + } + } else { + // For jwt's from keycloak, get the username from the email claim + username = (String) claims.get("email"); + } if (user instanceof AuthenticatedUser) { AuthenticatedUser auser = (AuthenticatedUser) user; @@ -117,72 +131,93 @@ private AuthResult auth(User user) { user = new AuthenticatedUser(username, auser.getConnectionId(), auser.getAddress(), auser.getCert(), username, "", "", auser.getConnectionType()); // no password or uid } - String auth = DistributedConfiguration.getInstance().getRemoteConfiguration().getAuth().getDefault(); + // if we have a groups claim in the token, use it to assign groups directly + ArrayList groupNames = (ArrayList)claims.get("groups"); + if (groupNames != null) { - if (auth.compareToIgnoreCase("file") == 0) { + Set groupNameSet = LdapAuthenticator.applyGroupPrefixFilter( + new HashSet(groupNames), groupPrefix); - for (UserAuthenticationFile.User fileUser : FileAuthenticator.getInstance().getAllUsers()) { - if (fileUser.getIdentifier().compareTo(username) == 0) { + Set groups = new ConcurrentSkipListSet<>(); + LdapAuthenticator.groupNamesToGroups( + groupManager, groupNameSet, groups, readOnlyGroup, readGroupSuffix, writeGroupSuffix); - // if the file user has a role, set it on the user - if (fileUser.getRole() != null) { - user.getAuthorities().add(fileUser.getRole().toString()); - } + // do the group updates based on this set of groups + groupManager.updateGroups(user, groups); - if (logger.isDebugEnabled() || !user.getConnectionType().equals(ConnectionType.WEB)) { - logger.debug("file user oauth client_id match for user " + fileUser.getIdentifier()); + } else { - logger.debug("groups: " + fileUser.getGroupList()); - } + String auth = DistributedConfiguration.getInstance().getRemoteConfiguration().getAuth().getDefault(); + if (auth.compareToIgnoreCase("file") == 0) { + + for (UserAuthenticationFile.User fileUser : FileAuthenticator.getInstance().getAllUsers()) { + if (fileUser.getIdentifier().compareTo(username) == 0) { + + // if the file user has a role, set it on the user + if (fileUser.getRole() != null) { + user.getAuthorities().add(fileUser.getRole().toString()); + } - Set groups = FileAuthenticator.getGroups(fileUser); + if (logger.isDebugEnabled() || !user.getConnectionType().equals(ConnectionType.WEB)) { + logger.debug("file user oauth client_id match for user " + fileUser.getIdentifier()); - groupManager.updateGroups(user, groups); + logger.debug("groups: " + fileUser.getGroupList()); + } + + Set groups = FileAuthenticator.getGroups(fileUser); + + groupManager.updateGroups(user, groups); + } } - } - } else if (auth.compareToIgnoreCase("ldap") == 0) { + } else if (auth.compareToIgnoreCase("ldap") == 0) { - // Assign LDAP groups for this users based on LDAP lookup by username. - // if the LDAP authenticator is configured, use it to assign groups for the user, using the service credentials. Can be disabled by setting the x509groups option to false. - try { - if (ldapConf != null && DistributedConfiguration.getInstance().getAuth().isX509Groups() && ldapConf.isX509Groups()) { + // Assign LDAP groups for this users based on LDAP lookup by username. + // if the LDAP authenticator is configured, use it to assign groups for the user, using the service credentials. Can be disabled by setting the x509groups option to false. + try { + if (ldapConf != null && DistributedConfiguration.getInstance().getAuth().isX509Groups() && ldapConf.isX509Groups()) { - try { - if (logger.isDebugEnabled()) { - logger.debug("username: " + username); - } + try { + if (logger.isDebugEnabled()) { + logger.debug("username: " + username); + } - Map groupInfo = LdapAuthenticator.getInstance() - .getGroupInfoBySearch(username); + Map groupInfo = LdapAuthenticator.getInstance() + .getGroupInfoBySearch(username); - if (logger.isDebugEnabled()) { - logger.debug("group info for " + username + " : " + groupInfo); - } + if (logger.isDebugEnabled()) { + logger.debug("group info for " + username + " : " + groupInfo); + } - boolean readOnly = false; - if (!groupInfo.isEmpty()) { - readOnly = LdapAuthenticator.getInstance().assignGroups(groupInfo, user); - } + boolean readOnly = false; + if (!groupInfo.isEmpty()) { + readOnly = LdapAuthenticator.getInstance().assignGroups(groupInfo, user); + } - if (logger.isDebugEnabled()) { - logger.debug("X509 / LDAP group assignment complete for " + user.getId()); - } + if (logger.isDebugEnabled()) { + logger.debug("X509 / LDAP group assignment complete for " + user.getId()); + } - if (ldapConf.isX509AddAnonymous() && - DistributedConfiguration.getInstance().getAuth().isX509AddAnonymous()) { - doAnonAssignment(user, readOnly); - } + if (ldapConf.isX509AddAnonymous() && + DistributedConfiguration.getInstance().getAuth().isX509AddAnonymous()) { + doAnonAssignment(user, readOnly); + } - } catch (NamingException e) { - logger.warn("exception connecting to LDAP", e); + } catch (NamingException e) { + logger.warn("exception connecting to LDAP", e); + } } + } catch (Exception e) { + logger.debug("exception searching for ldap groups for user " + user, e); } - } catch (Exception e) { - logger.debug("exception searching for ldap groups for user " + user, e); } } + Oauth oauthConfig = DistributedConfiguration.getInstance().getAuth().getOauth(); + if (oauthConfig != null && oauthConfig.isOauthAddAnonymous()) { + doAnonAssignment(user, false); + } + try { if (groupManager.getGroups(user).isEmpty()) { String msg = "no groups assigned to user " + user + " doing anonymous assignment"; @@ -202,6 +237,9 @@ private AuthResult auth(User user) { } catch (OAuth2Exception oAuth2Exception) { logger.error("rethrowing OAuth2Exception in OAuthAuthenticator!", oAuth2Exception); throw oAuth2Exception; + } catch (JwtException jwtException) { + logger.error("rethrowing JwtException as InvalidTokenException!", jwtException); + throw new InvalidTokenException(jwtException.getMessage()); } catch (Exception e) { logger.error("Exception in OAuthAuthenticator!", e); authStatus = AuthStatus.FAILURE; diff --git a/src/takserver-core/src/main/java/com/bbn/marti/groups/X509Authenticator.java b/src/takserver-core/src/main/java/com/bbn/marti/groups/X509Authenticator.java index 39b539bb..883b811c 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/groups/X509Authenticator.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/groups/X509Authenticator.java @@ -4,23 +4,28 @@ import java.io.Serializable; import java.security.cert.CertificateParsingException; -import java.util.*; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentSkipListSet; + import javax.naming.NamingException; import javax.naming.ldap.LdapName; import javax.naming.ldap.Rdn; -import com.google.common.base.Strings; -import com.google.common.collect.Sets; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.security.oauth2.common.exceptions.OAuth2Exception; import com.bbn.marti.config.Auth.Ldap; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.Input; import com.bbn.marti.remote.exception.NotFoundException; -import com.bbn.marti.remote.exception.TakException; import com.bbn.marti.remote.exception.RevokedException; +import com.bbn.marti.remote.exception.TakException; import com.bbn.marti.remote.groups.AuthCallback; import com.bbn.marti.remote.groups.AuthResult; import com.bbn.marti.remote.groups.AuthStatus; @@ -39,6 +44,8 @@ import com.bbn.marti.xml.bindings.UserAuthenticationFile; import com.bbn.tak.tls.TakCert; import com.bbn.tak.tls.repository.TakCertRepository; +import com.google.common.base.Strings; +import com.google.common.collect.Sets; import tak.server.cache.ActiveGroupCacheHelper; @@ -117,14 +124,35 @@ private User auth(User user, Input input) { String certFingerprint = RemoteUtil.getInstance().getCertSHA256Fingerprint(user.getCert()); - if (DistributedConfiguration.getInstance().getAuth().isX509CheckRevocation()) { - TakCert cert = takCertRepository.findOneByHash(certFingerprint); + TakCert cert = null; + if (DistributedConfiguration.getInstance().getAuth().isX509CheckRevocation() || + DistributedConfiguration.getInstance().getAuth().isX509TokenAuth()) { + cert = takCertRepository.findOneByHash(certFingerprint); if (cert != null && cert.getRevocationDate() != null) { throw new RevokedException("Attempt to use revoked certificate : " + cert.getSubjectDn()); } } + if (DistributedConfiguration.getInstance().getAuth().isX509TokenAuth() && + cert != null && cert.token != null && cert.token.length() > 0) { + user.setName(cert.token); + try { + groupManager.authenticate("oauth", user); + } catch (OAuth2Exception e) { + if (logger.isDebugEnabled()) { + logger.debug("found expired token in certificate: " + cert.token); + } + throw new TakException(cert.token); + } + + if (input == null) { + AuthenticatorUtil.setUserRolesBasedOnRequestPort(user, logger); + } + + return user; + } + if (logger.isDebugEnabled()) { logger.debug("cert fingerprint: " + certFingerprint); } diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/channel/ChannelHandler.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/channel/ChannelHandler.java index 22310d7e..f50bd549 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/channel/ChannelHandler.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/channel/ChannelHandler.java @@ -6,7 +6,7 @@ import java.nio.ByteBuffer; import java.nio.channels.SelectableChannel; -import com.bbn.marti.config.Network; +import com.bbn.marti.config.Input; import com.bbn.marti.nio.protocol.ChannelListener; import com.bbn.marti.nio.server.Server; import com.bbn.marti.util.concurrent.future.AsyncFuture; @@ -103,7 +103,7 @@ public interface ChannelHandler { // property queries public ChannelListener listener(); - public boolean isMatchingInput(Network.Input input); + public boolean isMatchingInput(Input input); public String identityHash(); } \ No newline at end of file diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/channel/base/AbstractBroadcastingChannelHandler.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/channel/base/AbstractBroadcastingChannelHandler.java index 48788bbd..23bb894e 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/channel/base/AbstractBroadcastingChannelHandler.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/channel/base/AbstractBroadcastingChannelHandler.java @@ -9,20 +9,18 @@ import java.nio.channels.SelectableChannel; import java.util.concurrent.atomic.AtomicLong; -import com.bbn.marti.remote.groups.ConnectionInfo; -import com.bbn.marti.util.Assertion; -import com.bbn.marti.util.MessageConversionUtil; - import org.apache.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import com.bbn.marti.config.Network; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.Input; import com.bbn.marti.nio.channel.ChannelHandler; import com.bbn.marti.nio.protocol.ChannelListener; import com.bbn.marti.nio.server.Server; import com.bbn.marti.nio.util.NetUtils; +import com.bbn.marti.remote.groups.ConnectionInfo; +import com.bbn.marti.util.Assertion; +import com.bbn.marti.util.MessageConversionUtil; import com.bbn.marti.util.concurrent.future.AsyncFuture; import com.google.common.base.Strings; @@ -52,10 +50,10 @@ public abstract class AbstractBroadcastingChannelHandler implements ChannelHandl protected ConnectionInfo connectionInfo; - protected Network.Input input; + protected Input input; private String inputId; - public Network.Input getInput() { + public Input getInput() { return input; } @@ -266,7 +264,9 @@ public ChannelHandler withInput(Input input) { throw new IllegalArgumentException("null input"); } - log.trace("set input " + input.getName() + " for channel handler " + this.getClass().getName() + " " + this + " " + this.hashCode()); + if (log.isTraceEnabled()) { + log.trace("set input " + input.getName() + " for channel handler " + this.getClass().getName() + " " + this + " " + this.hashCode()); + } this.input = input; diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/codec/impls/AnonymousAuthCodec.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/codec/impls/AnonymousAuthCodec.java index 10f849a4..7e895ae1 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/codec/impls/AnonymousAuthCodec.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/codec/impls/AnonymousAuthCodec.java @@ -13,7 +13,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.Input; import com.bbn.marti.groups.DummyAuthenticator; import com.bbn.marti.groups.GroupFederationUtil; import com.bbn.marti.groups.PeriodicUpdateCancellationException; diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/codec/impls/X509AuthCodec.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/codec/impls/X509AuthCodec.java index 009de658..9792afac 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/codec/impls/X509AuthCodec.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/codec/impls/X509AuthCodec.java @@ -7,21 +7,21 @@ import java.util.Date; import java.util.List; -import com.bbn.marti.remote.exception.TakException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.Input; import com.bbn.marti.groups.DummyAuthenticator; import com.bbn.marti.groups.GroupFederationUtil; import com.bbn.marti.groups.X509Authenticator; +import com.bbn.marti.nio.channel.ChannelHandler; import com.bbn.marti.nio.channel.base.AbstractBroadcastingChannelHandler; import com.bbn.marti.nio.codec.ByteCodec; import com.bbn.marti.nio.codec.ByteCodecFactory; import com.bbn.marti.nio.codec.Codec; import com.bbn.marti.nio.codec.PipelineContext; import com.bbn.marti.nio.util.CodecSource; -import com.bbn.marti.remote.exception.RevokedException; +import com.bbn.marti.remote.exception.TakException; import com.bbn.marti.remote.groups.AuthStatus; import com.bbn.marti.remote.groups.AuthenticatedUser; import com.bbn.marti.remote.groups.ConnectionInfo; @@ -221,6 +221,10 @@ protected synchronized void doTlsAuth() { return; } + if (cert.getNotAfter().before(new Date())) { + throw new TakException("found expired certificate : " + cert.getSubjectDN()); + } + logger.debug("cert: " + cert); if (cert.getSubjectDN() != null) { @@ -263,7 +267,10 @@ protected synchronized void doTlsAuth() { } if (e instanceof TakException) { - throw e; + logger.error("TakException in doTlsAuth {}", e.getMessage(), e); + if (connectionInfo != null && connectionInfo.getHandler() != null) { + ((ChannelHandler) connectionInfo.getHandler()).forceClose(); + } } } } diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/grpc/GrpcStreamingServer.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/grpc/GrpcStreamingServer.java index fbb2fd48..2005f8e7 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/grpc/GrpcStreamingServer.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/grpc/GrpcStreamingServer.java @@ -19,7 +19,7 @@ import com.atakmap.Tak.ClientVersion; import com.atakmap.Tak.InputChannelGrpc; import com.atakmap.Tak.ServerVersion; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.Input; import com.bbn.marti.remote.exception.TakException; import com.bbn.marti.service.Resources; import com.google.common.primitives.Longs; diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/grpc/NioGrpcChannelHandler.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/grpc/NioGrpcChannelHandler.java index 5f10891c..75180613 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/grpc/NioGrpcChannelHandler.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/grpc/NioGrpcChannelHandler.java @@ -3,10 +3,9 @@ import java.net.InetSocketAddress; import java.security.cert.X509Certificate; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.bbn.marti.config.Network.Input; +import com.bbn.cot.filter.DataFeedFilter; +import com.bbn.marti.config.DataFeed; +import com.bbn.marti.config.Input; import com.bbn.marti.nio.channel.base.AbstractBroadcastingChannelHandler; import com.bbn.marti.nio.netty.handlers.NioNettyTlsServerHandler; import com.bbn.marti.remote.groups.ConnectionInfo; @@ -21,7 +20,6 @@ /* */ public class NioGrpcChannelHandler extends NioNettyTlsServerHandler { - private final static Logger logger = LoggerFactory.getLogger(NioGrpcChannelHandler.class); private final StreamObserver stream; private final X509Certificate clientCertificate; @@ -37,7 +35,13 @@ public NioGrpcChannelHandler(Input input, StreamObserver stream, X50 public void submitTakMessage(TakMessage message) { CotEventContainer cotEventContainer = StreamingProtoBufHelper.getInstance().proto2cot(message); - protocolListeners.forEach(listener -> listener.onDataReceived(cotEventContainer, channelHandler, protocol)); + + if (isNotDOSLimited(cotEventContainer) && isNotReadLimited(cotEventContainer)) { + if (isDataFeedInput()) { + DataFeedFilter.getInstance().filter(cotEventContainer, (DataFeed) input); + } + protocolListeners.forEach(listener -> listener.onDataReceived(cotEventContainer, channelHandler, protocol)); + } } @Override diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/NioNettyBuilder.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/NioNettyBuilder.java index c0cdaef8..9b1c568b 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/NioNettyBuilder.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/NioNettyBuilder.java @@ -3,7 +3,10 @@ import java.io.Serializable; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.NetworkInterface; import java.net.UnknownHostException; +import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; @@ -12,12 +15,13 @@ import org.apache.log4j.Logger; -import com.bbn.marti.config.Filter; import com.bbn.marti.config.Federation.FederationOutgoing; import com.bbn.marti.config.Federation.FederationServer; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.Filter; +import com.bbn.marti.config.Input; import com.bbn.marti.nio.grpc.GrpcStreamingServer; import com.bbn.marti.nio.netty.handlers.NioNettyTcpStaticSubHandler; +import com.bbn.marti.nio.netty.handlers.NioNettyUdpHandler; import com.bbn.marti.nio.netty.initializers.NioNettyInitializer; import com.bbn.marti.remote.ConnectionStatus; import com.bbn.marti.remote.ConnectionStatusValue; @@ -28,6 +32,7 @@ import io.micrometer.core.lang.NonNull; import io.netty.bootstrap.Bootstrap; +import io.netty.channel.ChannelFactory; import io.netty.bootstrap.ServerBootstrap; import io.netty.buffer.PooledByteBufAllocator; import io.netty.channel.ChannelFuture; @@ -39,9 +44,13 @@ import io.netty.channel.epoll.EpollServerSocketChannel; import io.netty.channel.epoll.EpollSocketChannel; import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.DatagramChannel; +import io.netty.channel.socket.InternetProtocolFamily; +import io.netty.channel.socket.nio.NioDatagramChannel; import io.netty.channel.socket.nio.NioServerSocketChannel; import io.netty.channel.socket.nio.NioSocketChannel; import io.netty.handler.ssl.SslContext; +import io.netty.util.internal.SocketUtils; import tak.server.cot.CotEventContainer; import tak.server.federation.DistributedFederationManager; @@ -56,6 +65,7 @@ public class NioNettyBuilder implements Serializable { public static int flushThreshold; public static int maxOptimalMessagesPerMinute; private final boolean isEpoll; + private EventLoopGroup udpBossGroup; private EventLoopGroup workerGroup; private EventLoopGroup bossGroup; private final DistributedConfiguration config = DistributedConfiguration.getInstance(); @@ -90,6 +100,88 @@ public NioNettyBuilder() { maxOptimalMessagesPerMinute = baselineMaxOptimalMessagesPerMinute * NUM_AVAIL_CORES; } + public void buildUdpServer(@NonNull Input input) { + checkAndCreateEventLoopGroups(); + + new Thread(() -> { + try { + Bootstrap bootstrap = new Bootstrap(); + bootstrap.group(udpBossGroup) + .channelFactory(new ChannelFactory() { + @Override + public NioDatagramChannel newChannel() { + return new NioDatagramChannel(InternetProtocolFamily.IPv4); + } + }) + .handler(new NioNettyUdpHandler(input)) + .option(ChannelOption.AUTO_CLOSE, true) + .option(ChannelOption.SO_BROADCAST, true); + + portToNettyServer.put(input.getPort(), bootstrap.bind(input.getPort()).sync().channel().closeFuture()); + + portToInput.put(input.getPort(), input); + + log.info("Successfully Started Netty UDP Server for " + input.getName() + " on Port " + input.getPort()); + + } catch (Exception e) { + log.error("Error initializing Netty UDP Server ", e); + } + }).start(); + } + + public void buildMulticastServer(@NonNull Input input, InetAddress group, List interfs) { + checkAndCreateEventLoopGroups(); + + if (group == null) { + log.info("Not starting Multicast Server " + input.getName() + " on " + input.getPort() + " because not group was defined"); + return; + } + + if (!group.isMulticastAddress()) { + log.info("Not starting Multicast Server " + input.getName() + " on " + input.getPort() + " because the group is not a multicast address"); + return; + } + + List validInterfs = NioNettyUtils.validateMulticastInterfaces(interfs); + + if (validInterfs.size() == 0) { + log.info("Could Not Initialize Multicast Server for " + input + " on Port " + input.getPort() + ". No multicast interfaces found."); + return; + } + + + new Thread(() -> { + try { + Bootstrap bootstrap = new Bootstrap(); + bootstrap.group(udpBossGroup) + .channelFactory(new ChannelFactory() { + @Override + public NioDatagramChannel newChannel() { + return new NioDatagramChannel(InternetProtocolFamily.IPv4); + } + }) + .handler(new NioNettyUdpHandler(input)) + .option(ChannelOption.AUTO_CLOSE, true); + + InetSocketAddress groupAddress = SocketUtils.socketAddress(group.getHostAddress(), input.getPort()); + + DatagramChannel dc = (DatagramChannel) bootstrap.bind(groupAddress).sync().channel(); + + validInterfs.forEach(interf -> { + dc.joinGroup(groupAddress, interf); + }); + + portToNettyServer.put(input.getPort(), dc.closeFuture()); + portToInput.put(input.getPort(), input); + + log.info("Successfully Started Netty Multicast Server for " + input.getName() + " on Port " + input.getPort()); + + } catch (Exception e) { + log.error("Error initializing Netty Multicast Server ", e); + } + }).start(); + } + public void buildFederationServer() { startServer(NioNettyInitializer.Pipeline.initializer.federationServer(federationServer), federationServer.getPort()); } @@ -225,6 +317,9 @@ private void startClient(NioNettyInitializer initializer, InetSocketAddress remo private void checkAndCreateEventLoopGroups() { if (bossGroup == null) { bossGroup = isEpoll ? new EpollEventLoopGroup(1) : new NioEventLoopGroup(1); + } + if (udpBossGroup == null) { + udpBossGroup = new NioEventLoopGroup(1); } if (workerGroup == null) { workerGroup = isEpoll ? new EpollEventLoopGroup() : new NioEventLoopGroup(); diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/NioNettyUtils.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/NioNettyUtils.java new file mode 100644 index 00000000..014cc88b --- /dev/null +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/NioNettyUtils.java @@ -0,0 +1,69 @@ +package com.bbn.marti.nio.netty; + +import java.io.IOException; +import java.net.NetworkInterface; +import java.util.Enumeration; +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.log4j.Logger; + +import com.bbn.marti.util.Assertion; + +public class NioNettyUtils { + + private final static Logger log = Logger.getLogger(NioNettyUtils.class); + + public static List validateMulticastInterfaces(List interfs) { + List validInterfs = interfs.stream().filter(interf -> isValidMulticastInterface(interf)).collect(Collectors.toList()); + + if (validInterfs.size() == 0) { + log.warn("Defined network interfaces do not support multicast -- attempting to fetch all local interfaces that do"); + validInterfs = allMulticastInterfs(); + } + + return validInterfs; + } + + private static boolean isValidMulticastInterface(NetworkInterface interf) { + Assertion.notNull(interf); + + try { + if (interf.supportsMulticast()) { + return true; + } else { + log.warn("Given network interface claims to not support multicast -- excluding from interfaces list: " + interf); + } + } catch (Exception e) { + log.error("Error encountered trying to determine multicast properties of network interface -- excluding from multicast list: " + interf, e); + } + + return false; + } + + public static List allMulticastInterfs() { + List result = new LinkedList(); + + Enumeration interfsEnum = null; + try { + interfsEnum = NetworkInterface.getNetworkInterfaces(); + while (interfsEnum.hasMoreElements()) { + NetworkInterface finger = interfsEnum.nextElement(); + try { + if (finger.supportsMulticast()) { + result.add(finger); + log.warn("Adding interface to multicast list: " + finger.getDisplayName()); + } + } catch (IOException e) { + log.warn("Error trying to inspect multicast properties of network interface -- excluding from multicast list: " + finger, e); + } + } + } catch (IOException e) { + log.warn("Error trying to inspect network interfaces", e); + return result; + } + + return result; + } +} diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyHandlerBase.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyHandlerBase.java index 4fc4c12e..a831deb8 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyHandlerBase.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyHandlerBase.java @@ -23,7 +23,8 @@ import org.apache.log4j.Logger; import com.bbn.cot.CotParserCreator; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.DataFeed; +import com.bbn.marti.config.Input; import com.bbn.marti.groups.GroupFederationUtil; import com.bbn.marti.groups.MessagingUtilImpl; import com.bbn.marti.nio.channel.ChannelHandler; @@ -73,12 +74,12 @@ public abstract class NioNettyHandlerBase extends SimpleChannelInboundHandler protocol; protected ConnectionInfo connectionInfo; @@ -99,14 +100,22 @@ public abstract class NioNettyHandlerBase extends SimpleChannelInboundHandler cotParser = new ThreadLocal<>(); + + protected CotParser cotParser() { + if (cotParser.get() == null) { + cotParser.set(new CotParser(false)); + } + + return cotParser.get(); } @FunctionalInterface @@ -415,6 +424,8 @@ protected void processProtocolRequest(CotEventContainer protocolRequest) { } protected void createSubscription() { + if (input == null) return; + submissionService.createSubscriptionFromConnection(channelHandler, protocol); } @@ -564,5 +575,19 @@ public void channelActive(ChannelHandlerContext ctx) { } } + + protected boolean isDataFeedInput() { + if (input == null) return false; + + if (isDataFeedInput == null) { + if (input instanceof DataFeed) { + isDataFeedInput = new AtomicBoolean(true); + } else { + isDataFeedInput = new AtomicBoolean(false); + } + } + + return isDataFeedInput.get(); + } } \ No newline at end of file diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyStcpServerHandler.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyStcpServerHandler.java index 87edbb74..2e62e471 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyStcpServerHandler.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyStcpServerHandler.java @@ -3,12 +3,13 @@ import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; import org.apache.log4j.Logger; +import com.bbn.cot.filter.DataFeedFilter; import com.bbn.marti.config.AuthType; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.DataFeed; +import com.bbn.marti.config.Input; import com.bbn.marti.nio.channel.base.AbstractBroadcastingChannelHandler; import com.bbn.marti.nio.channel.connections.TcpChannelHandler; import com.bbn.marti.nio.codec.impls.AbstractAuthCodec; @@ -21,7 +22,6 @@ import com.bbn.marti.util.MessageConversionUtil; import com.google.common.base.Charsets; - import io.netty.channel.ChannelHandlerContext; import io.netty.channel.socket.SocketChannel; import tak.server.ignite.IgniteHolder; @@ -29,7 +29,6 @@ /* */ public class NioNettyStcpServerHandler extends NioNettyHandlerBase { - private final static Logger log = Logger.getLogger(NioNettyStcpServerHandler.class); private AbstractAuthCodec authCodec; private AuthType authenticationType; protected ScheduledFuture flushFuture; @@ -97,10 +96,14 @@ protected void setReader() { if(msgBuf == null || msgBuf.remaining() == 0) return; StreamingCotProtocol - .add(builder, Charsets.UTF_8.decode(msgBuf), parser, channelHandler) + .add(builder, Charsets.UTF_8.decode(msgBuf), cotParser(), channelHandler) .forEach(c -> { - if (isNotDOSLimited(c) && isNotReadLimited(c)) - protocolListeners.forEach(listener -> listener.onDataReceived(c, channelHandler, protocol)); + if (isNotDOSLimited(c) && isNotReadLimited(c)) { + if (isDataFeedInput()) { + DataFeedFilter.getInstance().filter(c, (DataFeed) input); + } + protocolListeners.forEach(listener -> listener.onDataReceived(c, channelHandler, protocol)); + } }); }; } diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyStcpStaticSubHandler.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyStcpStaticSubHandler.java index ac8ce9a8..3042f69e 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyStcpStaticSubHandler.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyStcpStaticSubHandler.java @@ -6,12 +6,8 @@ import org.apache.log4j.Logger; import com.bbn.marti.config.Filter; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.Input; import com.bbn.marti.groups.GroupFederationUtil; - -import tak.server.Constants; -import tak.server.ignite.IgniteHolder; - import com.bbn.marti.nio.channel.base.AbstractBroadcastingChannelHandler; import com.bbn.marti.nio.channel.connections.TcpChannelHandler; import com.bbn.marti.nio.netty.NioNettyBuilder; @@ -25,6 +21,8 @@ import io.micrometer.core.instrument.Metrics; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.socket.SocketChannel; +import tak.server.Constants; +import tak.server.ignite.IgniteHolder; /* diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyTcpServerHandler.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyTcpServerHandler.java index 032d905e..a0962dcf 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyTcpServerHandler.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyTcpServerHandler.java @@ -2,7 +2,7 @@ import java.net.InetSocketAddress; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.Input; import com.bbn.marti.nio.channel.base.AbstractBroadcastingChannelHandler; import io.netty.channel.ChannelHandlerContext; diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyTlsServerHandler.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyTlsServerHandler.java index 2bc2f6ec..b4890655 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyTlsServerHandler.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyTlsServerHandler.java @@ -11,8 +11,10 @@ import org.apache.log4j.Logger; import org.dom4j.DocumentException; +import com.bbn.cot.filter.DataFeedFilter; import com.bbn.marti.config.AuthType; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.DataFeed; +import com.bbn.marti.config.Input; import com.bbn.marti.groups.GroupFederationUtil; import com.bbn.marti.nio.channel.base.AbstractBroadcastingChannelHandler; import com.bbn.marti.nio.channel.connections.TcpChannelHandler; @@ -112,9 +114,7 @@ public void operationComplete(Future future) throws Exception { setNegotiator(); buildCallbacks(); setupFlushHandler(); - createSubscription(); - - + createSubscription(); } else { ctx.close(); } @@ -280,20 +280,32 @@ protected void convertAndSubmitBytesAsCot(byte[] msg) { } protected void convertAndSubmitBytesAsCot(ByteBuffer msg) { - StreamingCotProtocol.add(builder, Charsets.UTF_8.decode(msg), parser, channelHandler).forEach(c -> { - if (isNotDOSLimited(c) && isNotReadLimited(c)) { - protocolListeners.forEach(listener -> { - try { - listener.onDataReceived(c, channelHandler, protocol); - } catch (RejectedExecutionException ree) { - // count how often full queue has blocked message send - - queueFullCounter.increment(); + try { + + StreamingCotProtocol.add(builder, Charsets.UTF_8.decode(msg), cotParser(), channelHandler).forEach(c -> { + if (isNotDOSLimited(c) && isNotReadLimited(c)) { + if (isDataFeedInput()) { + DataFeedFilter.getInstance().filter(c, (DataFeed) input); } - }); + + protocolListeners.forEach(listener -> { + try { + listener.onDataReceived(c, channelHandler, protocol); + } catch (RejectedExecutionException ree) { + // count how often full queue has blocked message send + + queueFullCounter.increment(); + + } + }); + } + }); + } catch (Exception e) { + if (log.isWarnEnabled()) { + log.warn("Exception receiving message", e); } - }); + } } protected void convertAndSubmitProtoBufBytesAsCot(byte[] msg) { @@ -360,6 +372,11 @@ protected void convertAndSubmitProtoBufBytesAsCot(ByteBuffer msg) { CotEventContainer cotEventContainer = StreamingProtoBufHelper.getInstance().proto2cot(takMessage); if (isNotDOSLimited(cotEventContainer) && isNotReadLimited(cotEventContainer)) { + + if (isDataFeedInput()) { + DataFeedFilter.getInstance().filter(cotEventContainer, (DataFeed) input); + } + for (ProtocolListener listener : protocolListeners) { listener.onDataReceived(cotEventContainer, channelHandler, protocol); } diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyUdpHandler.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyUdpHandler.java new file mode 100644 index 00000000..c7954ed8 --- /dev/null +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/handlers/NioNettyUdpHandler.java @@ -0,0 +1,143 @@ +package com.bbn.marti.nio.netty.handlers; + +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.log4j.Logger; +import org.dom4j.Document; + +import com.bbn.cot.CotParserCreator; +import com.bbn.marti.config.Input; +import com.bbn.marti.groups.GroupFederationUtil; +import com.bbn.marti.groups.MessagingUtilImpl; +import com.bbn.marti.nio.channel.ChannelHandler; +import com.bbn.marti.nio.channel.connections.UdpDataChannelHandler; +import com.bbn.marti.nio.channel.connections.UdpServerChannelHandler; +import com.bbn.marti.nio.listener.ProtocolListener; +import com.bbn.marti.nio.protocol.Protocol; +import com.bbn.marti.nio.protocol.base.AbstractBroadcastingProtocol; +import com.bbn.marti.nio.protocol.connections.SingleCotProtocol; +import com.bbn.marti.nio.protocol.connections.SingleProtobufOrCotProtocol; +import com.bbn.marti.remote.groups.ConnectionInfo; +import com.bbn.marti.remote.groups.GroupManager; +import com.bbn.marti.service.DistributedConfiguration; +import com.bbn.marti.service.DistributedSubscriptionManager; +import com.bbn.marti.service.SubmissionService; +import com.bbn.marti.service.TransportCotEvent; +import com.bbn.marti.util.MessageConversionUtil; +import com.bbn.marti.util.MessagingDependencyInjectionProxy; +import com.bbn.marti.util.concurrent.future.AsyncFuture; +import com.google.common.base.Strings; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.socket.DatagramPacket; +import io.netty.handler.codec.MessageToMessageDecoder; +import io.netty.util.CharsetUtil; +import tak.server.cot.CotEventContainer; +import tak.server.cot.CotParser; +import tak.server.federation.DistributedFederationManager; +import tak.server.qos.MessageDeliveryStrategy; + +public class NioNettyUdpHandler extends MessageToMessageDecoder { + private final static Logger log = Logger.getLogger(NioNettyUdpHandler.class); + protected SubmissionService submissionService; + protected DistributedSubscriptionManager subscriptionManager; + protected DistributedFederationManager federationManager; + protected MessagingUtilImpl messagingUtil; + protected DistributedConfiguration config; + protected Protocol protocol; + protected volatile CotParser parser; + protected Input input; + protected ConcurrentLinkedQueue> protocolListeners; + protected AtomicBoolean protoSupported = new AtomicBoolean(false); + + public NioNettyUdpHandler(Input input) { + this.input = input; + this.submissionService = SubmissionService.getInstance(); + this.subscriptionManager = DistributedSubscriptionManager.getInstance(); + this.federationManager = DistributedFederationManager.getInstance(); + this.messagingUtil = MessagingUtilImpl.getInstance(); + this.config = DistributedConfiguration.getInstance(); + this.parser = CotParserCreator.newInstance(); + + TransportCotEvent transport = TransportCotEvent.findByID(input.getProtocol()); + if (transport == TransportCotEvent.COTPROTOMUDP) + this.protoSupported.set(true); + } + + @Override + protected void decode(ChannelHandlerContext ctx, DatagramPacket packet, List out) throws Exception { + try { + InetSocketAddress sender = packet.sender(); + + ConnectionInfo connectionInfo = new ConnectionInfo(); + + connectionInfo.setConnectionId(MessageConversionUtil.getConnectionId(input)); + connectionInfo.setAddress(sender.getAddress().toString()); + connectionInfo.setTls(false); + + UdpDataChannelHandler handler = (UdpDataChannelHandler) new UdpDataChannelHandler() + .withAddress(sender) + .withLocalPort(input.getPort()) + .withConnectionInfo(connectionInfo); + + handler.withInput(input); + + if (protoSupported.get()) { + CotEventContainer cot = SingleProtobufOrCotProtocol.byteBufToCot(packet.content().nioBuffer(), handler, parser); + if (cot != null) + createAdaptedNettyProtocol(handler).getProtocolListeners().forEach(listener -> listener.onDataReceived(cot, handler, protocol)); + } else { + CotEventContainer cot = SingleCotProtocol.byteBufToCot(packet.content().nioBuffer(), handler, parser); + if (cot != null) + createAdaptedNettyProtocol(handler).getProtocolListeners().forEach(listener -> listener.onDataReceived(cot, handler, protocol)); + } + } catch (Exception e) { + log.error("cot error",e); + } + } + + protected AbstractBroadcastingProtocol createAdaptedNettyProtocol(ChannelHandler channelHandler) { + AbstractBroadcastingProtocol protocol = new AbstractBroadcastingProtocol() { + @Override + public void negotiate() {} + + @Override + public void onConnect(ChannelHandler handler) {} + + @Override + public void onDataReceived(ByteBuffer buffer, ChannelHandler handler) {} + + @Override + public AsyncFuture write(CotEventContainer data, ChannelHandler handler) { + return null; + } + + @Override + public void onInboundClose(ChannelHandler handler) {} + + @Override + public void onOutboundClose(ChannelHandler handler) {} + + }; + + if (input.isArchiveOnly()) { + protocol.addProtocolListener( + SubmissionService.InputListenerAuxillaryRouter.onArchiveOnlyDataReceivedCallback + .newInstance(channelHandler, protocol)); + } + + if (!input.isArchive()) { + protocol.addProtocolListener(SubmissionService.InputListenerAuxillaryRouter.onNoArchiveDataReceivedCallback + .newInstance(channelHandler, protocol)); + } + + protocol.addProtocolListener(submissionService.onDataReceivedCallback.newInstance(channelHandler, protocol)); + + return protocol; + } +} diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/initializers/NioNettyInitializer.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/initializers/NioNettyInitializer.java index eb70e07b..319cc2b4 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/initializers/NioNettyInitializer.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/netty/initializers/NioNettyInitializer.java @@ -14,15 +14,15 @@ import com.bbn.marti.config.Federation.FederationOutgoing; import com.bbn.marti.config.Federation.FederationServer; import com.bbn.marti.config.Federation.FederationServer.V1Tls; -import com.bbn.marti.config.Network.Input; import com.bbn.marti.config.Filter; +import com.bbn.marti.config.Input; import com.bbn.marti.config.Tls; import com.bbn.marti.nio.netty.handlers.NioNettyFederationClientHandler; import com.bbn.marti.nio.netty.handlers.NioNettyFederationServerHandler; import com.bbn.marti.nio.netty.handlers.NioNettyStcpServerHandler; import com.bbn.marti.nio.netty.handlers.NioNettyStcpStaticSubHandler; -import com.bbn.marti.nio.netty.handlers.NioNettyTcpStaticSubConnectionHandler; import com.bbn.marti.nio.netty.handlers.NioNettyTcpServerHandler; +import com.bbn.marti.nio.netty.handlers.NioNettyTcpStaticSubConnectionHandler; import com.bbn.marti.nio.netty.handlers.NioNettyTlsServerHandler; import com.bbn.marti.remote.ConnectionStatus; import com.bbn.marti.remote.groups.User; diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/SingleCotProtocol.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/SingleCotProtocol.java index 7c8c8547..d4b6ca77 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/SingleCotProtocol.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/SingleCotProtocol.java @@ -10,7 +10,11 @@ import org.dom4j.DocumentException; import com.bbn.cot.CotParserCreator; +import com.bbn.cot.filter.DataFeedFilter; +import com.bbn.marti.config.DataFeed; +import com.bbn.marti.config.Input; import com.bbn.marti.nio.channel.ChannelHandler; +import com.bbn.marti.nio.channel.base.AbstractBroadcastingChannelHandler; import com.bbn.marti.nio.protocol.ProtocolInstantiator; import com.bbn.marti.nio.protocol.base.AbstractBroadcastingProtocol; import com.bbn.marti.util.concurrent.future.AsyncFuture; @@ -42,38 +46,15 @@ public class SingleCotProtocol extends AbstractBroadcastingProtocol msgs = add(builder, newData, parser, handler); + List msgs = add(messageStringBuilder, newData, parser, handler); @@ -156,12 +153,12 @@ public AsyncFuture write(CotEventContainer data, ChannelHandler handler @Override public void onInboundClose(ChannelHandler handler) { - if (this.builder != null && this.builder.length() > 0) { - log.warn("Received EOS notification with partial message (" + builder.length() + " bytes) in decode buffer for " + handler + ". Dumping data mercilessly."); + if (this.messageStringBuffer != null && this.messageStringBuffer.length() > 0) { + log.warn("Received EOS notification with partial message (" + messageStringBuffer.length() + " bytes) in decode buffer for " + handler + ". Dumping data mercilessly."); } // void out parser and builder - this.builder = null; + this.messageStringBuffer = null; this.parser = null; if (log.isTraceEnabled()) { @@ -196,7 +193,7 @@ public void onOutboundClose(ChannelHandler handler) { // notify listeners super.broadcastOutboundClose(handler); } - + /** * A static function that appends the given data to an existing * buffer, and checks to see if any new messages can be parsed with @@ -204,7 +201,7 @@ public void onOutboundClose(ChannelHandler handler) { * * Parses as many complete messages as possible out of the buffer. */ - public static List add(StringBuilder builder, CharSequence newData, CotParser parser, ChannelHandler handler) { + public static List add(StringBuffer messageStringBuffer, CharSequence newData, CotParser parser, ChannelHandler handler) { List results = new LinkedList(); @@ -214,18 +211,18 @@ public static List add(StringBuilder builder, CharSequence ne // init search pointer (0 for all other iterations, but we start searching after the end of the message) // note: subtract the end-tag length to cover end tag spanning multiple frames - int prevLen = Math.max(0, builder.length() - END_OF_COT_MSG_STR.length()); - builder.append(newData); + int prevLen = Math.max(0, messageStringBuffer.length() - END_OF_COT_MSG_STR.length()); + messageStringBuffer.append(newData); int indexOfEnd = -1; - while ((indexOfEnd = builder.indexOf(END_OF_COT_MSG_STR, prevLen)) >= 0) { + while ((indexOfEnd = messageStringBuffer.indexOf(END_OF_COT_MSG_STR, prevLen)) >= 0) { // skip anything received ahead of the first cot event (such as messages sent to anonymous ports) - int openIndex = builder.indexOf(START_OF_COT_MSG_STR, 0); + int openIndex = messageStringBuffer.indexOf(START_OF_COT_MSG_STR, 0); // have some end tag to seek to int closeIndex = indexOfEnd + END_OF_COT_MSG_STR.length(); - final String msg = builder.substring(openIndex, closeIndex); + final String msg = messageStringBuffer.substring(openIndex, closeIndex); if (msg.length() <= INDIVIDUAL_COT_MSG_SIZE_LIMIT) { // have a message window to parse, within size limits @@ -246,25 +243,25 @@ public static List add(StringBuilder builder, CharSequence ne } else { // message closure is too long, even if we could parse it -- chuck out if (log.isTraceEnabled()) { - log.trace("Error parsing CoT message: message too long to parse: " + builder.substring(0, closeIndex)); + log.trace("Error parsing CoT message: message too long to parse: " + messageStringBuffer.substring(0, closeIndex)); } } // TODO: change so that we don't delete parsed data until the end of our parse attempt... // string builder implementation will move all of the contained data to the zero index - builder.delete(0, closeIndex); + messageStringBuffer.delete(0, closeIndex); // END OF WHILE -- have cleared or parsed past the end tag that we found prevLen = 0; // reset search finger } - if (builder.length() > INDIVIDUAL_COT_MSG_SIZE_LIMIT) { - builder.delete(0, builder.length()); + if (messageStringBuffer.length() > INDIVIDUAL_COT_MSG_SIZE_LIMIT) { + messageStringBuffer.delete(0, messageStringBuffer.length()); } - if (builder.capacity() > BUFFER_TRIM_SIZE) { + if (messageStringBuffer.capacity() > BUFFER_TRIM_SIZE) { // trim buffer size down if we've grown beyond a meg -- don't want to hold on to a giant buffer if we only occasionally get a large message - builder.trimToSize(); + messageStringBuffer.trimToSize(); } return results; diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/StreamingProtoBufProtocol.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/StreamingProtoBufProtocol.java index cad8284d..36d4deb1 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/StreamingProtoBufProtocol.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/protocol/connections/StreamingProtoBufProtocol.java @@ -158,11 +158,14 @@ private static TakMessage createFileTransferRequest(CotEventContainer data) { try { // set the download url Network network = DistributedConfiguration.getInstance().getNetwork(); - if (network == null || network.getTakServerHost() == null || network.getTakServerHost().length() == 0) { - log.error("createFileTransferRequest failed, need to set takServerHost in CoreConfig "); + if (network == null || network.getTakServerHost() == null || network.getTakServerHost().length() == 0 + || network.getConnector() == null || network.getConnector().size() == 0) { + log.error("createFileTransferRequest failed, need to set takServerHost and connector in CoreConfig "); return null; } - String url = "https://" + network.getTakServerHost() + ":8443/Marti/api/cot/xml/" + data.getUid(); + + Network.Connector connector = network.getConnector().get(0); + String url = "https://" + network.getTakServerHost() + ":" + connector.getPort() + "/Marti/api/cot/xml/" + data.getUid(); // compute the hash of the cot that will be downloaded String dataXml = Constants.XML_HEADER + data.toCotElement().toCotXml(); diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/server/NioServer.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/server/NioServer.java index 3d4f2eba..d63007b7 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/server/NioServer.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/server/NioServer.java @@ -17,7 +17,7 @@ import org.apache.log4j.Logger; import org.jetbrains.annotations.NotNull; -import com.bbn.marti.config.Network; +import com.bbn.marti.config.Input; import com.bbn.marti.nio.binder.ServerBinder; import com.bbn.marti.nio.channel.ChannelHandler; import com.bbn.marti.nio.channel.base.AbstractBroadcastingChannelHandler; @@ -241,7 +241,7 @@ private void openSelector() throws IOException { * @param input Input object * @throws IOException */ - public void bind(@NotNull ServerBinder binder, @NotNull Network.Input input) throws IOException { + public void bind(@NotNull ServerBinder binder, @NotNull Input input) throws IOException { synchronized (wrapperMap) { try { ChannelWrapper wrapper = binder.handleBind(this); diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/server/Server.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/server/Server.java index 09b015cb..10eafeee 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/server/Server.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/server/Server.java @@ -6,7 +6,7 @@ import java.nio.channels.SelectableChannel; import java.util.EnumSet; -import com.bbn.marti.config.Network; +import com.bbn.marti.config.Input; import com.bbn.marti.nio.binder.ServerBinder; import com.bbn.marti.nio.channel.ChannelHandler; import com.bbn.marti.nio.util.IOEvent; @@ -28,7 +28,7 @@ * */ public interface Server { - public void bind(ServerBinder binder, Network.Input input) throws IOException; + public void bind(ServerBinder binder, Input input) throws IOException; // called to start the server listening public void listen(); diff --git a/src/takserver-core/src/main/java/com/bbn/marti/nio/websockets/NioWebSocketHandler.java b/src/takserver-core/src/main/java/com/bbn/marti/nio/websockets/NioWebSocketHandler.java index 02208d4b..f954f448 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/nio/websockets/NioWebSocketHandler.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/nio/websockets/NioWebSocketHandler.java @@ -10,7 +10,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.Input; import com.bbn.marti.groups.GroupFederationUtil; import com.bbn.marti.nio.channel.base.AbstractBroadcastingChannelHandler; import com.bbn.marti.nio.netty.handlers.NioNettyTlsServerHandler; diff --git a/src/takserver-core/src/main/java/com/bbn/marti/service/BrokerService.java b/src/takserver-core/src/main/java/com/bbn/marti/service/BrokerService.java index 697d8d87..5148c706 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/service/BrokerService.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/service/BrokerService.java @@ -12,6 +12,7 @@ import org.springframework.beans.factory.annotation.Autowired; import com.bbn.cot.filter.StreamingEndpointRewriteFilter; +import com.bbn.cot.filter.VBMSASharingFilter; import com.bbn.marti.nio.protocol.connections.StreamingProtoBufProtocol; import com.bbn.marti.remote.QueueMetric; import com.bbn.marti.remote.groups.ConnectionInfo; @@ -64,11 +65,10 @@ public boolean addToInputQueue(final CotEventContainer c) { executorService.execute(() -> { try { - streamendpointFilter.filter(c); Collection hits = subMgr.getMatches(c); - + c.setContextValue(Constants.SUBSCRIBER_HITS_KEY, subscriptionStore.subscriptionCollectionToConnectionIdSet(hits)); inputQueue.add(c); @@ -147,21 +147,30 @@ protected void processNextEvent() { } public void processMessage(CotEventContainer cot) { - try { + if(VBMSASharingFilter.getInstance().filter(cot) == null) return; + long hitTime = System.currentTimeMillis(); @SuppressWarnings("unchecked") Set hits = (Set) cot.getContextValue(Constants.SUBSCRIBER_HITS_KEY); Set websocketHits = new ConcurrentSkipListSet<>(); + String senderConnectionId = null; + if (cot.getContextValue(Constants.CONNECTION_ID_KEY) != null) { + senderConnectionId = (String) cot.getContextValue(Constants.CONNECTION_ID_KEY); + } // pre-convert to protobuf cot.setProtoBufBytes(StreamingProtoBufProtocol.convertCotToProtoBufBytes(cot)); for (String connectionId : hits) { - + // if the message was injected by a plugin, the list of hits may contain the original sender. + // we can filter them out by tracking the connection id the message originally came from + if (cot.getContextValue(Constants.PLUGIN_PROVENANCE) != null && senderConnectionId != null + && senderConnectionId.equals(connectionId)) continue; + Subscription subscription = null; try { diff --git a/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedConfiguration.java b/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedConfiguration.java index 04369a27..737407b7 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedConfiguration.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedConfiguration.java @@ -32,12 +32,14 @@ import com.bbn.marti.config.CertificateSigning; import com.bbn.marti.config.Configuration; import com.bbn.marti.config.ContactApi; +import com.bbn.marti.config.DataFeed; import com.bbn.marti.config.Dissemination; import com.bbn.marti.config.Federation; import com.bbn.marti.config.Federation.FederationServer; import com.bbn.marti.config.Ferry; import com.bbn.marti.config.Filter; import com.bbn.marti.config.Geocache; +import com.bbn.marti.config.Input; import com.bbn.marti.config.Network; import com.bbn.marti.config.Qos; import com.bbn.marti.config.Repeater; @@ -46,6 +48,7 @@ import com.bbn.marti.config.Submission; import com.bbn.marti.config.Subscription; import com.bbn.marti.config.Tls; +import com.bbn.marti.config.Vbm; import com.bbn.marti.remote.CoreConfig; import com.bbn.marti.remote.groups.ConnectionModifyResult; import com.bbn.marti.remote.groups.NetworkInputAddResult; @@ -148,18 +151,37 @@ public static DistributedConfiguration getInstance() { public synchronized void removeInputAndSave(String name) throws RemoteException { @SuppressWarnings({ "unchecked", "rawtypes" }) - List inputList = new ArrayList(getRemoteConfiguration().getNetwork().getInput()); - for (Network.Input input : inputList) { + List inputList = new ArrayList(getRemoteConfiguration().getNetwork().getInput()); + for (Input input : inputList) { if (input.getName().equals(name)) { getRemoteConfiguration().getNetwork().getInput().remove(input); saveChangesAndUpdateCache(); } } + @SuppressWarnings({ "unchecked", "rawtypes" }) + List dataFeedList = new ArrayList(getRemoteConfiguration().getNetwork().getDatafeed()); + for (DataFeed dataFeed: dataFeedList) { + if (dataFeed.getName().equals(name)) { + getRemoteConfiguration().getNetwork().getDatafeed().remove(dataFeed); + saveChangesAndUpdateCache(); + } + } + } + + public synchronized void removeDataFeedAndSave(String name) throws RemoteException { + @SuppressWarnings({ "unchecked", "rawtypes" }) + List DataFeed = new ArrayList(getRemoteConfiguration().getNetwork().getDatafeed()); + for (DataFeed dataFeed : DataFeed) { + if (dataFeed.getName().equals(name)) { + getRemoteConfiguration().getNetwork().getDatafeed().remove(dataFeed); + saveChangesAndUpdateCache(); + } + } } @SuppressWarnings("rawtypes") public synchronized ConnectionModifyResult updateInputGroupsNoSave(String inputName, String[] groupList) throws RemoteException { - Network.Input input = getInputByName(inputName); + Input input = getInputByName(inputName); if (input == null) { return ConnectionModifyResult.FAIL_NONEXISTENT; } @@ -185,7 +207,7 @@ public synchronized ConnectionModifyResult updateInputGroupsNoSave(String inputN } public ConnectionModifyResult setArchiveFlagNoSave(String inputName, boolean desiredState) { - Network.Input input = getInputByName(inputName); + Input input = getInputByName(inputName); if (input == null) { return ConnectionModifyResult.FAIL_NONEXISTENT; } @@ -194,7 +216,7 @@ public ConnectionModifyResult setArchiveFlagNoSave(String inputName, boolean des } public ConnectionModifyResult setArchiveOnlyFlagNoSave(String inputName, boolean desiredState) { - Network.Input input = getInputByName(inputName); + Input input = getInputByName(inputName); if (input == null) { return ConnectionModifyResult.FAIL_NONEXISTENT; } @@ -202,6 +224,47 @@ public ConnectionModifyResult setArchiveOnlyFlagNoSave(String inputName, boolean return ConnectionModifyResult.SUCCESS; } + @SuppressWarnings("rawtypes") + public synchronized ConnectionModifyResult updateTagsNoSave(String inputName, List newTagList) throws RemoteException { + Input input = getInputByName(inputName); + if (input == null) { + return ConnectionModifyResult.FAIL_NONEXISTENT; + } + if (!(input instanceof DataFeed)) { + return ConnectionModifyResult.FAIL_NOMOD_DFEED; + } + DataFeed dataFeed = (DataFeed) input; + + @SuppressWarnings("unchecked") + List currentTagList = new ArrayList(dataFeed.getTag()); + + for (String newTagName : newTagList) { + if (!currentTagList.contains(newTagName)) { + dataFeed.getTag().add(newTagName); + } + } + + for (String existingTag : currentTagList) { + if (!newTagList.contains(existingTag)) { + dataFeed.getTag().remove(existingTag); + } + } + return ConnectionModifyResult.SUCCESS; + } + + public ConnectionModifyResult setSyncFlagNoSave(String dataFeedName, boolean desiredState) { + Input input = getInputByName(dataFeedName); + if (input == null) { + return ConnectionModifyResult.FAIL_NONEXISTENT; + } + if (!(input instanceof DataFeed)) { + return ConnectionModifyResult.FAIL_NOMOD_DFEED; + } + DataFeed dataFeed = (DataFeed) input; + dataFeed.setSync(desiredState); + return ConnectionModifyResult.SUCCESS; + } + @Override public void saveChanges() { // if TAK Server processes are local to each other and we're not on the messaging process, don't save @@ -239,16 +302,26 @@ public void saveChangesAndUpdateCache() { saveChanges(); } - public synchronized NetworkInputAddResult addInputAndSave(@NotNull Network.Input input) { - for (Network.Input loopInput : getRemoteConfiguration().getNetwork().getInput()) { - if (loopInput.getName().equals(input.getName())) { - return NetworkInputAddResult.FAIL_INPUT_NAME_EXISTS; + public synchronized NetworkInputAddResult addInputAndSave(@NotNull Input input) { + if (input instanceof DataFeed) { + for (Input feedLoop : getRemoteConfiguration().getNetwork().getDatafeed()) { + if (((DataFeed) feedLoop).getName().equals(((DataFeed) input).getName())) { + return NetworkInputAddResult.FAIL_INPUT_NAME_EXISTS; + } + if (feedLoop.getName().equals(input.getName())) { + return NetworkInputAddResult.FAIL_INPUT_NAME_EXISTS; + } + } + getRemoteConfiguration().getNetwork().getDatafeed().add((DataFeed) input); + } else { + for (Input loopInput : getRemoteConfiguration().getNetwork().getInput()) { + if (loopInput.getName().equals(input.getName())) { + return NetworkInputAddResult.FAIL_INPUT_NAME_EXISTS; + } } + getRemoteConfiguration().getNetwork().getInput().add(input); } - // add the new input to the local configuration object, not the object from the cache - getRemoteConfiguration().getNetwork().getInput().add(input); - saveChangesAndUpdateCache(); if (logger.isDebugEnabled()) { @@ -259,13 +332,19 @@ public synchronized NetworkInputAddResult addInputAndSave(@NotNull Network.Input } - public synchronized Network.Input getInputByName(@NotNull String inputName) { - List inputList = getRemoteConfiguration().getNetwork().getInput(); - for (Network.Input input : inputList) { + public synchronized Input getInputByName(@NotNull String inputName) { + List inputList = getRemoteConfiguration().getNetwork().getInput(); + for (Input input : inputList) { if (input.getName().equals(inputName)) { return input; } } + List dataFeedList = getRemoteConfiguration().getNetwork().getDatafeed(); + for (Input input : dataFeedList) { + if (input.getName().equals(inputName)) { + return input; + } + } return null; } @@ -280,7 +359,7 @@ private synchronized Subscription.Static getStaticSubscriptionByName(@NotNull St } public synchronized void setInputArchiveFlagAndSave(@NotNull String inputName, boolean desiredState) { - Network.Input input = getInputByName(inputName); + Input input = getInputByName(inputName); if (input != null && input.isArchive() != desiredState) { input.setArchive(desiredState); saveChangesAndUpdateCache(); @@ -301,10 +380,14 @@ public synchronized void removeStaticSubscriptionAndSave(@NotNull String subscri } @NotNull - public synchronized List getNetworkInputs() { + public synchronized List getNetworkInputs() { return getRemoteConfiguration().getNetwork().getInput(); } + @NotNull + public synchronized List getNetworkDataFeeds() { + return getRemoteConfiguration().getNetwork().getDatafeed(); + } public Network getNetwork() { return getRemoteConfiguration().getNetwork(); @@ -368,7 +451,11 @@ public Async getAsync() { } public Geocache getGeocache() { - return getRemoteConfiguration().getGeocache(); + return getRemoteConfiguration().getGeocache(); + } + + public Vbm getVbm() { + return getRemoteConfiguration().getVbm(); } @Override diff --git a/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedRetentionQueryManager.java b/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedRetentionQueryManager.java index fdbe5d8f..37c75b94 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedRetentionQueryManager.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedRetentionQueryManager.java @@ -179,8 +179,8 @@ public boolean restoreMission(Map files, Map pro } missionService().createMission(properties.get("mission_name"), properties.get("creatorUid"), groupVector, - properties.get("description"), properties.get("chatroom"), properties.get("tool"), - properties.get("password_hash"), missionRole, expiration); + properties.get("description"), properties.get("chatroom"), properties.get("baseLayer"), + properties.get("bbox"), properties.get("path"), properties.get("classification"), properties.get("tool"), properties.get("password_hash"), missionRole, expiration, properties.get("bounding_polygon")); return true; } diff --git a/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedSubscriptionManager.java b/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedSubscriptionManager.java index b00aaae9..f1525ca6 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedSubscriptionManager.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/service/DistributedSubscriptionManager.java @@ -495,8 +495,9 @@ public boolean doExplicitBrokering(CotEventContainer c) { List callsignList = (List) c.getContextValue(StreamingEndpointRewriteFilter.EXPLICIT_CALLSIGN_KEY); List publishList = (List) c.getContextValue(StreamingEndpointRewriteFilter.EXPLICIT_PUBLISH_KEY); List uidList = (List) c.getContextValue(StreamingEndpointRewriteFilter.EXPLICIT_UID_KEY); + List feedUidList = (List) c.getContextValue(StreamingEndpointRewriteFilter.EXPLICIT_FEED_UID_KEY); Set missionSet = (Set) c.getContextValue(StreamingEndpointRewriteFilter.EXPLICIT_MISSION_KEY); - + if (missionSet != null && !missionSet.isEmpty()) { if (logger.isDebugEnabled()) { logger.debug("invalidate mission cache"); @@ -510,7 +511,8 @@ public boolean doExplicitBrokering(CotEventContainer c) { return ((callsignList != null && !callsignList.isEmpty()) || (publishList != null && !publishList.isEmpty())) || (uidList != null && !uidList.isEmpty()) || - (missionSet != null && !missionSet.isEmpty()); + (missionSet != null && !missionSet.isEmpty()) || + feedUidList != null; } /** @@ -544,6 +546,10 @@ private Collection getExplicitMatches(CotEventContainer c) { matches.addAll(getExplicitUidMatches(c, destList)); } + if ((destList = (List) c.getContextValue(StreamingEndpointRewriteFilter.EXPLICIT_FEED_UID_KEY)) != null) { + matches.addAll(getExplicitUidMatches(c, destList)); + } + try { if (c.getContextValue(StreamingEndpointRewriteFilter.EXPLICIT_MISSION_KEY) != null) { matches.addAll(getFederatedMissionMatches(c)); @@ -1579,6 +1585,7 @@ synchronized public Subscription setUserForSubscription(User user, Subscription private static final String detailXPath = "/event/detail"; private static final String linkXPath = detailXPath + "/link"; private static final String missionXPath = detailXPath + "/mission"; + private static final String missionContentXPath = detailXPath + "/mission/MissionChanges/MissionChange/content"; private static Document deleteMessageSeed = null; @@ -1647,9 +1654,20 @@ private static void addMissionXml(Document document, String xml) { } } + private static void addMissionChangeContentXml(Document document, String xml) { + try { + SAXReader reader = new SAXReader(); + Document xmlDoc = reader.read(new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))); + Element missionElem = DocumentHelper.makeElement(document, missionContentXPath); + missionElem.add(xmlDoc.getRootElement()); + } catch (DocumentException ex) { + logger.error("Exception attaching xml to mission change!", ex); + } + } + private static CotEventContainer createMissionMessage( String missionName, String cotType, String msgType, String authorUid, String tool, String changes, - String uid, String token, String roleXml) { + String uid, String token, String roleXml, String xmlContentForNotification) { // make mission change message with the given mission name Document mcMessage = (Document) missionChangeMessageSeed.clone(); @@ -1689,16 +1707,20 @@ private static CotEventContainer createMissionMessage( if (changes != null) { addMissionXml(mcMessage, changes); + if (xmlContentForNotification != null) { + addMissionChangeContentXml(mcMessage, xmlContentForNotification); + } } if (roleXml != null) { addMissionXml(mcMessage, roleXml); } + return new CotEventContainer(mcMessage); } - public CotEventContainer createMissionChangeMessage(String missionName, ChangeType changeType, String authorUid, String tool, String changes) { + public CotEventContainer createMissionChangeMessage(String missionName, ChangeType changeType, String authorUid, String tool, String changes, String xmlContentForNotification) { String cotType; switch (changeType) { @@ -1712,23 +1734,23 @@ public CotEventContainer createMissionChangeMessage(String missionName, ChangeTy case CONTENT: { cotType = "t-x-m-c"; break; } } - return createMissionMessage(missionName, cotType, "CHANGE", authorUid, tool, changes, null, null, null); + return createMissionMessage(missionName, cotType, "CHANGE", authorUid, tool, changes, null, null, null, xmlContentForNotification); } public CotEventContainer createMissionCreateMessage(String missionName, String authorUid, String tool) { - return createMissionMessage(missionName, "t-x-m-n", "CREATE", authorUid, tool, null, null, null, null); + return createMissionMessage(missionName, "t-x-m-n", "CREATE", authorUid, tool, null, null, null, null, null); } public CotEventContainer createMissionDeleteMessage(String missionName, String authorUid, String tool) { - return createMissionMessage(missionName, "t-x-m-d", "DELETE", authorUid, tool, null, null, null, null); + return createMissionMessage(missionName, "t-x-m-d", "DELETE", authorUid, tool, null, null, null, null, null); } public CotEventContainer createMissionInviteMessage(String missionName, String authorUid, String tool, String token, String roleXml) { - return createMissionMessage(missionName, "t-x-m-i", "INVITE", authorUid, tool,null, null, token, roleXml); + return createMissionMessage(missionName, "t-x-m-i", "INVITE", authorUid, tool,null, null, token, roleXml, null); } public CotEventContainer createMissionRoleChangeMessage(String missionName, String authorUid, String tool, String roleXml) { - return createMissionMessage(missionName, "t-x-m-r", "INVITE", authorUid, tool,null, null, null, roleXml); + return createMissionMessage(missionName, "t-x-m-r", "INVITE", authorUid, tool,null, null, null, roleXml, null); } @Override @@ -1754,7 +1776,7 @@ public void missionDisconnect(String missionName, String clientUid) { } @Override - public void missionUnsubscribe(String missionName, String clientUid) { + public void missionUnsubscribe(String missionName, String clientUid, String username, boolean disconnectOnly) { if (logger.isDebugEnabled()) { logger.debug("unsubscribe from mission " + missionName + " for uid " + clientUid); @@ -1762,8 +1784,16 @@ public void missionUnsubscribe(String missionName, String clientUid) { missionDisconnect(missionName, clientUid); + if (disconnectOnly) { + return; + } + if (config().getRepository().isEnable()) { - missionSubscriptionRepository().deleteByMissionNameAndClientUid(missionName, clientUid); + if (username != null) { + missionSubscriptionRepository().deleteByMissionNameAndClientUidAndUsername(missionName, clientUid, username); + } else { + missionSubscriptionRepository().deleteByMissionNameAndClientUid(missionName, clientUid); + } } } @@ -1775,7 +1805,7 @@ public void removeAllMissionSubscriptions(String missionName) { } for (String uid : subscriptionStore().getUidsByMission(missionName)) { - missionUnsubscribe(missionName, uid); + missionUnsubscribe(missionName, uid, null, false); } } @@ -1804,18 +1834,22 @@ public void announceMissionChange(String missionName, String creatorUid, String private AtomicInteger changeCount = new AtomicInteger(); private AtomicInteger changeHitCount = new AtomicInteger(); - + @Override - public void announceMissionChange(String missionName, SubscriptionManagerLite.ChangeType changeType, String creatorUid, String tool, String changes) { + public void announceMissionChange(String missionName, ChangeType changeType, String creatorUid, String tool, String changes) { + announceMissionChange(missionName, changeType, creatorUid, tool, changes, null); + } - CotEventContainer changeMessage = createMissionChangeMessage(missionName, changeType, creatorUid, tool, changes); - - if (DistributedConfiguration.getInstance().getRemoteConfiguration().getCluster().isEnabled()) { - MessagingDependencyInjectionProxy.getInstance().clusterManager().onAnnounceMissionChangeMessage(changeMessage, missionName); - } else { - submitAnnounceMissionChangeCot(missionName, changeMessage); - } - } + @Override + public void announceMissionChange(String missionName, ChangeType changeType, String creatorUid, String tool, String changes, String xmlContentForNotification) { + CotEventContainer changeMessage = createMissionChangeMessage(missionName, changeType, creatorUid, tool, changes, xmlContentForNotification); + + if (DistributedConfiguration.getInstance().getRemoteConfiguration().getCluster().isEnabled()) { + MessagingDependencyInjectionProxy.getInstance().clusterManager().onAnnounceMissionChangeMessage(changeMessage, missionName); + } else { + submitAnnounceMissionChangeCot(missionName, changeMessage); + } + } public void submitAnnounceMissionChangeCot(String missionName, CotEventContainer changeMessage) { if (changeLogger.isDebugEnabled()) { @@ -1894,7 +1928,7 @@ public void broadcastMissionAnnouncement( message = createMissionDeleteMessage(missionName, creatorUid, tool); } else if (changeType == ChangeType.KEYWORD || changeType == ChangeType.METADATA) { message = createMissionChangeMessage( - missionName, changeType, creatorUid, tool, null); + missionName, changeType, creatorUid, tool, null, null); } else { logger.error("attempt to broadcast unsupported change type: " + changeType); return; diff --git a/src/takserver-core/src/main/java/com/bbn/marti/service/LocalConfiguration.java b/src/takserver-core/src/main/java/com/bbn/marti/service/LocalConfiguration.java index bdd56d1b..6fc2a4a7 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/service/LocalConfiguration.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/service/LocalConfiguration.java @@ -54,6 +54,7 @@ import com.bbn.marti.config.Thumbnail; import com.bbn.marti.config.Tls; import com.bbn.marti.config.Urladd; +import com.bbn.marti.config.Vbm; import com.bbn.marti.remote.exception.NotFoundException; import com.bbn.marti.remote.exception.TakException; import com.bbn.marti.util.MessageConversionUtil; @@ -486,6 +487,10 @@ public boolean setDefaults() { configuration.setCluster(new Cluster()); changed = true; } + + if (configuration.getVbm() == null) { + configuration.setVbm(new Vbm()); + } return changed; } diff --git a/src/takserver-core/src/main/java/com/bbn/marti/service/PluginStore.java b/src/takserver-core/src/main/java/com/bbn/marti/service/PluginStore.java new file mode 100644 index 00000000..cc9ef053 --- /dev/null +++ b/src/takserver-core/src/main/java/com/bbn/marti/service/PluginStore.java @@ -0,0 +1,40 @@ +package com.bbn.marti.service; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.bbn.marti.util.spring.SpringContextBeanForApi; + +public class PluginStore { + private static final Logger logger = LoggerFactory.getLogger(SubscriptionStore.class); + + private final AtomicInteger interceptorsActive = new AtomicInteger(0); + + private static PluginStore instance; + + public static synchronized PluginStore getInstance() { + if (instance == null) { + synchronized (SubscriptionStore.class) { + if (instance == null) { + instance = SpringContextBeanForApi.getSpringContext().getBean(PluginStore.class); + } + } + } + + return instance; + } + + public void disableInterception() { + interceptorsActive.getAndSet(0); + } + + public void addInterceptorPluginsActive(int n) { + interceptorsActive.getAndAdd(n); + } + + public int getInterceptorPluginsActive() { + return interceptorsActive.get(); + } +} diff --git a/src/takserver-core/src/main/java/com/bbn/marti/service/RepositoryService.java b/src/takserver-core/src/main/java/com/bbn/marti/service/RepositoryService.java index 74f74594..c2f7db19 100755 --- a/src/takserver-core/src/main/java/com/bbn/marti/service/RepositoryService.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/service/RepositoryService.java @@ -9,7 +9,17 @@ import java.sql.SQLException; import java.sql.Timestamp; import java.text.SimpleDateFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.NavigableSet; +import java.util.SimpleTimeZone; +import java.util.TimeZone; +import java.util.UUID; import java.util.concurrent.RejectedExecutionException; import javax.annotation.PostConstruct; @@ -29,6 +39,7 @@ import com.bbn.cot.filter.Filter; import com.bbn.cot.filter.ImageFormattingFilter; +import com.bbn.marti.config.DataFeed; import com.bbn.marti.config.Repository; import com.bbn.marti.remote.ConnectionEventTypeValue; import com.bbn.marti.remote.ImagePref; @@ -128,9 +139,15 @@ public boolean addToInputQueue(CotEventContainer c) { } } + // This is the case where c is a message from a plugin and the plugin archive is disabled if (c.getContextValue(Constants.ARCHIVE_EVENT_KEY) != null && !((Boolean) c.getContextValue(Constants.ARCHIVE_EVENT_KEY)).booleanValue()) { return false; } + + // This is the case where c is a datafeed message, and the datafeed archive is disabled + if (c.getContextValue(Constants.DATA_FEED_KEY) != null && ((DataFeed) c.getContextValue(Constants.DATA_FEED_KEY)).isArchive() == false) { + return false; + } } catch (Exception e) { log.debug("exception checking archive flag in message", e); } @@ -233,7 +250,7 @@ protected void processNextEvent() { public void insertBatchCotData(List events) { try (Connection connection = dataSource.getConnection()) { - + LinkedList dataFeedEvents = new LinkedList<>(); try (PreparedStatement cotRouterInsert = connection.prepareStatement("INSERT INTO " + cotRouterTableName + " (uid, event_pt, cot_type, " @@ -242,13 +259,12 @@ public void insertBatchCotData(List events) { + "how, point_hae, point_ce, point_le, groups, " + "id, servertime) VALUES " + "(?,ST_GeometryFromText(?, 4326),?,?,?,?,?,?,?,?,?,?,?,?,(?)::bit(" + RemoteUtil.GROUPS_BIT_VECTOR_LEN + "), nextval('cot_router_seq'),?) ")) { - + // formats each cot message as we iterate over it Iterable imageFormattedEvents = Iterables.filter(events, imageFilter); LinkedList toRemove = new LinkedList<>(); for (CotEventContainer event : imageFormattedEvents) { try { - boolean[] groupsBitVector = new boolean[RemoteUtil.GROUPS_BIT_VECTOR_LEN]; if (event.getContextValue(Constants.GROUPS_KEY) != null) { @@ -288,42 +304,16 @@ public void insertBatchCotData(List events) { } continue; } - - - cotRouterInsert.setString(1, event.getUid()); - cotRouterInsert.setString(2, "POINT(" + event.getLon() + " " - + event.getLat() + ")"); - cotRouterInsert.setString(3, event.getType()); - - cotRouterInsert.setTimestamp(4, new Timestamp(DatatypeConverter - .parseDateTime(event.getStart()).getTimeInMillis()), utcCalendar); - cotRouterInsert.setTimestamp(5, new Timestamp(DatatypeConverter - .parseDateTime(event.getTime()).getTimeInMillis()), utcCalendar); - cotRouterInsert.setTimestamp(6, new Timestamp(DatatypeConverter - .parseDateTime(event.getStale()).getTimeInMillis()), utcCalendar); - - cotRouterInsert.setString(7, event.getDetailXml()); - cotRouterInsert.setString(8, event.getAccess()); - cotRouterInsert.setString(9, event.getQos()); - cotRouterInsert.setString(10, event.getOpex()); - cotRouterInsert.setString(11, event.getHow()); - cotRouterInsert.setDouble(12, event.getHae()); - cotRouterInsert.setDouble(13, event.getCe()); - cotRouterInsert.setDouble(14, event.getLe()); - - cotRouterInsert.setString(15, RemoteUtil.getInstance().bitVectorToString(groupsBitVector)); - - // - // check to see if this event has serverTime (set by SubmissionService.processNextEvent) - // - String serverTime; - if (event.hasServerTime()) { - serverTime = event.getTime(); - } else { - serverTime = DateUtil.toCotTime(new Date().getTime()); + + event.setContext(Constants.GROUPS_BIT_VECTOR_KEY, groupsBitVector); + + // CoT is valid, but came from a data feed. add it to the list and let {#archiveBatchDataFeedCot()} handle it + if (event.getContextValue(Constants.DATA_FEED_KEY) != null) { + dataFeedEvents.add(event); + continue; } - cotRouterInsert.setTimestamp(16, new Timestamp(DatatypeConverter - .parseDateTime(serverTime).getTimeInMillis()), utcCalendar); + + setCotQueryParams(cotRouterInsert, event); cotRouterInsert.addBatch(); } catch (Exception e) { @@ -343,6 +333,8 @@ public void insertBatchCotData(List events) { log.debug("exception executing CoT insert batch ", e); } } + + archiveBatchDataFeedCot(dataFeedEvents, connection); } catch (SQLException eee) { if (log.isWarnEnabled()) { @@ -350,6 +342,80 @@ public void insertBatchCotData(List events) { } } } + + private void setCotQueryParams(PreparedStatement dataFeedInsert, CotEventContainer event) throws SQLException { + dataFeedInsert.setString(1, event.getUid()); + dataFeedInsert.setString(2, "POINT(" + event.getLon() + " " + + event.getLat() + ")"); + dataFeedInsert.setString(3, event.getType()); + + dataFeedInsert.setTimestamp(4, new Timestamp(DatatypeConverter + .parseDateTime(event.getStart()).getTimeInMillis()), utcCalendar); + dataFeedInsert.setTimestamp(5, new Timestamp(DatatypeConverter + .parseDateTime(event.getTime()).getTimeInMillis()), utcCalendar); + dataFeedInsert.setTimestamp(6, new Timestamp(DatatypeConverter + .parseDateTime(event.getStale()).getTimeInMillis()), utcCalendar); + + dataFeedInsert.setString(7, event.getDetailXml()); + dataFeedInsert.setString(8, event.getAccess()); + dataFeedInsert.setString(9, event.getQos()); + dataFeedInsert.setString(10, event.getOpex()); + dataFeedInsert.setString(11, event.getHow()); + dataFeedInsert.setDouble(12, event.getHae()); + dataFeedInsert.setDouble(13, event.getCe()); + dataFeedInsert.setDouble(14, event.getLe()); + + dataFeedInsert.setString(15, RemoteUtil.getInstance().bitVectorToString((boolean[]) event.getContext(Constants.GROUPS_BIT_VECTOR_KEY))); + + // + // check to see if this event has serverTime (set by SubmissionService.processNextEvent) + // + String serverTime; + if (event.hasServerTime()) { + serverTime = event.getTime(); + } else { + serverTime = DateUtil.toCotTime(new Date().getTime()); + } + dataFeedInsert.setTimestamp(16, new Timestamp(DatatypeConverter + .parseDateTime(serverTime).getTimeInMillis()), utcCalendar); + } + + // link the Cot UID to the data feed it came from + private void archiveBatchDataFeedCot(List events, Connection connection) { + + if (events.size() != 0) { + try (PreparedStatement dataFeedInsert = connection.prepareStatement("" + + "WITH inserted_row as (INSERT INTO " + + cotRouterTableName + + " (uid, event_pt, cot_type, " + + "start, time, stale, detail, " + + "access, qos, opex, " + + "how, point_hae, point_ce, point_le, groups, " + + "id, servertime) VALUES " + + "(?,ST_GeometryFromText(?, 4326),?,?,?,?,?,?,?,?,?,?,?,?,(?)::bit(" + + RemoteUtil.GROUPS_BIT_VECTOR_LEN + "), nextval('cot_router_seq'),?) returning id)" + + " INSERT INTO data_feed_cot (cot_router_id, data_feed_id) VALUES ((SELECT id FROM inserted_row), (SELECT id FROM data_feed WHERE uuid = ?))")) { + + for (CotEventContainer event : events) { + try { + setCotQueryParams(dataFeedInsert, event); + + dataFeedInsert.setString(17, ((DataFeed) event.getContext(Constants.DATA_FEED_KEY)).getUuid()); + + dataFeedInsert.execute(); + } catch (Exception e) { + log.error("Error with datafeed batch insert: " + e.toString(), e); + dataFeedInsert.clearBatch(); + } + } + } catch (SQLException e) { + log.error("exception executing datafeed insert batch ", e); + if (log.isDebugEnabled()) { + log.debug("exception executing datafeed insert batch ", e); + } + } + } + } private void extractGroupContacts(org.w3c.dom.Node node, HashMap callsignUidMap) { NodeList nodeList = node.getChildNodes(); @@ -697,7 +763,7 @@ public boolean hasRoomInQueueFor(CotEventContainer c) { * @param eventType ConnectionEventTypeValue (required) * @return boolean indicating success */ - public void auditCallsignUIDEventAsync(String callsign, String uid, ConnectionEventTypeValue eventType, String groupVector) { + public void auditCallsignUIDEventAsync(String callsign, String uid, String username, ConnectionEventTypeValue eventType, String groupVector) { if (!config.getRepository().isEnable()) { return; @@ -714,22 +780,28 @@ public void auditCallsignUIDEventAsync(String callsign, String uid, ConnectionEv if (uid != null && uid.trim().length() > 0 && callsign != null && callsign.trim().length() > 0 && eventType != null) { - String ep_sql = new String("insert into client_endpoint (callsign, uid) " + - "select ?, ? where not exists " + - "(select * from client_endpoint ce where ce.callsign = ? and ce.uid = ?)"); + String ep_sql = new String("insert into client_endpoint (callsign, uid, username) " + + "select ?, ?, ? where not exists " + + "(select * from client_endpoint ce where ce.callsign = ? and ce.uid = ? and username = ?)"); String epe_sql = new String("insert into client_endpoint_event (client_endpoint_id, connection_event_type_id, created_ts, groups) " + "select ce.id, cet.id, current_timestamp, (?)::bit(" + RemoteUtil.GROUPS_BIT_VECTOR_LEN + ") " + "from client_endpoint ce join connection_event_type cet on cet.event_name = ? " + - "where ce.callsign = ? and ce.uid = ?"); + "where ce.callsign = ? and ce.uid = ? and username = ?"); + + if (log.isDebugEnabled()) { + log.debug("Insert client endpoint callsign: " + callsign + " uid: " + uid); + } try (Connection conn = dataSource.getConnection()) { try (PreparedStatement ps_ep = conn.prepareStatement(ep_sql)) { ps_ep.setString(1, callsign); ps_ep.setString(2, uid); - ps_ep.setString(3, callsign); - ps_ep.setString(4, uid); + ps_ep.setString(3, username); + ps_ep.setString(4, callsign); + ps_ep.setString(5, uid); + ps_ep.setString(6, username); ps_ep.executeUpdate(); // Insert a client endpoint row @@ -738,6 +810,7 @@ public void auditCallsignUIDEventAsync(String callsign, String uid, ConnectionEv ps_epe.setString(2, eventType.value()); ps_epe.setString(3, callsign); ps_epe.setString(4, uid); + ps_epe.setString(5, username); ps_epe.executeUpdate(); } } diff --git a/src/takserver-core/src/main/java/com/bbn/marti/service/Resources.java b/src/takserver-core/src/main/java/com/bbn/marti/service/Resources.java index 4a13d3b3..f2fb61d9 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/service/Resources.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/service/Resources.java @@ -27,7 +27,6 @@ import io.netty.channel.EventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.util.concurrent.DefaultThreadFactory; public class Resources { diff --git a/src/takserver-core/src/main/java/com/bbn/marti/service/SubmissionService.java b/src/takserver-core/src/main/java/com/bbn/marti/service/SubmissionService.java index f290914f..d8fdc34b 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/service/SubmissionService.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/service/SubmissionService.java @@ -32,6 +32,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; import javax.xml.XMLConstants; import javax.xml.bind.JAXBContext; @@ -39,6 +40,7 @@ import javax.xml.bind.Unmarshaller; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.EnumUtils; import org.apache.ignite.Ignite; import org.dom4j.Document; import org.dom4j.DocumentException; @@ -49,10 +51,12 @@ import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.context.event.EventListener; import org.xml.sax.SAXException; +import com.bbn.cot.filter.DataFeedFilter; import com.bbn.cot.filter.DropEventFilter; import com.bbn.cot.filter.FlowTagFilter; import com.bbn.cot.filter.GeospatialEventFilter; @@ -66,15 +70,15 @@ import com.bbn.marti.config.CertificateConfig; import com.bbn.marti.config.CertificateSigning; import com.bbn.marti.config.Configuration; +import com.bbn.marti.config.DataFeed; import com.bbn.marti.config.Dropfilter; import com.bbn.marti.config.Federation.FederationServer; import com.bbn.marti.config.Filter; +import com.bbn.marti.config.Input; import com.bbn.marti.config.MicrosoftCAConfig; import com.bbn.marti.config.NameEntries; import com.bbn.marti.config.NameEntry; -import com.bbn.marti.config.Network; import com.bbn.marti.config.Network.Connector; -import com.bbn.marti.config.Network.Input; import com.bbn.marti.config.Repository; import com.bbn.marti.config.TAKServerCAConfig; import com.bbn.marti.config.Tls; @@ -121,6 +125,8 @@ import com.bbn.marti.remote.groups.User; import com.bbn.marti.remote.util.DateUtil; import com.bbn.marti.remote.util.RemoteUtil; +import com.bbn.marti.sync.model.DataFeedDao; +import com.bbn.marti.sync.repository.DataFeedRepository; import com.bbn.marti.util.FixedSizeBlockingQueue; import com.bbn.marti.util.MessageConversionUtil; import com.bbn.marti.util.MessagingDependencyInjectionProxy; @@ -134,11 +140,14 @@ import tak.server.CommonConstants; import tak.server.Constants; import tak.server.cache.ActiveGroupCacheHelper; +import tak.server.cache.PluginDatafeedCacheHelper; import tak.server.cluster.ClusterManager; import tak.server.cot.CotElement; import tak.server.cot.CotEventContainer; import tak.server.cot.CotParser; import tak.server.federation.DistributedFederationManager; +import tak.server.feeds.DataFeed.DataFeedType; +import tak.server.feeds.PluginDataFeed; import tak.server.ignite.IgniteHolder; import tak.server.messaging.MessageConverter; @@ -206,6 +215,11 @@ public static enum ChannelType {FED_V1, FED_V2, COT}; private AtomicBoolean isIntercept = null; + private DataFeedRepository dataFeedRepository; + + @Autowired + private PluginDatafeedCacheHelper pluginDatafeedCacheHelper; + public static SubmissionService getInstance() { if (instance == null) { synchronized (SubmissionService.class) { @@ -244,7 +258,7 @@ public SubmissionService(DistributedFederationManager dfm, NioNettyBuilder nb, M GroupManager gm, ScrubInvalidValues siv, MessageConversionUtil mcu, GroupFederationUtil gfu, InjectionManager im, RepositoryService rs, Ignite i, SubscriptionManager sm, SubscriptionStore store, FlowTagFilter flowTag, ContactManager contactManager, ServerInfo serverInfo, CoreConfig coreConfig, - MessageConverter messageConverter, ActiveGroupCacheHelper agch, RemoteUtil remoteUtil) { + MessageConverter messageConverter, ActiveGroupCacheHelper agch, RemoteUtil remoteUtil, DataFeedRepository dfr) { this.federationManager = dfm; this.nettyBuilder = nb; this.messagingUtil = mui; @@ -265,6 +279,7 @@ public SubmissionService(DistributedFederationManager dfm, NioNettyBuilder nb, M this.messageConverter = messageConverter; this.activeGroupCacheHelper = agch; this.remoteUtil = remoteUtil; + this.dataFeedRepository = dfr; DistributedConfiguration dc = DistributedConfiguration.getInstance(); Auth.Ldap ldapConfig = dc.getAuth().getLdap(); @@ -305,7 +320,7 @@ public void init() throws IOException, DocumentException { postMissionEventsAsPublic = config.getAuth() != null && config.getAuth().getLdap() != null && config.getAuth().getLdap().isPostMissionEventsAsPublic(); - List inputs = config.getNetwork().getInput(); + List inputs = config.getNetwork().getInput(); if (logger.isDebugEnabled()) { @@ -317,7 +332,7 @@ public void init() throws IOException, DocumentException { } try { - for (Network.Input input : inputs) { + for (Input input : inputs) { if (CollectionUtils.isNotEmpty(input.getFiltergroup()) && input.getAuth().equals(AuthType.X_509)) { if (logger.isErrorEnabled()) { logger.error("You have configured an input with both x509 auth and filter groups, the filter groups will be ignored. " + input.getFiltergroup()); @@ -329,6 +344,113 @@ public void init() throws IOException, DocumentException { logger.error("Configuration or setup of network inputs failed: " + e.getMessage(), e); System.exit(-1); } + + List feeds = config.getNetwork().getDatafeed(); + + if (!feeds.isEmpty()) { + logger.info("listening for " + feeds.size() + " data feed(s)"); + } + + try { + Set feedUids = new HashSet<>(); + + for (DataFeed feed : feeds) { + if (CollectionUtils.isNotEmpty(feed.getFiltergroup()) && feed.getAuth().equals(AuthType.X_509)) { + if (logger.isErrorEnabled()) { + logger.error("You have configured a datafeed with both x509 auth and filter groups, the filter groups will be ignored. " + feed.getFiltergroup()); + } + } + if (Strings.isNullOrEmpty(feed.getUuid())) { + logger.info("Failed to initialize Data Feed: " + feed.getName() + + " because no uuid tag was specified. Here is an auto-generated uuid you can use: " + + UUID.randomUUID().toString() + ""); + continue; + } + + if (feedUids.contains(feed.getUuid())) { + logger.info("Failed to initialize Data Feed: " + feed.getName() + " because a data feed with that uuid already exists."); + continue; + } + + feedUids.add(feed.getUuid()); + + DataFeedType feedType = EnumUtils.getEnumIgnoreCase(DataFeedType.class, feed.getType()); + + if (feedType == null) { + feedType = DataFeedType.Streaming; + } + + feed.setType(feedType.toString()); + + AuthType feedAuthType = feed.getAuth(); + + if (feedAuthType == null) { + feedAuthType = AuthType.X_509; + } + + feed.setAuth(feedAuthType); + + // Determine the anongroup value + boolean anonGroup; + + if (feed.isAnongroup() == null) { + List groupList = feed.getFiltergroup(); + boolean hasGroups = (groupList != null && !groupList.isEmpty()); + anonGroup = !hasGroups; + } else { + anonGroup = feed.isAnongroup(); + } + + feed.setAnongroup(anonGroup); + + Long dataFeedId = null; + + if (dataFeedRepository.getDataFeedByUUID(feed.getUuid()).size() > 0) { + dataFeedId = dataFeedRepository.updateDataFeed(feed.getUuid(), feed.getName(), feedType.ordinal(), + feed.getAuth().toString(), feed.getPort(), feed.isAuthRequired(), feed.getProtocol(), + feed.getGroup(), feed.getIface(), feed.isArchive(), feed.isAnongroup(), + feed.isArchiveOnly(), feed.getCoreVersion(), feed.getCoreVersion2TlsVersions(), + feed.isSync()); + if (feed.getTag().size() > 0) { + dataFeedRepository.removeAllDataFeedTagsById(dataFeedId); + for (String tag : feed.getTag()) { + dataFeedRepository.addDataFeedTag(dataFeedId, tag); + } + } + if (feed.getFiltergroup().size() > 0) { + dataFeedRepository.removeAllDataFeedFilterGroupsById(dataFeedId); + for (String filterGroup : feed.getFiltergroup()) { + dataFeedRepository.addDataFeedFilterGroup(dataFeedId, filterGroup); + } + } + } else { + + if (dataFeedRepository.getDataFeedByName(feed.getName()).size() != 1) { + dataFeedId = dataFeedRepository.addDataFeed(feed.getUuid(), feed.getName(), feedType.ordinal(), + feed.getAuth().toString(), feed.getPort(), feed.isAuthRequired(), feed.getProtocol(), + feed.getGroup(), feed.getIface(), feed.isArchive(), feed.isAnongroup(), + feed.isArchiveOnly(), feed.getCoreVersion(), feed.getCoreVersion2TlsVersions(), feed.isSync()); + if (feed.getTag().size() > 0) { + dataFeedRepository.removeAllDataFeedTagsById(dataFeedId); + for (String tag : feed.getTag()) { + dataFeedRepository.addDataFeedTag(dataFeedId, tag); + } + } + if (feed.getFiltergroup().size() > 0) { + dataFeedRepository.removeAllDataFeedFilterGroupsById(dataFeedId); + for (String filterGroup : feed.getFiltergroup()) { + dataFeedRepository.addDataFeedFilterGroup(dataFeedId, filterGroup); + } + } + } + } + + addInput(feed); + } + } catch (Exception e) { + logger.error("Configuration or setup of network feeds failed: " + e.getMessage(), e); + } + if (config.getFilter().getDropfilter() != null) { for (Dropfilter.Typefilter t : config.getFilter().getDropfilter().getTypefilter()) { @@ -458,29 +580,95 @@ public void init() throws IOException, DocumentException { try { Message m = Message.parseFrom((byte[]) message); - - if (m != null && m.getProvenanceCount() > 0) { - // do not republish the message if it is marked with provenance - - if (logger.isDebugEnabled()) { - logger.debug("has provenance - processing in service queue"); - } - - // this is only necessary if interception is enabled - if (remoteUtil.getIsIntercept(isIntercept).get()) { + + if (m != null) { + boolean isInterceptorMessage = false; + List provenance = m.getProvenanceList(); + if (provenance != null && provenance.contains(Constants.PLUGIN_INTERCEPTOR_PROVENANCE)) { + isInterceptorMessage = true; + } + + if (isInterceptorMessage) { + // do not republish the message if it is marked as intercepted, submit it directly SubmissionService.this.addToInputQueue(messageConverter.dataMessageToCot(m, false)); - } - - } else { - - if (logger.isDebugEnabled()) { - logger.debug("no provenance - processing through plugin pipeline"); - } - - CotEventContainer pluginCotEvent = messageConverter.dataMessageToCot(m, false); - - MessagingDependencyInjectionProxy.getInstance().cotMessenger().send(pluginCotEvent); + } else { + if (logger.isDebugEnabled()) { + logger.debug("processing through plugin pipeline"); + } + + // Check if the message is a datafeed + boolean isDataFeedMessage = false; + if (m.getFeedUuid() != null && !m.getFeedUuid().isEmpty()) { + isDataFeedMessage = true; + } + + CotEventContainer pluginCotEvent = messageConverter.dataMessageToCot(m, false); + + if (isDataFeedMessage) { + + List cacheResult = pluginDatafeedCacheHelper.getPluginDatafeed(m.getFeedUuid()); + + if (cacheResult == null) { // Does not have in cache + + List dataFeedInfo = dataFeedRepository.getDataFeedByUUID(m.getFeedUuid()); + if (dataFeedInfo.size() == 0) { + + // update cache with empty result + pluginDatafeedCacheHelper.cachePluginDatafeed(m.getFeedUuid(), new ArrayList<>()); + + logger.warn("Datafeed with UUID {} does not exist. Ignore the message.", m.getFeedUuid()); + + } else { + + List tags = dataFeedRepository.getDataFeedTagsById(dataFeedInfo.get(0).getId()); + + // update cache + List pluginDatafeeds = new ArrayList<>(); + tak.server.plugins.PluginDataFeed pluginDataFeed = new tak.server.plugins.PluginDataFeed(m.getFeedUuid(), dataFeedInfo.get(0).getName(), tags, dataFeedInfo.get(0).getArchive(), dataFeedInfo.get(0).isSync()); + pluginDatafeeds.add(pluginDataFeed); + pluginDatafeedCacheHelper.cachePluginDatafeed(m.getFeedUuid(), pluginDatafeeds); + + com.bbn.marti.config.DataFeed dataFeed = new com.bbn.marti.config.DataFeed(); + dataFeed.setUuid(m.getFeedUuid()); + dataFeed.setName(dataFeedInfo.get(0).getName()); + dataFeed.getTag().addAll(tags); + dataFeed.setArchive(dataFeedInfo.get(0).getArchive()); + dataFeed.setSync(dataFeedInfo.get(0).isSync()); + logger.debug("Retrieve Datafeed info from dataFeedRepository: uuid: {}, name: {}, tags: {}, archive: {}, sync: {}", dataFeed.getUuid(), dataFeed.getName(), dataFeed.getTag(), dataFeed.isArchive(), dataFeed.isSync()); + + DataFeedFilter.getInstance().filter(pluginCotEvent, dataFeed); + + MessagingDependencyInjectionProxy.getInstance().cotMessenger().send(pluginCotEvent); + } + + } else { // exist in cache + + if (cacheResult.size() == 0) { // datafeed with this uuid does not exist + + logger.warn("-Datafeed with UUID {} does not exist. Ignore the message.", m.getFeedUuid()); + + } else { + com.bbn.marti.config.DataFeed dataFeed = new com.bbn.marti.config.DataFeed(); + dataFeed.setUuid(m.getFeedUuid()); + dataFeed.setName(cacheResult.get(0).getName()); + dataFeed.getTag().addAll(cacheResult.get(0).getTags()); + dataFeed.setArchive(cacheResult.get(0).isArchive()); + dataFeed.setSync(cacheResult.get(0).isSync()); + logger.debug("Retrieve Datafeed info from cache: uuid: {}, name: {}, tags: {}, archive: {}, sync: {}", dataFeed.getUuid(), dataFeed.getName(), dataFeed.getTag(), dataFeed.isArchive(), dataFeed.isSync()); + + DataFeedFilter.getInstance().filter(pluginCotEvent, dataFeed); + + MessagingDependencyInjectionProxy.getInstance().cotMessenger().send(pluginCotEvent); + } + } + + } else { + MessagingDependencyInjectionProxy.getInstance().cotMessenger().send(pluginCotEvent); + } + + } } + } catch (Exception e) { if (logger.isDebugEnabled()) { @@ -601,7 +789,7 @@ public void createSubscriptionFromConnection(ChannelHandler handler, Protocol readOnlyGroups = groupManager + .getGroups(user) + .stream() + .filter(g->g.getDirection() == Direction.IN).collect(Collectors.toSet()); + + groupManager.updateGroups(user, readOnlyGroups); + } + Subscription subscription = subMgr.getSubscription(handler.netProtocolName() + ":" + uid); subscription.isWebsocket.set(websocketApiNode != null); subscription.websocketApiNode = websocketApiNode; + if (logger.isDebugEnabled()) { logger.debug("subscription in subscriptionLifecycleCallback " + subscription); } @@ -654,7 +852,7 @@ private String getGroupVectorFromHandler(ChannelHandler handler) { NavigableSet groups = groupManager.getGroups(user); return RemoteUtil.getInstance().bitVectorToString(RemoteUtil.getInstance().getBitVectorForGroups(groups)); } catch (Exception e) { - logger.error("exception in getGroupVectorFromHandler!"); + logger.error("exception in getGroupVectorFromHandler!", e); return RemoteUtil.getInstance().getBitStringNoGroups(); } } @@ -685,8 +883,10 @@ public void handleChannelDisconnect(ChannelHandler handler) { if (config.getRepository().isEnable()) { + String username = subscription.getUser() != null ? subscription.getUser().getName() : ""; + //Audit disconnected event for callsign/uid pair - repositoryService.auditCallsignUIDEventAsync(subscription.callsign, subscription.clientUid, ConnectionEventTypeValue.DISCONNECTED, + repositoryService.auditCallsignUIDEventAsync(subscription.callsign, subscription.clientUid, username, ConnectionEventTypeValue.DISCONNECTED, getGroupVectorFromHandler(handler)); } } @@ -839,7 +1039,7 @@ public void onDataReceived(final CotEventContainer data, ChannelHandler handler, } // process per-input filters - Network.Input input = ((AbstractBroadcastingChannelHandler) handler).getInput(); + Input input = ((AbstractBroadcastingChannelHandler) handler).getInput(); Filter filter = input.getFilter(); if (filter != null) { @@ -1069,8 +1269,9 @@ public void onDataReceived(CotEventContainer data, ChannelHandler handler, Proto } try { + String username = sub.getUser() != null ? sub.getUser().getName() : ""; repositoryService.auditCallsignUIDEventAsync( - callsign, data.getUid(), ConnectionEventTypeValue.CONNECTED, groupVector); + callsign, data.getUid(), username, ConnectionEventTypeValue.CONNECTED, groupVector); } catch (Exception e) { if (logger.isDebugEnabled()) { logger.debug("error recording connection event", e); @@ -1093,8 +1294,27 @@ public void addInput(Input input) throws IOException { // add input business logic, optional clustering of the input add public void addInput(Input input, boolean cluster) throws IOException { + + logger.info("input class type: " + input.getClass().getSimpleName()); + String name = input.getName(); - logger.info("Configuring " + (cluster ? "clustered " : "") + "network input " + name + ": "); + logger.info("Configuring " + (cluster ? "clustered " : "") + "network input " + input.getClass().getName() + " " + name + ": "); + + if (input instanceof DataFeed) { + DataFeed feed = (DataFeed) input; + + if (Strings.isNullOrEmpty(feed.getUuid())) { + feed.setUuid(UUID.randomUUID().toString()); + } + if (feed.getType() == null) { + feed.setType("Streaming"); + } + if (feed.getProtocol() == null || feed.getType().equals("Plugin")) { + addMetric(input, new InputMetric(input)); + config.addInputAndSave(input); + return; + } + } TransportCotEvent transport = TransportCotEvent.findByID(input.getProtocol()); boolean isTls = (transport == TransportCotEvent.TLS || transport == TransportCotEvent.SSL @@ -1119,41 +1339,44 @@ else if (!isUdp) { if (config.getRemoteConfiguration().getSecurity() == null) { throw new IllegalArgumentException("Security section of CoreConfig is required"); } + + int localPort = input.getPort(); + + // Multicast options + InetAddress group = null; + List interfs = new LinkedList(); + + if (input.getGroup() != null) { + group = InetAddress.getByName(input.getGroup()); + // only set the interface if we're using multicast + if (input.getIface() != null) { + interfs.add(NetworkInterface.getByName(input.getIface())); + } + } + + if (input.getCoreVersion() == 2) { + if (transport == TransportCotEvent.UDP) + nettyBuilder.buildUdpServer(input); + if (transport == TransportCotEvent.MUDP || transport == TransportCotEvent.COTPROTOMUDP) + nettyBuilder.buildMulticastServer(input, group, interfs); + } else { - int localPort = input.getPort(); - - // Multicast options - InetAddress group = null; - List interfs = new LinkedList(); - - if (input.getGroup() != null) { - group = InetAddress.getByName(input.getGroup()); - // only set the interface if we're using multicast - if (input.getIface() != null) { - interfs.add(NetworkInterface.getByName(input.getIface())); - } - } - - // Get codec sources, such as SSL and LDAPAUTH - List codecSources = getCodecSources(input); - LinkedBlockingQueue> protocolListenerInstantiators = new LinkedBlockingQueue>(); - addProtocolListeners(protocolListenerInstantiators, input); + // Get codec sources, such as SSL and LDAPAUTH + List codecSources = getCodecSources(input); + LinkedBlockingQueue> protocolListenerInstantiators = new LinkedBlockingQueue>(); + addProtocolListeners(protocolListenerInstantiators, input); - ServerBinder binder = transport.binder( - localPort, - protocolListenerInstantiators, - codecSources, - interfs, - group - ); + ServerBinder binder = transport.binder(localPort, protocolListenerInstantiators, codecSources, interfs, + group); - server.bind(binder, input); + server.bind(binder, input); + } - if (!TransportCotEvent.isStreaming(input.getProtocol())) { - groupFederationUtil.updateInputGroups(input); - } + if (!TransportCotEvent.isStreaming(input.getProtocol())) { + groupFederationUtil.updateInputGroups(input); + } - } + } addMetric(input, new InputMetric(input)); config.addInputAndSave(input); @@ -1652,7 +1875,7 @@ private List getCodecSources(Input input) { } // keep track of metrics based on the Network.Input object - private Map inputMetrics = new ConcurrentHashMap<>(); + private Map inputMetrics = new ConcurrentHashMap<>(); @Override public NetworkInputAddResult addInputAndSave(Input newInput) { @@ -1680,6 +1903,7 @@ public NetworkInputAddResult addInputAndSave(Input newInput) { } else { List currentInputList = config.getNetworkInputs(); + List currentDataFeedList = config.getNetworkDataFeeds(); String protocol = newInput.getProtocol(); boolean isUdp = protocol.equals(TransportCotEvent.UDP.configID); @@ -1707,6 +1931,38 @@ public NetworkInputAddResult addInputAndSave(Input newInput) { break; } } + + for (DataFeed loopfeed : currentDataFeedList) { + if (newInput.getName().equals(loopfeed.getName())) { + returnResult = NetworkInputAddResult.FAIL_INPUT_NAME_EXISTS; + } else { + boolean isPlugin = false; + if (newInput instanceof DataFeed) { + DataFeed newDataFeed = (DataFeed) newInput; + if (newDataFeed.getType().equals("Plugin")) { + isPlugin = true; + } + } + + if (!isPlugin && newInput.getPort() == loopfeed.getPort()) { + String loopProtocol = loopfeed.getProtocol(); + boolean loopIsUdp = loopProtocol.equals(TransportCotEvent.UDP.configID); + boolean loopIsMcast = loopProtocol.equals(TransportCotEvent.MUDP.configID); + + if (loopIsUdp && isUdp) { + returnResult = NetworkInputAddResult.FAIL_UDP_PORT_ALREADY_IN_USE; + } else if (loopIsMcast && isMcast) { + returnResult = NetworkInputAddResult.FAIL_MCAST_PORT_ALREADY_IN_USE; + } else if (!loopIsUdp && !loopIsMcast && !isUdp && !isMcast) { + returnResult = NetworkInputAddResult.FAIL_TCP_PORT_ALREADY_IN_USE; + } + } + } + + if (returnResult != NetworkInputAddResult.SUCCESS) { + break; + } + } } if (logger.isDebugEnabled()) { @@ -1742,20 +1998,19 @@ public void removeInputAndSave(String name) { } // untrack metrics - Input input = null; + InputMetric inputMetric = inputMetrics.get(name); - for (Input inp : inputMetrics.keySet()) { - if (inp.getName().equalsIgnoreCase(name)) { - input = inp; - break; - } - } + if (inputMetric == null) { + throw new IllegalStateException("input named " + name + " not found in input metric map for deletion"); + } - if (input == null) { - throw new IllegalStateException("input named " + name + " not found in input -> metric map for deletion"); - } + Input input = inputMetric.getInput(); + + if (input == null) { + throw new IllegalStateException("input metric contained null input"); + } - inputMetrics.remove(input); + inputMetrics.remove(input.getName()); logger.info("Stopping server for input: " + input.getName()); TransportCotEvent transport = TransportCotEvent.findByID(input.getProtocol()); @@ -1775,12 +2030,19 @@ public void removeInputAndSave(String name) { } @Override - public Collection getInputMetrics() { + public Collection getInputMetrics(boolean excludeDataFeeds) { if (logger.isDebugEnabled()) { logger.debug("getInputMetrics: " + inputMetrics.values()); } + if (excludeDataFeeds) { + Collection onlyInputsMetrics = inputMetrics.values().stream() + .filter(input -> !(input.getInput() instanceof DataFeed)) + .collect(Collectors.toList()); + return onlyInputsMetrics; + } + return inputMetrics.values(); } @@ -1805,7 +2067,8 @@ public ConnectionModifyResult modifyInputAndSave(String inputName, Input modifie } else if (currentState.getAuth() != modifiedInput.getAuth()) { return ConnectionModifyResult.FAIL_NOMOD_AUTH_TYPE; - } else if (!currentState.getProtocol().toLowerCase(Locale.ENGLISH).equals(modifiedInput.getProtocol().toLowerCase(Locale.ENGLISH))) { + } else if (currentState.getProtocol() != null && modifiedInput.getProtocol() != null && + !currentState.getProtocol().toLowerCase(Locale.ENGLISH).equals(modifiedInput.getProtocol().toLowerCase(Locale.ENGLISH))) { return ConnectionModifyResult.FAIL_NOMOD_PROTOCOL; } else if (currentState.getPort() != modifiedInput.getPort()) { @@ -1855,11 +2118,30 @@ public ConnectionModifyResult modifyInputAndSave(String inputName, Input modifie } } - // Save the flag changes + // Save the flag changes; updateProtocolListeners(config.getInputByName(inputName)); nettyBuilder.modifyServerInput(modifiedInput); config.saveChangesAndUpdateCache(); } + + // Modify data feed attributes + if (modifiedInput instanceof DataFeed && currentState instanceof DataFeed) { + DataFeed modifiedDataFeed = (DataFeed) modifiedInput; + DataFeed currentDataFeed = (DataFeed) currentState; + + config.updateTagsNoSave(inputName, modifiedDataFeed.getTag()); + + if (modifiedDataFeed.isSync() != currentDataFeed.isSync()) { + result = config.setSyncFlagNoSave(inputName, modifiedDataFeed.isSync()); + if (result != ConnectionModifyResult.SUCCESS) { + return result; + } + } + config.saveChangesAndUpdateCache(); + } + + // Update input in inputMetric + updateMetric(config.getInputByName(inputName)); } catch (RemoteException e) { throw new TakException(e); // will not happen @@ -1870,18 +2152,78 @@ public ConnectionModifyResult modifyInputAndSave(String inputName, Input modifie } + @Override + public void removeDataFeedAndSave(String name) { + if (logger.isDebugEnabled()) { + logger.debug("Removing datafeed '" + name + "' if it exists."); + } + + try { + config.removeDataFeedAndSave(name); + } catch (RemoteException e) { + throw new TakException(e); + } + + // untrack metrics + InputMetric inputMetric = inputMetrics.get(name); + + if (inputMetric == null) { + throw new IllegalStateException("input named " + name + " not found in input metric map for deletion"); + } + + Input input = inputMetric.getInput(); + + if (input == null) { + throw new IllegalStateException("input metric contained null input"); + } + + inputMetrics.remove(input.getName()); + logger.info("Stopping server for input: " + input.getName()); + + if (input.getProtocol() != null) { + TransportCotEvent transport = TransportCotEvent.findByID(input.getProtocol()); + + boolean isUdp = transport == TransportCotEvent.MUDP || transport == TransportCotEvent.UDP + || transport == TransportCotEvent.COTPROTOMUDP; + + try { + if (input.getProtocol().equals("grpc") || !isUdp) { + nettyBuilder.stopServer(input.getPort()); + } else { + server.unbind(name); + } + } catch (Exception e) { + logger.warn("exception unbinding server port for input " + name); + } + } + } + @Override - public void addMetric(Network.Input input, InputMetric metric) { + public void addMetric(Input input, InputMetric metric) { if (input == null || metric == null) { throw new IllegalArgumentException("null input or metric"); } - inputMetrics.put(input, metric); + inputMetrics.put(input.getName(), metric); } + @Override + public void updateMetric(Input input) { + if (input == null) { + throw new IllegalArgumentException("null input"); + } + if (!inputMetrics.containsKey(input.getName())) { + throw new IllegalArgumentException("input to update not found"); + } + + InputMetric inputMetric = inputMetrics.get(input.getName()); + inputMetric.setInput(input); + inputMetrics.put(input.getName(), inputMetric); + } + @Override - public InputMetric getMetric(Network.Input input) { - return inputMetrics.get(input); + public InputMetric getMetric(Input input) { + return inputMetrics.get(input.getName()); } @Override @@ -2139,5 +2481,5 @@ public HashMap verifyConfiguration() { ret.put("truststoreFile", truststoreFile.exists()); return ret; } - + } diff --git a/src/takserver-core/src/main/java/com/bbn/marti/service/SubscriptionManager.java b/src/takserver-core/src/main/java/com/bbn/marti/service/SubscriptionManager.java index 0503a42d..a6668d18 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/service/SubscriptionManager.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/service/SubscriptionManager.java @@ -111,7 +111,7 @@ Subscription addSubscription( CotEventContainer makeDeleteMessage(String linkUid, String linkType); - CotEventContainer createMissionChangeMessage(String missionName, ChangeType changeType, String authorUid, String tool, String changes); + CotEventContainer createMissionChangeMessage(String missionName, ChangeType changeType, String authorUid, String tool, String changes, String xmlContentForNotification); CotEventContainer createMissionCreateMessage(String missionName, String authorUid, String tool); diff --git a/src/takserver-core/src/main/java/com/bbn/marti/util/MessageConversionUtil.java b/src/takserver-core/src/main/java/com/bbn/marti/util/MessageConversionUtil.java index 252ba50e..26221bfd 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/util/MessageConversionUtil.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/util/MessageConversionUtil.java @@ -37,12 +37,11 @@ import org.springframework.context.event.EventListener; import com.bbn.cot.filter.StreamingEndpointRewriteFilter; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.Input; import com.bbn.marti.nio.channel.ChannelHandler; import com.bbn.marti.nio.channel.base.AbstractBroadcastingChannelHandler; import com.bbn.marti.remote.ContactManager; import com.bbn.marti.remote.InputMetric; -import com.bbn.marti.remote.MessagingConfigurator; import com.bbn.marti.remote.RemoteContact; import com.bbn.marti.remote.exception.TakException; import com.bbn.marti.remote.socket.ChatMessage; diff --git a/src/takserver-core/src/main/java/com/bbn/marti/util/MessagingDependencyInjectionProxy.java b/src/takserver-core/src/main/java/com/bbn/marti/util/MessagingDependencyInjectionProxy.java index 95232bc6..c0aaeb28 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/util/MessagingDependencyInjectionProxy.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/util/MessagingDependencyInjectionProxy.java @@ -19,6 +19,8 @@ import com.bbn.marti.service.DistributedConfiguration; import com.bbn.marti.service.SubmissionService; import com.bbn.marti.service.SubscriptionStore; +import com.bbn.marti.sync.EnterpriseSyncService; +import com.bbn.marti.sync.repository.DataFeedRepository; import com.bbn.marti.sync.repository.FederationEventRepository; import com.bbn.marti.sync.repository.MissionSubscriptionRepository; import com.bbn.marti.sync.service.MissionService; @@ -377,4 +379,32 @@ public ApplicationEventPublisher eventPublisher() { return aep; } + + private DataFeedRepository dataFeedRepository = null; + + public DataFeedRepository dataFeedRepository() { + if (dataFeedRepository == null) { + synchronized (this) { + if (dataFeedRepository == null) { + dataFeedRepository = springContext.getBean(DataFeedRepository.class); + } + } + } + + return dataFeedRepository; + } + + private EnterpriseSyncService esyncService = null; + + public EnterpriseSyncService esyncService() { + if (esyncService == null) { + synchronized (this) { + if (esyncService == null) { + esyncService = springContext.getBean(EnterpriseSyncService.class); + } + } + } + + return esyncService; + } } diff --git a/src/takserver-core/src/main/java/com/bbn/marti/util/spring/TakAuthenticationProvider.java b/src/takserver-core/src/main/java/com/bbn/marti/util/spring/TakAuthenticationProvider.java index e0bbde64..d238609a 100644 --- a/src/takserver-core/src/main/java/com/bbn/marti/util/spring/TakAuthenticationProvider.java +++ b/src/takserver-core/src/main/java/com/bbn/marti/util/spring/TakAuthenticationProvider.java @@ -155,10 +155,6 @@ protected User authenticateCore(Authentication authentication, String authentica // authenticate using authenticator registered in group manager try { authResult = groupManager.authenticate(authenticatorName, user); - - if (logger.isDebugEnabled()) { - logger.debug("auth result: " + authResult); - } } catch (RemoteLookupFailureException e) { throw new CoreCommunicationException("Unable to establish connection with TAK Server core services", e); } catch (OAuth2Exception e) { diff --git a/src/takserver-core/src/main/java/com/bbn/vbm/VBMConfigurationApi.java b/src/takserver-core/src/main/java/com/bbn/vbm/VBMConfigurationApi.java new file mode 100644 index 00000000..a8bd3bd4 --- /dev/null +++ b/src/takserver-core/src/main/java/com/bbn/vbm/VBMConfigurationApi.java @@ -0,0 +1,57 @@ +package com.bbn.vbm; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import com.bbn.marti.config.Vbm; +import com.bbn.marti.service.DistributedConfiguration; + +@Validated +@RestController +@RequestMapping(value = "/vbm/api") +public class VBMConfigurationApi { + + Logger logger = LoggerFactory.getLogger(VBMConfigurationApi.class); + + @RequestMapping(value = "/config", method = RequestMethod.GET) + public VBMConfigurationModel getVBMConfiguration() { + + try { + Vbm vbm = DistributedConfiguration.getInstance().getVbm(); + VBMConfigurationModel config = new VBMConfigurationModel(); + config.setVbmEnabled(vbm.isEnabled()); + config.setSADisabled(vbm.isDisableSASharing()); + config.setChatDisabled(vbm.isDisableChatSharing()); + + return config; + + } catch (Exception e) { + logger.error("Error in getVBMConfiguration: ", e); + throw new RuntimeException(e); + } + } + + @RequestMapping(value = "/config", method = RequestMethod.POST) + public void setVBMConfiguration(@RequestBody VBMConfigurationModel vbmConfigurationModel) { + + if (logger.isDebugEnabled()) { + logger.debug("setVBMConfiguration to: {}", vbmConfigurationModel.isVbmEnabled()); + } + + try { + DistributedConfiguration.getInstance().getVbm().setEnabled(vbmConfigurationModel.isVbmEnabled()); + DistributedConfiguration.getInstance().getVbm().setDisableSASharing(vbmConfigurationModel.isSADisabled()); + DistributedConfiguration.getInstance().getVbm().setDisableChatSharing(vbmConfigurationModel.isChatDisabled()); + DistributedConfiguration.getInstance().saveChangesAndUpdateCache(); + + } catch (Exception e) { + logger.error("Error in setVBMConfiguration: ", e); + throw new RuntimeException(e); + } + } +} diff --git a/src/takserver-core/src/main/java/com/bbn/vbm/VBMConfigurationModel.java b/src/takserver-core/src/main/java/com/bbn/vbm/VBMConfigurationModel.java new file mode 100644 index 00000000..662b2db4 --- /dev/null +++ b/src/takserver-core/src/main/java/com/bbn/vbm/VBMConfigurationModel.java @@ -0,0 +1,39 @@ +package com.bbn.vbm; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class VBMConfigurationModel { + + private boolean vbmEnabled; + private boolean saDisabled; + private boolean chatDisabled; + + public VBMConfigurationModel() { + this.vbmEnabled = false; + } + + public boolean isVbmEnabled() { + return vbmEnabled; + } + + public void setVbmEnabled(boolean vbmEnabled) { + this.vbmEnabled = vbmEnabled; + } + + public boolean isChatDisabled() { + return chatDisabled; + } + + public void setChatDisabled(boolean chatDisabled) { + this.chatDisabled = chatDisabled; + } + + public boolean isSADisabled() { + return saDisabled; + } + + public void setSADisabled(boolean saDisabled) { + this.saDisabled = saDisabled; + } +} diff --git a/src/takserver-core/src/main/java/tak/server/ServerConfiguration.java b/src/takserver-core/src/main/java/tak/server/ServerConfiguration.java index d654e1b1..e3e7b5b9 100644 --- a/src/takserver-core/src/main/java/tak/server/ServerConfiguration.java +++ b/src/takserver-core/src/main/java/tak/server/ServerConfiguration.java @@ -68,6 +68,7 @@ import com.bbn.marti.config.Tls; import com.bbn.marti.config.Tls.Crl; import com.bbn.marti.dao.kml.JDBCCachingKMLDao; +import com.bbn.marti.feeds.DataFeedService; import com.bbn.marti.groups.DummyAuthenticator; import com.bbn.marti.groups.FileAuthenticator; import com.bbn.marti.groups.FileAuthenticatorAgent; @@ -75,6 +76,7 @@ import com.bbn.marti.groups.OAuthAuthenticator; import com.bbn.marti.groups.X509Authenticator; import com.bbn.marti.logging.AuditLogUtil; +import com.bbn.marti.maplayer.MapLayerService; import com.bbn.marti.remote.CoreConfig; import com.bbn.marti.remote.SubmissionInterface; import com.bbn.marti.remote.SubscriptionManagerLite; @@ -91,12 +93,15 @@ import com.bbn.marti.service.kml.KMLServiceImpl; import com.bbn.marti.service.kml.KmlIconStrategyJaxb; import com.bbn.marti.swaggerconfig.SwaggerConfig; +import com.bbn.marti.sync.cache.AllCopMissionsCacheKeyGenerator; import com.bbn.marti.sync.cache.AllMissionsCacheKeyGenerator; import com.bbn.marti.sync.cache.MethodNameMultiStringArgCacheKeyGenerator; import com.bbn.marti.sync.model.MissionChange; +import com.bbn.marti.sync.repository.DataFeedRepository; import com.bbn.marti.sync.repository.ExternalMissionDataRepository; import com.bbn.marti.sync.repository.LogEntryRepository; import com.bbn.marti.sync.repository.MissionChangeRepository; +import com.bbn.marti.sync.repository.MissionFeedRepository; import com.bbn.marti.sync.repository.MissionInvitationRepository; import com.bbn.marti.sync.repository.MissionRepository; import com.bbn.marti.sync.repository.MissionRoleRepository; @@ -508,6 +513,11 @@ public KeyGenerator allMissionsCacheKeyGenerator() { return new AllMissionsCacheKeyGenerator(); } + @Bean + public KeyGenerator allCopsMissionsCacheKeyGenerator() { + return new AllCopMissionsCacheKeyGenerator(); + } + @Bean public KeyGenerator methodNameMultiStringArgCacheKeyGenerator() { return new MethodNameMultiStringArgCacheKeyGenerator(); @@ -840,6 +850,9 @@ RemoteUtil remoteUtil() { return new RemoteUtil(); } + @Bean + MapLayerService mapLayerService() { return new MapLayerService(); } + @Bean public SubscriptionManagerProxyHandler subscriptionManagerProxyHandler(CoreConfig config) { return new SubscriptionManagerProxyHandler(config); @@ -864,6 +877,9 @@ MissionService missionService( ExternalMissionDataRepository externalMissionDataRepository, MissionInvitationRepository missionInvitationRepository, MissionSubscriptionRepository missionSubscriptionRepository, + MissionFeedRepository missionFeedRepository, + DataFeedRepository dataFeedRepository, + MapLayerService mapLayerService, GroupManager groupManager, CacheManager cacheManager, CommonUtil commonUtil, @@ -888,6 +904,9 @@ MissionService missionService( externalMissionDataRepository, missionInvitationRepository, missionSubscriptionRepository, + missionFeedRepository, + dataFeedRepository, + mapLayerService, groupManager, cacheManager, commonUtil, @@ -895,6 +914,11 @@ MissionService missionService( cotCacheHelper, missionCacheHelper); } + + @Bean + DataFeedService dataFeedService(DataSource dataSource, DataFeedRepository dataFeedRepository) { + return new DataFeedService(dataSource, dataFeedRepository); + } @Bean("kmlDao") public JDBCCachingKMLDao kmlDao() { diff --git a/src/takserver-core/src/main/java/tak/server/cluster/ClusterControlRolVisitor.java b/src/takserver-core/src/main/java/tak/server/cluster/ClusterControlRolVisitor.java index 5ea34993..133fd8fc 100644 --- a/src/takserver-core/src/main/java/tak/server/cluster/ClusterControlRolVisitor.java +++ b/src/takserver-core/src/main/java/tak/server/cluster/ClusterControlRolVisitor.java @@ -1,16 +1,9 @@ package tak.server.cluster; -import java.io.IOException; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.commons.lang3.tuple.ImmutablePair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.bbn.marti.config.Network.Input; -import com.bbn.marti.remote.exception.TakException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Strings; import mil.af.rl.rol.ResourceOperationParameterEvaluator; import mil.af.rl.rol.RolBaseVisitor; diff --git a/src/takserver-core/src/main/java/tak/server/cluster/DistributedInputManager.java b/src/takserver-core/src/main/java/tak/server/cluster/DistributedInputManager.java index 48c3d499..4c1c91d4 100644 --- a/src/takserver-core/src/main/java/tak/server/cluster/DistributedInputManager.java +++ b/src/takserver-core/src/main/java/tak/server/cluster/DistributedInputManager.java @@ -6,7 +6,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.DataFeed; +import com.bbn.marti.config.Input; import com.bbn.marti.remote.InputMetric; import com.bbn.marti.remote.MessagingConfigInfo; import com.bbn.marti.remote.groups.ConnectionModifyResult; @@ -14,8 +15,6 @@ import com.bbn.marti.remote.service.InputManager; import com.bbn.marti.util.MessagingDependencyInjectionProxy; -import tak.server.Constants; - /** */ public class DistributedInputManager implements InputManager, org.apache.ignite.services.Service { @@ -48,6 +47,11 @@ public NetworkInputAddResult createInput(Input input) { return MessagingDependencyInjectionProxy.getInstance().submissionService().addInputAndSave(input); } + @Override + public NetworkInputAddResult createDataFeed(DataFeed dataFeed) { + return MessagingDependencyInjectionProxy.getInstance().submissionService().addInputAndSave(dataFeed); + } + @Override public ConnectionModifyResult modifyInput(String id, Input input) { return MessagingDependencyInjectionProxy.getInstance().submissionService().modifyInputAndSave(id, input); @@ -59,8 +63,13 @@ public void deleteInput(String name) { } @Override - public Collection getInputMetrics() { - return MessagingDependencyInjectionProxy.getInstance().submissionService().getInputMetrics(); + public void deleteDataFeed(String name) { + MessagingDependencyInjectionProxy.getInstance().submissionService().removeDataFeedAndSave(name); + } + + @Override + public Collection getInputMetrics(boolean excludeDataFeeds) { + return MessagingDependencyInjectionProxy.getInstance().submissionService().getInputMetrics(excludeDataFeeds); } @Override diff --git a/src/takserver-core/src/main/java/tak/server/cluster/DistributedSubmissionService.java b/src/takserver-core/src/main/java/tak/server/cluster/DistributedSubmissionService.java index bc117c71..d8c512d4 100644 --- a/src/takserver-core/src/main/java/tak/server/cluster/DistributedSubmissionService.java +++ b/src/takserver-core/src/main/java/tak/server/cluster/DistributedSubmissionService.java @@ -36,7 +36,7 @@ public class DistributedSubmissionService implements SubmissionInterface { @Autowired Messenger cotMessenger; - + public DistributedSubmissionService() { } @Override @@ -154,7 +154,7 @@ public boolean submitMissionPackageCotAtTime(String cotMessage, String missionNa } @Override - public boolean submitCot(String cotMessage, List uids, List callsigns, NavigableSet groups, boolean federate) { + public boolean submitCot(String cotMessage, List uids, List callsigns, NavigableSet groups, boolean federate, boolean resubmission) { requireNonNull(uids, "submitCot uids"); requireNonNull(callsigns, "submitCot callsigns"); requireNonNull(groups, "submitCot groups"); @@ -182,7 +182,18 @@ public boolean submitCot(String cotMessage, List uids, List call // only set groups, not the user cot.setContext(Constants.NOFEDV2_KEY, "true"); } - + + if (resubmission) { + // trim off existing flow tags so we can resend the message + Node flowTags = cot.getDocument().selectSingleNode("/event/detail/_flow-tags_"); + if (flowTags != null) { + flowTags.detach(); + } + + // turn off message archiving so we dont save the message again + cot.setContext(Constants.ARCHIVE_EVENT_KEY, Boolean.FALSE); + } + // send message cotMessenger.send(cot); @@ -196,7 +207,7 @@ public boolean submitCot(String cotMessage, List uids, List call return false; } - + private CotParser getCotParser() { if (parser.get() == null) { diff --git a/src/takserver-core/src/main/java/tak/server/config/ApiConfiguration.java b/src/takserver-core/src/main/java/tak/server/config/ApiConfiguration.java index cfc043b0..80c91cbb 100644 --- a/src/takserver-core/src/main/java/tak/server/config/ApiConfiguration.java +++ b/src/takserver-core/src/main/java/tak/server/config/ApiConfiguration.java @@ -60,6 +60,7 @@ import com.bbn.marti.device.profile.service.ProfileService; import com.bbn.marti.excheck.ExCheckAPI; import com.bbn.marti.excheck.ExCheckService; +import com.bbn.marti.feeds.DataFeedApi; import com.bbn.marti.groups.CustomExceptionHandler; import com.bbn.marti.groups.GroupsApi; import com.bbn.marti.injector.InjectionApi; @@ -67,6 +68,7 @@ import com.bbn.marti.kml.icon.service.IconsetUploadProcessor; import com.bbn.marti.kml.icon.service.IconsetUploadProcessorImpl; import com.bbn.marti.logs.LogServlet; +import com.bbn.marti.maplayer.MapLayerService; import com.bbn.marti.maplayer.api.MapLayersApi; import com.bbn.marti.network.ContactManagerApi; import com.bbn.marti.network.ContactManagerService; @@ -102,6 +104,7 @@ import com.bbn.marti.sync.SearchServlet; import com.bbn.marti.sync.UploadServlet; import com.bbn.marti.sync.api.ContactsApi; +import com.bbn.marti.sync.api.CopViewApi; import com.bbn.marti.sync.api.CotApi; import com.bbn.marti.sync.api.MissionApi; import com.bbn.marti.sync.api.PropertiesApi; @@ -118,6 +121,7 @@ import com.bbn.marti.util.spring.SpringContextBeanForApi; import com.bbn.marti.util.spring.TakAuthSessionDestructionListener; import com.bbn.marti.video.VideoConnectionManager; +import com.bbn.marti.video.VideoConnectionManagerV2; import com.bbn.marti.video.VideoConnectionSender; import com.bbn.marti.video.VideoConnectionUploader; import com.bbn.marti.video.VideoManagerService; @@ -137,6 +141,7 @@ import com.bbn.user.registration.RegistrationApi; import com.bbn.user.registration.service.UserRegistrationService; import com.bbn.useraccountmanagement.FileUserAccountManagementApi; +import com.bbn.vbm.VBMConfigurationApi; import com.fasterxml.jackson.databind.ObjectMapper; import tak.server.Constants; @@ -146,11 +151,13 @@ import tak.server.grid.MissionArchiveManagerProxyFactory; import tak.server.grid.PluginManagerProxyFactory; import tak.server.grid.RetentionPolicyConfigProxyFactory; +import tak.server.plugins.PluginDataApi; import tak.server.plugins.PluginManagerApi; import tak.server.qos.QoSApi; import tak.server.qos.QoSManager; import tak.server.retention.RetentionApi; import tak.server.system.ApiDependencyProxy; +import tak.server.util.ExecutorSource; import tak.server.util.LoginAccessController; /* @@ -491,6 +498,11 @@ public ExCheckService exCheckService() { return new ExCheckService(); } + @Bean + public MapLayerService mapLayerService() { + return new MapLayerService(); + } + @Bean("errorLogPersistenceStore") public com.bbn.marti.logs.PersistenceStore errorLogPersistenceStore() { return new com.bbn.marti.logs.PersistenceStore(); @@ -633,6 +645,11 @@ public LDAPApi ldapAPI() { public SubmissionApi submissionApi() { return new SubmissionApi(); } + + @Bean + public DataFeedApi dataFeedApi() { + return new DataFeedApi(); + } @Bean public FederationApi federationApi() { @@ -719,6 +736,11 @@ public MissionApi missionApi() { return new MissionApi(); } + @Bean + public CopViewApi copViewApi() { + return new CopViewApi(); + } + @Bean public PropertiesApi propertiesApi() { return new PropertiesApi(); @@ -773,6 +795,11 @@ public UserRegistrationService userRegistrationService() { public PluginManagerApi pluginManagerApi() { return new PluginManagerApi(); } + + @Bean + public PluginDataApi pluginDataApi() { + return new PluginDataApi(); + } @Bean public RetentionApi retentionApi() { @@ -801,7 +828,7 @@ public RetentionPolicyConfigProxyFactory retentionPolicyConfigurationProxyFactor } @Bean - public MissionArchiveManagerProxyFactory misisonArchiveManagerProxyFactory() { + public MissionArchiveManagerProxyFactory missionArchiveManagerProxyFactory() { return new MissionArchiveManagerProxyFactory(); } @@ -836,11 +863,6 @@ public LoginAccessController loginAcccessController() { return new LoginAccessController(); } - @Bean - public ContactCacheHelper contactCacheHelper() { - return new ContactCacheHelper(); - } - @Bean public FileUserAccountManagementApi fileUserAccountManagementApi() { return new FileUserAccountManagementApi(); @@ -852,4 +874,25 @@ public FileUserManagementInterface distributedUserManager(Ignite ignite) { return ignite.services(ClusterGroupDefinition.getMessagingClusterDeploymentGroup(ignite)).serviceProxy(Constants.DISTRIBUTED_USER_FILE_MANAGER, FileUserManagementInterface.class, false); } + + @Bean + public VBMConfigurationApi vbmConfigurationApi() { + return new VBMConfigurationApi(); + } + + @Bean + public VideoConnectionManagerV2 videoConnectionManagerV2() { + return new VideoConnectionManagerV2(); + } + + @Bean + public ContactCacheHelper contactCacheHelper(CoreConfig conf) { + return new ContactCacheHelper(); + } + + @Bean + public ExecutorSource executorSource(CoreConfig conf) { + return new ExecutorSource(conf); + } + } diff --git a/src/takserver-core/src/main/java/tak/server/config/DistributedSystemInfoApi.java b/src/takserver-core/src/main/java/tak/server/config/DistributedSystemInfoApi.java new file mode 100644 index 00000000..f7a08b16 --- /dev/null +++ b/src/takserver-core/src/main/java/tak/server/config/DistributedSystemInfoApi.java @@ -0,0 +1,49 @@ +package tak.server.config; + +import org.apache.ignite.services.Service; +import org.apache.ignite.services.ServiceContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.bbn.marti.service.DistributedConfiguration; + +import tak.server.plugins.SystemInfoApi; + +/** + */ +public class DistributedSystemInfoApi implements Service, SystemInfoApi { + + private static final long serialVersionUID = -900700616308028885L; + private static final Logger logger = LoggerFactory.getLogger(DistributedSystemInfoApi.class); + + @Override + public void cancel(ServiceContext ctx) { + if (logger.isDebugEnabled()) { + logger.debug(getClass().getSimpleName() + " service cancelled"); + } + } + + @Override + public void init(ServiceContext ctx) throws Exception { + if (logger.isDebugEnabled()) { + logger.debug("init method " + getClass().getSimpleName()); + } + } + + @Override + public void execute(ServiceContext ctx) throws Exception { + if (logger.isDebugEnabled()) { + logger.debug("execute method " + getClass().getSimpleName()); + } + } + + @Override + public String getTAKServerUrl() { + try { + return DistributedConfiguration.getInstance() + .getRemoteConfiguration().getFederation().getFederationServer().getWebBaseUrl(); + } catch (Exception e) { + return null; + } + } +} diff --git a/src/takserver-core/src/main/java/tak/server/config/MessagingConfiguration.java b/src/takserver-core/src/main/java/tak/server/config/MessagingConfiguration.java index 6100aee2..5b0c2e31 100644 --- a/src/takserver-core/src/main/java/tak/server/config/MessagingConfiguration.java +++ b/src/takserver-core/src/main/java/tak/server/config/MessagingConfiguration.java @@ -36,10 +36,12 @@ import org.springframework.web.socket.messaging.SubProtocolHandler; import com.bbn.cluster.ClusterGroupDefinition; +import com.bbn.cot.filter.DataFeedFilter; import com.bbn.cot.filter.FlowTagFilter; import com.bbn.cot.filter.ScrubInvalidValues; import com.bbn.cot.filter.StreamingEndpointRewriteFilter; import com.bbn.cot.filter.UrlAddingFilter; +import com.bbn.cot.filter.VBMSASharingFilter; import com.bbn.marti.groups.DistributedPersistentGroupManager; import com.bbn.marti.groups.DistributedUserManager; import com.bbn.marti.groups.FileAuthenticator; @@ -52,6 +54,7 @@ import com.bbn.marti.injector.ClusterUidCotTagInjector; import com.bbn.marti.injector.InjectionManager; import com.bbn.marti.injector.UidCotTagInjector; +import com.bbn.marti.network.PluginDataFeedJdbc; import com.bbn.marti.nio.netty.NioNettyBuilder; import com.bbn.marti.nio.server.NioServer; import com.bbn.marti.remote.ContactManager; @@ -75,6 +78,7 @@ import com.bbn.marti.service.LocalConfiguration; import com.bbn.marti.service.MessagingInitializer; import com.bbn.marti.service.MissionPackageExtractor; +import com.bbn.marti.service.PluginStore; import com.bbn.marti.service.RepeaterService; import com.bbn.marti.service.RepositoryService; import com.bbn.marti.service.SubmissionService; @@ -83,6 +87,7 @@ import com.bbn.marti.sync.EnterpriseSyncService; import com.bbn.marti.sync.federation.FederationROLHandler; import com.bbn.marti.sync.federation.MissionActionROLConverter; +import com.bbn.marti.sync.repository.DataFeedRepository; import com.bbn.marti.sync.repository.FederationEventRepository; import com.bbn.marti.sync.service.MissionService; import com.bbn.marti.util.MessageConversionUtil; @@ -99,6 +104,7 @@ import tak.server.Constants; import tak.server.cache.ActiveGroupCacheHelper; import tak.server.cache.MissionCacheResolver; +import tak.server.cache.PluginDatafeedCacheHelper; import tak.server.cluster.DistributedInjectionService; import tak.server.cluster.DistributedInputManager; import tak.server.cluster.DistributedSecurityManager; @@ -109,9 +115,16 @@ import tak.server.federation.TakFigClient; import tak.server.grid.CoreConfigProxyFactoryForMessaging; import tak.server.messaging.DistributedCotMessenger; +import tak.server.messaging.DistributedPluginApi; +import tak.server.messaging.DistributedPluginDataFeedApi; +import tak.server.messaging.DistributedPluginSelfStopApi; import tak.server.messaging.DistributedTakMessenger; import tak.server.messaging.MessageConverter; import tak.server.messaging.Messenger; +import tak.server.plugins.PluginApi; +import tak.server.plugins.PluginDataFeedApi; +import tak.server.plugins.PluginSelfStopApi; +import tak.server.plugins.SystemInfoApi; import tak.server.profile.DistributedServerInfo; import tak.server.qos.DistributedQoSManager; import tak.server.qos.MessageDOSStrategy; @@ -141,9 +154,9 @@ public LocalConfiguration localConfiguration() { public SubmissionService submissionService(DistributedFederationManager dfm, NioNettyBuilder nb, MessagingUtilImpl mui, NioServer ns, GroupManager gm, ScrubInvalidValues siv, MessageConversionUtil mcu, GroupFederationUtil gfu, InjectionManager im, RepositoryService rs, Ignite ig, SubscriptionManager sm, SubscriptionStore ss, FlowTagFilter flowTag, ContactManager contactManager, ServerInfo serverInfo, @Qualifier(Constants.MESSAGING_CORE_CONFIG_PROXY_BEAN) CoreConfig coreConfig, - MessageConverter messageConverter, ActiveGroupCacheHelper activeGroupCacheHelper, RemoteUtil remoteUtil) { + MessageConverter messageConverter, ActiveGroupCacheHelper activeGroupCacheHelper, RemoteUtil remoteUtil, DataFeedRepository dfr) { - return new SubmissionService(dfm, nb, mui, ns, gm, siv, mcu, gfu, im, rs, ig, sm, ss, flowTag, contactManager, serverInfo, coreConfig, messageConverter, activeGroupCacheHelper, remoteUtil); + return new SubmissionService(dfm, nb, mui, ns, gm, siv, mcu, gfu, im, rs, ig, sm, ss, flowTag, contactManager, serverInfo, coreConfig, messageConverter, activeGroupCacheHelper, remoteUtil, dfr); } @Bean @@ -158,7 +171,13 @@ public RepeaterService repeaterService(CoreConfig coreConfig, BrokerService brok @Order(Ordered.LOWEST_PRECEDENCE) @Bean - public DistributedFederationManager distributedFederationManager(NioNettyBuilder nettyBuilder, Ignite ignite, CoreConfig coreConfig, CoreConfigProxyFactoryForMessaging coreConfigProxy, DistributedConfiguration distConf, FederationEventRepository federationEventRepository) throws RemoteException { + public DistributedFederationManager distributedFederationManager( + NioNettyBuilder nettyBuilder, + Ignite ignite, + CoreConfig coreConfig, + CoreConfigProxyFactoryForMessaging coreConfigProxy, + DistributedConfiguration distConf, + FederationEventRepository federationEventRepository) throws RemoteException { DistributedFederationManager distributedFederationManager = new DistributedFederationManager(nettyBuilder, ignite, coreConfig); ignite.services(ClusterGroupDefinition.getMessagingClusterDeploymentGroup(ignite)).deployNodeSingleton(Constants.DISTRIBUTED_FEDERATION_MANAGER, distributedFederationManager); return (DistributedFederationManager) ignite.services(ClusterGroupDefinition.getMessagingLocalClusterDeploymentGroup(ignite)) @@ -173,6 +192,15 @@ public FileUserManagementInterface distributedUserManager(Ignite ignite, FileAut return ignite.services(ClusterGroupDefinition.getMessagingLocalClusterDeploymentGroup(ignite)) .serviceProxy(Constants.DISTRIBUTED_USER_FILE_MANAGER, FileUserManagementInterface.class, false); } + + @Bean + public SystemInfoApi distributedSystemInfoApi(Ignite ignite) { + DistributedSystemInfoApi distributedSystemInfoApi = new DistributedSystemInfoApi(); + ignite.services(ClusterGroupDefinition.getMessagingClusterDeploymentGroup(ignite)).deployNodeSingleton(Constants.DISTRIBUTED_SYSTEM_INFO_API, distributedSystemInfoApi); + + return ignite.services(ClusterGroupDefinition.getMessagingLocalClusterDeploymentGroup(ignite)) + .serviceProxy(Constants.DISTRIBUTED_SYSTEM_INFO_API, SystemInfoApi.class, false); + } @Bean @Scope("prototype") @@ -196,8 +224,8 @@ public FederationServer federationServer() { } @Bean(Constants.DISTRIBUTED_COT_MESSENGER) - public Messenger distributedCotMessenger(Ignite ignite, SubscriptionStore subscriptionStore, ServerInfo serverInfo, MessageConverter messageConverter, SubmissionService submissionService, CoreConfig config, RemoteUtil remoteUtil) { - return new DistributedCotMessenger(ignite, subscriptionStore, serverInfo, messageConverter, submissionService, config, remoteUtil); + public Messenger distributedCotMessenger(Ignite ignite, SubscriptionStore subscriptionStore, ServerInfo serverInfo, MessageConverter messageConverter, SubmissionService submissionService, CoreConfig config) { + return new DistributedCotMessenger(ignite, subscriptionStore, serverInfo, messageConverter, submissionService, config); } @Bean(Constants.DISTRIBUTED_TAK_MESSENGER) @@ -219,7 +247,17 @@ public MessagingInitializer coreMessagingInitializer(SubmissionService submissio public StreamingEndpointRewriteFilter streamingEnpointRewriteFilter(@Lazy MissionService missionService) { return new StreamingEndpointRewriteFilter(missionService); } + + @Bean + public DataFeedFilter dataFeedFilter() { + return new DataFeedFilter(); + } + @Bean + public VBMSASharingFilter vbmSASharingFilter() { + return new VBMSASharingFilter(); + } + @Bean public NioServer nioServer() throws IOException { return new NioServer(); @@ -505,4 +543,52 @@ public QoSManager qosManager(Ignite ignite) { return qosManager; } + + @Bean + public PluginDataFeedApi distributedPluginDataFeedApi(Ignite ignite) { + + DistributedPluginDataFeedApi distributedPluginDataFeedApi = new DistributedPluginDataFeedApi(); + + ignite.services(ClusterGroupDefinition.getMessagingClusterDeploymentGroup(ignite)).deployNodeSingleton(Constants.DISTRIBUTED_PLUGIN_DATA_FEED_API, distributedPluginDataFeedApi); + + return ignite.services(ClusterGroupDefinition.getMessagingLocalClusterDeploymentGroup(ignite)).serviceProxy(Constants.DISTRIBUTED_PLUGIN_DATA_FEED_API, PluginDataFeedApi.class, false); + + } + + @Bean + public PluginDatafeedCacheHelper pluginDatafeedCacheHelper() { + return new PluginDatafeedCacheHelper(); + } + + @Bean + public PluginDataFeedJdbc pluginDataFeedJdbc() { + return new PluginDataFeedJdbc(); + } + + @Bean + public PluginStore pluginStore() { + return new PluginStore(); + } + + @Bean + public PluginApi pluginApi(Ignite ignite) { + + DistributedPluginApi distributedPluginApi = new DistributedPluginApi(); + + ignite.services(ClusterGroupDefinition.getMessagingClusterDeploymentGroup(ignite)).deployNodeSingleton(Constants.DISTRIBUTED_PLUGIN_API, distributedPluginApi); + + return distributedPluginApi; + + } + + @Bean + public PluginSelfStopApi pluginSelfStopApi(Ignite ignite) { + + DistributedPluginSelfStopApi distributedPluginSelfStopApi = new DistributedPluginSelfStopApi(); + + ignite.services(ClusterGroupDefinition.getMessagingClusterDeploymentGroup(ignite)).deployNodeSingleton(Constants.DISTRIBUTED_PLUGIN_SELF_STOP_API, distributedPluginSelfStopApi); + + return ignite.services(ClusterGroupDefinition.getMessagingLocalClusterDeploymentGroup(ignite)).serviceProxy(Constants.DISTRIBUTED_PLUGIN_SELF_STOP_API, PluginSelfStopApi.class, false); + + } } diff --git a/src/takserver-core/src/main/java/tak/server/federation/DistributedFederationManager.java b/src/takserver-core/src/main/java/tak/server/federation/DistributedFederationManager.java index 3cf3c598..9814b6d1 100644 --- a/src/takserver-core/src/main/java/tak/server/federation/DistributedFederationManager.java +++ b/src/takserver-core/src/main/java/tak/server/federation/DistributedFederationManager.java @@ -4,6 +4,7 @@ import static com.google.common.base.Preconditions.checkNotNull; +import java.io.ByteArrayInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.security.KeyStore; @@ -52,6 +53,7 @@ import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.context.event.EventListener; +import com.atakmap.Tak.BinaryBlob; import com.atakmap.Tak.CRUD; import com.atakmap.Tak.ContactListEntry; import com.atakmap.Tak.FederatedEvent; @@ -97,6 +99,8 @@ import com.bbn.marti.service.SSLConfig; import com.bbn.marti.service.Subscription; import com.bbn.marti.service.SubscriptionStore; +import com.bbn.marti.sync.EnterpriseSyncService; +import com.bbn.marti.util.CommonUtil; import com.bbn.marti.util.MessageConversionUtil; import com.bbn.marti.util.MessagingDependencyInjectionProxy; import com.bbn.marti.util.spring.SpringContextBeanForApi; @@ -105,6 +109,7 @@ import com.google.common.collect.Multimap; import com.google.common.collect.SortedSetMultimap; import com.google.common.collect.TreeMultimap; +import com.google.protobuf.ByteString; import io.micrometer.core.instrument.Metrics; import tak.server.Constants; @@ -138,7 +143,7 @@ public DistributedFederationManager(NioNettyBuilder nettyBuilder, Ignite ignite, private final AtomicReference> cotMessenger = new AtomicReference<>(); private static AtomicBoolean outgoingsInitiated = new AtomicBoolean(); - + @SuppressWarnings("unchecked") private Messenger messenger() { if (cotMessenger.get() == null) { @@ -2034,12 +2039,60 @@ public List getConfiguredFederates() { return coreConfig.getRemoteConfiguration().getFederation().getFederate(); } - + @Override public void submitFederateROL(final ROL rol, final NavigableSet groups) { + + submitFederateROL(rol, groups, null); + } + + @Override + public void submitFederateROL(ROL rol, final NavigableSet groups, String fileHash) { if (logger.isDebugEnabled()) { logger.debug("Federated ROL: " + rol.getProgram() + " groups: " + groups); } + + + // Populate the file contents here in messaging if no content provided. Avoids serializing file over ignite + try { + if (rol.getPayloadList().isEmpty() && !Strings.isNullOrEmpty(fileHash)) { + + String groupVector = RemoteUtil.getInstance().bitVectorToString(RemoteUtil.getInstance().getBitVectorForGroups(groups)); + + // use the file hash to load the file from db / cache + byte[] fileBytes = MessagingDependencyInjectionProxy.getInstance().esyncService().getContentByHash(fileHash, groupVector); + + if (logger.isDebugEnabled()) { + if (fileBytes == null) { + logger.debug("null bytes for file " + fileHash); + } else { + + logger.debug("fetched " + fileBytes.length + " for hash " + fileHash); + + } + } + + BinaryBlob filePayload = BinaryBlob.newBuilder().setData(ByteString.readFrom(new ByteArrayInputStream(fileBytes))).build(); + + ROL.Builder rolBuilder = rol.toBuilder(); + + rolBuilder.addPayload(filePayload); + + if (logger.isDebugEnabled()) { + logger.debug("Added file payload size " + fileBytes.length + " bytes to rol"); + } + + rol = rolBuilder.build(); + + } + } catch (Exception e) { + if (logger.isWarnEnabled()) { + logger.warn("exception fetching file for federation from data layer", e); + } + } + + final ROL finalRol = rol; + try { @@ -2068,7 +2121,7 @@ public void run() { continue; } - ROL.Builder builder = rol.toBuilder(); + ROL.Builder builder = finalRol.toBuilder(); if (federate.isFederatedGroupMapping()) { Set outGroups = GroupFederationUtil.getInstance().filterFedOutboundGroups( diff --git a/src/takserver-core/src/main/java/tak/server/federation/FederationServer.java b/src/takserver-core/src/main/java/tak/server/federation/FederationServer.java index 0eee685d..161fc50c 100644 --- a/src/takserver-core/src/main/java/tak/server/federation/FederationServer.java +++ b/src/takserver-core/src/main/java/tak/server/federation/FederationServer.java @@ -2,6 +2,9 @@ import static java.util.Objects.requireNonNull; +import com.bbn.marti.config.Federation.Federate; +import com.bbn.marti.config.Federation.FederateCA; + import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -70,9 +73,7 @@ import com.atakmap.Tak.Subscription; import com.bbn.marti.config.Configuration; import com.bbn.marti.config.Federation; -import com.bbn.marti.config.Federation.Federate; -import com.bbn.marti.config.Federation.FederateCA; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.Input; import com.bbn.marti.config.Tls; import com.bbn.marti.groups.CommonGroupDirectedReachability; import com.bbn.marti.groups.GroupFederationUtil; @@ -280,7 +281,7 @@ private void init() { serverConfig.setTruststoreFile(fedServerConfig.getTls().getTruststoreFile()); serverConfig.setTruststorePass(fedServerConfig.getTls().getTruststorePass()); serverConfig.setSkipGateway(true); // eliminate this - serverConfig.setMaxMessageSizeBytes(134217728); // put in coreconfig + serverConfig.setMaxMessageSizeBytes(fedConfig().getFederationServer().getMaxMessageSizeBytes()); // put in coreconfig serverConfig.setMetricsLogIntervalSeconds(60); // put in coreconfig serverConfig.setClientTimeoutTime(15); // put in coreconfig serverConfig.setClientRefreshTime(5); // put in coreconfig @@ -339,10 +340,6 @@ private void start() { try { requireNonNull(config, "v2 fed configuration object"); - if (logger.isDebugEnabled()) { - logger.debug("v2 fed configuration: " + config); - } - sslConfig = new SSLConfig(); if (config.isEnableHealthCheck()) { @@ -1461,7 +1458,7 @@ public void process(ROL clientROL) { logger.debug("submitting federated mission package announcement: " + announceCotResult + " groups " + groups + " to submission service."); } - submission.submitCot(announceCotResult.getAnnoucement(), announceCotResult.getUids(), announceCotResult.getCallsigns(), groups, false); + submission.submitCot(announceCotResult.getAnnoucement(), announceCotResult.getUids(), announceCotResult.getCallsigns(), groups, false, false); } catch (Exception e) { logger.warn("exception processing federated mission package announcement.", e); diff --git a/src/takserver-core/src/main/java/tak/server/federation/TakFigClient.java b/src/takserver-core/src/main/java/tak/server/federation/TakFigClient.java index d597d0a3..db8cb99c 100644 --- a/src/takserver-core/src/main/java/tak/server/federation/TakFigClient.java +++ b/src/takserver-core/src/main/java/tak/server/federation/TakFigClient.java @@ -68,7 +68,7 @@ import com.bbn.marti.config.Federation; import com.bbn.marti.config.Federation.Federate; import com.bbn.marti.config.Federation.FederationOutgoing; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.Input; import com.bbn.marti.config.Tls; import com.bbn.marti.groups.CommonGroupDirectedReachability; import com.bbn.marti.groups.GroupFederationUtil; @@ -103,8 +103,8 @@ import com.bbn.marti.util.concurrent.future.AsyncCallback; import com.bbn.marti.util.concurrent.future.AsyncFuture; import com.bbn.marti.util.spring.SpringContextBeanForApi; -import com.bbn.roger.fig.FigProtocolNegotiator; import com.bbn.roger.fig.FederationUtils; +import com.bbn.roger.fig.FigProtocolNegotiator; import com.bbn.roger.fig.Propagator; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; @@ -247,8 +247,6 @@ public String getClientName() { // track last server health check message private final AtomicReference serverLastHealth = new AtomicReference<>(null); - private static final int SERVER_HEALTH_OFFSET_SEC = 5; - private final AtomicBoolean running = new AtomicBoolean(true); private String federateId; @@ -1400,7 +1398,7 @@ public void process(ROL rol) { logger.debug("submitting federated mission package announcement: " + announceCotResult + " groups " + groups + " to submission service."); } - submission.submitCot(announceCotResult.getAnnoucement(), announceCotResult.getUids(), announceCotResult.getCallsigns(), groups, false); + submission.submitCot(announceCotResult.getAnnoucement(), announceCotResult.getUids(), announceCotResult.getCallsigns(), groups, false, false); } catch (Exception e) { logger.warn("exception processing federated mission package announcement.", e); diff --git a/src/takserver-core/src/main/java/tak/server/messaging/DistributedCotMessenger.java b/src/takserver-core/src/main/java/tak/server/messaging/DistributedCotMessenger.java index a542aa0d..af5621ea 100644 --- a/src/takserver-core/src/main/java/tak/server/messaging/DistributedCotMessenger.java +++ b/src/takserver-core/src/main/java/tak/server/messaging/DistributedCotMessenger.java @@ -1,6 +1,5 @@ package tak.server.messaging; -import java.util.concurrent.atomic.AtomicBoolean; import org.apache.ignite.Ignite; import org.slf4j.Logger; @@ -8,7 +7,7 @@ import com.bbn.marti.remote.CoreConfig; import com.bbn.marti.remote.ServerInfo; -import com.bbn.marti.remote.util.RemoteUtil; +import com.bbn.marti.service.PluginStore; import com.bbn.marti.service.SubmissionService; import com.bbn.marti.service.SubscriptionStore; import com.bbn.marti.util.MessagingDependencyInjectionProxy; @@ -23,7 +22,7 @@ public class DistributedCotMessenger implements Messenger { private boolean isPlugins = false; private boolean isCluster = false; - public DistributedCotMessenger(Ignite ignite, SubscriptionStore subscriptionStore, ServerInfo serverInfo, MessageConverter messageConverter, SubmissionService submissionService, CoreConfig config, RemoteUtil remoteUtil) { + public DistributedCotMessenger(Ignite ignite, SubscriptionStore subscriptionStore, ServerInfo serverInfo, MessageConverter messageConverter, SubmissionService submissionService, CoreConfig config) { this.ignite = ignite; this.subscriptionStore = subscriptionStore; this.serverInfo = serverInfo; @@ -33,7 +32,6 @@ public DistributedCotMessenger(Ignite ignite, SubscriptionStore subscriptionStor this.isCluster = config.getRemoteConfiguration().getCluster() != null && config.getRemoteConfiguration().getCluster().isEnabled(); this.isPlugins = config.getRemoteConfiguration().getPlugins().isUsePluginMessageQueue(); - this.remoteUtil = remoteUtil; } private final Ignite ignite; @@ -53,25 +51,24 @@ public DistributedCotMessenger(Ignite ignite, SubscriptionStore subscriptionStor private static final Logger logger = LoggerFactory.getLogger(DistributedCotMessenger.class); - private AtomicBoolean isIntercept = null; - - private final RemoteUtil remoteUtil; - - @Override public void send(CotEventContainer message) { - if (logger.isTraceEnabled()) { logger.trace("sending message " + message); } - - if (!remoteUtil.getIsIntercept(isIntercept).get()) { + + boolean isControlMessage = submissionService.isControlMessage(message.getType()); + if (isControlMessage) { + submissionService.addToInputQueue(message); + return; + } + + if (PluginStore.getInstance().getInterceptorPluginsActive() == 0 || !isPlugins) { submissionService.addToInputQueue(message); } // push the message to the plugin queue, if plugins and plugin message queue are enabled if (isPlugins) { - try { byte[] rawMessage = messageConverter.cotToDataMessage(new CotEventContainer(message, true, ImmutableSet.of(Constants.SOURCE_TRANSPORT_KEY, Constants.SOURCE_PROTOCOL_KEY, Constants.USER_KEY)), true); diff --git a/src/takserver-core/src/main/java/tak/server/messaging/DistributedPluginApi.java b/src/takserver-core/src/main/java/tak/server/messaging/DistributedPluginApi.java new file mode 100644 index 00000000..2a0d6af3 --- /dev/null +++ b/src/takserver-core/src/main/java/tak/server/messaging/DistributedPluginApi.java @@ -0,0 +1,71 @@ +package tak.server.messaging; + +import org.apache.ignite.events.DiscoveryEvent; +import org.apache.ignite.events.EventType; +import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.services.ServiceContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import com.bbn.marti.remote.exception.TakException; +import com.bbn.marti.service.PluginStore; + +import tak.server.Constants; +import tak.server.PluginManager; +import tak.server.ignite.IgniteHolder; +import tak.server.plugins.PluginApi; +import tak.server.plugins.PluginManagerConstants; + +/* + */ +public class DistributedPluginApi implements PluginApi, org.apache.ignite.services.Service { + private static final long serialVersionUID = -8643385239608114377L; + private static final Logger logger = LoggerFactory.getLogger(DistributedPluginApi.class); + + @Override + public void cancel(ServiceContext ctx) { + if (logger.isDebugEnabled()) { + logger.debug(getClass().getSimpleName() + " service cancelled"); + } + } + + @Override + public void init(ServiceContext ctx) throws Exception { + if (logger.isDebugEnabled()) { + logger.debug("init method " + getClass().getSimpleName()); + } + setupPluginProcessListener(); + } + + @Override + public void execute(ServiceContext ctx) throws Exception { + if (logger.isDebugEnabled()) { + logger.debug("execute method " + getClass().getSimpleName()); + } + } + + @Override + public void addInterceptorPluginsActive(int n) { + PluginStore.getInstance().addInterceptorPluginsActive(n); + } + + private void setupPluginProcessListener() { + // we need to track when the plugin process exits so we can turn off interceptions + IgnitePredicate ignitePredicate = new IgnitePredicate() { + @Override + public boolean apply(DiscoveryEvent event) { + if (PluginManagerConstants.PLUGIN_MANAGER_IGNITE_PROFILE.equals(event.eventNode().attribute(Constants.TAK_PROFILE_KEY))) { + PluginStore.getInstance().disableInterception(); + } + return true; + } + }; + + IgniteHolder.getInstance() + .getIgnite() + .events() + .localListen(ignitePredicate, EventType.EVT_NODE_LEFT, EventType.EVT_NODE_FAILED); + } + +} diff --git a/src/takserver-core/src/main/java/tak/server/messaging/DistributedPluginDataFeedApi.java b/src/takserver-core/src/main/java/tak/server/messaging/DistributedPluginDataFeedApi.java new file mode 100644 index 00000000..f221134c --- /dev/null +++ b/src/takserver-core/src/main/java/tak/server/messaging/DistributedPluginDataFeedApi.java @@ -0,0 +1,279 @@ +package tak.server.messaging; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.ignite.services.ServiceContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import com.bbn.marti.network.PluginDataFeedJdbc; +import com.bbn.marti.remote.groups.NetworkInputAddResult; +import com.bbn.marti.remote.service.InputManager; +import com.bbn.marti.sync.model.DataFeedDao; +import com.bbn.marti.sync.repository.DataFeedRepository; +import com.bbn.marti.config.AuthType; +import com.bbn.marti.config.DataFeed; +import com.bbn.marti.util.MessagingDependencyInjectionProxy; + +import tak.server.Constants; +import tak.server.cache.PluginDatafeedCacheHelper; +import tak.server.feeds.DataFeed.DataFeedType; +import tak.server.ignite.MessagingIgniteBroker; +import tak.server.plugins.PluginDataFeed; +import tak.server.plugins.PluginDataFeedApi; + +public class DistributedPluginDataFeedApi implements PluginDataFeedApi, org.apache.ignite.services.Service { + + private static final long serialVersionUID = 2276211741137405196L; + + private static final Logger logger = LoggerFactory.getLogger(DistributedPluginDataFeedApi.class); + + @Autowired + private PluginDatafeedCacheHelper pluginDatafeedCacheHelper; + + @Autowired + private PluginDataFeedJdbc pluginDataFeedJdbc; + + private DataFeedRepository dataFeedRepository() { + return MessagingDependencyInjectionProxy.getInstance().dataFeedRepository(); + } + + + @Override + public PluginDataFeed create(String uuid, String name, List tags, boolean archive, boolean sync) { + + logger.info("Calling create() method in DistributedPluginDataFeedApi, uuid: {}, name: {}, tags: {}, archive: {}, sync: {}", uuid, name, tags, archive, sync); + + DataFeedRepository dataFeedRepository = dataFeedRepository(); + + Long dataFeedId; + if (dataFeedRepository.getDataFeedByUUID(uuid).size() > 0) { + + logger.info("Updating datafeed uuid {}", uuid); + + try { + // Updating dataFeed in metrics for UI + DataFeed dataFeed = new DataFeed(); + dataFeed.setName(name); + dataFeed.setType("Plugin"); + dataFeed.setUuid(uuid); + for (String tag: tags) { + dataFeed.getTag().add(tag); + } + dataFeed.setAuth(AuthType.ANONYMOUS); + dataFeed.setPort(0); + dataFeed.setProtocol("Plugin"); + dataFeed.setAuthRequired(false); + dataFeed.setGroup(""); + dataFeed.setIface(""); + dataFeed.setArchive(archive); + dataFeed.setAnongroup(false); + dataFeed.setArchiveOnly(false); + dataFeed.setCoreVersion(0); + dataFeed.setCoreVersion2TlsVersions(""); + dataFeed.setSync(sync); + + MessagingIgniteBroker.brokerServiceCalls(service -> ((InputManager) service) + .modifyInput(name, dataFeed), Constants.DISTRIBUTED_INPUT_MANAGER, InputManager.class); + + } catch (Exception e) { + logger.warn("Exception updating plugin data feed for UI", e.getMessage()); + } + + dataFeedId = dataFeedRepository.updateDataFeed(uuid, name, DataFeedType.Plugin.ordinal(), + AuthType.ANONYMOUS.toString(), 0, false, "", + null, null, archive, false, false ,0, "", sync); + + logger.info("Updated datafeed uuid {}, row id", uuid, dataFeedId); + + try { + dataFeedRepository.removeAllDataFeedTagsById(dataFeedId); + logger.info("Removed old tags for datafeed row id {}", dataFeedId); + }catch(Exception e) { + logger.warn("Exception when removing all tags for datafeed row id {}. Reason: {}", dataFeedId, e.getMessage()); + } + + if (tags.size() > 0) { + for (String tag : tags) { + dataFeedRepository.addDataFeedTag(dataFeedId, tag); + } + logger.info("Added new tags for datafeed row id {}", dataFeedId); + } + + // update cache + PluginDataFeed re = new PluginDataFeed(uuid, name, tags, archive, sync); + List pluginDataFeeds = new ArrayList(); + pluginDataFeeds.add(re); + pluginDatafeedCacheHelper.cachePluginDatafeed(uuid, pluginDataFeeds); + + pluginDatafeedCacheHelper.invalidate(PluginDatafeedCacheHelper.ALL_PLUGIN_DATAFEED_KEY); + + return re; + + } else { + + logger.info("Adding datafeed uuid {}", uuid); + + try { + // Add dataFeed to metrics for UI + DataFeed dataFeed = new DataFeed(); + dataFeed.setName(name); + dataFeed.setType("Plugin"); + dataFeed.setUuid(uuid); + for (String tag: tags) { + dataFeed.getTag().add(tag); + } + dataFeed.setAuth(AuthType.ANONYMOUS); + dataFeed.setPort(0); + dataFeed.setProtocol("Plugin"); + dataFeed.setAuthRequired(false); + dataFeed.setGroup(""); + dataFeed.setIface(""); + dataFeed.setArchive(archive); + dataFeed.setAnongroup(false); + dataFeed.setArchiveOnly(false); + dataFeed.setCoreVersion(0); + dataFeed.setCoreVersion2TlsVersions(""); + dataFeed.setSync(sync); + + MessagingIgniteBroker.brokerServiceCalls(service -> ((InputManager) service) + .createDataFeed(dataFeed), Constants.DISTRIBUTED_INPUT_MANAGER, InputManager.class); + + } catch (Exception e) { + logger.warn("Exception adding plugin data feed for UI", e.getMessage()); + } + + dataFeedId = dataFeedRepository.addDataFeed(uuid, name, DataFeedType.Plugin.ordinal(), + AuthType.ANONYMOUS.toString(), 0, false, "", + null, null, archive, false, false ,0, "", sync); + + logger.info("Added datafeed uuid {}, row id", uuid, dataFeedId); + + try { + dataFeedRepository.removeAllDataFeedTagsById(dataFeedId); + logger.info("Removed old tags for datafeed row id {}", dataFeedId); + }catch(Exception e) { + logger.warn("Exception when removing all tags for datafeed row id {}. Reason: {}", dataFeedId, e.getMessage()); + } + + if (tags.size() > 0) { + for (String tag : tags) { + dataFeedRepository.addDataFeedTag(dataFeedId, tag); + } + logger.info("Added new tags for datafeed row id {}", dataFeedId); + } + + // update cache + PluginDataFeed re = new PluginDataFeed(uuid, name, tags, archive, sync); + List pluginDataFeeds = new ArrayList(); + pluginDataFeeds.add(re); + pluginDatafeedCacheHelper.cachePluginDatafeed(uuid, pluginDataFeeds); + + pluginDatafeedCacheHelper.invalidate(PluginDatafeedCacheHelper.ALL_PLUGIN_DATAFEED_KEY); + + return re; + } + } + + @Override + public PluginDataFeed create(String uuid, String name, List tags) { + + return create(uuid, name, tags, true, false); + + } + + @Override + public void delete(String uuid) { + + logger.info("Calling delete method in DistributedPluginDataFeedApi, uuid: {}", uuid); + + DataFeedRepository dataFeedRepository = dataFeedRepository(); + + List dataFeedDaos = dataFeedRepository.getDataFeedByUUID(uuid); + + if (dataFeedDaos.size() == 0) { + logger.warn("There is no datafeed with uuid {} to delete", uuid); + return; + } + + if (dataFeedDaos.size() > 1) { + logger.warn("There are {} datafeeds with uuid {}", dataFeedDaos.size(), uuid); + return; + } + + for (DataFeedDao dataFeedDao: dataFeedDaos) { + + logger.info("Removing all tags for datafeed row id {}", dataFeedDao.getId()); + try { + dataFeedRepository.removeAllDataFeedTagsById(dataFeedDao.getId()); + logger.info("Removed all tags for datafeed row id {}", dataFeedDao.getId()); + }catch(Exception e) { + logger.warn("Exception when removing all tags for datafeed row id {}. Reason: {}", dataFeedDao.getId(), e.getMessage()); + } + + logger.info("Deleting datafeed row id {}", dataFeedDao.getId()); + dataFeedRepository.deleteDataFeedById(dataFeedDao.getId()); + logger.info("Deleted datafeed row id {}", dataFeedDao.getId()); + } + + // update cache + pluginDatafeedCacheHelper.cachePluginDatafeed(uuid, new ArrayList()); + + pluginDatafeedCacheHelper.invalidate(PluginDatafeedCacheHelper.ALL_PLUGIN_DATAFEED_KEY); + + } + + @Override + public Collection getAllPluginDataFeeds() { + + logger.info("Calling getAllPluginDataFeeds() in DistributedPluginDataFeedApi"); + + List allPluginDatafeeds = pluginDatafeedCacheHelper.getAllPluginDatafeeds(); + + if (allPluginDatafeeds == null) { // not in cache + + if (logger.isDebugEnabled()) { + logger.debug("Result not in cache"); + } + + allPluginDatafeeds = pluginDataFeedJdbc.getPluginDataFeeds(); + + // cache the result + pluginDatafeedCacheHelper.cacheAllPluginDatafeeds(allPluginDatafeeds); + }else { + + if (logger.isDebugEnabled()) { + logger.debug("Get result from cache"); + } + } + + return allPluginDatafeeds; + } + + + @Override + public void cancel(ServiceContext ctx) { + if (logger.isDebugEnabled()) { + logger.debug(getClass().getSimpleName() + " service cancelled"); + } + } + + @Override + public void init(ServiceContext ctx) throws Exception { + + if (logger.isDebugEnabled()) { + logger.debug("init method " + getClass().getSimpleName()); + } + } + + @Override + public void execute(ServiceContext ctx) throws Exception { + if (logger.isDebugEnabled()) { + logger.debug("execute method " + getClass().getSimpleName()); + } + } + +} diff --git a/src/takserver-core/src/main/java/tak/server/messaging/DistributedPluginSelfStopApi.java b/src/takserver-core/src/main/java/tak/server/messaging/DistributedPluginSelfStopApi.java new file mode 100644 index 00000000..ff31060f --- /dev/null +++ b/src/takserver-core/src/main/java/tak/server/messaging/DistributedPluginSelfStopApi.java @@ -0,0 +1,72 @@ +package tak.server.messaging; + +import org.apache.ignite.Ignite; +import org.apache.ignite.services.ServiceContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import com.bbn.cluster.ClusterGroupDefinition; + +import tak.server.PluginManager; +import tak.server.plugins.PluginSelfStopApi; +import tak.server.Constants; + +public class DistributedPluginSelfStopApi implements PluginSelfStopApi, org.apache.ignite.services.Service { + + private static final long serialVersionUID = 2276211741137405196L; + + private static final Logger logger = LoggerFactory.getLogger(DistributedPluginSelfStopApi.class); + + @Autowired + Ignite ignite; + + private PluginManager pluginManager() throws Exception { + + if (logger.isDebugEnabled()) { + logger.debug("get PluginManager from ignite"); + } + + return ignite.services(ClusterGroupDefinition.getPluginManagerClusterDeploymentGroup(ignite)).serviceProxy(Constants.DISTRIBUTED_PLUGIN_MANAGER, PluginManager.class, false); + } + + @Override + public void cancel(ServiceContext ctx) { + if (logger.isDebugEnabled()) { + logger.debug(getClass().getSimpleName() + " service cancelled"); + } + } + + @Override + public void init(ServiceContext ctx) throws Exception { + + if (logger.isDebugEnabled()) { + logger.debug("init method " + getClass().getSimpleName()); + } + } + + @Override + public void execute(ServiceContext ctx) throws Exception { + if (logger.isDebugEnabled()) { + logger.debug("execute method " + getClass().getSimpleName()); + } + } + + @Override + public void pluginSelfStop(String pluginName) { + + PluginManager pluginManager; + try { + pluginManager = pluginManager(); + if (pluginManager == null) { + logger.error("Plugin Manager is NULL"); + }else { + pluginManager.stopPluginByName(pluginName); + } + } catch (Exception e) { + logger.error("Error in pluginSelfStop in DistributedSelfStopApi", e); + } + + } + +} diff --git a/src/takserver-core/src/main/java/tak/server/messaging/MessageConverter.java b/src/takserver-core/src/main/java/tak/server/messaging/MessageConverter.java index 77268cfc..60446ab9 100644 --- a/src/takserver-core/src/main/java/tak/server/messaging/MessageConverter.java +++ b/src/takserver-core/src/main/java/tak/server/messaging/MessageConverter.java @@ -19,7 +19,8 @@ import com.atakmap.Tak.ROL; import com.bbn.cluster.ClusterGroupDefinition; import com.bbn.cot.CotParserCreator; -import com.bbn.marti.config.Network.Input; +import com.bbn.cot.filter.StreamingEndpointRewriteFilter; +import com.bbn.marti.config.Input; import com.bbn.marti.remote.ServerInfo; import com.bbn.marti.remote.exception.NotFoundException; import com.bbn.marti.remote.exception.TakException; @@ -119,12 +120,32 @@ public byte[] cotToDataMessage(CotEventContainer message, boolean padEmptyGroups if (clientId != null) { mb.setClientId(clientId); } + + String connectionId = (String)message.getContext().get(Constants.CONNECTION_ID_KEY); + if (connectionId != null) { + mb.setConnectionId(connectionId); + } @SuppressWarnings("unchecked") List provenance = (List)message.getContext().get(Constants.PLUGIN_PROVENANCE); if (provenance != null) { mb.addAllProvenance(provenance); } + + Boolean archivedEnabled = (Boolean)message.getContext().get(Constants.ARCHIVE_EVENT_KEY); + if (archivedEnabled != null) { + mb.setArchive(archivedEnabled.booleanValue()); + } + + List destUids = (List) message.getContextValue(StreamingEndpointRewriteFilter.EXPLICIT_UID_KEY); + if (destUids != null && !destUids.isEmpty()) { + mb.addAllDestClientUids(destUids); + } + + List destCallsigns = (List) message.getContextValue(StreamingEndpointRewriteFilter.EXPLICIT_CALLSIGN_KEY); + if (destCallsigns != null && !destCallsigns.isEmpty()) { + mb.addAllDestCallsigns(destCallsigns); + } if (logger.isTraceEnabled()) { logger.trace("TAK proto message converted: " + mb); @@ -231,11 +252,30 @@ public CotEventContainer dataMessageToCot(Message m, boolean setClusterKey) thro if (clientId != null) { cot.setContext(Constants.CLIENT_UID_KEY, clientId); } + + String connectionId = m.getConnectionId(); + if (connectionId != null) { + cot.setContext(Constants.CONNECTION_ID_KEY, connectionId); + } List provenance = m.getProvenanceList(); if (provenance != null) { cot.setContextValue(Constants.PLUGIN_PROVENANCE, provenance); } + + if (provenance != null && provenance.contains(Constants.PLUGIN_MANAGER_PROVENANCE)) { + cot.setContextValue(Constants.ARCHIVE_EVENT_KEY, m.getArchive()); + } + + List destUids = m.getDestClientUidsList(); + if (destUids != null && !destUids.isEmpty()) { + cot.setContext(StreamingEndpointRewriteFilter.EXPLICIT_UID_KEY, destUids); + } + + List destCallsigns = m.getDestCallsignsList(); + if (destCallsigns != null && !destCallsigns.isEmpty()) { + cot.setContext(StreamingEndpointRewriteFilter.EXPLICIT_CALLSIGN_KEY, destCallsigns); + } if (logger.isTraceEnabled()) { logger.trace("CoT from data message: " + cot.asXml()); diff --git a/src/takserver-core/src/main/resources/app-context.xml b/src/takserver-core/src/main/resources/app-context.xml index ac22b78d..b5c4de78 100644 --- a/src/takserver-core/src/main/resources/app-context.xml +++ b/src/takserver-core/src/main/resources/app-context.xml @@ -54,7 +54,7 @@ + com.bbn.marti.maplayer.model, com.bbn.user.registration.model, com.bbn.marti.video"/> @@ -66,7 +66,7 @@ +com.bbn.marti.maplayer.repository, com.bbn.user.registration.repository, com.bbn.marti.video" /> diff --git a/src/takserver-core/src/main/resources/security-context.xml b/src/takserver-core/src/main/resources/security-context.xml index cae29c06..b11d8cd9 100644 --- a/src/takserver-core/src/main/resources/security-context.xml +++ b/src/takserver-core/src/main/resources/security-context.xml @@ -74,9 +74,15 @@ + + + + + + @@ -173,6 +179,7 @@ + @@ -194,7 +201,8 @@ - + + @@ -221,6 +229,7 @@ + @@ -332,6 +341,7 @@ + @@ -344,6 +354,8 @@ + + diff --git a/src/takserver-core/src/main/webapp/Marti/MissionSend.jsp b/src/takserver-core/src/main/webapp/Marti/MissionSend.jsp index 3cddd6d0..980f3089 100644 --- a/src/takserver-core/src/main/webapp/Marti/MissionSend.jsp +++ b/src/takserver-core/src/main/webapp/Marti/MissionSend.jsp @@ -29,6 +29,7 @@ addCell(row, cell++, document.createTextNode(_mission.description), 'top'); addCell(row, cell++, createDiv(createContents(_mission.contents)), 'top'); addCell(row, cell++, createDiv(createUids(_mission.uids)), 'top'); + addCell(row, cell++, createDiv(createGroups(_mission.groups)), 'top'); addCell(row, cell++, createDiv(createKeywords(_mission.keywords)), 'top'); addCell(row, cell++, document.createTextNode(_mission.creatorUid), 'top'); addCell(row, cell++, document.createTextNode(_mission.createTime), 'top'); @@ -173,6 +174,7 @@ Description Contents UIDs + Groups Keywords Creator Uid Create Time diff --git a/src/takserver-core/src/main/webapp/Marti/Missions.js b/src/takserver-core/src/main/webapp/Marti/Missions.js index 1ef43972..d985099b 100644 --- a/src/takserver-core/src/main/webapp/Marti/Missions.js +++ b/src/takserver-core/src/main/webapp/Marti/Missions.js @@ -86,6 +86,16 @@ function displayInUnits ( seconds ) { return 'less than a second'; //'just now' //or other string you like; } +function createGroups(groups) { + var html = ""; + for (var i = 0; i < groups.length - 1; i++) { + html += sanitize(groups[i]); + html += ", "; + } + html += sanitize(groups[groups.length - 1]) + return html; +} + function createKeywords(keywords) { var html = ""; for (var i = 0; i < keywords.length; i++) { @@ -114,6 +124,7 @@ function createMissionTable(missions) { addCell(row, cell++, document.createTextNode(missions[i].description), 'top'); addCell(row, cell++, createDiv(createContents(missions[i].contents)), 'top'); addCell(row, cell++, createDiv(createUids(missions[i].uids)), 'top'); + addCell(row, cell++, createDiv(createGroups(missions[i].groups)), 'top'); addCell(row, cell++, createDiv(createKeywords(missions[i].keywords)), 'top'); addCell(row, cell++, document.createTextNode(missions[i].creatorUid), 'top'); addCell(row, cell++, document.createTextNode(missions[i].createTime), 'top'); diff --git a/src/takserver-core/src/main/webapp/Marti/Missions.jsp b/src/takserver-core/src/main/webapp/Marti/Missions.jsp index 7ebf53d8..bb611c7b 100644 --- a/src/takserver-core/src/main/webapp/Marti/Missions.jsp +++ b/src/takserver-core/src/main/webapp/Marti/Missions.jsp @@ -97,6 +97,7 @@ addCell(row, cell++, document.createTextNode(missions[i].description), 'top'); addCell(row, cell++, createDiv(createContents(missions[i].contents)), 'top'); addCell(row, cell++, createDiv(createUids(missions[i].uids)), 'top'); + addCell(row, cell++, createDiv(createGroups(missions[i].groups)), 'top'); addCell(row, cell++, createDiv(createKeywords(missions[i].keywords)), 'top'); addCell(row, cell++, document.createTextNode(missions[i].creatorUid), 'top'); addCell(row, cell++, document.createTextNode(missions[i].createTime), 'top'); @@ -193,6 +194,7 @@ Description Contents UIDs + Groups Keywords Creator Uid Create Time diff --git a/src/takserver-core/src/main/webapp/Marti/VideoManager.jsp b/src/takserver-core/src/main/webapp/Marti/VideoManager.jsp index 3af7441e..5f0fb4d0 100644 --- a/src/takserver-core/src/main/webapp/Marti/VideoManager.jsp +++ b/src/takserver-core/src/main/webapp/Marti/VideoManager.jsp @@ -6,6 +6,7 @@ <%@ page import="com.bbn.marti.video.VideoManagerService" %> <%@ page import="com.bbn.marti.video.VideoConnections" %> <%@ page import="org.owasp.esapi.ESAPI" %> +<%@ page import="com.bbn.marti.util.CommonUtil" %> @@ -176,7 +177,9 @@ <% - VideoConnections videoConnections = videoService.getVideoConnections(false); + CommonUtil martiUtil = SpringContextBeanForApi.getSpringContext().getBean(CommonUtil.class); + String groupVector = martiUtil.getGroupBitVector(request); + VideoConnections videoConnections = videoService.getVideoConnections(false, true, groupVector); for (Feed feed : videoConnections.getFeeds()) { %> diff --git a/src/takserver-core/src/main/webapp/Marti/VideoSend.jsp b/src/takserver-core/src/main/webapp/Marti/VideoSend.jsp index 0f6caac1..18df7ae6 100644 --- a/src/takserver-core/src/main/webapp/Marti/VideoSend.jsp +++ b/src/takserver-core/src/main/webapp/Marti/VideoSend.jsp @@ -14,6 +14,7 @@ <%@ page import="com.bbn.marti.remote.RemoteSubscription" %> <%@ page import="org.owasp.esapi.ESAPI" %> <%@ page import="com.bbn.marti.util.spring.SpringContextBeanForApi" %> +<%@ page import="com.bbn.marti.util.CommonUtil" %> <%@ page language="java" contentType="text/html; charset=UTF-8" pageEncoding="UTF-8"%> @@ -80,9 +81,12 @@ <% + CommonUtil martiUtil = SpringContextBeanForApi.getSpringContext().getBean(CommonUtil.class); + String groupVector = martiUtil.getGroupBitVector(request); + String[] feedIds = request.getParameter("feedId").split("\\|"); for (String feedId : feedIds) { - Feed feed = com.bbn.marti.util.spring.SpringContextBeanForApi.getSpringContext().getBean(com.bbn.marti.video.VideoManagerService.class).getFeed(Integer.parseInt(feedId)); + Feed feed = com.bbn.marti.util.spring.SpringContextBeanForApi.getSpringContext().getBean(com.bbn.marti.video.VideoManagerService.class).getFeed(Integer.parseInt(feedId), groupVector); %> <%=feed.getType().toString()%> diff --git a/src/takserver-core/src/main/webapp/Marti/VideoUpload.jsp b/src/takserver-core/src/main/webapp/Marti/VideoUpload.jsp index 793eebd9..5115ce94 100644 --- a/src/takserver-core/src/main/webapp/Marti/VideoUpload.jsp +++ b/src/takserver-core/src/main/webapp/Marti/VideoUpload.jsp @@ -3,10 +3,15 @@ <%@ page import="java.lang.*" %> <%@ page import="org.owasp.esapi.ESAPI" %> <%@ page import="com.bbn.marti.video.Feed" %> +<%@ page import="com.bbn.marti.util.CommonUtil" %> +<%@ page import="com.bbn.marti.util.spring.SpringContextBeanForApi" %> <% - String title = "Add"; + CommonUtil martiUtil = SpringContextBeanForApi.getSpringContext().getBean(CommonUtil.class); + String groupVector = martiUtil.getGroupBitVector(request); + + String title = "Add"; String action = "Add"; String active = "true"; String uuid = UUID.randomUUID().toString(); @@ -33,7 +38,7 @@ if (feedId != null) { title = "Edit"; action = "Save"; - Feed feed = com.bbn.marti.util.spring.SpringContextBeanForApi.getSpringContext().getBean(com.bbn.marti.video.VideoManagerService.class).getFeed(Integer.parseInt(feedId)); + Feed feed = com.bbn.marti.util.spring.SpringContextBeanForApi.getSpringContext().getBean(com.bbn.marti.video.VideoManagerService.class).getFeed(Integer.parseInt(feedId), groupVector); uuid = feed.getUuid(); active = Boolean.toString(feed.getActive()); alias = feed.getAlias(); diff --git a/src/takserver-core/src/main/webapp/Marti/data_retention/js/controllers.js b/src/takserver-core/src/main/webapp/Marti/data_retention/js/controllers.js index 67ee2a24..cba59095 100644 --- a/src/takserver-core/src/main/webapp/Marti/data_retention/js/controllers.js +++ b/src/takserver-core/src/main/webapp/Marti/data_retention/js/controllers.js @@ -309,7 +309,7 @@ dataRetentionControllers.controller('MissionArchiveCtrl', [ ) { $scope.missions = {} - $scope.missions.missonArchiveStoreEntries = [] + $scope.missions.missionArchiveStoreEntries = [] $scope.missions.queryMissionName = '' $scope.missions.queryCreateTime = '' @@ -319,11 +319,11 @@ dataRetentionControllers.controller('MissionArchiveCtrl', [ $window.location.href = '/Marti/data_retention/index.html#!/'; } - $scope.restoreMission = function(missonArchiveStoreEntry) { - $scope.missions.missonArchiveStoreEntries = $scope.missions.missonArchiveStoreEntries.filter(function(entry, index, arr) { - return entry.id !== missonArchiveStoreEntry.id; + $scope.restoreMission = function(missionArchiveStoreEntry) { + $scope.missions.missionArchiveStoreEntries = $scope.missions.missionArchiveStoreEntries.filter(function(entry, index, arr) { + return entry.id !== missionArchiveStoreEntry.id; }); - MissionRestoreService.save(missonArchiveStoreEntry.id, + MissionRestoreService.save(missionArchiveStoreEntry.id, function(apiResponse) { alert(apiResponse.data) } @@ -332,11 +332,11 @@ dataRetentionControllers.controller('MissionArchiveCtrl', [ MissionArchiveService.query(res =>{ let data = JSON.parse(res.data) - if (data && data.missonArchiveStoreEntries) { - $scope.missions.missonArchiveStoreEntries = data.missonArchiveStoreEntries; - console.log($scope.missonArchiveStoreEntries) + if (data && data.missionArchiveStoreEntries) { + $scope.missions.missionArchiveStoreEntries = data.missionArchiveStoreEntries; + console.log($scope.missionArchiveStoreEntries) } else { - $scope.missions.missonArchiveStoreEntries = [] + $scope.missions.missionArchiveStoreEntries = [] } }) @@ -373,7 +373,7 @@ dataRetentionControllers.controller('MissionArchiveCtrl', [ $scope.archiveMissionByNoContentActivity = false; $scope.archiveMissionByNoSubscriptionActivity = false; - $scope.archiveAfterNoActivityDays = 365; + $scope.timeToArchiveAfterNoActivityDays = 365; $scope.removeFromArchiveAfterDays = 1065; MissionArchiveConfigService.query( diff --git a/src/takserver-core/src/main/webapp/Marti/data_retention/partials/missionArchive.html b/src/takserver-core/src/main/webapp/Marti/data_retention/partials/missionArchive.html index 5bcb7696..342d16df 100644 --- a/src/takserver-core/src/main/webapp/Marti/data_retention/partials/missionArchive.html +++ b/src/takserver-core/src/main/webapp/Marti/data_retention/partials/missionArchive.html @@ -11,7 +11,7 @@

Mission Archive Configuration



Archive Missions By Subscription Activity

- Number of days to archive the mission after no activity + Number of days to archive the mission after no activity

Number of days to remove an archived mission from the archive

@@ -94,15 +94,15 @@

Mission Archive Entry

- - {{missonArchiveStoreEntry.missionName}} + + {{missionArchiveStoreEntry.missionName}} - {{missonArchiveStoreEntry.archiveTime}} + {{missionArchiveStoreEntry.archiveTime}} - {{missonArchiveStoreEntry.createTime}} + {{missionArchiveStoreEntry.createTime}} - + diff --git a/src/takserver-core/src/main/webapp/Marti/injectors/partials/new.html b/src/takserver-core/src/main/webapp/Marti/injectors/partials/new.html index abbee771..a5bc7084 100644 --- a/src/takserver-core/src/main/webapp/Marti/injectors/partials/new.html +++ b/src/takserver-core/src/main/webapp/Marti/injectors/partials/new.html @@ -78,7 +78,7 @@

Create Injector

- +
Provide a name (e.g., __video) follow by one or more name=value pairs, each pair separated by a space
Example: __video url="udp://224.10.10.1:1234" test="value" diff --git a/src/takserver-core/src/main/webapp/Marti/inputs/index.html b/src/takserver-core/src/main/webapp/Marti/inputs/index.html index ca6c7fb3..eb4229cd 100644 --- a/src/takserver-core/src/main/webapp/Marti/inputs/index.html +++ b/src/takserver-core/src/main/webapp/Marti/inputs/index.html @@ -3,7 +3,7 @@ - Input and QoS Configuration + Input and Data Feed Configuration diff --git a/src/takserver-core/src/main/webapp/Marti/inputs/js/app.js b/src/takserver-core/src/main/webapp/Marti/inputs/js/app.js index 9bf131c6..d5b1211c 100644 --- a/src/takserver-core/src/main/webapp/Marti/inputs/js/app.js +++ b/src/takserver-core/src/main/webapp/Marti/inputs/js/app.js @@ -21,6 +21,22 @@ app.config(['$routeProvider', templateUrl: 'partials/modify.html', controller: 'InputModificationCtrl' }). + when('/createStreamingDataFeed', { + templateUrl: 'partials/newStreamingDataFeed.html', + controller: 'DataFeedCreationCtrl' + }). + when('/modifyStreamingDataFeed/:name', { + templateUrl: 'partials/modifyStreamingDataFeed.html', + controller: 'DataFeedModificationCtrl' + }). + when('/createPluginDataFeed', { + templateUrl: 'partials/newPluginDataFeed.html', + controller: 'DataFeedCreationCtrl' + }). + when('/modifyPluginDataFeed/:name', { + templateUrl: 'partials/modifyPluginDataFeed.html', + controller: 'DataFeedModificationCtrl' + }). otherwise({ redirectTo: '/' }); diff --git a/src/takserver-core/src/main/webapp/Marti/inputs/js/controllers.js b/src/takserver-core/src/main/webapp/Marti/inputs/js/controllers.js index 60ef6411..ea6e6209 100644 --- a/src/takserver-core/src/main/webapp/Marti/inputs/js/controllers.js +++ b/src/takserver-core/src/main/webapp/Marti/inputs/js/controllers.js @@ -3,8 +3,8 @@ var inputManagerControllers = angular.module('inputManagerControllers', []); inputManagerControllers.controller('InputListCtrl', ['$rootScope', '$window', '$scope', '$http', '$location', '$timeout', '$interval', - 'InputManagerService', 'MessagingConfigService', 'securityConfigService', 'MetricsService', 'QosService', - function ($rootScope, $window, $scope, $http, $location, $timeout, $interval, InputManagerService, MessagingConfigService, securityConfigService, MetricsService, QosService) { + 'InputManagerService', 'DataFeedManagerService', 'MessagingConfigService', 'securityConfigService', 'MetricsService', 'QosService', + function ($rootScope, $window, $scope, $http, $location, $timeout, $interval, InputManagerService, DataFeedManagerService, MessagingConfigService, securityConfigService, MetricsService, QosService) { $scope.hasAdminRole = false; $scope.dbIsConnected = true; $scope.maxConnections = 0; @@ -138,10 +138,28 @@ inputManagerControllers.controller('InputListCtrl', ['$rootScope', '$window', '$ $scope.showRmiError = false; (function refreshMetrics() {InputManagerService.query( - function(apiResponse) { $scope.inputMetrics = apiResponse.data; + function(apiResponse) { $scope.inputs = apiResponse.data; + $scope.inputMetrics = []; + $scope.streamingDataFeeds = []; + $scope.pluginDataFeeds = []; + let inputLength = apiResponse.data.length; + for (let i = 0; i < inputLength; i ++) { + let loopInput = apiResponse.data[i]; + if (loopInput.input.type != null){ + if (loopInput.input.type === 'Streaming') { + $scope.streamingDataFeeds.push(loopInput); + } else if (loopInput.input.type === 'Plugin') { + $scope.pluginDataFeeds.push(loopInput); + } + } else { + $scope.inputMetrics.push(loopInput); + } + } + $timeout(refreshMetrics,2000)}, function() {$scope.showRmiError = true;})})(); + $scope.deleteObject = function (inputName) { if (inputName.trim() != '') { @@ -157,6 +175,21 @@ inputManagerControllers.controller('InputListCtrl', ['$rootScope', '$window', '$ } }; + $scope.deleteDataFeed = function (dataFeedName) { + + if (dataFeedName.trim() != '') { + if (confirm('Proceed to delete the selected data feed? (Data Feed Name: ' + dataFeedName + ')')) { + var dms = new DataFeedManagerService(); + dms.$delete({name:dataFeedName}, + function(data) { + $scope.dataFeeds = DataFeedManagerService.query(function(apiResponse) {$scope.dataFeeds = apiResponse.data});}, + function(data) { + alert('An error occurred deleting the data feed.')} + ); + } + } + }; + $scope.modifyObject = function() { //Placeholder if we need preprocessing }; @@ -165,6 +198,11 @@ inputManagerControllers.controller('InputListCtrl', ['$rootScope', '$window', '$ //Placeholder if we need preprocessing }; + $scope.createDataFeed = function () { + //Placeholder if we need preprocessing + }; + + $scope.actualNum = 0; function getMsgConfig() { MessagingConfigService.query( @@ -246,6 +284,11 @@ inputManagerControllers.controller('InputCreationCtrl', function(apiResponse) {$scope.inputNameDuplicate = false;}); } } + + $scope.cancelInput = function() { + $location.path("/"); + }; + }]); inputManagerControllers.controller('InputModificationCtrl', ['$scope', '$location', 'InputManagerService', '$routeParams', @@ -297,3 +340,119 @@ inputManagerControllers.controller('InputModificationCtrl', ['$scope', '$locatio }]); +inputManagerControllers.controller('DataFeedCreationCtrl', + ['$scope', '$location', 'DataFeedManagerService', '$routeParams', + function ($scope, $location, DataFeedManagerService, $routeParams) { + $scope.dataFeed = new DataFeedManagerService(); + $scope.dataFeed.name = ''; + $scope.dataFeed.type = 'Streaming'; + $scope.dataFeed.tags = ''; + $scope.dataFeed.auth = 'X_509'; + $scope.dataFeed.protocol = 'tls'; + $scope.dataFeed.archive = 'true'; + $scope.dataFeed.anongroup = 'false'; + $scope.dataFeed.archiveOnly = 'false'; + $scope.dataFeed.coreVersion = "2"; + $scope.dataFeed.protocol = 'tls'; + $scope.dataFeed.sync = 'false'; + $scope.inputNameDuplicate = false; + $scope.serviceReportedMessages = false; + $scope.messages = []; + $scope.submitInProgress = false; + $scope.tls = { "TLSv1": false, "TLSv1.1": false, "TLSv1.2": true, "TLSv1.3": true }; + + $scope.saveStreamingDataFeed = function (dataFeed) { + $scope.submitInProgress = true; + $scope.dataFeed.type = 'Streaming'; + DataFeedManagerService.save(dataFeed, + function(apiResponse) { + $location.path('/');}, + function(apiResponse) { + $scope.serviceReportedMessages = true; + $scope.messages = apiResponse.data.messages; + alert('An error occurred saving the input definition. Please correct the errors and resubmit.'); + $scope.submitInProgress = false;} + ); + } + + $scope.savePluginDataFeed = function (dataFeed) { + $scope.submitInProgress = true; + $scope.dataFeed.type = 'Plugin'; + $scope.dataFeed.auth = 'ANONYMOUS'; + $scope.dataFeed.protocol = 'tls'; + $scope.dataFeed.anongroup = 'false'; + $scope.dataFeed.archiveOnly = 'false'; + $scope.dataFeed.coreVersion = "2"; + $scope.dataFeed.protocol = ''; + DataFeedManagerService.save(dataFeed, + function(apiResponse) { + $location.path('/');}, + function(apiResponse) { + $scope.serviceReportedMessages = true; + $scope.messages = apiResponse.data.messages; + alert('An error occurred saving the input definition. Please correct the errors and resubmit.'); + $scope.submitInProgress = false;} + ); + } + + $scope.isInputNameUnique = function(inputName) { + + if (inputName != null && inputName.trim() != '') { + DataFeedManagerService.get({name:inputName}, + function(apiResponse) {if (apiResponse.data != null) {$scope.inputNameDuplicate = true;} else {$scope.inputNameDuplicate = false};}, + function(apiResponse) {$scope.inputNameDuplicate = false;}); + } + } + + + $scope.cancelInput = function() { + $location.path("/"); + } + +}]); + +inputManagerControllers.controller('DataFeedModificationCtrl', ['$scope', '$location', 'DataFeedManagerService', '$routeParams', + function ($scope, $location, DataFeedManagerService, $routeParams) { + + $scope.boolToStr = function(arg) {return arg ? 'True' : 'False'}; + + $scope.queryDataFeed = function(dataFeedName) { + DataFeedManagerService.query( + {name: dataFeedName}, + function(apiResponse) { + $scope.dataFeed = apiResponse.data; + $scope.hideArchive = false; + $scope.hideArchiveOnly = false; + var protocol = apiResponse.data.protocol.toLowerCase(); + $scope.hideFilterGroupList = (apiResponse.data.auth.toLowerCase() != 'anonymous'); + }, + function(apiResponse) { + $scope.showRmiError = true; + } + ); + } + + $scope.queryDataFeed($routeParams.name); + + $scope.updateDataFeed = function(updatedDataFeed) { + updatedDataFeed.filtergroup = updatedDataFeed.filterGroups; + updatedDataFeed.tag = updatedDataFeed.tags; + DataFeedManagerService.update({name: updatedDataFeed.name}, updatedDataFeed, + function(apiResponse) { + $location.path('/'); + }, + function(apiResponse) { + if (apiResponse.data && apiResponse.data.data && apiResponse.data.data['displayMessage']) { + alert(apiResponse.data.data['displayMessage']); + } else { + alert(apiResponse.statusText); + } + } + ); + } + + $scope.cancelDataFeed = function() { + $location.path("/"); + } + +}]); diff --git a/src/takserver-core/src/main/webapp/Marti/inputs/js/services.js b/src/takserver-core/src/main/webapp/Marti/inputs/js/services.js index 9036d6d8..8f27226d 100644 --- a/src/takserver-core/src/main/webapp/Marti/inputs/js/services.js +++ b/src/takserver-core/src/main/webapp/Marti/inputs/js/services.js @@ -1,11 +1,19 @@ var services = angular.module('inputManagerServices', []).factory('InputManagerService', function($resource) { return $resource('/Marti/api/inputs/:id', { id: '@_id' }, { - 'query': {method: "GET", isArray: false}, + 'query': {method: "GET", isArray: false, params: {excludeDataFeeds: false}}, 'update': {method: "PUT", params: {id: '@id'}, isArray: false, cache: false} } ); }); +services.factory('DataFeedManagerService', function($resource){ + return $resource('/Marti/api/datafeeds/:name', {name: '@_name'}, { + 'query': {method: "GET", isArray: false}, + 'update': {method: "PUT", params: {name: '@name'}, isArray: false, cache: false} + } + ); +}); + services.factory('MessagingConfigService', function($resource){ return $resource('/Marti/api/inputs/config', {}, { 'query': {method: "GET", isArray: false}, diff --git a/src/takserver-core/src/main/webapp/Marti/inputs/partials/list.html b/src/takserver-core/src/main/webapp/Marti/inputs/partials/list.html index 606dc318..be7384d7 100644 --- a/src/takserver-core/src/main/webapp/Marti/inputs/partials/list.html +++ b/src/takserver-core/src/main/webapp/Marti/inputs/partials/list.html @@ -58,7 +58,107 @@

Inputs (Server Ports)

+

Streaming Data Feeds

+ Add Streaming Data Feed +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Auth TypeNameTypeSyncTagsProtocolPortCore Messaging VersionTLSGroupInterfaceArchiveFilter GroupAnonymous GroupArchive OnlyReads ReceivedMessages ReceivedNumber Connected  
{{x.input.protocol === 'tls' ? (x.input.coreVersion == 2 + ? x.input.coreVersion2TlsVersions : secConfig.tlsVersion) : ''}}DeleteModify
+
+

Plugin Data Feeds

+ Add Plugin Data Feed +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Auth TypeNameTypeSyncTagsArchiveReads ReceivedMessages ReceivedNumber Connected  
DeleteModify
+

QoS (Quality of Service)

When enabled, rate limiters will be applied based on the number of connected clients, as shown in the tables.
@@ -129,7 +229,6 @@

QoS (Quality of Service)

-

Store and Forward

Enable Store and Forward for Chat Messages  diff --git a/src/takserver-core/src/main/webapp/Marti/inputs/partials/modifyPluginDataFeed.html b/src/takserver-core/src/main/webapp/Marti/inputs/partials/modifyPluginDataFeed.html new file mode 100644 index 00000000..daa97296 --- /dev/null +++ b/src/takserver-core/src/main/webapp/Marti/inputs/partials/modifyPluginDataFeed.html @@ -0,0 +1,103 @@ + + +
+

Modify Plugin Data Feed Definition

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
Must be unique and newline-separated + 		 
+ +
+   + +  
+
+ +
+

Data Feed Access is Unavailable

+

The data feed cannot be displayed because the Web application failed to connect to the TakServer broker.

+

Please ensure the broker is running and reload this page.

+
+
+ +
diff --git a/src/takserver-core/src/main/webapp/Marti/inputs/partials/modifyStreamingDataFeed.html b/src/takserver-core/src/main/webapp/Marti/inputs/partials/modifyStreamingDataFeed.html new file mode 100644 index 00000000..1c3b50a0 --- /dev/null +++ b/src/takserver-core/src/main/webapp/Marti/inputs/partials/modifyStreamingDataFeed.html @@ -0,0 +1,120 @@ + + +
+

Modify Streaming Data Feed Definition

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
Must be unique and newline-separated + 		 
+ +
Must be unique and newline-separated + 		 
+ +
+   + +  
+
+ +
+

Data Feed Access is Unavailable

+

The data feed cannot be displayed because the Web application failed to connect to the TakServer broker.

+

Please ensure the broker is running and reload this page.

+
+
+ +
diff --git a/src/takserver-core/src/main/webapp/Marti/inputs/partials/newPluginDataFeed.html b/src/takserver-core/src/main/webapp/Marti/inputs/partials/newPluginDataFeed.html new file mode 100644 index 00000000..2ea0115c --- /dev/null +++ b/src/takserver-core/src/main/webapp/Marti/inputs/partials/newPluginDataFeed.html @@ -0,0 +1,121 @@ + + +
+

Create Plugin Data Feed

+ +
+
    +
  • {{message}}
    • +
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
Name may contain upper and lower case letters, numbers, spaces and underscores up to 30 characters. + 		+ + Data Feed name is required. + Data Feed name must be between 1 and 30 characters + +
+ +
Provide 0 or more tags separated by newlines. + Example: red,green,blue + 		 
+ + + + Archive is required + +
+ +
*Required +   + +  
+
+
diff --git a/src/takserver-core/src/main/webapp/Marti/inputs/partials/newStreamingDataFeed.html b/src/takserver-core/src/main/webapp/Marti/inputs/partials/newStreamingDataFeed.html new file mode 100644 index 00000000..0f088ac7 --- /dev/null +++ b/src/takserver-core/src/main/webapp/Marti/inputs/partials/newStreamingDataFeed.html @@ -0,0 +1,275 @@ + + +
+

Create Streaming Data Feed

+ +
+
    +
  • {{message}}
    • +
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
Name may contain upper and lower case letters, numbers, spaces and underscores up to 30 characters. + 		+ + Data Feed name is required. + Data Feed name must be between 1 and 30 characters + +
+ +
Provide 0 or more tags separated by newlines. + Example: red,green,blue + 		 
+ + Protocol is required + +
+ + + + Authentication Type is required + +
+ +
Port must be an integer between 1 and 65535. + 		+ + Port value is required. + Port value must be an integer between 1 and 65535. + Port value must be an integer. + Port value may only contain digits. + +
+ +
+ 		+ + Core Version is required + +
+ Core Messaging 2 TLS Version(s): + + + + + + + +
TLSv1: TLSv1.1: TLSv1.2: TLSv1.3:
+ 		+ At least one TLS Version is required +
+ +
Provide an optional valid IPv4 multicast address. + 		+ + Provide a valid IPv4 multicast address containing only digits and periods. + +
+ +
Provide an optional valid network interface name (e.g., en0). + 		+ + Provide a valid network interface name containing only letters and numbers. + +
+ + + + Archive is required + +
+ + + + Anonymous Group is required + +
+ + + + Archive Only is required + +
+ +
Provide 0 or more LDAP distinguished names separated by newlines. + Example: cn=John Smith,ou=test,dc=xyz,dc=gov + 		 
+ +
*Required +   + +  
+
+
diff --git a/src/takserver-core/src/main/webapp/Marti/menubar.html b/src/takserver-core/src/main/webapp/Marti/menubar.html index 8da345ed..4325b7a3 100644 --- a/src/takserver-core/src/main/webapp/Marti/menubar.html +++ b/src/takserver-core/src/main/webapp/Marti/menubar.html @@ -148,7 +148,7 @@
  • Configuration
    • @@ -258,7 +259,9 @@ console.log(data.data); if (data.data != 'true' && !isSecCookie) { $('#securityAlert').show(); - $('#securityAlert span').html("ALERT! The following ports support unsecure connections: " + data.data); + $('#securityAlert span').html("ALERT! The following ports support unsecure connections: " + data.data + + ". Unencrypted ports may be exploited allowing unauthorized access to the system and sensitive information. " + + "See section 4.4 Use Setup Wizard to Configure TAK Server in the TAK Server Configuration Guide for how to secure the configuration."); } }) })(jQuery); diff --git a/src/takserver-core/src/main/webapp/Marti/plugins/js/controllers.js b/src/takserver-core/src/main/webapp/Marti/plugins/js/controllers.js index b588c950..09272744 100644 --- a/src/takserver-core/src/main/webapp/Marti/plugins/js/controllers.js +++ b/src/takserver-core/src/main/webapp/Marti/plugins/js/controllers.js @@ -6,9 +6,11 @@ pluginManagerControllers.controller('PluginListCtrl', [ '$scope', '$timeout', 'AllPluginsInfoService', - 'PluginStatusService', - 'AllPluginStatusService', - function($scope, $timeout, AllPluginsInfoService, PluginStatusService, AllPluginStatusService) { + 'PluginEnabledService', + 'PluginStartedService', + 'AllPluginStartedService', + 'PluginArchiveService', + function($scope, $timeout, AllPluginsInfoService, PluginEnabledService, PluginStartedService, AllPluginStartedService, PluginArchiveService) { var unsorted_class = "glyphicon glyphicon-sort"; var ascend_class = "glyphicon glyphicon-arrow-up"; @@ -38,9 +40,16 @@ pluginManagerControllers.controller('PluginListCtrl', [ plugin['status'] = "Startup Error: " + plugin.exceptionMessage; plugin['status-class'] = "bg-danger"; } else { - plugin['status'] = "Started"; - plugin['status-class'] = ""; - } + if (plugin['started']){ + plugin['status'] = "Started"; + plugin['status-class'] = ""; + plugin['start_stop_command'] = "Stop"; + }else{ + plugin['status'] = "Stopped"; + plugin['status-class'] = ""; + plugin['start_stop_command'] = "Start"; + } + } } $scope.pluginMetadata = res.data; sort(); @@ -88,16 +97,42 @@ pluginManagerControllers.controller('PluginListCtrl', [ sort(); } - $scope.handleStatus = function (plugin, input) { + $scope.handlePluginEnabledUpdate = function (plugin, input) { input.currentTarget.disabled = true - PluginStatusService.save({ + PluginEnabledService.save({ name: plugin.name, status: !plugin.enabled }, function (apiResponse) {}, function () { - alert('An unexpected error occurred changing the plugin status.'); + alert('An unexpected error occurred changing the plugin enabled status.'); + }); + } + + $scope.handlePluginStartedUpdate = function (plugin, input) { + input.currentTarget.disabled = true + + PluginStartedService.save({ + name: plugin.name, + status: !plugin.started + }, + function (apiResponse) {}, + function () { + alert('An unexpected error occurred changing the plugin started status.'); + }); + } + + $scope.handleArchiveUpdate = function (plugin, input) { + input.currentTarget.disabled = true + + PluginArchiveService.save({ + name: plugin.name, + archiveEnabled: !plugin.archiveEnabled + }, + function (apiResponse) {}, + function () { + alert('An unexpected error occurred changing the plugin archive setting.'); }); } @@ -105,7 +140,7 @@ pluginManagerControllers.controller('PluginListCtrl', [ document.getElementById('startButton').disabled = true document.getElementById('stopButton').disabled = true - AllPluginStatusService.save({ + AllPluginStartedService.save({ status: status }, function (apiResponse) {}, diff --git a/src/takserver-core/src/main/webapp/Marti/plugins/js/services.js b/src/takserver-core/src/main/webapp/Marti/plugins/js/services.js index f3d028ad..f19b00cf 100644 --- a/src/takserver-core/src/main/webapp/Marti/plugins/js/services.js +++ b/src/takserver-core/src/main/webapp/Marti/plugins/js/services.js @@ -4,10 +4,18 @@ services.factory('AllPluginsInfoService', function($resource) { return $resource('/Marti/api/plugins/info/all'); }); -services.factory('PluginStatusService', function($resource) { - return $resource('/Marti/api/plugins/info/status', {name:'@name', status: '@status'}, {}); +services.factory('PluginEnabledService', function($resource) { + return $resource('/Marti/api/plugins/info/enabled', {name:'@name', status: '@status'}, {}); }); -services.factory('AllPluginStatusService', function($resource) { - return $resource('/Marti/api/plugins/info/all/status', {status: '@status'}, {}); +services.factory('PluginStartedService', function($resource) { + return $resource('/Marti/api/plugins/info/started', {name:'@name', status: '@status'}, {}); +}); + +services.factory('AllPluginStartedService', function($resource) { + return $resource('/Marti/api/plugins/info/all/started', {status: '@status'}, {}); +}); + +services.factory('PluginArchiveService', function($resource) { + return $resource('/Marti/api/plugins/info/archive', {name:'@name', archiveEnabled: '@archiveEnabled'}, {}); }); diff --git a/src/takserver-core/src/main/webapp/Marti/plugins/partials/list.html b/src/takserver-core/src/main/webapp/Marti/plugins/partials/list.html index 79adb46d..8e079cf7 100644 --- a/src/takserver-core/src/main/webapp/Marti/plugins/partials/list.html +++ b/src/takserver-core/src/main/webapp/Marti/plugins/partials/list.html @@ -22,24 +22,36 @@

    Plugins

    Name + Version + Tag Description Plugin Type Class Name - Startup Status + Status + Command Enabled + Archive Enabled {{x.name}} + {{x.version}} + {{x.tag}} {{x.description}} {{x.className}} {{x.status}} - + + + + + + + diff --git a/src/takserver-core/src/main/webapp/Marti/vbm/css/main.css b/src/takserver-core/src/main/webapp/Marti/vbm/css/main.css new file mode 100644 index 00000000..69f0a8b3 --- /dev/null +++ b/src/takserver-core/src/main/webapp/Marti/vbm/css/main.css @@ -0,0 +1,6 @@ + +.content { + padding: 10px; + height: 200px; +} + diff --git a/src/takserver-core/src/main/webapp/Marti/vbm/index.html b/src/takserver-core/src/main/webapp/Marti/vbm/index.html new file mode 100644 index 00000000..d9e28e74 --- /dev/null +++ b/src/takserver-core/src/main/webapp/Marti/vbm/index.html @@ -0,0 +1,46 @@ + + + + + + VBM Configuration + + + + + + + + + + + + + + + + + +
    +
    +

    VBM Controller

    +
    + + +
    + +
    + + +
    + +
    + + +
    + +
    +
    + + + diff --git a/src/takserver-core/src/main/webapp/Marti/vbm/js/app.js b/src/takserver-core/src/main/webapp/Marti/vbm/js/app.js new file mode 100644 index 00000000..e6f9b641 --- /dev/null +++ b/src/takserver-core/src/main/webapp/Marti/vbm/js/app.js @@ -0,0 +1,4 @@ +'use strict'; + +var app = angular.module('vbmConfiguration', []); + diff --git a/src/takserver-core/src/main/webapp/Marti/vbm/js/controllers.js b/src/takserver-core/src/main/webapp/Marti/vbm/js/controllers.js new file mode 100644 index 00000000..0ed9fb04 --- /dev/null +++ b/src/takserver-core/src/main/webapp/Marti/vbm/js/controllers.js @@ -0,0 +1,62 @@ + +app.controller('vbmController', + function($scope, $http) { + + $scope.get_current_config = function(){ + $http({ + method : "GET", + url : "/vbm/api/config" + }).then(function mySuccess(response) { + + console.log("current_config: "+ response.data); + + var current_config = angular.fromJson(response.data); + + $scope.vbm_enabled = current_config.vbmEnabled; + $scope.sa_disabled = current_config.sadisabled; + $scope.chat_disabled = current_config.chatDisabled; + + }, function myError(response) { + alert("Error fetching data from server"); + console.error("response status: "+response.status); + console.error("response text: "+response.statusText); + }); + } + + $scope.get_current_config(); + + $scope.save_changes = function() { + + data = { + vbmEnabled: $scope.vbm_enabled, + sadisabled: $scope.sa_disabled, + chatDisabled: $scope.chat_disabled, + }; + + $http({ + method : "POST", + url : "/vbm/api/config", + data: JSON.stringify(data) + }).then(function mySuccess(response) { + + if (response.status == 200){ + alert("Successfully changed VBM Configuration"); + }else{ + alert("response.status: "+ response.status); + } + + }, function myError(response) { + if (response.data != null){ + alert("Error changing VBM Configuration. " + response.data.message); + } else { + alert("Error changing VBM Configuration."); + } + + console.error("response status: "+response.status); + console.error("response text: "+response.statusText); + }); + + }; + + } +); diff --git a/src/takserver-core/src/main/webapp/user-management/js/controllers/newUsersController.js b/src/takserver-core/src/main/webapp/user-management/js/controllers/newUsersController.js index 3de6b429..98b8395e 100644 --- a/src/takserver-core/src/main/webapp/user-management/js/controllers/newUsersController.js +++ b/src/takserver-core/src/main/webapp/user-management/js/controllers/newUsersController.js @@ -1,4 +1,4 @@ -app.controller('newUsersController', function($scope, $http) { +app.controller('newUsersController', function($scope, $http, ngDialog) { $scope.reset_form = function(){ // Reset the form model. @@ -32,36 +32,52 @@ app.controller('newUsersController', function($scope, $http) { $scope.create_users = function(){ - data = { - usernameExpression: $scope.new_users.username_expression, - startN: $scope.new_users.startN, - endN: $scope.new_users.endN, - groupListIN: $scope.list_groups_for_user.groupListIN, - groupListOUT: $scope.list_groups_for_user.groupListOUT, - groupList: $scope.list_groups_for_user.groupList - }; + var confirmDialog = ngDialog.openConfirm({ + template:'\ +

    After new users are created, you will only have one chance to save the password file. Do you want to proceed?

    \ +
    \ + \ + \ +
    ', + plain: true + }).then(function (confirm) { + + data = { + usernameExpression: $scope.new_users.username_expression, + startN: $scope.new_users.startN, + endN: $scope.new_users.endN, + groupListIN: $scope.list_groups_for_user.groupListIN, + groupListOUT: $scope.list_groups_for_user.groupListOUT, + groupList: $scope.list_groups_for_user.groupList + }; - $http({ - method : "POST", - url : "api/new-users", - data: JSON.stringify(data) - }).then(function mySuccess(response) { - // Start file download. - download("tak_users.txt", JSON.stringify(response.data)); + $http({ + method : "POST", + url : "api/new-users", + data: JSON.stringify(data) + }).then(function mySuccess(response) { + // Start file download. + download("tak_users.txt", JSON.stringify(response.data)); - // alert("Created new users successfully"); - $scope.reset_form(); - $scope.list_users(); + // alert("Created new users successfully"); + $scope.reset_form(); + $scope.list_users(); + + }, function myError(response) { + if (response.data != null){ + alert("Error creating users. " + response.data.message); + } else { + alert("Error creating users."); + } + console.error("response status: "+response.status); + console.error("response text: "+response.statusText); + }); - }, function myError(response) { - if (response.data != null){ - alert("Error creating users. " + response.data.message); - } else { - alert("Error creating users."); - } - console.error("response status: "+response.status); - console.error("response text: "+response.statusText); }); + + + + } diff --git a/src/takserver-core/src/main/webapp/user-management/menubar_modified.html b/src/takserver-core/src/main/webapp/user-management/menubar_modified.html index 7218db79..60aa9f40 100644 --- a/src/takserver-core/src/main/webapp/user-management/menubar_modified.html +++ b/src/takserver-core/src/main/webapp/user-management/menubar_modified.html @@ -164,7 +164,8 @@
  • Client Certificates
  • Tokens
  • Device Logs
    • -
  • Device Profiles
    • +
  • Device Profiles
    • +
  • VBM Configuration
    • @@ -259,7 +260,9 @@ console.log(data.data); if (data.data != 'true' && !isSecCookie) { $('#securityAlert').show(); - $('#securityAlert span').html("ALERT! The following ports support unsecure connections: " + data.data); + $('#securityAlert span').html("ALERT! The following ports support unsecure connections: " + data.data + + ". Unencrypted ports may be exploited allowing unauthorized access to the system and sensitive information. " + + "See section 4.4 Use Setup Wizard to Configure TAK Server in the TAK Server Configuration Guide for how to secure the configuration."); } }) })(jQuery); diff --git a/src/takserver-core/src/test/java/tak/server/DistributedMessageTests.java b/src/takserver-core/src/test/java/tak/server/DistributedMessageTests.java index e47cae0f..d4550f26 100644 --- a/src/takserver-core/src/test/java/tak/server/DistributedMessageTests.java +++ b/src/takserver-core/src/test/java/tak/server/DistributedMessageTests.java @@ -32,7 +32,7 @@ import com.bbn.marti.nio.protocol.connections.StreamingProtoBufProtocol; import com.bbn.marti.remote.groups.Direction; import com.bbn.marti.remote.groups.Group; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.Input; import com.bbn.marti.util.concurrent.future.AsyncFuture; import tak.server.cot.CotEventContainer; diff --git a/src/takserver-core/src/test/java/tak/server/InputControlMessageTests.java b/src/takserver-core/src/test/java/tak/server/InputControlMessageTests.java index 08e97794..807e2c10 100644 --- a/src/takserver-core/src/test/java/tak/server/InputControlMessageTests.java +++ b/src/takserver-core/src/test/java/tak/server/InputControlMessageTests.java @@ -22,7 +22,7 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import com.atakmap.Tak.ROL; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.Input; import com.bbn.marti.remote.ServerInfo; import com.bbn.marti.remote.exception.TakException; import com.google.common.base.Charsets; diff --git a/src/takserver-core/src/test/java/tak/server/TakServerTestApplicationConfig.java b/src/takserver-core/src/test/java/tak/server/TakServerTestApplicationConfig.java index daeacbb2..e8d4a8c0 100644 --- a/src/takserver-core/src/test/java/tak/server/TakServerTestApplicationConfig.java +++ b/src/takserver-core/src/test/java/tak/server/TakServerTestApplicationConfig.java @@ -1,17 +1,9 @@ package tak.server; import java.io.File; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.UUID; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.Logger; -import ch.qos.logback.classic.LoggerContext; -import ch.qos.logback.classic.PatternLayout; -import ch.qos.logback.classic.log4j.XMLLayout; -import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.core.FileAppender; import org.apache.ignite.Ignite; import org.apache.ignite.Ignition; import org.apache.ignite.configuration.IgniteConfiguration; @@ -23,7 +15,7 @@ import org.springframework.oxm.Marshaller; import org.springframework.oxm.jaxb.Jaxb2Marshaller; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.Input; import com.bbn.marti.remote.ServerInfo; import com.bbn.marti.remote.groups.GroupManager; import com.bbn.marti.service.SubscriptionStore; @@ -33,6 +25,11 @@ import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.PatternLayout; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.FileAppender; import tak.server.ignite.IgniteConfigurationHolder; import tak.server.messaging.MessageConverter; diff --git a/src/takserver-core/takserver-war/build.gradle b/src/takserver-core/takserver-war/build.gradle index 84f81011..e229f43e 100644 --- a/src/takserver-core/takserver-war/build.gradle +++ b/src/takserver-core/takserver-war/build.gradle @@ -94,16 +94,12 @@ dependencies { compile group: 'com.sun.xml.ws', name: 'jaxws-rt', version: jaxwsrt_version compile group: 'org.json', name: 'json', version: json_org_version compile group: 'io.jsonwebtoken', name: 'jjwt', version: jsonwebtoken_version - + compile group: 'org.igniterealtime.whack', name: 'core', version: whack_version + compile 'javax.xml.bind:jaxb-api:' + jaxb_api_version compile('javax.activation:activation:1.1') compile('org.glassfish.jaxb:jaxb-runtime:' + jaxb_api_version) - - // some issue with this dependency - //compile group: 'org.igniterealtime', name: 'whack', version: whack_version, ext: 'pom' - compile name: 'whack_2_0_1' - compile name: 'xml-apis-2.11.0' compile name: 'JavaAPIforKml-2.2.2' } diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/locate/LocateApi.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/locate/LocateApi.java index 2329475f..58dd20dd 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/locate/LocateApi.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/locate/LocateApi.java @@ -126,7 +126,7 @@ public void locate( // create the mission and make sure that the we're subscribed missionService.createMission(missionName, creatorUid, groupVector, null, - null, "public", null, null, null); + null, null, null, null, null,"public", null, null, null, null); missionService.missionSubscribe(missionName, creatorUid, groupVector); // submit the marker to the mission diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/EsapiServlet.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/EsapiServlet.java index b07dd545..43f07747 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/EsapiServlet.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/EsapiServlet.java @@ -13,6 +13,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.lang3.StringUtils; import org.owasp.esapi.Validator; import org.owasp.esapi.errors.IntrusionException; import org.owasp.esapi.errors.ValidationException; @@ -139,7 +140,7 @@ public String getParameterValue(Map httpParameters, String giv safeKey = "(unsafe parameter name)"; } if (values.length > 1) { - log.warning("Multiple values detected for HTTP parameter " + safeKey + "; ignoring extras." ); + log.warning("Multiple values detected for HTTP parameter " + StringUtils.normalizeSpace(safeKey) + "; ignoring extras." ); } if (!values[0].isEmpty()) { result = values[0]; diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/LatestKMLServlet.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/LatestKMLServlet.java index ba9f0c85..f08ccfef 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/LatestKMLServlet.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/LatestKMLServlet.java @@ -157,9 +157,17 @@ private void processHttpQuery(HttpServletRequest request, HttpServletResponse re } OutputStream servletReponseOutputStream = null; + String contentDisposition = "filename=" + DEFAULT_FILENAME_BASE + "-" + cotType + ".kml"; + try { + contentDisposition = validator.getValidInput("Content Disposition", contentDisposition, "Filename", MartiValidator.DEFAULT_STRING_CHARS, false); + } catch (ValidationException ex) { + log.severe(ex.getMessage()); + response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Unsafe filename value detected."); + return; + } response.setContentType("application/vnd.google-earth.kml+xml"); - response.setHeader("Content-Disposition", "filename=" + DEFAULT_FILENAME_BASE + "-" + cotType + ".kml"); + response.setHeader("Content-Disposition", contentDisposition); try { diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/MissionKMLServlet.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/MissionKMLServlet.java index 0e387bc0..ef6afe27 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/MissionKMLServlet.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/MissionKMLServlet.java @@ -34,6 +34,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.zip.ZipEntry; import java.util.zip.ZipException; @@ -46,6 +47,8 @@ import javax.servlet.http.HttpServletResponse; import javax.sql.DataSource; +import org.owasp.esapi.ESAPI; +import org.owasp.esapi.errors.EncodingException; import org.owasp.esapi.errors.IntrusionException; import org.owasp.esapi.errors.ValidationException; import org.slf4j.LoggerFactory; @@ -244,7 +247,7 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) try { interval = Double.parseDouble(intervalParam); } catch (NumberFormatException ex) { - log.warning("Invalid numeric string for parameter 'interval': " + intervalParam); + log.warning("Invalid numeric string for parameter 'interval', double value is required"); response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Invalid numeric format for time interval."); } } @@ -254,7 +257,7 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) try { multiTrackThresholdAsInteger = Integer.parseInt(multiTrackThreshold, 10); } catch (NumberFormatException ex) { - log.warning("Invalid numeric string for parameter 'multiTrachThreshold': " + multiTrackThreshold); + log.warning("Invalid numeric string for parameter 'multiTrachThreshold', integer value required"); response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Invalid numeric format for time interval."); } } @@ -752,14 +755,15 @@ protected void buildMissionFeatures(LinkedList qrs, Document doc, Se try { String imageUrl = baseUrl + URLEncoder.encode(last.uid, "UTF-8"); - - log.finest("key: " + imageUrl + " images: " + images); + if (log.isLoggable(Level.FINEST)) { + log.finest("key: " + ESAPI.encoder().encodeForURL(imageUrl)); + } if (format.equals(AllowedFormat.kmz) && !images.containsKey(imageUrl)) { imageTag = false; } - } catch (UnsupportedEncodingException e) { } - + } catch (UnsupportedEncodingException | EncodingException e) { } + if (cotType.startsWith("b-m-r")) { // route List ps = buildTimeseriesLineString(qrs, maxStartTime); diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/citrap/CITrapReportAPI.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/citrap/CITrapReportAPI.java index 89d73e15..797b19e3 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/citrap/CITrapReportAPI.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/citrap/CITrapReportAPI.java @@ -5,11 +5,11 @@ import java.text.SimpleDateFormat; import java.util.Date; import java.util.List; -import java.util.SimpleTimeZone; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.lang3.StringUtils; import org.owasp.esapi.Validator; import org.owasp.esapi.errors.IntrusionException; import org.postgresql.geometric.PGbox; @@ -184,7 +184,9 @@ ResponseEntity searchReports( missionService.missionSubscribe(reportType.getId(), clientUid, groupVector); } catch (JpaSystemException e) { } // DuplicateKeyException comes through as JpaSystemException due to transaction catch (NotFoundException e) { - logger.error("missionSubscribe couldn't find mission for report id : " + reportType.getId()); + if (logger.isErrorEnabled()) { + logger.error("missionSubscribe couldn't find mission for report id : " + StringUtils.normalizeSpace(reportType.getId())); + } } } } diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/citrap/CITrapReportService.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/citrap/CITrapReportService.java index c1542563..565ff0e5 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/citrap/CITrapReportService.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/citrap/CITrapReportService.java @@ -165,8 +165,8 @@ else if (reportExists(report.getId(), groupVector)) { missionService.addMissionContent(CI_TRAP_MISSION, content, clientUid, groupVector); // create a new mission for this report and add the report - missionService.createMission(report.getId(), clientUid, groupVector, null, null, - CI_TRAP_MISSION, null, null, null); + missionService.createMission(report.getId(), clientUid, groupVector, null, null, null, null, null, null, + CI_TRAP_MISSION, null, null, null, null); missionService.addMissionContent(report.getId(), content, clientUid, groupVector); // subscribe for notifications to the new report diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/config/ConfigAPI.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/config/ConfigAPI.java index 51f7fc83..401b518a 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/config/ConfigAPI.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/config/ConfigAPI.java @@ -12,7 +12,6 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; -import com.bbn.marti.config.Network.Input; import com.bbn.marti.network.BaseRestController; import com.bbn.marti.remote.CoreConfig; import com.bbn.marti.util.CommonUtil; diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/device/profile/api/ProfileAPI.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/device/profile/api/ProfileAPI.java index 5b406e7a..2118ec0f 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/device/profile/api/ProfileAPI.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/device/profile/api/ProfileAPI.java @@ -7,8 +7,12 @@ import com.bbn.marti.device.profile.repository.ProfileFileRepository; import com.bbn.marti.device.profile.service.ProfileService; import com.bbn.marti.network.BaseRestController; +import com.bbn.marti.remote.CoreConfig; +import com.bbn.marti.remote.RemoteSubscription; +import com.bbn.marti.remote.SubscriptionManagerLite; import com.bbn.marti.remote.exception.NotFoundException; import com.bbn.marti.remote.exception.TakException; +import com.bbn.marti.remote.groups.GroupManager; import com.bbn.marti.util.CommonUtil; import com.bbn.security.web.MartiValidator; @@ -54,6 +58,36 @@ public class ProfileAPI extends BaseRestController { @Autowired private ProfileFileRepository profileFileRepository; + @Autowired + SubscriptionManagerLite subscriptionManager; + + @Autowired + private GroupManager groupManager; + + @Autowired + CoreConfig coreConfig; + + + private String getGroupVectorFromStreamingClient(String clientUid, String groupVector) { + try { + if (coreConfig.getRemoteConfiguration().getProfile() != null && + coreConfig.getRemoteConfiguration().getProfile().isUseStreamingGroup()) { + RemoteSubscription subscription = subscriptionManager.getRemoteSubscriptionByClientUid(clientUid); + if (subscription != null) { + String groupVectorTmp = groupManager.getCachedOutboundGroupVectorByConnectionId( + subscription.getUser().getConnectionId()); + if (groupVectorTmp != null && groupVectorTmp.length() > 0) { + groupVector = groupVectorTmp; + } + } + } + } catch (Exception e) { + logger.error("exception getting groupVector from streaming user", e); + } + + return groupVector; + } + @RequestMapping(value = "/tls/profile/enrollment", method = RequestMethod.GET) public byte[] getEnrollmentTimeProfiles( @RequestParam(value = "clientUid") String clientUid) @@ -62,6 +96,8 @@ public byte[] getEnrollmentTimeProfiles( String groupVector = martiUtil.getGroupVectorBitString(request); String host = new URI(request.getRequestURL().toString()).getHost(); + groupVector = getGroupVectorFromStreamingClient(clientUid, groupVector); + List files = profileService.getProfileFiles(host, groupVector, true, false, -1); @@ -92,6 +128,8 @@ public byte[] getConnectionTimeProfiles( String groupVector = martiUtil.getGroupVectorBitString(request); String host = new URI(request.getRequestURL().toString()).getHost(); + groupVector = getGroupVectorFromStreamingClient(clientUid, groupVector); + List files = profileService.getProfileFiles(host, groupVector, false, true, syncSecago); @@ -122,6 +160,8 @@ public byte[] getToolProfiles( try { String groupVector = martiUtil.getGroupVectorBitString(request); + groupVector = getGroupVectorFromStreamingClient(clientUid, groupVector); + List files = profileService.getProfileFilesForTool(groupVector, toolName, syncSecago); diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/excheck/ExCheckAPI.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/excheck/ExCheckAPI.java index bebf0405..ca0b5eec 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/excheck/ExCheckAPI.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/excheck/ExCheckAPI.java @@ -1,6 +1,7 @@ package com.bbn.marti.excheck; import java.io.IOException; +import java.io.InputStream; import java.net.URLDecoder; import java.rmi.RemoteException; import java.text.SimpleDateFormat; @@ -35,6 +36,7 @@ import com.bbn.marti.sync.service.MissionService; import com.bbn.marti.util.CommonUtil; import com.bbn.marti.util.cert.exception.NotFoundException; +import com.google.common.io.ByteStreams; import tak.server.Constants; @@ -76,7 +78,7 @@ ResponseEntity postTemplate( Checklist checklist = null; if (contentType != null && contentType.compareToIgnoreCase("application/xml") == 0) { - payload = UploadServlet.readByteArray(request.getInputStream()); + payload = readByteArray(request.getInputStream()); String xml = new String(payload); checklist = exCheckService.checklistFromXml(xml); @@ -90,7 +92,7 @@ ResponseEntity postTemplate( } else { if (contentType == null || !contentType.contains("multipart/form-data")) { - payload = UploadServlet.readByteArray(request.getInputStream()); + payload = readByteArray(request.getInputStream()); } else { MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request; MultipartFile mpf = multipartRequest.getFile("assetfile"); @@ -553,4 +555,19 @@ ResponseEntity removeMissionReferenceFromChecklist( checklistUid, missionService.trimName(missionName), clientUid, martiUtil.getGroupBitVector(request)); return new ResponseEntity(HttpStatus.OK); } + + /** + * Utility method that reads a byte array from an input stream using Guava. + * + * @param in + * any InputStream containing some data + * @return the InputStream's contents as a byte array; may be size 0 but + * will not be null. + * @throws IOException + * if a read error occurs + */ + private byte[] readByteArray(InputStream in) throws IOException { + + return ByteStreams.toByteArray(in); + } } \ No newline at end of file diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/excheck/ExCheckService.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/excheck/ExCheckService.java index 04294631..2288f121 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/excheck/ExCheckService.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/excheck/ExCheckService.java @@ -510,17 +510,19 @@ public String startChecklist(String id, String clientUid, String callsign, public String addEditChecklistTask( ChecklistTask task, Checklist checklist, String clientUid, String groupVector) { + Date now = new Date(); + // add the task to esync String xml = toXml(task); List keywords = new ArrayList(); keywords.add("Task"); - Metadata metadata = addToEnterpriseSync(xml.getBytes(), groupVector, task.getUid(), keywords, new Date()); + Metadata metadata = addToEnterpriseSync(xml.getBytes(), groupVector, task.getUid(), keywords, now); String hash = metadata.getHash(); // add the task doc to the checklist mission MissionContent content = new MissionContent(); content.getHashes().add(hash); - missionService.addMissionContent(checklist.getChecklistDetails().getUid(), content, clientUid, groupVector); + missionService.addMissionContentAtTime(checklist.getChecklistDetails().getUid(), content, clientUid, groupVector, now, xml); return hash; } @@ -546,7 +548,7 @@ public void createChecklistMission(Checklist checklist, String clientUid, String // create a new mission for the checklist Mission checklistMission = missionService.createMission(checklistId, EXCHECK_TOOL, groupVector, - checklist.getChecklistDetails().getDescription(), null, EXCHECK_TOOL, null, null, null); + checklist.getChecklistDetails().getDescription(), null, null, null, null, null, EXCHECK_TOOL, null, null, null, null); // add the new checklist to the checklist mission MissionContent content = new MissionContent(); diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/excheck/checklist/ChecklistTask.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/excheck/checklist/ChecklistTask.java index 052740a2..117f7766 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/excheck/checklist/ChecklistTask.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/excheck/checklist/ChecklistTask.java @@ -2,7 +2,7 @@ // This file was generated by the JavaTM Architecture for XML Binding(JAXB) Reference Implementation, v2.2.8-b130911.1802 // See http://java.sun.com/xml/jaxb // Any modifications to this file will be lost upon recompilation of the source schema. -// Generated on: 2019.11.18 at 10:16:56 AM EST +// Generated on: 2022.03.15 at 08:36:44 AM EDT // @@ -29,6 +29,7 @@ * <element name="uid" type="{http://www.w3.org/2001/XMLSchema}string"/> * <element name="value" type="{http://www.w3.org/2001/XMLSchema}string" maxOccurs="unbounded"/> * <element name="status" type="{}checklistTaskStatus" minOccurs="0"/> + * <element name="customStatus" type="{http://www.w3.org/2001/XMLSchema}string" minOccurs="0"/> * <element name="completeBy" type="{http://www.w3.org/2001/XMLSchema}string" minOccurs="0"/> * <element name="completeDTG" type="{http://www.w3.org/2001/XMLSchema}string" minOccurs="0"/> * <element name="notes" type="{http://www.w3.org/2001/XMLSchema}string" minOccurs="0"/> @@ -51,6 +52,7 @@ "uid", "value", "status", + "customStatus", "completeBy", "completeDTG", "notes", @@ -71,6 +73,7 @@ public class ChecklistTask { protected List value; @XmlSchemaType(name = "string") protected ChecklistTaskStatus status = ChecklistTaskStatus.PENDING; + protected String customStatus; protected String completeBy; protected String completeDTG; protected String notes; @@ -211,6 +214,30 @@ public void setStatus(ChecklistTaskStatus value) { this.status = value; } + /** + * Gets the value of the customStatus property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getCustomStatus() { + return customStatus; + } + + /** + * Sets the value of the customStatus property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setCustomStatus(String value) { + this.customStatus = value; + } + /** * Gets the value of the completeBy property. * diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/excheck/checklist/excheck.xsd b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/excheck/checklist/excheck.xsd index cbd67276..2ba0bf42 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/excheck/checklist/excheck.xsd +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/excheck/checklist/excheck.xsd @@ -88,6 +88,7 @@ + diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/feeds/DataFeedApi.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/feeds/DataFeedApi.java new file mode 100644 index 00000000..e3411a4a --- /dev/null +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/feeds/DataFeedApi.java @@ -0,0 +1,73 @@ +package com.bbn.marti.feeds; + +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; + +import com.bbn.marti.cot.search.model.ApiResponse; +import com.bbn.marti.network.BaseRestController; + +import tak.server.Constants; +import tak.server.feeds.DataFeed; + +/* + */ +public class DataFeedApi extends BaseRestController { + Logger logger = LoggerFactory.getLogger(DataFeedApi.class); + + @Autowired + DataFeedService dataFeedService; + + @RequestMapping(value = "/datafeeds/bounds/{bbox}", method = RequestMethod.GET) + public ResponseEntity>> getDataFeedsInBbox(@PathVariable("bbox") String bbox) { + List dataFeeds = new ArrayList<>(); + try { + String[] bboxArr = bbox.split(","); + if (bboxArr.length == 4) { + double maxLat, minLat, maxLong, minLong; + maxLat = Double.valueOf(bboxArr[0]); + minLong = Double.valueOf(bboxArr[1]); + minLat = Double.valueOf(bboxArr[2]); + maxLong = Double.valueOf(bboxArr[3]); + + dataFeeds = dataFeedService.getDataFeedsWithinBbox(minLat, maxLat, minLong, maxLong); + } + } catch (Exception e) { + logger.error("Failed getting data feeds", e); + return new ResponseEntity>>( + new ApiResponse>(Constants.API_VERSION, DataFeed.class.getName(), null), + HttpStatus.INTERNAL_SERVER_ERROR); + } + + return new ResponseEntity>>( + new ApiResponse>(Constants.API_VERSION, DataFeed.class.getName(), dataFeeds), + HttpStatus.OK); + } + + @RequestMapping(value = "/datafeeds/bounds/polygon", method = RequestMethod.GET) + public ResponseEntity>> getDataFeedsInPolygon(@RequestBody List points) { + List dataFeeds = new ArrayList<>(); + try { + dataFeeds = dataFeedService.getDataFeedsWithinPolyBounds(points); + } catch (Exception e) { + logger.error("Failed getting data feeds", e); + return new ResponseEntity>>( + new ApiResponse>(Constants.API_VERSION, DataFeed.class.getName(), null), + HttpStatus.INTERNAL_SERVER_ERROR); + } + + return new ResponseEntity>>( + new ApiResponse>(Constants.API_VERSION, DataFeed.class.getName(), dataFeeds), + HttpStatus.OK); + } + +} diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/feeds/DataFeedService.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/feeds/DataFeedService.java new file mode 100644 index 00000000..8aaa93d7 --- /dev/null +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/feeds/DataFeedService.java @@ -0,0 +1,147 @@ +package com.bbn.marti.feeds; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import javax.sql.DataSource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.bbn.marti.sync.repository.DataFeedRepository; +import com.bbn.marti.util.spring.SpringContextBeanForApi; +import com.google.common.base.Strings; + +/* + */ +public class DataFeedService { + + private static final Logger logger = LoggerFactory.getLogger(DataFeedService.class); + + private final DataSource dataSource; + private final DataFeedRepository dataFeedRepository; + + private static DataFeedService dataFeedService; + private synchronized DataFeedService getDataFeedService() { + if (dataFeedService != null) { + return dataFeedService; + } + + try { + dataFeedService = SpringContextBeanForApi.getSpringContext().getBean(DataFeedService.class); + return dataFeedService; + } catch (Exception e) { + logger.error("exception trying to get DataFeedService bean!", e); + return this; + } + } + + public DataFeedService(DataSource dataSource, DataFeedRepository dataFeedRepository) { + this.dataSource = dataSource; + this.dataFeedRepository = dataFeedRepository; + } + + + public List getDataFeedsWithinBbox(double minLat, double maxLat, double minLong, double maxLong) { + List dataFeedsInBounds = new ArrayList<>(); + + String minLatS = String.valueOf(minLat); + String maxLatS = String.valueOf(maxLat); + String minLongS = String.valueOf(minLong); + String maxLongS = String.valueOf(maxLong); + try (Connection connection = dataSource.getConnection()) { + try (PreparedStatement ps = connection.prepareStatement("WITH feeds_in_bounds as (SELECT DISTINCT data_feed_cot.data_feed_id FROM data_feed_cot " + + "INNER JOIN cot_router ON cot_router.id=data_feed_cot.cot_router_id AND st_within(cot_router.event_pt, " + + "ST_GeomFromText('POLYGON((' || ? || '))', 4326))) " + + "SELECT data_feed.* FROM data_feed INNER JOIN feeds_in_bounds ON feeds_in_bounds.data_feed_id=data_feed.id;")) { + + String polyString = + minLatS + " " + minLongS + "," + + minLatS + " " + maxLongS + "," + + maxLatS + " " + maxLongS + "," + + maxLatS + " " + minLongS + "," + + minLatS + " " + minLongS; + + ps.setString(1, polyString); + + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + String uuid = rs.getString("uuid"); + if (!Strings.isNullOrEmpty(uuid)) + dataFeedsInBounds.add(uuid); + } + } + } catch (Exception e) { + logger.info("Exception with query in getDataFeedsWithinBounds! " + e.getMessage(), e); + } + } catch (Exception e) { + logger.info("Exception with connection in getDataFeedsWithinBounds! " + e.getMessage(), e); + } + + return dataFeedsInBounds; + } + + public List getDataFeedsWithinPolyBounds(List points) { + List dataFeedsInBounds = new ArrayList<>(); + // poly needs 3 points minimum + if (points.size() < 3) { + logger.info("Polygon requires at least 3 points. Found size " + points.size()); + return dataFeedsInBounds; + } + + try { + // check that all points are valid numbers + for (String point : points) { + String[] xy = point.split(","); + if (xy.length != 2) { + logger.info("Point is not in the format of ," + Arrays.deepToString(xy)); + return dataFeedsInBounds; + } + Double.parseDouble(xy[0]); + Double.parseDouble(xy[1]); + } + } catch (Exception e) { + logger.error("Error parsing points for data feed bounds",e); + return dataFeedsInBounds; + } + + // first != last, so append first to end to close the poly + if (!points.get(0).replace(" ", "").equals(points.get(points.size()-1).replace(" ", ""))) { + points.add(points.get(0)); + } + + String polyString = points.stream() + .map(p->p.replace(" ", "")) + .map(p->p.replace(",", " ")) + .collect(Collectors.joining(",")); + + try (Connection connection = dataSource.getConnection()) { + try (PreparedStatement ps = connection.prepareStatement("WITH feeds_in_bounds as (SELECT DISTINCT data_feed_cot.data_feed_id FROM data_feed_cot " + + "INNER JOIN cot_router ON cot_router.id=data_feed_cot.cot_router_id AND st_within(cot_router.event_pt, " + + "ST_GeomFromText('POLYGON((' || ? || '))', 4326))) " + + "SELECT data_feed.* FROM data_feed INNER JOIN feeds_in_bounds ON feeds_in_bounds.data_feed_id=data_feed.id;")) { + + ps.setString(1, polyString); + + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + String uuid = rs.getString("uuid"); + if (!Strings.isNullOrEmpty(uuid)) + dataFeedsInBounds.add(uuid); + } + } + } catch (Exception e) { + logger.info("Exception with query in getDataFeedsWithinBounds! " + e.getMessage(), e); + } + } catch (Exception e) { + logger.info("Exception with connection in getDataFeedsWithinBounds! " + e.getMessage(), e); + } + + return dataFeedsInBounds; + } +} diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/groups/GroupsApi.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/groups/GroupsApi.java index 4ba3fa92..4670361d 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/groups/GroupsApi.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/groups/GroupsApi.java @@ -240,7 +240,7 @@ public ResponseEntity>> getAllGroups( try { if (coreConfig.getRemoteConfiguration().getAuth().getDefault().equalsIgnoreCase("ldap")) { for (Group group : groups) { - List ldapGroups = groupManager.searchGroups(group.getName()); + List ldapGroups = groupManager.searchGroups(group.getName(), true); if (ldapGroups.size() == 0) { logger.debug("unable to find description for group! " + group.getName()); continue; @@ -255,6 +255,25 @@ public ResponseEntity>> getAllGroups( } catch (Exception e) { logger.error("exception getting group description", e); } + + // + // Ensure that all groups meet the ATAK ServerGroup.isValid check + // + for (Group g : groups) { + if (g.getBitpos() == null || g.getBitpos() < 0) { + g.setBitpos(0); + } + if (g.getCreated() == null) { + g.setCreated(new Date()); + } + if (g.getType() == null) { + g.setType(Group.Type.SYSTEM); + } + if (g.getDirection() == null) { + g.setDirection(Direction.OUT); + } + } + } return new ResponseEntity>>(new ApiResponse>( diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/jwt/JwtUtils.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/jwt/JwtUtils.java index a2a1876e..4793abfc 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/jwt/JwtUtils.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/jwt/JwtUtils.java @@ -1,18 +1,22 @@ package com.bbn.marti.jwt; +import com.bbn.marti.config.Oauth; import com.bbn.marti.remote.CoreConfig; import com.bbn.marti.util.spring.SpringContextBeanForApi; -import io.jsonwebtoken.Claims; -import io.jsonwebtoken.JwtParser; -import io.jsonwebtoken.Jwts; -import io.jsonwebtoken.SignatureAlgorithm; +import io.jsonwebtoken.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.FileInputStream; +import java.nio.file.Files; +import java.nio.file.Paths; import java.security.*; import java.security.cert.Certificate; +import java.security.interfaces.RSAPublicKey; +import java.security.spec.X509EncodedKeySpec; +import java.util.ArrayList; import java.util.Enumeration; +import java.util.List; public class JwtUtils { @@ -37,7 +41,6 @@ private void loadKeys() { // // load keys from tls keystore // - String keyStoreType = coreConfig().getRemoteConfiguration().getSecurity().getTls().getKeystore(); String keyStoreFile = coreConfig().getRemoteConfiguration().getSecurity().getTls().getKeystoreFile(); String keyStorePass = coreConfig().getRemoteConfiguration().getSecurity().getTls().getKeystorePass(); @@ -118,21 +121,21 @@ public static JwtUtils getInstanceGenerateKeys() { public PublicKey getPublicKey() { return publicKey; } public PrivateKey getPrivateKey() { return privateKey; } - private JwtParser getParser(SignatureAlgorithm signatureAlgorithm) { + private JwtParser getParser(SignatureAlgorithm signatureAlgorithm, Key key) { JwtParser parser = null; if (signatureAlgorithm == SignatureAlgorithm.RS256) { parser = jwtRsaParser.get(); if (parser == null) { parser = Jwts.parser(); - parser.setSigningKey(privateKey); + parser.setSigningKey(key); jwtRsaParser.set(parser); } } else if (signatureAlgorithm == SignatureAlgorithm.HS256) { parser = jwtHmacParser.get(); if (parser == null) { parser = Jwts.parser(); - parser.setSigningKey(privateKey.getEncoded()); + parser.setSigningKey(key.getEncoded()); jwtHmacParser.set(parser); } } @@ -140,8 +143,76 @@ private JwtParser getParser(SignatureAlgorithm signatureAlgorithm) { return parser; } + public List getExternalVerifiers() { + try { + Oauth oAuth = coreConfig.getRemoteConfiguration().getAuth().getOauth(); + if (oAuth == null) { + logger.error("OAuth config not found"); + return null; + } + + if (oAuth.getAuthServer() == null || oAuth.getAuthServer().size() == 0) { + logger.error("No auth servers configured"); + return null; + } + + // + // iterate across our configured authorization servers and add their issuer public keys + // + List rsaPublicKeys = new ArrayList<>(); + for (Oauth.AuthServer authServer : oAuth.getAuthServer()) { + byte[] keyBytes = Files.readAllBytes(Paths.get(authServer.getIssuer())); + X509EncodedKeySpec spec = new X509EncodedKeySpec(keyBytes); + KeyFactory kf = KeyFactory.getInstance("RSA"); + rsaPublicKeys.add((RSAPublicKey) kf.generatePublic(spec)); + } + + return rsaPublicKeys; + } catch (Exception e) { + logger.error("exception in getExternalVerifiers!"); + return null; + } + } + + private List getExternalParsers(SignatureAlgorithm signatureAlgorithm) { + List jwtParsers = new ArrayList<>(); + for (RSAPublicKey rsaPublicKey : getExternalVerifiers()) { + jwtParsers.add(Jwts.parser().setSigningKey(rsaPublicKey)); + } + return jwtParsers; + } + + public Claims parseClaim(String token, JwtParser jwtParser) { + try { + Claims claims = jwtParser.parseClaimsJws(token).getBody(); + if (claims != null) { + return claims; + } + } catch (Exception e) { + if (logger.isDebugEnabled()) { + logger.debug("exception parsing token " + token, e); + } + } + + return null; + } + public Claims parseClaims(String token, SignatureAlgorithm signatureAlgorithm) { - return getParser(signatureAlgorithm).parseClaimsJws(token).getBody(); + // try to verify the claims using the tls keystore + Claims claims = parseClaim(token, getParser(signatureAlgorithm, privateKey)); + if (claims != null) { + return claims; + } + + // try to verify the claims using the external verifiers + for (JwtParser jwtParser : getExternalParsers(signatureAlgorithm)) { + claims = parseClaim(token, jwtParser); + if (claims != null) { + return claims; + } + } + + return null; } private CoreConfig coreConfig() { diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/logs/Log.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/logs/Log.java index 4e821e40..31bed654 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/logs/Log.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/logs/Log.java @@ -24,6 +24,7 @@ public class Log { private String majorVersion; private String minorVersion; private String log; + private byte[] contents; private String filename; private List callstacks; private Timestamp time; @@ -73,12 +74,20 @@ public void setMinorVersion(String minorVersion) { this.minorVersion = minorVersion; } - public String getLog() { - return log; - } +// public String getLog() { +// return log; +// } public void setLog(String log) { this.log = log; - } + } + + public byte[] getContents() { + return contents; + } + + public void setContents(byte[] contents) { + this.contents = contents; + } public String getFilename() { return filename; diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/logs/LogServlet.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/logs/LogServlet.java index 8a10e0cf..7c8b66f1 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/logs/LogServlet.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/logs/LogServlet.java @@ -107,7 +107,6 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) while ((count = zis.read(data, 0, BUFFER)) != -1) { bos.write(data, 0, count); } - String fileContents = new String(bos.toByteArray(), "UTF-8"); // create a Log object and store in the db Log log = new Log(); @@ -116,7 +115,7 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) log.setPlatform(platform); log.setMajorVersion(majorVersion); log.setMinorVersion(minorVersion); - log.setLog(fileContents); + log.setContents(bos.toByteArray()); log.setFilename(entry.getName()); persistenceStore.addLog(log); @@ -159,14 +158,12 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) Log log = persistenceStore.getLog(logId); response.setContentType("text/plain"); - int contentLength = log.getLog().length(); + int contentLength = log.getContents().length; response.setContentLength(contentLength); - - response.addHeader( - "Content-Disposition", - "attachment; filename=" + ids[0] + "_" + log.getFilename()); - - response.getWriter().write(log.getLog()); + String contentDisposition = "attachment; filename=" + ids[0] + "_" + log.getFilename(); + contentDisposition = validator.getValidInput("Content Disposition", contentDisposition, "Filename", MartiValidator.DEFAULT_STRING_CHARS, false); + response.addHeader("Content-Disposition", contentDisposition); + response.getOutputStream().write(log.getContents()); } // @@ -188,7 +185,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) } zos.putNextEntry(new ZipEntry(nextId + "_" + log.getFilename())); - zos.write(log.getLog().getBytes()); + zos.write(log.getContents()); zos.closeEntry(); } zos.close(); diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/logs/PersistenceStore.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/logs/PersistenceStore.java index 569a19dd..6572f8a2 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/logs/PersistenceStore.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/logs/PersistenceStore.java @@ -38,7 +38,7 @@ public boolean addLog(Log errorLog) { try (Connection connection = ds.getConnection(); PreparedStatement insert = wrapper.prepareStatement( "insert into error_logs ( uid, callsign, platform, " + - "major_version, minor_version, filename, log ) " + + "major_version, minor_version, filename, contents ) " + "values ( ?, ?, ?, ?, ?, ?, ? )", connection)) { validator.getValidInput("errorLog", errorLog.getUid(), @@ -60,7 +60,7 @@ public boolean addLog(Log errorLog) { insert.setString(4, errorLog.getMajorVersion()); insert.setString(5, errorLog.getMinorVersion()); insert.setString(6, errorLog.getFilename()); - insert.setString(7, errorLog.getLog()); + insert.setBytes(7, errorLog.getContents()); insert.executeUpdate(); insert.close(); @@ -97,11 +97,17 @@ public static Log logFromResultSet(ResultSet results, boolean includeLogContents { logz = null; } - log.setLog(logz); + + if (logz != null && logz.length() > 0) { + log.setContents(logz.getBytes()); + } else { + log.setContents(results.getBytes("contents")); + } if (onlyCallstack) { log.storeCallstacks(); log.setLog(null); + log.setContents(null); } } @@ -165,7 +171,7 @@ public List getLogs(String query, boolean metrics, boolean includeLogConten String sql = "select id, uid, callsign, platform, major_version, minor_version, "; if (includeLogContents) { - sql += "log, "; + sql += "log, contents, "; } sql += "filename, time " + diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/maplayer/MapLayerService.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/maplayer/MapLayerService.java new file mode 100644 index 00000000..9e28a429 --- /dev/null +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/maplayer/MapLayerService.java @@ -0,0 +1,100 @@ +package com.bbn.marti.maplayer; + + +import java.util.List; +import java.util.Date; +import java.util.UUID; +import com.google.common.base.Strings; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Sort; +import com.bbn.marti.maplayer.model.MapLayer; +import com.bbn.marti.maplayer.repository.MapLayerRepository; +import com.bbn.marti.remote.exception.NotFoundException; +import com.bbn.marti.remote.exception.TakException; + + +public class MapLayerService { + + @Autowired + MapLayerRepository mapLayerRepository; + + public List getAllMapLayers() { + return mapLayerRepository.findAllByMissionIsNull(Sort.by("name")); + } + + public MapLayer createMapLayer(MapLayer mapLayer) { + String uid = UUID.randomUUID().toString().replace("-", ""); + mapLayer.setUid(uid); + mapLayer.setCreateTime(new Date()); + mapLayer.setModifiedTime(new Date()); + MapLayer newMapLayer; + + if (mapLayer.isDefaultLayer()) { + mapLayerRepository.unsetDefault(); + } + try { + + newMapLayer = mapLayerRepository.save(mapLayer); + + } catch (Exception e) { + throw new TakException("exception in createMapLayer", e); + } + + return newMapLayer; + + } + + public MapLayer getMapLayerForUid(String uid) { + if (Strings.isNullOrEmpty(uid)) { + throw new IllegalArgumentException("UID must be specified"); + } + MapLayer mapLayer = mapLayerRepository.findByUidNoMission(uid); + + if (mapLayer == null) { + throw new NotFoundException("no map layer stored for uid " + uid); + } + + return mapLayer; + } + + public void deleteMapLayer(String uid) { + + if (Strings.isNullOrEmpty(uid)) { + throw new IllegalArgumentException("UID must be specified"); + } + try { + mapLayerRepository.deleteByUid(uid); + } catch (Exception e) { + throw new TakException("exception in deleteMapLayer", e); + } + } + + public MapLayer updateMapLayer(MapLayer modMapLayer) { + MapLayer updatedMapLayer; // result set returned + String uid = modMapLayer.getUid(); + try { + MapLayer record = mapLayerRepository.findByUid(uid); + if (record == null) { + throw new NotFoundException("no map layer stored for uid " + uid); + } + // if the new layer is the default, unset all the others + if (modMapLayer.isDefaultLayer()) { + mapLayerRepository.unsetDefault(); + } + record.setCreatorUid(modMapLayer.getCreatorUid()); + record.setName(modMapLayer.getName()); + record.setDescription(modMapLayer.getDescription()); + record.setType(modMapLayer.getType()); + record.setUrl(modMapLayer.getUrl()); + record.setModifiedTime(new Date()); + record.setDefaultLayer(modMapLayer.isDefaultLayer()); + record.setEnabled(modMapLayer.isEnabled()); + + updatedMapLayer = mapLayerRepository.save(record); + } catch (Exception e) { + throw new TakException("exception in updateMapLayer", e); + } + + return updatedMapLayer; + } +} \ No newline at end of file diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/maplayer/api/MapLayersApi.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/maplayer/api/MapLayersApi.java index 00c58cb0..c1cfb5ac 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/maplayer/api/MapLayersApi.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/maplayer/api/MapLayersApi.java @@ -1,13 +1,12 @@ package com.bbn.marti.maplayer.api; + import java.rmi.RemoteException; import java.util.Collection; -import java.util.Date; -import java.util.UUID; +import com.bbn.marti.maplayer.MapLayerService; import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.Sort; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; @@ -18,15 +17,12 @@ import com.bbn.marti.cot.search.model.ApiResponse; import com.bbn.marti.maplayer.model.MapLayer; -import com.bbn.marti.maplayer.repository.MapLayerRepository; import com.bbn.marti.network.BaseRestController; -import com.bbn.marti.remote.exception.NotFoundException; -import com.bbn.marti.remote.exception.TakException; -import com.google.common.base.Strings; import io.swagger.annotations.Api; import tak.server.Constants; + /* * * REST API for adding and retrieving Map Layers. @@ -37,7 +33,7 @@ public class MapLayersApi extends BaseRestController { @Autowired - private MapLayerRepository mapLayerRepository; + private MapLayerService mapLayerService; /* * Get all Map Layers @@ -47,53 +43,26 @@ public class MapLayersApi extends BaseRestController { ApiResponse> getAllMapLayers() throws RemoteException { return new ApiResponse>(Constants.API_VERSION, MapLayer.class.getName(), - mapLayerRepository.findAll(Sort.by("name"))); + mapLayerService.getAllMapLayers()); } - /* * Create a Map Layer */ @RequestMapping(value = "/maplayers", method = RequestMethod.POST) @ResponseStatus(HttpStatus.OK) ApiResponse createMapLayer(@RequestBody MapLayer mapLayer) { - String uid = UUID.randomUUID().toString().replace("-", ""); - mapLayer.setUid(uid); - mapLayer.setCreateTime(new Date()); - mapLayer.setModifiedTime(new Date()); - MapLayer newMapLayer; - - if (mapLayer.isDefaultLayer()) { - mapLayerRepository.unsetDefault(); - } - try { - - newMapLayer = mapLayerRepository.save(mapLayer); - - } catch (Exception e) { - throw new TakException("exception in createMapLayer", e); - } - - return new ApiResponse<>(Constants.API_VERSION, "MapLayer", newMapLayer); + return new ApiResponse<>(Constants.API_VERSION, "MapLayer", mapLayerService.createMapLayer(mapLayer)); } /* * Get Map Layer per uid */ - @RequestMapping(value = "/maplayers/{uid}", method = RequestMethod.GET) @ResponseStatus(HttpStatus.OK) ApiResponse getMapLayerForUid(@PathVariable("uid") @NotNull String uid) throws RemoteException { - if (Strings.isNullOrEmpty(uid)) { - throw new IllegalArgumentException("UID must be specified"); - } - MapLayer mapLayer = mapLayerRepository.findByUid(uid); - - if (mapLayer == null) { - throw new NotFoundException("no map layer stored for uid " + uid); - } - - return new ApiResponse<>(Constants.API_VERSION, "MapLayer", mapLayer); + return new ApiResponse<>(Constants.API_VERSION, "MapLayer", + mapLayerService.getMapLayerForUid(uid)); } /* @@ -101,15 +70,7 @@ ApiResponse getMapLayerForUid(@PathVariable("uid") @NotNull String uid */ @RequestMapping(value = "/maplayers/{uid}", method = RequestMethod.DELETE) public void deleteMapLayer(@PathVariable("uid") @NotNull String uid) throws RemoteException { - - if (Strings.isNullOrEmpty(uid)) { - throw new IllegalArgumentException("UID must be specified"); - } - try { - mapLayerRepository.deleteByUid(uid); - } catch (Exception e) { - throw new TakException("exception in deleteMapLayer", e); - } + mapLayerService.deleteMapLayer(uid); } /** @@ -120,31 +81,6 @@ public void deleteMapLayer(@PathVariable("uid") @NotNull String uid) throws Remo */ @RequestMapping(value = "/maplayers", method = RequestMethod.PUT) ApiResponse updateMapLayer(@RequestBody MapLayer modMapLayer) throws RemoteException { - - MapLayer updatedMapLayer; // result set returned - String uid = modMapLayer.getUid(); - try { - MapLayer record = mapLayerRepository.findByUid(uid); - if (record == null) { - throw new NotFoundException("no map layer stored for uid " + uid); - } - // if the new layer is the default, unset all the others - if (modMapLayer.isDefaultLayer()) { - mapLayerRepository.unsetDefault(); - } - record.setCreatorUid(modMapLayer.getCreatorUid()); - record.setName(modMapLayer.getName()); - record.setDescription(modMapLayer.getDescription()); - record.setType(modMapLayer.getType()); - record.setUrl(modMapLayer.getUrl()); - record.setModifiedTime(new Date()); - record.setDefaultLayer(modMapLayer.isDefaultLayer()); - record.setEnabled(modMapLayer.isEnabled()); - - updatedMapLayer = mapLayerRepository.save(record); - } catch (Exception e) { - throw new TakException("exception in updateMapLayer", e); - } - return new ApiResponse<>(Constants.API_VERSION, "MapLayer", updatedMapLayer); + return new ApiResponse<>(Constants.API_VERSION, "MapLayer", mapLayerService.updateMapLayer(modMapLayer)); } } \ No newline at end of file diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/maplayer/model/MapLayer.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/maplayer/model/MapLayer.java index 5e3081b1..734104d5 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/maplayer/model/MapLayer.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/maplayer/model/MapLayer.java @@ -4,20 +4,15 @@ import java.util.Date; import java.util.Objects; -import javax.persistence.Cacheable; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; -import javax.persistence.Temporal; -import javax.persistence.TemporalType; +import javax.persistence.*; +import javax.xml.bind.annotation.XmlTransient; +import com.bbn.marti.sync.model.Mission; import org.jetbrains.annotations.NotNull; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; import io.swagger.annotations.ApiModel; import tak.server.Constants; @@ -26,6 +21,7 @@ @Table(name = "maplayer") @Cacheable @ApiModel +@JsonInclude(JsonInclude.Include.NON_NULL) public class MapLayer implements Serializable, Comparable { /** @@ -33,25 +29,74 @@ public class MapLayer implements Serializable, Comparable { */ private static final long serialVersionUID = 1L; private Long id; + private Integer minZoom; + private Integer maxZoom; + private Double north; + private Double south; + private Double east; + private Double west; private String uid; private String creatorUid; private String name; private String description; - private String type; // MapTile or WMS + private String type = "MapTile"; // MapTile or WMS private String url; + private String tileType; + private String serverParts; + private String backgroundColor; + private String tileUpdate; + private String additionalParameters; + private String coordinateSystem; + private String version; + private String layers; + private Integer opacity; private Date createTime; private Date modifiedTime; private boolean defaultLayer; private boolean enabled; + private boolean ignoreErrors; + private boolean invertYCoordinate; + protected Mission mission; public MapLayer() { } + public MapLayer(MapLayer other) { + this.minZoom = other.minZoom; + this.maxZoom = other.maxZoom; + this.north = other.north; + this.south = other.south; + this.east = other.east; + this.west = other.west; + //this.uid = other.uid; + this.creatorUid = other.creatorUid; + this.name = other.name; + this.description = other.description; + this.type = other.type; + this.url = other.url; + this.tileType = other.tileType; + this.serverParts = other.serverParts; + this.backgroundColor = other.backgroundColor; + this.tileUpdate = other.tileUpdate; + this.additionalParameters = other.additionalParameters; + this.coordinateSystem = other.coordinateSystem; + this.layers = other.layers; + this.createTime = other.createTime; + this.modifiedTime = other.modifiedTime; + this.defaultLayer = other.defaultLayer; + this.enabled = other.enabled; + this.ignoreErrors = other.ignoreErrors; + this.invertYCoordinate = other.invertYCoordinate; + this.opacity = other.opacity; + this.version = other.version; + } + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(name = "id", unique = true, nullable = false) @JsonIgnore + @XmlTransient public Long getId() { return id; } @@ -154,15 +199,201 @@ public void setDefaultLayer(boolean defaultLayer) { this.defaultLayer = defaultLayer; } + @Column(name = "min_zoom") + public Integer getMinZoom() { + return minZoom; + } + + public void setMinZoom(Integer minZoom) { + this.minZoom = minZoom; + } + + @Column(name = "max_zoom") + public Integer getMaxZoom() { + return maxZoom; + } + + public void setMaxZoom(Integer maxZoom) { + this.maxZoom = maxZoom; + } + + @Column(name = "tile_type") + public String getTileType() { + return tileType; + } + + public void setTileType(String tileType) { + this.tileType = tileType; + } + + @Column(name = "server_parts") + public String getServerParts() { + return serverParts; + } + + public void setServerParts(String serverParts) { + this.serverParts = serverParts; + } + + @Column(name = "background_color") + public String getBackgroundColor() { + return backgroundColor; + } + + public void setBackgroundColor(String backgroundColor) { + this.backgroundColor = backgroundColor; + } + + @Column(name = "tile_update") + public String getTileUpdate() { + return tileUpdate; + } + + public void setTileUpdate(String tileUpdate) { + this.tileUpdate = tileUpdate; + } + + @Column(name = "ignore_errors") + public boolean isIgnoreErrors() { + return ignoreErrors; + } + + public void setIgnoreErrors(boolean ignoreErrors) { + this.ignoreErrors = ignoreErrors; + } + + @Column(name = "invert_y_coordinate") + public boolean isInvertYCoordinate() { + return invertYCoordinate; + } + + public void setInvertYCoordinate(boolean invertYCoordinate) { + this.invertYCoordinate = invertYCoordinate; + } + + @Column(name = "north") + public Double getNorth() { + return north; + } + + public void setNorth(Double north) { + this.north = north; + } + + @Column(name = "south") + public Double getSouth() { + return south; + } + + public void setSouth(Double south) { + this.south = south; + } + + @Column(name = "east") + public Double getEast() { + return east; + } + + public void setEast(Double east) { + this.east = east; + } + + @Column(name = "west") + public Double getWest() { + return west; + } + + public void setWest(Double west) { + this.west = west; + } + + @Column(name = "additional_parameters") + public String getAdditionalParameters() { + return additionalParameters; + } + + public void setAdditionalParameters(String additionalParameters) { + this.additionalParameters = additionalParameters; + } + + @Column(name = "coordinate_system") + public String getCoordinateSystem() { + return coordinateSystem; + } + + public void setCoordinateSystem(String coordinateSystem) { + this.coordinateSystem = coordinateSystem; + } + + @Column(name = "layers") + public String getLayers() { + return layers; + } + + public void setLayers(String layers) { + this.layers = layers; + } + + @Column(name = "version") + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + @Column(name = "opacity") + public Integer getOpacity() { + return opacity; + } + + public void setOpacity(Integer opacity) { + this.opacity = opacity; + } + + + @JsonIgnore + @XmlTransient + @ManyToOne + @JoinColumn(name="mission_id") + public Mission getMission() { return mission; } + public void setMission(Mission mission) { this.mission = mission; } + + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; MapLayer mapLayer = (MapLayer) o; - return Objects.equals(uid, mapLayer.uid) && - Objects.equals(url, mapLayer.url) && + return + Objects.equals(minZoom, mapLayer.minZoom) && + Objects.equals(maxZoom, mapLayer.maxZoom) && + Objects.equals(north, mapLayer.north) && + Objects.equals(south, mapLayer.south) && + Objects.equals(east, mapLayer.east) && + Objects.equals(west, mapLayer.west) && + Objects.equals(uid, mapLayer.uid) && + Objects.equals(creatorUid, mapLayer.creatorUid) && Objects.equals(name, mapLayer.name) && - Objects.equals(type, mapLayer.type); + Objects.equals(description, mapLayer.description) && + Objects.equals(type, mapLayer.type) && + Objects.equals(url, mapLayer.url) && + Objects.equals(tileType, mapLayer.tileType) && + Objects.equals(serverParts, mapLayer.serverParts) && + Objects.equals(backgroundColor, mapLayer.backgroundColor) && + Objects.equals(tileUpdate, mapLayer.tileUpdate) && + Objects.equals(additionalParameters, mapLayer.additionalParameters) && + Objects.equals(coordinateSystem, mapLayer.coordinateSystem) && + Objects.equals(layers, mapLayer.layers) && + Objects.equals(createTime, mapLayer.createTime) && + Objects.equals(modifiedTime, mapLayer.modifiedTime) && + Objects.equals(defaultLayer, mapLayer.defaultLayer) && + Objects.equals(enabled, mapLayer.enabled) && + Objects.equals(ignoreErrors, mapLayer.ignoreErrors) && + Objects.equals(invertYCoordinate, mapLayer.invertYCoordinate) && + Objects.equals(opacity, mapLayer.opacity) && + Objects.equals(version, mapLayer.version); } @Override @@ -191,5 +422,4 @@ public String toString() { public int compareTo(@NotNull MapLayer mapLayer) { return 0; } - } diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/maplayer/repository/MapLayerRepository.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/maplayer/repository/MapLayerRepository.java index fe03c114..a8852764 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/maplayer/repository/MapLayerRepository.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/maplayer/repository/MapLayerRepository.java @@ -8,6 +8,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; import com.bbn.marti.maplayer.model.MapLayer; @@ -15,7 +16,15 @@ public interface MapLayerRepository extends JpaRepository { MapLayer findByUid(String uid); - List findAll(Sort sort); + + @Query(value = " select id, create_time, modified_time, uid, creator_uid, name, description, type, url, " + + "default_layer, enabled, min_zoom, max_zoom, tile_type, server_parts, background_color, tile_update, " + + "ignore_errors, invert_y_coordinate, north, south, east, west, coordinate_system, " + + "additional_parameters, opacity, version, layers, " + + "null as mission_id from maplayer where uid = :uid ", nativeQuery = true) + MapLayer findByUidNoMission(@Param("uid") String uid); + + List findAllByMissionIsNull(Sort sort); @Transactional void deleteByUid(String uid); @@ -24,5 +33,10 @@ public interface MapLayerRepository extends JpaRepository { @Transactional @Query(value = "update maplayer set default_layer = false") void unsetDefault(); + + @Modifying + @Transactional + @Query(value = "delete from maplayer where mission_id = :mission_id", nativeQuery = true) + void deleteAllByMissionId(@Param("mission_id") Long mission_id); } diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/network/ContactManagerApi.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/network/ContactManagerApi.java index a5c37967..5c8bf3f2 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/network/ContactManagerApi.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/network/ContactManagerApi.java @@ -1,9 +1,9 @@ - package com.bbn.marti.network; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.Callable; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -41,50 +41,45 @@ public class ContactManagerApi extends BaseRestController { @Autowired private CommonUtil martiUtil; - @RequestMapping(value = "/clientEndPoints", method = RequestMethod.GET) - public ResponseEntity>> getClientEndpoints(HttpServletRequest request, HttpServletResponse response, - @RequestParam(value="secAgo", required=false, defaultValue="-1") long secAgo, - @RequestParam(value="showCurrentlyConnectedClients", required=false, defaultValue="false") String showCurrentlyConnectedClients, - @RequestParam(value="showMostRecentOnly", required=false, defaultValue="false") String showMostRecentOnly) { + @RequestMapping(value = "/clientEndPoints", method = RequestMethod.GET) + public Callable>>> getClientEndpoints(HttpServletRequest request, HttpServletResponse response, + @RequestParam(value="secAgo", required=false, defaultValue="0") long secAgo, + @RequestParam(value="showCurrentlyConnectedClients", required=false, defaultValue="false") String showCurrentlyConnectedClients, + @RequestParam(value="showMostRecentOnly", required=false, defaultValue="false") String showMostRecentOnly) { + + if (logger.isDebugEnabled()) { + logger.debug("Received REST call for clientEndPoints with params: secAgo = " + secAgo + ", showCurrentlyConnectedClients = " + showCurrentlyConnectedClients); + } + + final String groupVector = martiUtil.getGroupVectorBitString(request, Direction.OUT); - if (logger.isDebugEnabled()) { - logger.debug("Received REST call for clientEndPoints with params: secAgo = " + secAgo + ", showCurrentlyConnectedClients = " + showCurrentlyConnectedClients); - } + return () -> { - setCacheHeaders(response); + setCacheHeaders(response); - List errors = new ArrayList(); + List errors = new ArrayList(); - ResponseEntity>> result = null; - boolean connected = Boolean.valueOf(showCurrentlyConnectedClients); - boolean recent = Boolean.valueOf(showMostRecentOnly); + ResponseEntity>> result = null; + boolean connected = Boolean.valueOf(showCurrentlyConnectedClients); + boolean recent = Boolean.valueOf(showMostRecentOnly); - List data = null; + if (secAgo < 0) { + throw new IllegalArgumentException("invalid secAgo parameter " + secAgo); + } - String groupVector = martiUtil.getGroupVectorBitString(request, Direction.OUT); + try { - try { - - if (secAgo >= 0) { - data = contactManagerService.getClientEndpointData(secAgo, connected, groupVector); - } else { - // use default limit from CoreConfig.buffer.queue.contactCacheRecencyLimitSeconds - data = contactManagerService.getCachedClientEndpointData(connected, recent, groupVector); - } - - result = new ResponseEntity>>(new ApiResponse>(Constants.API_VERSION, ClientEndpoint.class.getName(), data), HttpStatus.OK); + return new ResponseEntity>>(new ApiResponse>(Constants.API_VERSION, ClientEndpoint.class.getName(), contactManagerService.getCachedClientEndpointData(connected, recent, groupVector, secAgo)), HttpStatus.OK); - } catch (Exception e) { - errors.add("Exception getting client endpoint search results."); - logger.error("Exception getting client endpoint search results.", e); - } + } catch (Exception e) { + errors.add("Exception getting client endpoint search results."); + logger.error("Exception getting client endpoint search results.", e); + } - if (result == null) { - //This would be an error condition (not an empty list) - result = new ResponseEntity>>(new ApiResponse>(Constants.API_VERSION, ClientEndpoint.class.getName(), null, errors), HttpStatus.INTERNAL_SERVER_ERROR); - } + - return result; - } + return result; + }; + } } diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/network/ContactManagerService.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/network/ContactManagerService.java index 879ea285..5c71b8ce 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/network/ContactManagerService.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/network/ContactManagerService.java @@ -1,5 +1,4 @@ - package com.bbn.marti.network; import java.sql.ResultSet; @@ -44,83 +43,6 @@ public class ContactManagerService { private AtomicLong lastUpdateMillis = new AtomicLong(-1); - public List getClientEndpointData(long secAgo, boolean connected, String groupVector) { - - try { - - if (logger.isDebugEnabled()) { - logger.debug("no cache query for contacts - secAgo: " + secAgo + " connected: " + connected); - } - - Object[] arguments = null; - - JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource); - - String query = null; - - //There is the remote possibility that the queries below return multiple values - //for a callsign/uid pair if there are multiple entries at exactly the same timestamp. - //However, the changes needed to address this condition would introduce a performance hit - //that's not worth the benefit. - - //Client has specified secAgo, which means he wants to see all endpoints that have connected within secAgo seconds, - //regardless of the endpoints' current status (connected or disconnected) - // this is harder to cache since it's a relative value - query = "select ce.callsign, ce.uid, cee.created_ts, cet.event_name, cee.groups " + - "from " + - " (select ce2.callsign, ce2.uid, max(cee2.created_ts) last_event_time " + - " from client_endpoint ce2 join client_endpoint_event cee2 on ce2.id = cee2.client_endpoint_id " + - " join (select distinct ce3.id " + - " from client_endpoint ce3 join client_endpoint_event cee3 on ce3.id = cee3.client_endpoint_id " + - " join connection_event_type cet3 on cee3.connection_event_type_id = cet3.id " + - " where " + (secAgo > 0 ? " cee3.created_ts >= (current_timestamp - (? || ' seconds')::interval) and " : "") + - " cet3.event_name = 'Connected') tt1 on ce2.id = tt1.id " + - " where " + RemoteUtil.getInstance().getGroupClause("cee2") + - " group by ce2.callsign, ce2.uid) t1 join client_endpoint ce on t1.callsign = ce.callsign and t1.uid = ce.uid " + - "join client_endpoint_event cee on ce.id = cee.client_endpoint_id and cee.created_ts = t1.last_event_time " + - "join connection_event_type cet on cee.connection_event_type_id = cet.id"; - - //Calling client is only interested in seeing currently connected clients; - //this filter may be added to the secAgo filter above - if (connected) { - query += " where cet.event_name = 'Connected' "; - } - - //Sorting is managed by the database and applies to all filter combinations - query += " order by callsign, uid "; - - if (secAgo > 0) { - arguments = new Object[] { secAgo, groupVector }; - } else { - arguments = new Object[] { groupVector }; - } - - if (logger.isDebugEnabled()) { - logger.debug("query: " + query + " args: "); - for (Object arg : arguments) { - logger.debug(arg + " "); - } - } - - @SuppressWarnings("deprecation") - List result = jdbcTemplate.query(query, arguments, new ClientEndpointResultSetExtractor()); - - if (logger.isDebugEnabled()) { - logger.debug("result size: " + result.size()); - } - - return result; - - } catch (Exception e) { - logger.error("exception getting non-cached contacts", e); - throw new TakException(e); - } - } - - public List getCachedClientEndpointData(boolean connected, boolean recent, String groupVector) { - return getCachedClientEndpointData(connected, recent, groupVector, config.getCachedConfiguration().getBuffer().getQueue().getContactCacheRecencyLimitSeconds()); - } - public List getCachedClientEndpointData(boolean connected, boolean recent, String groupVector, long secAgo) { try { @@ -130,7 +52,7 @@ public List getCachedClientEndpointData(boolean connected, boole List result = null; - String key = contactCache.getKeyGetCachedClientEndpointData(connected, recent); + String key = contactCache.getKeyGetCachedClientEndpointData(connected, recent, secAgo); result = (List) contactCache.getContactsCache().getIfPresent(key); @@ -211,7 +133,7 @@ private List queryClientEndpointData(boolean connected, boolean if (recent) { // only show the most recent uid for a given callsign - query = "select ce.callsign, ce.uid, cee.created_ts, cet.event_name, cee.groups " + + query = "select ce.callsign, ce.uid, ce.username, cee.created_ts, cet.event_name, cee.groups " + "from " + " (select ce2.callsign, max(cee2.id) client_endpoint_event_id " + " from client_endpoint ce2 join client_endpoint_event cee2 on ce2.id = cee2.client_endpoint_id" + (secAgo > 0 ? " where cee2.created_ts >= (current_timestamp - (? || ' seconds')::interval) " : "") + @@ -220,11 +142,11 @@ private List queryClientEndpointData(boolean connected, boolean " join client_endpoint ce on cee.client_endpoint_id = ce.id " + " join connection_event_type cet on cee.connection_event_type_id = cet.id "; } else { - query = "select ce.callsign, ce.uid, cee.created_ts, cet.event_name, cee.groups " + + query = "select ce.callsign, ce.uid, ce.username, cee.created_ts, cet.event_name, cee.groups " + "from " + - " (select ce2.callsign, ce2.uid, max(cee2.created_ts) last_event_time " + + " (select ce2.callsign, ce2.uid, ce2.username, max(cee2.created_ts) last_event_time " + " from client_endpoint ce2 join client_endpoint_event cee2 on ce2.id = cee2.client_endpoint_id" + (secAgo > 0 ? " where cee2.created_ts >= (current_timestamp - (? || ' seconds')::interval) " : "") + - " group by ce2.callsign, ce2.uid ) t1 join client_endpoint ce on t1.callsign = ce.callsign and t1.uid = ce.uid " + + " group by ce2.callsign, ce2.uid, ce2.username ) t1 join client_endpoint ce on t1.callsign = ce.callsign and t1.uid = ce.uid and t1.username = ce.username " + "join client_endpoint_event cee on ce.id = cee.client_endpoint_id and cee.created_ts = t1.last_event_time " + "join connection_event_type cet on cee.connection_event_type_id = cet.id "; } @@ -234,11 +156,15 @@ private List queryClientEndpointData(boolean connected, boolean } //Sorting is managed by the database and applies to all filter combinations - query += " order by callsign, uid "; + query += " order by callsign, uid, username "; if (logger.isDebugEnabled()) { logger.debug("executing client endpoints query"); } + + if (logger.isTraceEnabled()) { + logger.trace("client endpoints query: " + query + " secAgo: " + secAgo); + } Object[] arguments = new Object[] { secAgo }; @@ -268,11 +194,13 @@ public List extractData(ResultSet resultSet) throws SQLException validator.isValidInput("Client Endpoint uid", uid, MartiValidator.Regex.MartiSafeString.name(), MartiValidator.DEFAULT_STRING_CHARS, false); String callsign = resultSet.getString("callsign"); validator.isValidInput("Client Endpoint callsign", callsign, MartiValidator.Regex.MartiSafeString.name(), MartiValidator.DEFAULT_STRING_CHARS, false); + String username = resultSet.getString("username"); + validator.isValidInput("Client Endpoint username", username, MartiValidator.Regex.MartiSafeString.name(), MartiValidator.DEFAULT_STRING_CHARS, true); java.sql.Timestamp ts = resultSet.getTimestamp("created_ts"); String lastEventName = resultSet.getString("event_name"); validator.isValidInput("Client Endpoint event name", lastEventName, MartiValidator.Regex.MartiSafeString.name(), MartiValidator.DEFAULT_STRING_CHARS, false); String groups = resultSet.getString("groups"); - list.add(new ClientEndpoint(callsign, uid, new Date(ts.getTime()), lastEventName, groups)); + list.add(new ClientEndpoint(callsign, uid, username, new Date(ts.getTime()), lastEventName, groups)); } catch (Throwable t) { logger.debug("Exception processing row in ClientEndpointResultSetExtractor: ", t); } @@ -283,7 +211,7 @@ public List extractData(ResultSet resultSet) throws SQLException } public String getCallsignForUid(String clientUid, String groupVector) { - List clientEndpoints = getCachedClientEndpointData(true, true, groupVector); + List clientEndpoints = getCachedClientEndpointData(true, true, groupVector, 0); for (ClientEndpoint clientEndpoint : clientEndpoints) { if (clientEndpoint.getUid().compareToIgnoreCase(clientUid) == 0) { return clientEndpoint.getCallsign(); diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/network/LDAPApi.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/network/LDAPApi.java index 08cf356c..fcb54667 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/network/LDAPApi.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/network/LDAPApi.java @@ -82,7 +82,7 @@ public int compare(LdapGroup thiz, LdapGroup that) { } }); - List groupsAsList = groupManager.searchGroups(groupNameFilter); + List groupsAsList = groupManager.searchGroups(groupNameFilter, false); groups.addAll(groupsAsList); @@ -135,7 +135,7 @@ public ResponseEntity> getLdapGroupMembers( if (errors.isEmpty()) { for (String filter : search) { - for (LdapGroup group : groupManager.searchGroups(filter)) { + for (LdapGroup group : groupManager.searchGroups(filter, false)) { if (group.getMembers() == null) { continue; } diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/network/PluginDataFeedJdbc.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/network/PluginDataFeedJdbc.java new file mode 100644 index 00000000..1380dc15 --- /dev/null +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/network/PluginDataFeedJdbc.java @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2013-2015 Raytheon BBN Technologies. Licensed to US Government with unlimited rights. + */ + +package com.bbn.marti.network; + +import java.io.Serializable; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import javax.sql.DataSource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; + +import tak.server.feeds.DataFeed.DataFeedType; +import tak.server.plugins.PluginDataFeed; + + +public class PluginDataFeedJdbc { + + Logger logger = LoggerFactory.getLogger(PluginDataFeedJdbc.class); + + @Autowired + private DataSource dataSource; + + public List getPluginDataFeeds(){ + + HashMap hashTable = new HashMap<>(); + + List dbItems = queryPluginDataFeedWithTagEntries(); + + for (PluginDataFeedWithTagEntry dbItem: dbItems) { + PluginDataFeed pluginFeed = hashTable.get(dbItem.getUuid()); + if (pluginFeed == null) { + List tags = new ArrayList(); + if (dbItem.getTag() != null) { + tags.add(dbItem.getTag()); + } + pluginFeed = new PluginDataFeed(dbItem.getUuid(), dbItem.getName(), tags, dbItem.isArchive(), dbItem.isSync()); + hashTable.put(dbItem.getUuid(),pluginFeed); + }else { + if (dbItem.getTag() != null) { + pluginFeed.getTags().add(dbItem.getTag()); + } + } + } + + List allPluginDatafeeds = new ArrayList(hashTable.values()); + return allPluginDatafeeds; + } + + private List queryPluginDataFeedWithTagEntries(){ + + JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource); + + String query = "select uuid, name, tag, archive, sync from data_feed left join data_feed_tag on data_feed_tag.data_feed_id = data_feed.id where type = " + DataFeedType.Plugin.ordinal(); + + List result = jdbcTemplate.query(query, new PluginDataFeedWithTagEntryMapper()); + + if (logger.isDebugEnabled()) { + logger.debug("result size: " + result.size()); + } + + return result; + } + + private class PluginDataFeedWithTagEntryMapper implements RowMapper { + + @Override + public PluginDataFeedWithTagEntry mapRow(ResultSet rs, int rowNum) throws SQLException { + PluginDataFeedWithTagEntry pluginDataFeedWithTagEntry = new PluginDataFeedWithTagEntry(); + pluginDataFeedWithTagEntry.setUuid(rs.getString("uuid")); + pluginDataFeedWithTagEntry.setName(rs.getString("name")); + pluginDataFeedWithTagEntry.setTag(rs.getString("tag")); + pluginDataFeedWithTagEntry.setArchive(rs.getBoolean("archive")); + pluginDataFeedWithTagEntry.setSync(rs.getBoolean("sync")); + + return pluginDataFeedWithTagEntry; + } + + } + + class PluginDataFeedWithTagEntry implements Serializable{ + + private static final long serialVersionUID = 5919115432876484174L; + + private String uuid; + + private String name; + + private String tag; + + private boolean archive; + + private boolean sync; + + public PluginDataFeedWithTagEntry() { + } + + public PluginDataFeedWithTagEntry(String uuid, String name, String tag, boolean archive, boolean sync) { + super(); + this.uuid = uuid; + this.name = name; + this.tag = tag; + this.archive = archive; + this.sync = sync; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getTag() { + return tag; + } + + public void setTag(String tag) { + this.tag = tag; + } + + public boolean isArchive() { + return archive; + } + + public void setArchive(boolean archive) { + this.archive = archive; + } + + public boolean isSync() { + return sync; + } + + public void setSync(boolean sync) { + this.sync = sync; + } + + @Override + public String toString() { + return "PluginDataFeedWithTagEntry [uuid=" + uuid + ", name=" + name + ", tag=" + tag + ", archive=" + + archive + ", sync=" + sync + "]"; + } + + } + +} diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/network/SubmissionApi.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/network/SubmissionApi.java index 0949d112..00997549 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/network/SubmissionApi.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/network/SubmissionApi.java @@ -11,8 +11,11 @@ import java.util.List; import java.util.Map; import java.util.SortedSet; +import java.util.UUID; import java.util.concurrent.ConcurrentSkipListSet; + import javax.servlet.http.HttpServletResponse; +import javax.sql.DataSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,11 +27,12 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import com.bbn.marti.CotImageBean; import com.bbn.marti.config.AuthType; -import com.bbn.marti.config.Network.Input; +import com.bbn.marti.config.Input; import com.bbn.marti.cot.search.model.ApiResponse; import com.bbn.marti.remote.CoreConfig; import com.bbn.marti.remote.InputMetric; @@ -36,9 +40,14 @@ import com.bbn.marti.remote.groups.ConnectionModifyResult; import com.bbn.marti.remote.groups.NetworkInputAddResult; import com.bbn.marti.remote.service.InputManager; +import com.bbn.marti.sync.model.DataFeedDao; +import com.bbn.marti.sync.repository.DataFeedRepository; import com.bbn.security.web.MartiValidator; import com.google.common.collect.ComparisonChain; + import tak.server.Constants; +import tak.server.feeds.DataFeed; +import tak.server.feeds.DataFeed.DataFeedType; import tak.server.ignite.MessagingIgniteBroker; /** @@ -62,9 +71,217 @@ public class SubmissionApi extends BaseRestController { @Autowired private CoreConfig coreConfig; + + @Autowired + DataFeedRepository dataFeedRepository; + + @Autowired + DataSource ds; + + @RequestMapping(value = "/datafeeds", method = RequestMethod.GET) + public ResponseEntity>> getDataFeeds(HttpServletResponse response) { + setCacheHeaders(response); + + List dataFeeds = new ArrayList<>(); + + try { + List dataFeedDaos = dataFeedRepository.getDataFeeds(); + for (DataFeedDao dao : dataFeedDaos) { + DataFeed dataFeed = convertDataFeedDao(dao); + dataFeeds.add(dataFeed); + } + } catch (Exception e) { + logger.error("Failed getting data feeds", e); + return new ResponseEntity>>(new ApiResponse>(Constants.API_VERSION, DataFeed.class.getName(), null), HttpStatus.INTERNAL_SERVER_ERROR); + } + + return new ResponseEntity>>(new ApiResponse>(Constants.API_VERSION, DataFeed.class.getName(), dataFeeds), HttpStatus.OK); + } + + @RequestMapping(value = "/datafeeds/{name}", method = RequestMethod.GET) + public ResponseEntity> getDataFeed(@PathVariable("name") String name) { + ResponseEntity> result = null; + try { + if (!getInputNameValidationErrors(name).isEmpty()) { + result = new ResponseEntity>(new ApiResponse(Constants.API_VERSION, + DataFeed.class.getName(), null), HttpStatus.BAD_REQUEST); + } else { + List dataFeeds = dataFeedRepository.getDataFeedByName(name); + if (dataFeeds == null || dataFeeds.size() != 1) { + result = new ResponseEntity>(new ApiResponse(Constants.API_VERSION, + DataFeed.class.getName(), null), HttpStatus.BAD_REQUEST); + } else { + DataFeedDao dataFeed = dataFeeds.get(0); + DataFeed returnDataFeed = this.convertDataFeedDao(dataFeed); + result = new ResponseEntity>( + new ApiResponse(Constants.API_VERSION, DataFeed.class.getName(), returnDataFeed), + HttpStatus.OK); + } + } + } catch (Exception e) { + logger.error("Exception getting data feed.", e); + result = new ResponseEntity>(new ApiResponse(Constants.API_VERSION, DataFeed.class.getName(), null), + HttpStatus.INTERNAL_SERVER_ERROR); + } + + return result; + } + + @RequestMapping(value = "/datafeeds/{name}", method = RequestMethod.DELETE) + public ResponseEntity> deleteDataFeed(@PathVariable("name") String name) { + + ResponseEntity> result = null; + + try { + if (!getInputNameValidationErrors(name).isEmpty()) { + return new ResponseEntity>(new ApiResponse(Constants.API_VERSION, + DataFeed.class.getName(), null), HttpStatus.BAD_REQUEST); + } else { + // Delete from config file + MessagingIgniteBroker.brokerVoidServiceCalls(service -> ((InputManager) service) + .deleteDataFeed(name), Constants.DISTRIBUTED_INPUT_MANAGER, InputManager.class); + } + } catch (Exception e) { + logger.error("Exception deleting data feed from config file.", e); + + // Shouldn't return error in case deleting from database is needed + // return new ResponseEntity>(new ApiResponse(Constants.API_VERSION, DataFeed.class.getName(), null), + // HttpStatus.INTERNAL_SERVER_ERROR); + } + + try { + // Delete from database + List dataFeeds = dataFeedRepository.getDataFeedByName(name); + + if (dataFeeds.size() >0 && dataFeeds.get(0) != null) { + Long dataFeedId = dataFeeds.get(0).getId(); + + dataFeedRepository.removeAllDataFeedTagsById(dataFeedId); + dataFeedRepository.removeAllDataFeedFilterGroupsById(dataFeedId); + dataFeedRepository.deleteDataFeed(name); + } + + result = new ResponseEntity>( + new ApiResponse(Constants.API_VERSION, DataFeed.class.getName(), null), + HttpStatus.OK); + } catch (Exception e) { + logger.error("Exception deleting data feed from database.", e); + result = new ResponseEntity>(new ApiResponse(Constants.API_VERSION, DataFeed.class.getName(), null), + HttpStatus.INTERNAL_SERVER_ERROR); + } + + return result; + } + + @RequestMapping(value = "/datafeeds/{name}", method = RequestMethod.PUT) + public ResponseEntity> modifyDataFeed(@PathVariable("name") String name, @RequestBody com.bbn.marti.config.DataFeed dataFeed) { + ResponseEntity> result = null; + + try { + List dataFeeds = dataFeedRepository.getDataFeedByName(name); + if (!getInputNameValidationErrors(name).isEmpty() || dataFeeds == null || dataFeeds.size() != 1) { + result = new ResponseEntity>(new ApiResponse(Constants.API_VERSION, + DataFeed.class.getName(), null), HttpStatus.BAD_REQUEST); + } else { + // Update config file + ConnectionModifyResult updateResult = MessagingIgniteBroker.brokerServiceCalls(service -> ((InputManager) service) + .modifyInput(name, dataFeed), Constants.DISTRIBUTED_INPUT_MANAGER, InputManager.class); + + if (updateResult.getHttpStatusCode() == ConnectionModifyResult.SUCCESS.getHttpStatusCode()) { + // Update data base + Long dataFeedId = dataFeeds.get(0).getId(); + int type = DataFeedType.valueOf(dataFeed.getType()).ordinal(); + dataFeedRepository.modifyDataFeed(dataFeed.getUuid(), dataFeed.getName(), type, dataFeed.isArchive(), dataFeed.isArchiveOnly(), dataFeed.isSync()); + + dataFeedRepository.removeAllDataFeedTagsById(dataFeedId); + for (String tag : dataFeed.getTag()) { + dataFeedRepository.addDataFeedTag(dataFeedId, tag); + } + + dataFeedRepository.removeAllDataFeedFilterGroupsById(dataFeedId); + for (String filterGroup: dataFeed.getFiltergroup()) { + dataFeedRepository.addDataFeedFilterGroup(dataFeedId, filterGroup); + } + + result = new ResponseEntity>( + new ApiResponse(Constants.API_VERSION, DataFeed.class.getName(), null), + HttpStatus.OK); + } else { + result = new ResponseEntity>(new ApiResponse(Constants.API_VERSION, DataFeed.class.getName(), null), + HttpStatus.INTERNAL_SERVER_ERROR); + } + } + } catch (Exception e) { + logger.error("Failed updating data feed", e); + result = new ResponseEntity>(new ApiResponse(Constants.API_VERSION, DataFeed.class.getName(), null), + HttpStatus.INTERNAL_SERVER_ERROR); + } + + return result; + } + @RequestMapping(value = "/datafeeds", method = RequestMethod.POST) + public ResponseEntity> createDataFeed(@RequestBody com.bbn.marti.config.DataFeed dataFeed) { + + ResponseEntity> result = null; + DataFeed returnDataFeed = new DataFeed(dataFeed); + List errors = new ArrayList<>(); + + try { + if (dataFeed.getUuid() == null) { + dataFeed.setUuid(UUID.randomUUID().toString()); + } + + errors = getDataFeedValidationErrors(dataFeed); + + if (errors.isEmpty()) { + // Add dataFeed to config file + NetworkInputAddResult addResult = MessagingIgniteBroker.brokerServiceCalls(service -> ((InputManager) service) + .createDataFeed(dataFeed), Constants.DISTRIBUTED_INPUT_MANAGER, InputManager.class); + + if (addResult == NetworkInputAddResult.SUCCESS) { + // Add dataFeed to database + int type = DataFeedType.valueOf(dataFeed.getType()).ordinal(); + String auth = dataFeed.getAuth().toString(); + + Long dataFeedId = dataFeedRepository.addDataFeed(dataFeed.getUuid(), dataFeed.getName(), type, auth, dataFeed.getPort(), + dataFeed.isAuthRequired(), dataFeed.getProtocol(), dataFeed.getGroup(), dataFeed.getIface(), dataFeed.isArchive(), + dataFeed.isAnongroup(), dataFeed.isArchiveOnly(), dataFeed.getCoreVersion(), dataFeed.getCoreVersion2TlsVersions(), + dataFeed.isSync()); + + for (String tag: dataFeed.getTag()) { + dataFeedRepository.addDataFeedTag(dataFeedId, tag); + } + for (String filterGroup: dataFeed.getFiltergroup()) { + dataFeedRepository.addDataFeedFilterGroup(dataFeedId, filterGroup); + } + result = new ResponseEntity>( + new ApiResponse(Constants.API_VERSION, DataFeed.class.getName(), returnDataFeed), + HttpStatus.OK); + } else { + logger.error("Error adding data feed to config file " + addResult.getDisplayMessage()); + errors.add(addResult.getDisplayMessage()); + } + } else { + result = new ResponseEntity>(new ApiResponse(Constants.API_VERSION, + DataFeed.class.getName(), returnDataFeed, errors), HttpStatus.BAD_REQUEST); + } + } catch (Exception e) { + logger.error("Exception adding input.", e); + errors.add(e.getMessage()); + } + + if (result == null) { + result = new ResponseEntity>(new ApiResponse(Constants.API_VERSION, DataFeed.class.getName(), null, errors), + HttpStatus.INTERNAL_SERVER_ERROR); + } + + return result; + } + @RequestMapping(value = "/inputs", method = RequestMethod.GET) - public ResponseEntity>> getInputMetrics(HttpServletResponse response) { + public ResponseEntity>> getInputMetrics( + @RequestParam(value = "excludeDataFeeds", defaultValue = "false") boolean excludeDataFeeds, HttpServletResponse response) { setCacheHeaders(response); @@ -82,7 +299,7 @@ public int compare(InputMetric thiz, InputMetric that) { } }); - Collection inputs = inputManager.getInputMetrics(); + Collection inputs = inputManager.getInputMetrics(excludeDataFeeds); if (logger.isDebugEnabled()) { logger.debug("inputs: " + inputs); @@ -229,7 +446,7 @@ public ResponseEntity> getInputMetric(@PathVariable("na result = new ResponseEntity>(new ApiResponse(Constants.API_VERSION, InputMetric.class.getName(), null), HttpStatus.BAD_REQUEST); } else { - Collection metrics = inputManager.getInputMetrics(); + Collection metrics = inputManager.getInputMetrics(false); if (metrics != null) { for (InputMetric m : metrics) { if (m.getInput().getName().equalsIgnoreCase(name)) { @@ -365,10 +582,10 @@ private List getValidationErrors(Input input) { } //Validate protocol - if (!PROTOCOLS.contains(input.getProtocol().toLowerCase())) { + if (!PROTOCOLS.contains(input.getProtocol().toString().toLowerCase())) { errors.add("Invalid protocol selection."); } else { - input.setProtocol(input.getProtocol().toLowerCase()); + input.setProtocol(input.getProtocol()); } //Validate port @@ -418,6 +635,93 @@ private List getValidationErrors(Input input) { return errors; } + + private List getDataFeedValidationErrors(com.bbn.marti.config.DataFeed dataFeed) { + List errors = new ArrayList<>(); + + if (dataFeed == null) { + //Should not occur + errors.add("Missing data feed definition."); + } else { + + //Validate input name + errors.addAll(getInputNameValidationErrors(dataFeed.getName())); + if (dataFeed.getName() != null) { + dataFeed.setName(dataFeed.getName().trim()); + } + + //Validate feed type + if (dataFeed.getType() == null) { + errors.add("Null data feed type provided"); + } + + //Validate multicast group + if (dataFeed.getGroup() != null && dataFeed.getGroup().trim().length() > 0) { + dataFeed.setGroup(dataFeed.getGroup().trim()); + if (dataFeed.getGroup().replaceFirst("^([0-9]{1,3}\\.){3}[0-9]{1,3}$", "").length() > 0) { + errors.add("Invalid multicast group value."); + } + } + + //Validate interface + if (dataFeed.getIface() != null && dataFeed.getIface().trim().length() > 0) { + dataFeed.setIface(dataFeed.getIface().trim()); + if (dataFeed.getIface().replaceFirst("^[A-Za-z0-9]+$", "").length() > 0) { + errors.add("Invalid interface value."); + } + } + + //Enforce (File v LDAP) <=> (stcp v tls v prototls v cottls)) + //If auth type is file or ldap, then protocol must be stcp or tls + if ((dataFeed.getAuth() == AuthType.FILE || dataFeed.getAuth() == AuthType.LDAP) && !(dataFeed.getProtocol().equals("stcp") || dataFeed.getProtocol().equals("tls") || dataFeed.getProtocol().equals("prototls") || dataFeed.getProtocol().equals("cottls"))) { + errors.add("If Authentication Type is set to File or LDAP, then Protocol should be be Streaming TCP or Secure Streaming TCP."); + } + + //If protocol is mcast, then multicast group must not be empty + if ((dataFeed.getProtocol().equals("mcast") || dataFeed.getProtocol().equals("cotmcast")) && (dataFeed.getGroup() == null || dataFeed.getGroup().trim().isEmpty())) { + errors.add("If Protocol is set to Multicast, then the Multicast Group must be provided."); + } + + //If multicast group must is not empty, then protocol must be mcast + if (dataFeed.getGroup() != null && !dataFeed.getGroup().trim().isEmpty() && !dataFeed.getProtocol().equals("mcast") && !dataFeed.getProtocol().equals("cotmcast")) { + errors.add("If the Multicast Group is provided, then Protocol should be set to Multicast."); + } + } + return errors; + } + + private DataFeed convertDataFeedDao(DataFeedDao dao) { + DataFeedType type = DataFeedType.values()[dao.getType()]; + List tags = dataFeedRepository.getDataFeedTagsById(dao.getId()); + List filterGroups = dataFeedRepository.getDataFeedFilterGroupsById(dao.getId()); + AuthType auth = AuthType.valueOf(dao.getAuth()); + + DataFeed dataFeed = new DataFeed(dao.getUUID(), dao.getName(), type, new ArrayList()); + + dataFeed.setAuth(auth); + dataFeed.setAnongroup(dao.getAnongroup()); + dataFeed.setAuthRequired(dao.getAuthRequired()); + dataFeed.setProtocol(dao.getProtocol()); + dataFeed.setGroup(dao.getFeedGroup()); + dataFeed.setIface(dao.getIface()); + dataFeed.setArchive(dao.getArchive()); + dataFeed.setAnongroup(dao.getAnongroup()); + dataFeed.setArchiveOnly(dao.getArchiveOnly()); + dataFeed.setCoreVersion(dao.getCoreVersion().intValue()); + dataFeed.setCoreVersion2TlsVersions(dao.getCoreVersion2TlsVersions()); + dataFeed.setSync(dao.isSync()); + dataFeed.setTags(tags); + dataFeed.setFilterGroups(filterGroups); + + + if (dao.getPort() == 0) { + dataFeed.setPort(null); + } else { + dataFeed.setPort(dao.getPort()); + } + + return dataFeed; + } private static final int INPUT_NAME_MAX_LENGTH = 30; private static final int PORT_RANGE_LOW = 1; diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/oauth/AuthCookieUtils.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/oauth/AuthCookieUtils.java index afa5be79..40565812 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/oauth/AuthCookieUtils.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/oauth/AuthCookieUtils.java @@ -1,18 +1,35 @@ package com.bbn.marti.oauth; -import org.springframework.security.oauth2.common.OAuth2AccessToken; -import org.springframework.security.oauth2.provider.token.DefaultTokenServices; - import javax.servlet.http.Cookie; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpSession; +import org.owasp.esapi.Validator; +import org.owasp.esapi.errors.ValidationException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.security.oauth2.common.OAuth2AccessToken; +import org.springframework.security.oauth2.provider.token.DefaultTokenServices; + +import com.bbn.security.web.MartiValidator; public class AuthCookieUtils { + private static Validator validator = new MartiValidator(); + private static final Logger logger = LoggerFactory.getLogger(AuthCookieUtils.class); + public static Cookie createCookie(final String content, final int expirationTimeSeconds) { - final Cookie cookie = new Cookie(OAuth2AccessToken.ACCESS_TOKEN, content); + String validatedContent; + try { + validatedContent = validator.getValidInput(AuthCookieUtils.class.getName(), content, + MartiValidator.Regex.MartiSafeString.name(), MartiValidator.LONG_STRING_CHARS, false); + } catch (ValidationException e) { + logger.error("ValidationException in createCookie!", e); + return null; + } + + final Cookie cookie = new Cookie(OAuth2AccessToken.ACCESS_TOKEN, validatedContent); cookie.setMaxAge(expirationTimeSeconds); cookie.setHttpOnly(true); cookie.setSecure(true); @@ -43,8 +60,7 @@ public static void logout(HttpServletRequest request, HttpServletResponse respon final Cookie copyCookie = createCookie(token, 0); response.addCookie(copyCookie); response.setStatus(HttpServletResponse.SC_MOVED_PERMANENTLY); - String referrer = request.getHeader("referrer"); - response.setHeader("Location", referrer != null ? referrer : "/webtak/index.html"); + response.setHeader("Location", "/webtak/index.html"); response.setHeader("Cache-Control", "no-store"); response.setHeader("Pragma", "no-cache"); return; diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/ContentServlet.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/ContentServlet.java index e6046864..9afffd92 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/ContentServlet.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/ContentServlet.java @@ -19,6 +19,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.lang3.StringUtils; import org.owasp.esapi.errors.IntrusionException; import org.owasp.esapi.errors.ValidationException; import org.slf4j.LoggerFactory; @@ -110,7 +111,9 @@ private void getResource(HttpServletRequest request, HttpServletResponse respons } if (content == null) { - logger.error("found null content for :" + query); + if (logger.isErrorEnabled()) { + logger.error("found null content for :" + StringUtils.normalizeSpace(query)); + } response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); return; } @@ -122,7 +125,7 @@ private void getResource(HttpServletRequest request, HttpServletResponse respons logger.debug("Metadata is: " + match.toJSONObject().toString()); } - String mimeType = match.getFirst(Metadata.Field.MIMEType); + String mimeType = match.getFirst(Metadata.Field.MIMEType); if(validator != null && validator.isValidInput("MIME Type", mimeType, "MartiSafeString", DEFAULT_PARAMETER_LENGTH, false)) { // Set MIME type of HTTP response if (response.containsHeader("Content-Type")) { diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/DeleteServlet.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/DeleteServlet.java index 6da30760..7d734a90 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/DeleteServlet.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/DeleteServlet.java @@ -9,6 +9,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.logging.Level; import java.util.logging.Logger; import javax.naming.NamingException; @@ -16,6 +17,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.lang3.StringUtils; import org.owasp.esapi.errors.IntrusionException; import org.owasp.esapi.errors.ValidationException; @@ -71,8 +73,9 @@ protected void doDelete(HttpServletRequest request, HttpServletResponse response List toDelete = new LinkedList(); // Parse the HTTP parameters and slurp out all valid integers for parameters matching ID_KEY - - log.finest("http params: " + httpParameters.keySet()); + if (log.isLoggable(Level.FINEST)) { + log.finest("http params: " + StringUtils.normalizeSpace(httpParameters.keySet().toString())); + } try { for (String key : httpParameters.keySet()) { if (key.compareToIgnoreCase(ID_KEY) == 0 ) { @@ -103,8 +106,9 @@ protected void doDelete(HttpServletRequest request, HttpServletResponse response if (!(httpParameters.get(key).length == 0)) { String value = httpParameters.get(key)[0]; - - log.finest("delete by hash " + value); + if (log.isLoggable(Level.FINEST)) { + log.finest("delete by hash " + StringUtils.normalizeSpace(value)); + } try { if (validator != null) { @@ -132,7 +136,9 @@ protected void doDelete(HttpServletRequest request, HttpServletResponse response // delete by primary key enterpriseSyncService.delete(toDelete, groupVector); } else { - log.finest("delete by hash" + hash); + if(log.isLoggable(Level.FINEST)) { + log.finest("delete by hash" + StringUtils.normalizeSpace(hash)); + } // delete by hash enterpriseSyncService.delete(hash, groupVector); } diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/JDBCEnterpriseSyncService.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/JDBCEnterpriseSyncService.java index 63a72f1e..077f0790 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/JDBCEnterpriseSyncService.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/JDBCEnterpriseSyncService.java @@ -1,5 +1,3 @@ - - package com.bbn.marti.sync; import java.io.ByteArrayInputStream; @@ -30,11 +28,13 @@ import java.util.SortedMap; import java.util.TreeMap; import java.util.UUID; +import java.util.logging.Level; import java.util.logging.Logger; import javax.naming.NamingException; import javax.sql.DataSource; +import org.apache.commons.lang.StringUtils; import org.owasp.esapi.Validator; import org.owasp.esapi.errors.IntrusionException; import org.owasp.esapi.errors.ValidationException; @@ -57,7 +57,6 @@ import com.bbn.marti.remote.service.RetentionPolicyConfig; import com.bbn.marti.remote.util.RemoteUtil; import com.bbn.marti.sync.Metadata.Field; -import com.bbn.marti.util.RestrictedInputStream; import com.google.common.base.Strings; import tak.server.Constants; @@ -258,7 +257,9 @@ public void delete(String hash, String groupVector) throws SQLException, NamingE + RemoteUtil.getInstance().getGroupAndClause(); // only allow delete where there is common group membership try (Connection connection = ds.getConnection(); PreparedStatement statement = wrapper.prepareStatement(sql, connection)) { - log.fine("Deleting resource hash =" + hash); + if (log.isLoggable(Level.FINE)) { + log.fine("Deleting resource hash =" + StringUtils.normalizeSpace(hash)); + } statement.setString(1, hash); statement.setString(2, groupVector); statement.execute(); @@ -302,12 +303,21 @@ public void delete(List primaryKeys, String groupVector) throws SQLExce statement.executeBatch(); } } + + + public Metadata insertResource(Metadata metadata, byte[] content, String groupVector) + throws SQLException, NamingException, IllegalArgumentException, + ValidationException, IntrusionException, IllegalStateException, IOException { + + return insertResource(metadata, content, groupVector, false); + } + /** * Inserts the given byte array into the database, with the given metadata mapped to columns. * */ - public Metadata insertResource(Metadata metadata, byte[] content, String groupVector) + public Metadata insertResource(Metadata metadata, byte[] content, String groupVector, boolean validate) throws SQLException, NamingException, IllegalArgumentException, ValidationException, IntrusionException, IllegalStateException, IOException { @@ -315,7 +325,7 @@ public Metadata insertResource(Metadata metadata, byte[] content, String groupVe throw new IllegalArgumentException("empty group vector"); } - if (validator != null) { + if (validate && validator != null) { content = validator.getValidFileContent("Storing resource content to DB", content, content.length, false); } @@ -467,7 +477,7 @@ private Metadata insertResource(Metadata metadata, InputStream contentStream, lo // bind input stream to database log.fine("binding binary stream with content length " + contentLen + " to the database query"); - statement.setBinaryStream(1, new RestrictedInputStream(contentStream, contentLen), contentLen); + statement.setBinaryStream(1, contentStream, contentLen); int columnIndex = 2; for (TypeValuePair toStore : metadataColumns) { diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/Metadata.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/Metadata.java index 392c96d1..1f81919b 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/Metadata.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/Metadata.java @@ -20,7 +20,6 @@ import org.owasp.esapi.errors.ValidationException; import com.bbn.security.web.MartiValidator; -import com.google.common.collect.ImmutableMap; /** * Metadata describing a specific resource in the Marti Enterprise Sync feature. @@ -517,6 +516,9 @@ public void validate(Validator validator) throws ValidationException, IntrusionE } } } - - + + @Override + public String toString() { + return "Metadata [fields=" + fields + "] json: " + toJSONObject(); + } } diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/MetadataServlet.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/MetadataServlet.java index a02a6fa6..5ddccf93 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/MetadataServlet.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/MetadataServlet.java @@ -10,6 +10,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; import javax.naming.NamingException; @@ -18,6 +19,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.lang3.StringUtils; import org.owasp.esapi.errors.IntrusionException; import org.owasp.esapi.errors.ValidationException; @@ -177,7 +179,9 @@ private String getMetadataAsJSON(HttpServletRequest request) } allResults.add(result); } else { - log.fine("Getting metadata for UID " + uid); + if (log.isLoggable(Level.FINE)) { + log.fine("Getting metadata for UID " + StringUtils.normalizeSpace(uid)); + } allResults = enterpriseSyncService.getMetadataByUid(uid, groupVector); Set badResults = new HashSet(); for (Metadata result : allResults) { diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/SearchServlet.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/SearchServlet.java index b325d44a..14677577 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/SearchServlet.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/SearchServlet.java @@ -15,6 +15,7 @@ import java.util.Map; import java.util.SimpleTimeZone; import java.util.SortedMap; +import java.util.logging.Level; import java.util.logging.Logger; import javax.naming.NamingException; @@ -24,6 +25,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.lang3.StringUtils; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.owasp.esapi.errors.IntrusionException; @@ -168,15 +170,18 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) for (String key : httpParameters.keySet()) { String[] values = httpParameters.get(key); if (values != null && values.length > 1) { - log.warning("Ignoring duplicate value(s) for HTTP request parameter " + key); + log.warning("Ignoring duplicate value(s) for HTTP request parameter " + StringUtils.normalizeSpace(key)); } RequestParameters searchParameter = RequestParameters.fromString(key); - log.fine("Setting search parameter " + searchParameter.toString() + "=" + values[0]); + if (log.isLoggable(Level.FINE)) { + log.fine("Setting search parameter " + StringUtils.normalizeSpace(searchParameter.toString()) + "=" + + StringUtils.normalizeSpace(values[0])); + } if (searchParameter != null) { if (values.length > 1) { - log.warning("Received " + values.length + " values for " + key + ". Ignoring all but first."); + log.warning("Received " + values.length + " values for " + StringUtils.normalizeSpace(key) + ". Ignoring all but first."); } switch (searchParameter) { case BBox: @@ -211,10 +216,12 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) default: Metadata.Field metadataField = Metadata.Field.fromString(key); if (metadataField != null) { - log.fine("Setting metadata constraint " + metadataField.toString() + "=" + values[0]); - metadataConstraints.put(metadataField, values[0]); + if (log.isLoggable(Level.FINE)) { + log.fine("Setting metadata constraint " + metadataField.toString() + "=" + StringUtils.normalizeSpace(values[0])); + } + metadataConstraints.put(metadataField, values[0]); } else { - log.warning("Unrecognized request parameter: " + key); + log.warning("Unrecognized request parameter: " + StringUtils.normalizeSpace(key)); } } } @@ -255,8 +262,10 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) if ( validator != null) { if (validator.isValidInput("doGet processing database results", uid, "MartiSafeString", MartiValidator.SHORT_STRING_CHARS, false)) { - log.fine("Processing search results for uid " + uid); - } else { + if (log.isLoggable(Level.FINE)) { + log.fine("Processing search results for uid " + StringUtils.normalizeSpace(uid)); + } + } else { log.warning("Invalid UID found in database!"); } } diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/UploadServlet.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/UploadServlet.java index 81fd54f3..21cabafe 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/UploadServlet.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/UploadServlet.java @@ -6,11 +6,8 @@ import java.io.IOException; import java.io.InputStream; import java.io.PrintWriter; -import java.rmi.RemoteException; import java.sql.SQLException; -import java.time.Duration; import java.util.Collection; -import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -34,6 +31,8 @@ import com.bbn.marti.remote.CoreConfig; import com.bbn.marti.sync.Metadata.Field; import com.google.common.base.Strings; +import com.google.common.io.ByteSource; +import com.google.common.io.ByteStreams; import io.micrometer.core.instrument.Metrics; @@ -49,6 +48,8 @@ public class UploadServlet extends EnterpriseSyncServlet { private static final long serialVersionUID = -8151782550681449153L; private static final int DEFAULT_PARAMETER_LENGTH = 1024; private static Set optionalParameters; + + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(UploadServlet.class); @Autowired private CoreConfig coreConfig; @@ -72,7 +73,7 @@ public void init(final ServletConfig config) throws ServletException { uploadSizeLimitMB = coreConfig.getRemoteConfiguration().getNetwork().getEnterpriseSyncSizeLimitMB(); - log.info("Enterprise Sync upload limit is " + uploadSizeLimitMB + " MB"); + logger.info("Enterprise Sync upload limit is " + uploadSizeLimitMB + " MB"); } @@ -113,9 +114,11 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) try { // Get group vector for the user associated with this session groupVector = martiUtil.getGroupBitVector(request); - log.finer("groups bit vector: " + groupVector); + if (logger.isDebugEnabled()) { + logger.debug("groups bit vector: " + groupVector); + } } catch (Exception e) { - log.fine("exception getting group membership for current web user " + e.getMessage()); + logger.error("exception getting group membership for current user ", e); } if (Strings.isNullOrEmpty(groupVector)) { @@ -174,24 +177,35 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) field.maximumLength, true); } } - log.finer("Added " + field.toString() + " (" + values.length + ") values"); + + if (logger.isDebugEnabled()) { + logger.debug("Added " + field.toString() + " (" + values.length + ") values"); + } metadataParameters.put(field, values); } } Metadata toStore = Metadata.fromMap(metadataParameters); toStore.validate(validator); - log.fine("Request is: " + toStore.toJSONObject().toJSONString()); - log.fine("Content length is " + request.getContentLength()); + + if (logger.isDebugEnabled()) { + logger.debug("Request is: " + toStore.toJSONObject().toJSONString()); + logger.debug("Content length is " + request.getContentLength()); + } if (request.getContentLength() > uploadSizeLimitMB * 1000000) { - String message = "Uploaded file exceeds server's size limit of " + uploadSizeLimitMB - + " MB! (limit is set in server's conf/context.xml)"; - log.warning(badRequestPrefix + message); + String message = "Uploaded file exceeds server's size limit of " + uploadSizeLimitMB + " MB! (limit is set in CoreConfig.xml network.enterpriseSyncSizeLimitMB"; + + if (logger.isWarnEnabled()) { + logger.warn(badRequestPrefix + message); + } response.sendError(HttpServletResponse.SC_BAD_REQUEST, message); return; } else if (request.getContentLength() < 1) { String message = "HTTP request body has no content."; - log.warning(badRequestPrefix + message); + + if (logger.isWarnEnabled()) { + logger.warn(badRequestPrefix + message); + } response.sendError(HttpServletResponse.SC_BAD_REQUEST, message); return; } @@ -204,11 +218,26 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) String mimeType = request.getHeader("Content-Type"); if (mimeType == null || !mimeType.contains("multipart/form-data")) { - log.fine("Uploading content."); + + if (logger.isDebugEnabled()) { + logger.debug("Uploading content."); + } + + if (logger.isDebugEnabled()) { + logger.debug("reading payload from request"); + } + payload = readByteArray(request.getInputStream()); + + if (logger.isDebugEnabled()) { + logger.debug("done reading payload from request - size: " + payload.length + " bytes"); + } } else { Collection parts = request.getParts(); - log.fine("Uploading multi-part content with " + parts.size() + " parts."); + + if (logger.isDebugEnabled()) { + logger.debug("Uploading multi-part content with " + parts.size() + " parts."); + } // ATAK sends a part called "assetfile" Part part = request.getPart("assetfile"); if (part == null) { @@ -218,13 +247,26 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) if (part != null) { try(InputStream is = part.getInputStream()) { + + if (logger.isDebugEnabled()) { + logger.debug("reading payload from request"); + } payload = readByteArray(is); + + if (logger.isDebugEnabled()) { + logger.debug("done reading payload from request - size: " + payload.length + " bytes"); + } } } else { for (Part myPart : parts) { - log.finer(myPart.getName() + ": " + myPart.getContentType() ); + if (logger.isDebugEnabled()) { + logger.debug(myPart.getName() + ": " + myPart.getContentType() ); + } + } + + if (logger.isErrorEnabled()) { + logger.error("Unable to find content in multi-part submission"); } - log.severe("Unable to find content in multi-part submission"); response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Upload request was not formatted in a way Marti can understand.\n" + "Please try a different browser."); @@ -233,7 +275,10 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) // Get the file name from the part's content-disposition header if possible String cd = part.getHeader("content-disposition"); - log.fine("content-disposition is: " + cd); + + if (logger.isDebugEnabled()) { + logger.debug("content-disposition is: " + cd); + } String filenameToken = "filename=\""; if (cd != null && cd.contains(filenameToken)) { int filenameIndex = cd.indexOf(filenameToken); @@ -271,17 +316,28 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) toStore.set(Metadata.Field.SubmissionUser, userName); } - uploadedMetadata = enterpriseSyncService.insertResource(toStore, payload, groupVector); + if (logger.isDebugEnabled()) { + logger.debug("inserting resource " + toStore + " payload length: " + payload.length); + } + uploadedMetadata = enterpriseSyncService.insertResource(toStore, payload, groupVector); // do not validate file length, as that's already checked above Metrics.counter("UploadMissionContent", "missions", "content").increment(); } catch (NamingException|SQLException ex) { - response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, - "Enterprise Sync database failed to process write operation."); - ex.printStackTrace(); + String msg = "Enterprise Sync database failed to process write operation."; + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, msg); + + if (logger.isErrorEnabled()) { + logger.error(msg, ex); + } return; } catch (ServletException srvex) { // Thrown by request.getPart if request is not a well-formed - response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Unparseable multi-part content in request."); - srvex.printStackTrace(); + + String msg = "Unparseable multi-part content in request."; + + response.sendError(HttpServletResponse.SC_BAD_REQUEST, msg); + if (logger.isErrorEnabled()) { + logger.error(msg, srvex); + } return; } catch (IOException ioex) { String errorMessage; @@ -289,27 +345,41 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) if (request.getContentLength() == 0) { responseCode = HttpServletResponse.SC_BAD_REQUEST; errorMessage = "POST request contained no data!"; - log.warning(badRequestPrefix + errorMessage); + if (logger.isWarnEnabled()) { + logger.warn(badRequestPrefix + errorMessage, ioex); + } } else { responseCode = HttpServletResponse.SC_INTERNAL_SERVER_ERROR; errorMessage = "Failed to read data from HTTP POST body"; - ioex.printStackTrace(); + + if (logger.isErrorEnabled()) { + logger.error(errorMessage, ioex); + } } response.sendError(responseCode, errorMessage); return; } catch (ValidationException ex) { - log.warning(badRequestPrefix + ex.getLogMessage()); + if (logger.isWarnEnabled()) { + logger.warn(badRequestPrefix + ex.getLogMessage()); + } response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Illegal characters detected. Accents and most punctuation characters are not allowed for security."); return; } catch (IntrusionException evilException) { - log.severe("Intrusion attempt from " + remoteHost + ": " + evilException.getLogMessage()); + if (logger.isErrorEnabled()) { + logger.error("Intrusion attempt from " + remoteHost + ": ", evilException); + } response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Intrusion attempt detected! HTTP request denied."); return; } PrintWriter writer = response.getWriter(); + + if (logger.isDebugEnabled()) { + logger.debug("response: " + response + " uploadedMetadata: " + uploadedMetadata); + } + writer.print(uploadedMetadata.toJSONObject()); writer.close(); if (response.containsHeader("Content-Type")) { @@ -329,8 +399,7 @@ public void doPut(HttpServletRequest request, HttpServletResponse response) } /** - * Utility method that reads a byte array from an input stream. This - * implementation is not efficient. + * Utility method that reads a byte array from an input stream using Guava. * * @param in * any InputStream containing some data @@ -339,18 +408,9 @@ public void doPut(HttpServletRequest request, HttpServletResponse response) * @throws IOException * if a read error occurs */ - public static byte[] readByteArray(InputStream in) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - int value = in.read(); - - while (value != -1) { - out.write(value); - value = in.read(); - } - - out.flush(); - out.close(); - return out.toByteArray(); + private byte[] readByteArray(InputStream in) throws IOException { + + return ByteStreams.toByteArray(in); } @Override diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/api/CopViewApi.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/api/CopViewApi.java new file mode 100644 index 00000000..81a95e91 --- /dev/null +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/api/CopViewApi.java @@ -0,0 +1,130 @@ +package com.bbn.marti.sync.api; + +import java.rmi.RemoteException; +import java.util.HashMap; +import java.util.List; +import java.util.NavigableSet; +import java.util.Set; +import java.util.concurrent.Callable; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.owasp.esapi.errors.IntrusionException; +import org.owasp.esapi.errors.ValidationException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import com.bbn.marti.config.Configuration; +import com.bbn.marti.cot.search.model.ApiResponse; +import com.bbn.marti.network.BaseRestController; +import com.bbn.marti.remote.CoreConfig; +import com.bbn.marti.remote.groups.Group; +import com.bbn.marti.sync.model.CopHierarchyNode; +import com.bbn.marti.sync.model.Mission; +import com.bbn.marti.sync.repository.MissionRepository; +import com.bbn.marti.sync.service.MissionService; +import com.bbn.marti.util.CommonUtil; +import com.google.common.base.Strings; + +import tak.server.Constants; + + +@RestController +public class CopViewApi extends BaseRestController { + + private static final Logger logger = LoggerFactory.getLogger(CopViewApi.class); + + // keep a reference to the currently active request + @Autowired + private HttpServletRequest request; + + @Autowired + private HttpServletResponse response; + + @Autowired + private MissionService missionService; + + @Autowired + private MissionRepository missionRepository; + + @Autowired + private CommonUtil martiUtil; + + @Autowired + private CoreConfig config; + + private static String mcsCopTool = ""; + + + /* + * get all cops + */ + @RequestMapping(value = "/cops", method = RequestMethod.GET) + Callable>> getAllCopMissions( + @RequestParam(value = "path", required = false) String path, + @RequestParam(value = "offset", required = false) Integer offset, + @RequestParam(value = "size", required = false) Integer size) throws RemoteException { + + if (logger.isDebugEnabled()) { + logger.debug("cop API getAllCops"); + } + + NavigableSet groups = martiUtil.getGroupsFromRequest(request); + + List missions = missionService.getAllCopsMissions(getDefaultMcsTool(), groups, path, offset, size); + + return () -> { + + return new ApiResponse>(Constants.API_VERSION, Mission.class.getSimpleName(), missions); + }; + } + + + @RequestMapping(value = "/cops/hierarchy", method = RequestMethod.GET) + public ApiResponse> getHierarchy(HttpServletRequest request) + throws ValidationException, IntrusionException, RemoteException { + + // get the set of paths that this user can see + List paths = missionRepository.getMissionPathsByTool( + getDefaultMcsTool(), martiUtil.getGroupVectorBitString(request)); + + // organize the paths into a cop hierarchy and return + HashMap nodeMap = new HashMap<>(); + CopHierarchyNode root = new CopHierarchyNode(""); + for (String path : paths) { + CopHierarchyNode parent = root; + String tmpPath = ""; + for (String level : path.split("/")) { + tmpPath += "/" + level; + CopHierarchyNode node = nodeMap.get(tmpPath); + if (node == null) { + node = new CopHierarchyNode(level); + nodeMap.put(tmpPath, node); + parent.addChild(node); + } + parent = node; + } + } + + return new ApiResponse>( + Constants.API_VERSION, CopHierarchyNode.class.getSimpleName(), root.getChildren()); + } + + public String getDefaultMcsTool() { + if (Strings.isNullOrEmpty(mcsCopTool)) { + try { + Configuration conf = config.getRemoteConfiguration(); + mcsCopTool = conf.getNetwork().getMissionCopTool(); + } catch(Exception ex) { + logger.info("Failed to get default mission cop tool. Using \"vbm\""); + mcsCopTool = "vbm"; + } + } + return mcsCopTool; + } +} diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/api/MissionApi.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/api/MissionApi.java index a293413a..a31398e3 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/api/MissionApi.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/api/MissionApi.java @@ -26,12 +26,14 @@ import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentSkipListSet; +import java.util.stream.Collectors; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import com.fasterxml.jackson.databind.DeserializationFeature; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; import org.owasp.esapi.Validator; import org.owasp.esapi.errors.IntrusionException; @@ -62,6 +64,8 @@ import com.bbn.marti.cot.search.model.ApiResponse; import com.bbn.marti.logging.AuditLogUtil; +import com.bbn.marti.maplayer.model.MapLayer; +import com.bbn.marti.maplayer.repository.MapLayerRepository; import com.bbn.marti.network.BaseRestController; import com.bbn.marti.remote.RemoteSubscription; import com.bbn.marti.remote.SubmissionInterface; @@ -82,6 +86,7 @@ import com.bbn.marti.sync.model.LogEntry; import com.bbn.marti.sync.model.Mission; import com.bbn.marti.sync.model.MissionChange; +import com.bbn.marti.sync.model.MissionFeed; import com.bbn.marti.sync.model.MissionInvitation; import com.bbn.marti.sync.model.MissionPermission; import com.bbn.marti.sync.model.MissionRole; @@ -89,6 +94,7 @@ import com.bbn.marti.sync.model.Resource; import com.bbn.marti.sync.repository.LogEntryRepository; import com.bbn.marti.sync.repository.MissionRepository; +import com.bbn.marti.sync.repository.MissionFeedRepository; import com.bbn.marti.sync.repository.MissionRoleRepository; import com.bbn.marti.sync.repository.MissionSubscriptionRepository; import com.bbn.marti.sync.service.MissionService; @@ -96,6 +102,7 @@ import com.bbn.marti.util.CommonUtil; import com.bbn.marti.util.KmlUtils; import com.bbn.marti.util.spring.RequestHolderBean; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; @@ -163,10 +170,16 @@ public class MissionApi extends BaseRestController { @Autowired private MissionSubscriptionRepository missionSubscriptionRepository; - - @Autowired + + @Autowired private RequestHolderBean requestHolderBean; + @Autowired + private MissionFeedRepository missionFeedRepository; + + @Autowired + private MapLayerRepository mapLayerRepository; + @Autowired(required = false) private RetentionPolicyConfig retentionPolicyConfig; @@ -291,15 +304,20 @@ Callable>> getMission( */ @RequestMapping(value = "/missions/{name:.+}", method = RequestMethod.PUT) public Callable>> createMission(@PathVariable("name") @NotNull String nameParam, - @RequestParam(value = "creatorUid", defaultValue = "") @ValidatedBy("MartiSafeString") String creatorUid, - @RequestParam(value = "group", defaultValue = "__ANON__") @ValidatedBy("MartiSafeString") String[] groupNames, - @RequestParam(value = "description", defaultValue = "") @ValidatedBy("MartiSafeString") String description, - @RequestParam(value = "chatRoom", defaultValue = "") @ValidatedBy("MartiSafeString") String chatRoom, - @RequestParam(value = "tool", defaultValue = "public") @ValidatedBy("MartiSafeString") String tool, - @RequestParam(value = "password", required = false) @ValidatedBy("MartiSafeString") String password, - @RequestParam(value = "defaultRole", required = false) @ValidatedBy("MartiSafeString") MissionRole.Role role, - @RequestParam(value = "expiration", required = false) Long expiration, - @RequestBody(required = false) byte[] missionPackage) + @RequestParam(value = "creatorUid", defaultValue = "") @ValidatedBy("MartiSafeString") String creatorUidParam, + @RequestParam(value = "group", defaultValue = "__ANON__") @ValidatedBy("MartiSafeString") String[] groupNamesParam, + @RequestParam(value = "description", defaultValue = "") @ValidatedBy("MartiSafeString") String descriptionParam, + @RequestParam(value = "chatRoom", defaultValue = "") @ValidatedBy("MartiSafeString") String chatRoomParam, + @RequestParam(value = "baseLayer", defaultValue = "") @ValidatedBy("MartiSafeString") String baseLayerParam, + @RequestParam(value = "bbox", defaultValue = "") @ValidatedBy("MartiSafeString") String bboxParam, + @RequestParam(value = "boundingPolygon", defaultValue = "") @ValidatedBy("MartiSafeString") List boundingPolygonParam, + @RequestParam(value = "path", defaultValue = "") @ValidatedBy("MartiSafeString") String pathParam, + @RequestParam(value = "classification", defaultValue = "") @ValidatedBy("MartiSafeString") String classificationParam, + @RequestParam(value = "tool", defaultValue = "public") @ValidatedBy("MartiSafeString") String toolParam, + @RequestParam(value = "password", required = false) @ValidatedBy("MartiSafeString") String passwordParam, + @RequestParam(value = "defaultRole", required = false) @ValidatedBy("MartiSafeString") MissionRole.Role roleParam, + @RequestParam(value = "expiration", required = false) Long expirationParam, + @RequestBody(required = false) byte[] requestBody) throws ValidationException, IntrusionException, RemoteException { if (Strings.isNullOrEmpty(nameParam)) { @@ -323,8 +341,54 @@ public Callable>> createMission(@PathVariable("name") @ final MissionRole adminRole = martiUtil.isAdmin() ? missionRoleRepository.findFirstByRole(MissionRole.Role.MISSION_OWNER) : null; + final String username = SecurityContextHolder.getContext().getAuthentication().getName(); + return () -> { + String creatorUid = creatorUidParam; + String[] groupNames = groupNamesParam; + String description = descriptionParam; + String chatRoom = chatRoomParam; + String baseLayer = baseLayerParam; + String bbox = bboxParam; + String boundingPolygon = boundingPolygonPointsToString(boundingPolygonParam); + String path = pathParam; + String classification = classificationParam; + String tool = toolParam; + String password = passwordParam; + MissionRole.Role role = roleParam; + Long expiration = expirationParam; + + byte[] missionPackage = null; + + Mission reqMission = null; + String contentType = request.getHeader("content-type"); + if (contentType != null && contentType.toLowerCase().contains("application/json")) { + try { + ObjectMapper objectMapper = new ObjectMapper().configure( + DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + reqMission = objectMapper.readValue(new String(requestBody), Mission.class); + if (reqMission != null) { + description = reqMission.getDescription() != null ? reqMission.getDescription() : description; + chatRoom = reqMission.getChatRoom() != null ? reqMission.getChatRoom() : chatRoom; + baseLayer = reqMission.getBaseLayer() != null ? reqMission.getBaseLayer() : baseLayer; + bbox = reqMission.getBbox() != null ? reqMission.getBbox() : bbox; + boundingPolygon = reqMission.getBoundingPolygon() != null ? reqMission.getBoundingPolygon() : boundingPolygon; + path = reqMission.getPath() != null ? reqMission.getPath() : path; + classification = reqMission.getClassification() != null ? reqMission.getClassification() : classification; + tool = reqMission.getTool() != null ? reqMission.getTool() : tool; + role = reqMission.getDefaultRole() != null ? reqMission.getDefaultRole().getRole() : role; + expiration = reqMission.getExpiration() != null ? reqMission.getExpiration() : expiration; + groupNames = reqMission.getGroups() != null ? reqMission.getGroups().toArray(new String[0]) : groupNames; + } + } catch (JsonProcessingException e) { + logger.error("exception parsing mission json!", e); + throw new IllegalArgumentException("exception parsing mission json!"); + } + } else { + missionPackage = requestBody; + } + BigInteger bitVectorUser = remoteUtil.bitVectorStringToInt(groupVectorUser); Set groups = groupManager.findGroups(Arrays.asList(groupNames)); @@ -378,6 +442,31 @@ public Callable>> createMission(@PathVariable("name") @ updated = true; } + if (!StringUtils.equals(baseLayer, mission.getBaseLayer())) { + mission.setBaseLayer(baseLayer); + updated = true; + } + + if (!StringUtils.equals(bbox, mission.getBbox())) { + mission.setBbox(bbox); + updated = true; + } + + if (!StringUtils.equals(boundingPolygon, mission.getBoundingPolygon())) { + mission.setBoundingPolygon(boundingPolygon); + updated = true; + } + + if (!StringUtils.equals(path, mission.getPath())) { + mission.setPath(path); + updated = true; + } + + if (!StringUtils.equals(classification, mission.getClassification())) { + mission.setClassification(classification); + updated = true; + } + if (!(expiration == null ? mission.getExpiration() == null || mission.getExpiration() == -1L : mission.getExpiration() != null && expiration.equals(mission.getExpiration()))) { @@ -387,9 +476,9 @@ public Callable>> createMission(@PathVariable("name") @ if (updated) { if (expiration != null) { - missionRepository.update(name, groupVectorUser, description, chatRoom, expiration); + missionRepository.update(name, groupVectorUser, description, chatRoom, baseLayer, bbox, path, classification, expiration, boundingPolygon); } else { - missionRepository.update(name, groupVectorUser, description, chatRoom, -1L); + missionRepository.update(name, groupVectorUser, description, chatRoom, baseLayer, bbox, path, classification, -1L, boundingPolygon); } } @@ -445,6 +534,10 @@ public Callable>> createMission(@PathVariable("name") @ } } + if (reqMission != null && createOrUpdateMissionRequestBody(mission, reqMission, creatorUid)) { + updated = true; + } + if (updated) { missionService.invalidateMissionCache(name); @@ -468,15 +561,15 @@ public Callable>> createMission(@PathVariable("name") @ // For now there are no properties to be concerted with besides name, so PUT missions doesn't need a JSON body yet if (expiration != null) { mission = missionService.createMission( - name, creatorUid, groupVectorMission, description, chatRoom, tool, passwordHash, defaultRole, expiration); + name, creatorUid, groupVectorMission, description, chatRoom, baseLayer, bbox, path, classification, tool, passwordHash, defaultRole, expiration, boundingPolygon); } else { mission = missionService.createMission( - name, creatorUid, groupVectorMission, description, chatRoom, tool, passwordHash, defaultRole, -1L); + name, creatorUid, groupVectorMission, description, chatRoom, baseLayer, bbox, path, classification, tool, passwordHash, defaultRole, -1L, boundingPolygon); } MissionRole ownerRole = missionRoleRepository.findFirstByRole(MissionRole.Role.MISSION_OWNER); MissionSubscription ownerSubscription = missionService.missionSubscribe( - name, mission.getId(), creatorUid, ownerRole, groupVectorUser); + name, mission.getId(), creatorUid, username, ownerRole, groupVectorUser); mission.setToken(ownerSubscription.getToken()); mission.setOwnerRole(ownerRole); @@ -487,6 +580,10 @@ public Callable>> createMission(@PathVariable("name") @ mission = missionService.getMission(name, groupVectorUser); } + if (reqMission != null) { + createOrUpdateMissionRequestBody(mission, reqMission, creatorUid); + } + response.setStatus(mission.getId() != 0 ? HttpServletResponse.SC_CREATED : HttpServletResponse.SC_INTERNAL_SERVER_ERROR); } @@ -496,17 +593,227 @@ public Callable>> createMission(@PathVariable("name") @ } - /* + private boolean createOrUpdateMissionRequestBody(Mission mission, Mission reqMission, String creatorUid) { + + if (reqMission == null) { + return false; + } + + boolean updated = false; + if (reqMission.getMapLayers() != null) { + + if (mission.getMapLayers() != null && mission.getMapLayers().size() > 0) { + mapLayerRepository.deleteAllByMissionId(mission.getId()); + mission.getMapLayers().clear(); + updated = true; + } + + for (MapLayer mapLayer : reqMission.getMapLayers()) { + mission.getMapLayers().add(mapLayer); + mapLayer.setMission(mission); + mapLayer.setCreatorUid(creatorUid); + missionService.addMapLayerToMission(mission.getName(), creatorUid, mission, mapLayer); + } + } + + if (reqMission.getFeeds() != null) { + + if (mission.getFeeds() != null && mission.getFeeds().size() > 0) { + missionFeedRepository.deleteAllByMissionId(mission.getId()); + mission.getFeeds().clear(); + updated = true; + } + + for (MissionFeed missionFeed : reqMission.getFeeds()) { + + missionFeed = missionService.addFeedToMission(mission.getName(), creatorUid, mission, + missionFeed.getDataFeedUid(), missionFeed.getFilterBbox(), + missionFeed.getFilterType(), missionFeed.getFilterCallsign()); + + mission.getFeeds().add(missionFeed); + } + } + + return updated; + } + + private void copyMissionContainers(Mission origMission, Mission missionCopy, String creatorUid, String groupVector) { + + try { + + String missionCopyName = missionCopy.getName(); + + for (String keyword : origMission.getKeywords()) { + missionRepository.addMissionKeyword(missionCopy.getId(), keyword); + } + if (logger.isTraceEnabled()) { + logger.trace("copy key words " + origMission.getKeywords()); + } + + for (MapLayer mapLayer : origMission.getMapLayers()) { + MapLayer mapLayerCopy = new MapLayer(mapLayer); + missionCopy.getMapLayers().add(mapLayerCopy); + mapLayerCopy.setMission(missionCopy); + mapLayerCopy.setCreatorUid(creatorUid); + mapLayerCopy.setMission(missionCopy); + missionService.addMapLayerToMission(missionCopy.getName(), creatorUid, missionCopy, mapLayerCopy); + } + if (logger.isTraceEnabled()) { + logger.trace("copy map layers " + origMission.getMapLayers()); + } + + for (MissionFeed missionFeed : origMission.getFeeds()) { + missionFeed = missionService.addFeedToMission(missionCopyName, creatorUid, missionCopy, + missionFeed.getDataFeedUid(), missionFeed.getFilterBbox(), + missionFeed.getFilterType(), missionFeed.getFilterCallsign()); + missionCopy.getFeeds().add(missionFeed); + } + if (logger.isTraceEnabled()) { + logger.trace("copy mission feeds " + origMission.getFeeds()); + } + + for (ExternalMissionData externalMissionData : origMission.getExternalData()) { + externalMissionData = missionService.setExternalMissionData(missionCopyName, creatorUid, externalMissionData, groupVector); + missionCopy.getExternalData().add(externalMissionData); + } + if (logger.isTraceEnabled()) { + logger.trace("copy external data" + origMission.getExternalData()); + } + + for (Resource resource : origMission.getContents()) { + MissionContent mc = new MissionContent(); + mc.getHashes().add(resource.getHash()); + missionService.addMissionContent(missionCopyName, mc, creatorUid, groupVector); + } + if (logger.isTraceEnabled()) { + logger.trace("copied mission content " + origMission.getContents()); + } + + for (String uid : origMission.getUids()) { + missionCopy.getUids().add(uid); + + MissionContent mc = new MissionContent(); + mc.getUids().add(uid); + missionService.addMissionContent(missionCopyName, mc, creatorUid, groupVector); + } + if (logger.isTraceEnabled()) { + logger.trace("copied mission uids " + origMission.getUids()); + } + + for (LogEntry logEntry : missionService.getLogEntriesForMission(origMission, null, null, null)) { + logEntry.getMissionNames().add(missionCopyName); + missionService.addUpdateLogEntry(logEntry, new Date(), groupVector); + } + + } catch (Exception e) { + logger.error("exception in copyMissionContainers", e); + throw e; + } + } + + @PreAuthorize("hasPermission(#request, 'MISSION_READ')") + @RequestMapping(value = "/missions/{missionName:.+}/copy", method = RequestMethod.PUT) + Callable>> copyMission( + @PathVariable("missionName") @NotNull String missionName, + @RequestParam(value = "creatorUid", required = true) @ValidatedBy("MartiSafeString") String creatorUidParam, + @RequestParam(value = "copyName", required = true) @ValidatedBy("MartiSafeString") String copyName, + @RequestParam(value = "copyPath", required = false) @ValidatedBy("MartiSafeString") String copyPath, + @RequestParam(value = "defaultRole", required = false) @ValidatedBy("MartiSafeString") MissionRole.Role roleParam, + @RequestParam(value = "password", required = false) String passwordParam) + { + + if (Strings.isNullOrEmpty(missionName)) { + throw new IllegalArgumentException("empty 'name' path parameter"); + } + + if (Strings.isNullOrEmpty(copyName)) { + throw new IllegalArgumentException("empty 'copyName' request parameter"); + } + + final HttpServletRequest request = requestHolderBean.getRequest(); + final String sessionId = requestHolderBean.sessionId(); + + if (logger.isDebugEnabled()) { + logger.debug("session id: " + requestHolderBean.sessionId()); + } + + if (logger.isDebugEnabled()) { + logger.debug("request: " + request); + } + + final String groupVector = martiUtil.getGroupVectorBitString(sessionId); + final String username = SecurityContextHolder.getContext().getAuthentication().getName(); + + return () -> { + String name = missionService.trimName(missionName); + String copy = missionService.trimName(copyName); + + if (logger.isDebugEnabled()) { + logger.debug("copyMission " + missionName + " copy name " + copy); + } + + // find the mission for which to copy + Mission mission = null; + mission = missionService.getMission(missionName, groupVector); + + if (mission == null) { + if (logger.isDebugEnabled()) { + logger.debug(" mission to copy was not found " + name); + } + } + + // see if the copy mission already exists + if (missionService.getMission(copy, false) != null) { + throw new IllegalArgumentException("mission named " + copy + " already exists"); + } + + MissionRole defaultRole = null; + if (roleParam != null) { + defaultRole = missionRoleRepository.findFirstByRole(roleParam); + } + + String passwordHash = null; + if (!Strings.isNullOrEmpty(passwordParam)) { + passwordHash = BCrypt.hashpw(passwordParam, BCrypt.gensalt()); + } + + // create a mission from the existing mission + Mission missionCopy = missionService.createMission(copy, creatorUidParam, mission.getGroupVector(), mission.getDescription(), + mission.getChatRoom(), mission.getBaseLayer(), mission.getBbox(), copyPath != null ? copyPath : mission.getPath(), mission.getClassification(), + mission.getTool(), passwordHash, defaultRole, mission.getExpiration(), mission.getBoundingPolygon()); + + if (logger.isDebugEnabled()) { + logger.debug("mission copy created " + missionCopy); + } + + MissionRole ownerRole = missionRoleRepository.findFirstByRole(MissionRole.Role.MISSION_OWNER); + MissionSubscription ownerSubscription = missionService.missionSubscribe(copy, missionCopy.getId(), + creatorUidParam, username, ownerRole, groupVector); + + if (missionCopy.getId() != null) { + copyMissionContainers(mission, missionCopy, creatorUidParam, groupVector); + } + + missionCopy.setToken(ownerSubscription.getToken()); + missionCopy.setOwnerRole(ownerRole); + + response.setStatus((missionCopy != null && missionCopy.getId() != 0 )? HttpServletResponse.SC_CREATED + : HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + return new ApiResponse>(Constants.API_VERSION, Mission.class.getSimpleName(), Sets.newHashSet(missionCopy)); + }; + } + + /* * Delete a mission by name. Respond with the deleted mission JSON. * */ - @PreAuthorize("hasPermission(#request, 'MISSION_WRITE')") - @RequestMapping(value = "/missions/{name:.+}", method = RequestMethod.DELETE) - @Transactional(noRollbackFor = Exception.class) - ApiResponse> deleteMission( - @PathVariable("name") @NotNull String name, + @PreAuthorize("hasPermission(#request, 'MISSION_WRITE')") + @RequestMapping(value = "/missions/{name:.+}", method = RequestMethod.DELETE) + @Transactional(noRollbackFor = Exception.class) + ApiResponse> deleteMission( + @PathVariable("name") @NotNull String name, @RequestParam(value = "creatorUid", defaultValue = "") @ValidatedBy("MartiSafeString") String creatorUid, - @RequestParam(value = "deepDelete", defaultValue = "false") boolean deepDelete, + @RequestParam(value = "deepDelete", defaultValue = "false") boolean deepDelete, HttpServletRequest request ) throws ValidationException, IntrusionException { @@ -546,12 +853,12 @@ ApiResponse> deleteMission( } } - logger.debug("archiving mission"); + logger.debug("archiving mission"); byte[] archive = missionService.archiveMission(mission.getName(), groupVector, request.getServerName()); missionService.addMissionArchiveToEsync(mission.getName(), archive, groupVector, true); - logger.debug("added archived mission to esync " + mission.getName()); + logger.debug("added archived mission to esync " + mission.getName()); mission = missionService.deleteMission(name, creatorUid, groupVector, deepDelete); @@ -759,6 +1066,7 @@ ApiResponse> getMissionChanges( @RequestParam(value = "squashed", required = false, defaultValue = "true") boolean squashed, HttpServletRequest request) { + try { Mission mission = missionService.getMissionByNameCheckGroups(missionService.trimName(name), martiUtil.getGroupVectorBitString(request)); missionService.validateMission(mission, missionService.trimName(name)); @@ -767,6 +1075,10 @@ ApiResponse> getMissionChanges( return new ApiResponse>(Constants.API_VERSION, MissionChange.class.getSimpleName(), new ConcurrentSkipListSet<>(changes)); + } catch (Exception e) { + logger.error("exception in getMissionChanges", e); + return null; + } } @@ -1238,9 +1550,10 @@ public void validateParameters(Method method) { if (request.getParameterValues(paramName) != null) { for (String paramValue : request.getParameterValues(paramName)) { - - logger.trace("param name: " + paramName + " param value: " + paramValue + " validation pattern: " + validatorPattern + " "); - + if (logger.isTraceEnabled()) { + logger.trace("param name: " + StringUtils.normalizeSpace(paramName) + " param value: " + StringUtils.normalizeSpace(paramValue) + + " validation pattern: " + StringUtils.normalizeSpace(validatorPattern) + " "); + } // do validation try { validator.getValidInput(context, paramValue, validatorPattern, DEFAULT_PARAMETER_LENGTH, false); @@ -1293,6 +1606,33 @@ public ApiResponse getReadOnlyAccessToken( Constants.API_VERSION, String.class.getName(), token); } + /* + * Returns the mission subscription for the current user + */ + @RequestMapping(value = "/missions/{missionName:.+}/subscription", method = RequestMethod.GET) + @ResponseStatus(HttpStatus.OK) + public ApiResponse getSubscriptionForUser( + @PathVariable("missionName") String missionName, + @RequestParam(value = "uid", defaultValue = "") String uid) + { + missionName = missionService.trimName(missionName); + + // validate existence of mission + Mission mission = missionService.getMissionByNameCheckGroups(missionName, martiUtil.getGroupVectorBitString(request)); + missionService.validateMission(mission, missionName); + + final String username = SecurityContextHolder.getContext().getAuthentication().getName(); + MissionSubscription missionSubscription = missionSubscriptionRepository + .findByMissionNameAndClientUidAndUsernameNoMission(missionName, uid, username); + + if (missionSubscription == null) { + throw new NotFoundException("Mission subscription not found"); + } + + return new ApiResponse( + Constants.API_VERSION, MissionSubscription.class.getName(), missionSubscription); + } + /* * subscribe to mission changes */ @@ -1313,6 +1653,8 @@ public Callable> createMissionSubscription( final String groupVector = martiUtil.getGroupVectorBitString(sessionId); + final String username = SecurityContextHolder.getContext().getAuthentication().getName(); + return () -> { String missionName = missionService.trimName(missionNameParam); @@ -1360,7 +1702,13 @@ public Callable> createMissionSubscription( } MissionSubscription missionSubscription = missionService.missionSubscribe(missionName, mission.getId(), - Strings.isNullOrEmpty(topic) ? uid : "topic:" + topic, role, groupVector); + Strings.isNullOrEmpty(topic) ? uid : "topic:" + topic, username, role, groupVector); + + if (mission.getFeeds() != null) { + for (MissionFeed missionFeed : mission.getFeeds()) { + missionService.sendLatestFeedEvents(mission, missionFeed, uid, groupVector); + } + } try { // clear out any uid invitations now that the uid has subscribed @@ -1443,11 +1791,14 @@ public void setSubscriptionRole( @ResponseStatus(HttpStatus.OK) public Callable deleteMissionSubscription( @RequestParam(value = "uid", defaultValue = "") String uid, - @RequestParam(value = "topic", defaultValue = "") String topic, + @RequestParam(value = "topic", defaultValue = "") String topic, + @RequestParam(value = "disconnectOnly", defaultValue = "false") boolean disconnectOnly, @PathVariable("missionName") String missionNameParam) { final String groupVector = martiUtil.getGroupVectorBitString(request); + final String username = SecurityContextHolder.getContext().getAuthentication().getName(); + return () -> { String missionName = missionService.trimName(missionNameParam); @@ -1460,7 +1811,7 @@ public Callable deleteMissionSubscription( throw new IllegalArgumentException("either 'uid' or 'topic' parameter must be specified"); } - missionService.missionUnsubscribe(missionName, Strings.isNullOrEmpty(topic) ? uid : topic, groupVector); + missionService.missionUnsubscribe(missionName, Strings.isNullOrEmpty(topic) ? uid : topic, username, groupVector, disconnectOnly); return null; }; @@ -1506,8 +1857,14 @@ public ApiResponse> getMissionSubscriptionRoles( Mission mission = missionService.getMissionByNameCheckGroups(missionService.trimName(missionName), martiUtil.getGroupVectorBitString(request)); missionService.validateMission(mission, missionName); + // ensure tokens are removed from the output + List missionSubscriptions = missionSubscriptionRepository.findAllByMissionNameNoMissionNoToken(missionName); + for (MissionSubscription missionSubscription : missionSubscriptions) { + missionSubscription.setToken(null); + } + return new ApiResponse>(Constants.API_VERSION, "MissionSubscription", - missionSubscriptionRepository.findAllByMissionNameNoMissionNoToken(missionName)); + missionSubscriptions); } @PreAuthorize("hasPermission(#request, 'MISSION_READ')") @@ -1531,7 +1888,8 @@ public ApiResponse getMissionRoleFromToken( @RequestMapping(value = "/missions/{missionName:.+}/role", method = RequestMethod.PUT) public void setMissionRole( @PathVariable("missionName") String missionName, - @RequestParam(value = "clientUid", defaultValue = "") @ValidatedBy("MartiSafeString") String clientUid, + @RequestParam(value = "clientUid", defaultValue = "") @ValidatedBy("MartiSafeString") String clientUid, + @RequestParam(value = "username", defaultValue = "") @ValidatedBy("MartiSafeString") String username, @RequestParam(value = "role", defaultValue = "") @ValidatedBy("MartiSafeString") MissionRole.Role newRole, HttpServletRequest request) { @@ -1553,7 +1911,7 @@ public void setMissionRole( "validateRoleAssignment failed! Illegal attempt to assign role " + newRole.name()); } - boolean result = missionService.setRole(mission, clientUid, role); + boolean result = missionService.setRole(mission, clientUid, username, role); response.setStatus(result ? HttpServletResponse.SC_OK : HttpServletResponse.SC_INTERNAL_SERVER_ERROR); } @@ -2256,5 +2614,140 @@ public void setExpiration( logger.debug("exception setting mission expiration " + e.getMessage(), e); } } -} + @PreAuthorize("hasPermission(#request, 'MISSION_MANAGE_FEEDS')") + @RequestMapping(value = "/missions/{missionName:.+}/feed", method = RequestMethod.POST) + @ResponseStatus(HttpStatus.OK) + public void addFeed( + @PathVariable(value = "missionName") String missionName, + @RequestParam(value = "creatorUid") @ValidatedBy("MartiSafeString") String creatorUid, + @RequestParam(value = "dataFeedUid") @ValidatedBy("MartiSafeString") String dataFeedUid, + @RequestParam(value = "filterBbox", required = false) @ValidatedBy("MartiSafeString") String filterBbox, + @RequestParam(value = "filterType", required = false) @ValidatedBy("MartiSafeString") String filterType, + @RequestParam(value = "filterCallsign", required = false) @ValidatedBy("MartiSafeString") String filterCallsign, + HttpServletRequest request) { + + missionName = missionService.trimName(missionName); + + String groupVector = martiUtil.getGroupVectorBitString(request); + Mission mission = missionService.getMission(missionName, groupVector); + + try { + missionService.addFeedToMission(mission.getName(), creatorUid, mission, dataFeedUid, filterBbox, filterType, filterCallsign); + } catch (Exception e) { + logger.error("exception in addFeed!", e); + } + } + + @PreAuthorize("hasPermission(#request, 'MISSION_MANAGE_FEEDS')") + @RequestMapping(value = "/missions/{missionName:.+}/feed/{uid:.+}", method = RequestMethod.DELETE) + @ResponseStatus(HttpStatus.OK) + public void removeFeed( + @PathVariable(value = "missionName") String missionName, + @PathVariable(value = "uid") String uid, + @RequestParam(value = "creatorUid") @ValidatedBy("MartiSafeString") String creatorUid, + HttpServletRequest request) { + + missionName = missionService.trimName(missionName); + + String groupVector = martiUtil.getGroupVectorBitString(request); + Mission mission = missionService.getMission(missionName, groupVector); + + try { + missionService.removeFeedFromMission(mission.getName(), creatorUid, mission, uid); + } catch (Exception e) { + logger.error("exception in removeFeed!", e); + } + } + + @PreAuthorize("hasPermission(#request, 'MISSION_WRITE')") + @RequestMapping(value = "/missions/{missionName:.+}/maplayers", method = RequestMethod.POST) + @ResponseStatus(HttpStatus.OK) + ApiResponse createMapLayer( + @PathVariable(value = "missionName") String missionName, + @RequestParam(value = "creatorUid") @ValidatedBy("MartiSafeString") String creatorUid, + @RequestBody MapLayer mapLayer) { + + missionName = missionService.trimName(missionName); + + String groupVector = martiUtil.getGroupVectorBitString(request); + Mission mission = missionService.getMission(missionName, groupVector); + + mapLayer.setMission(mission); + + MapLayer newMapLayer = missionService.addMapLayerToMission(missionName, creatorUid, mission, mapLayer); + + return new ApiResponse<>(Constants.API_VERSION, "MapLayer", newMapLayer); + } + + @PreAuthorize("hasPermission(#request, 'MISSION_WRITE')") + @RequestMapping(value = "/missions/{missionName:.+}/maplayers/{uid}", method = RequestMethod.DELETE) + @ResponseStatus(HttpStatus.OK) + void deleteMapLayer( + @PathVariable(value = "missionName") String missionName, + @RequestParam(value = "creatorUid") @ValidatedBy("MartiSafeString") String creatorUid, + @PathVariable("uid") @NotNull String uid) { + + missionName = missionService.trimName(missionName); + + String groupVector = martiUtil.getGroupVectorBitString(request); + Mission mission = missionService.getMission(missionName, groupVector); + + missionService.removeMapLayerFromMission(missionName, creatorUid, mission, uid); + } + + @PreAuthorize("hasPermission(#request, 'MISSION_WRITE')") + @RequestMapping(value = "/missions/{missionName:.+}/maplayers", method = RequestMethod.PUT) + @ResponseStatus(HttpStatus.OK) + ApiResponse updateMapLayer( + @PathVariable(value = "missionName") String missionName, + @RequestParam(value = "creatorUid") @ValidatedBy("MartiSafeString") String creatorUid, + @RequestBody MapLayer mapLayer) { + + missionName = missionService.trimName(missionName); + + String groupVector = martiUtil.getGroupVectorBitString(request); + Mission mission = missionService.getMission(missionName, groupVector); + + mapLayer.setMission(mission); + + MapLayer newMapLayer = missionService.updateMapLayer(missionName, creatorUid, mission, mapLayer); + + return new ApiResponse<>(Constants.API_VERSION, "MapLayer", newMapLayer); + } + + private static String boundingPolygonPointsToString(List points) { + if (points == null || points.size() == 0) { + return null; + } + + // poly needs 3 points minimum + if (points.size() < 3) { + logger.info("Polygon requires at least 3 points. Found size " + points.size()); + return null; + } + + try { + // check that all points are valid numbers + for (String point : points) { + String[] xy = point.split(","); + if (xy.length != 2) { + logger.info("Point is not in the format of ," + Arrays.deepToString(xy)); + return null; + } + Double.parseDouble(xy[0]); + Double.parseDouble(xy[1]); + } + } catch (Exception e) { + logger.error("Error parsing points for data feed bounds", e); + return null; + } + + // first != last, so append first to end to close the poly + if (!points.get(0).replace(" ", "").equals(points.get(points.size() - 1).replace(" ", ""))) { + points.add(points.get(0)); + } + + return points.stream().map(p -> p.replace(" ", "")).map(p -> p.replace(",", " ")).collect(Collectors.joining(",")); + } +} \ No newline at end of file diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/cache/AllCopMissionsCacheKeyGenerator.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/cache/AllCopMissionsCacheKeyGenerator.java new file mode 100644 index 00000000..dc29b15d --- /dev/null +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/cache/AllCopMissionsCacheKeyGenerator.java @@ -0,0 +1,54 @@ +package com.bbn.marti.sync.cache; + +import java.lang.reflect.Method; +import java.util.NavigableSet; + +import org.springframework.cache.interceptor.KeyGenerator; + +import com.bbn.marti.remote.groups.Group; +import com.bbn.marti.remote.util.RemoteUtil; + + +public class AllCopMissionsCacheKeyGenerator implements KeyGenerator { + + public Object generate(Object target, Method method, Object... params) { + + if (!(params[1] instanceof NavigableSet)) { + throw new IllegalArgumentException("can't construct cache key for all mission - invalid group collection type"); + } + + // method name + StringBuilder keyBuilder = new StringBuilder(); + keyBuilder.append(method.getName()); + + // tool + keyBuilder.append("_"); + keyBuilder.append(String.valueOf(params[0])); + + // groups + NavigableSet groups = ((NavigableSet)params[1]); + String groupVector = RemoteUtil.getInstance().bitVectorToString(RemoteUtil.getInstance().getBitVectorForGroups(groups)); + keyBuilder.append("_"); + keyBuilder.append(groupVector); + + // path + if (params[2] != null) { + keyBuilder.append("_"); + keyBuilder.append((String) params[2]); + } + + // page + if (params[3] != null) { + keyBuilder.append("_"); + keyBuilder.append((String.valueOf(params[3]))); + } + + //size + if (params[4] != null) { + keyBuilder.append("_"); + keyBuilder.append((String.valueOf(params[4]))); + } + + return keyBuilder.toString(); + } +} \ No newline at end of file diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/federation/EnterpriseSyncFederationAspect.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/federation/EnterpriseSyncFederationAspect.java index 3df675a4..51a9452b 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/federation/EnterpriseSyncFederationAspect.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/federation/EnterpriseSyncFederationAspect.java @@ -3,6 +3,8 @@ import java.rmi.RemoteException; import org.aspectj.lang.JoinPoint; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; import org.aspectj.lang.annotation.Aspect; import org.aspectj.lang.annotation.Before; import org.slf4j.Logger; @@ -14,6 +16,8 @@ import com.bbn.marti.remote.groups.GroupManager; import com.bbn.marti.sync.Metadata; +import tak.server.util.ExecutorSource; + @Aspect @Configurable public class EnterpriseSyncFederationAspect { @@ -28,46 +32,71 @@ public class EnterpriseSyncFederationAspect { @Autowired private CoreConfig coreConfig; + + @Autowired + private ExecutorSource executorSource; public EnterpriseSyncFederationAspect() { if (logger.isDebugEnabled()) { logger.debug("EnterpriseSyncFederationAspect constructor"); } } - - // If we were using AspectJ instead of Spring AOP - something like !adviceexecution() would avoid the stack tracing - // or name this pointcut and use !within(A) - @Before("execution(* com.bbn.marti.sync.EnterpriseSyncService.insertResource(..))") - public void insertResource(JoinPoint jp) throws RemoteException { - - if (!coreConfig.getRemoteConfiguration().getFederation().isEnableFederation()) { - return; - } - - if (!coreConfig.getRemoteConfiguration().getFederation().isAllowMissionFederation()) { - if (logger.isDebugEnabled()) { - logger.debug("mission federation disabled in config"); - } - return; + + @Around("execution(* com.bbn.marti.sync.EnterpriseSyncService.insertResource(..)))") + public Object insertResource(ProceedingJoinPoint jp) throws Throwable { + + if (logger.isDebugEnabled()) { + logger.debug("EnterpriseSyncService.insertResource() : " + jp.getSignature().getName() + ": Before Method Execution"); } - - try { - - if (isCyclic()) { - if (logger.isDebugEnabled()) { - logger.debug("skipping cyclic esync aspect execution"); - } - return; + try { + Object result = jp.proceed(); + + if (logger.isDebugEnabled()) { + logger.debug("result: " + result); + logger.debug("EnterpriseSyncService.insertResource() : " + jp.getSignature().getName() + ": After Method Execution"); + } + + if (!coreConfig.getRemoteConfiguration().getFederation().isEnableFederation()) { + return result; } - - if (logger.isDebugEnabled()) { - logger.debug("esync advice " + jp.getSignature().getName() + " " + jp.getKind()); - } - - mfm.insertResource((Metadata) jp.getArgs()[0], (byte[]) jp.getArgs()[1], gm.groupVectorToGroupSet((String) jp.getArgs()[2])); - } catch (Exception e) { - logger.debug("exception executing create mission advice: " + e); - } + + if (!coreConfig.getRemoteConfiguration().getFederation().isAllowMissionFederation()) { + if (logger.isDebugEnabled()) { + logger.debug("mission federation disabled in config"); + } + return result; + } + + try { + + if (isCyclic()) { + if (logger.isDebugEnabled()) { + logger.debug("skipping cyclic esync aspect execution"); + } + return result; + } + + if (logger.isDebugEnabled()) { + logger.debug("esync advice " + jp.getSignature().getName() + " " + jp.getKind()); + } + + executorSource.missionRepositoryProcessor.submit(() -> { + try { + // federate only file metadata (no content) + mfm.insertResource((Metadata) jp.getArgs()[0], (byte[]) jp.getArgs()[1], gm.groupVectorToGroupSet((String) jp.getArgs()[2])); + } catch (Exception e) { + logger.error("exception federating file", e); + } + }); + + + } catch (Exception e) { + logger.debug("exception executing create mission advice: " + e); + } + + + return result; + } finally { } } @Before("execution(* com.bbn.marti.sync.EnterpriseSyncService.delete(..))") @@ -132,7 +161,7 @@ public void updateMetadata(JoinPoint jp) throws RemoteException { private boolean isCyclic() { StackTraceElement[] stack = new Exception().getStackTrace(); - + for (StackTraceElement el : stack) { if (logger.isTraceEnabled()) { diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/federation/FederationROLHandler.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/federation/FederationROLHandler.java index 3e2bf6d1..33ad5344 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/federation/FederationROLHandler.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/federation/FederationROLHandler.java @@ -184,7 +184,7 @@ private void processCreate(ROL rol) { MissionMetadata md = (MissionMetadata) parameters; // TODO - missionService.createMission(md.getName(), md.getCreatorUid(), remoteUtil.bitVectorToString(remoteUtil.getBitVectorForGroups(groups)), md.getDescription(), md.getChatRoom(), md.getTool(), null, null, null); + missionService.createMission(md.getName(), md.getCreatorUid(), remoteUtil.bitVectorToString(remoteUtil.getBitVectorForGroups(groups)), md.getDescription(), md.getChatRoom(), null, null, null, null, md.getTool(), null, null, null, null); } private void processDelete(ROL rol) { @@ -243,7 +243,7 @@ private void processUpdate(ROL rol) { case ADD_CONTENT: if (!missionService.exists(mud.getMissionName(), remoteUtil.bitVectorToString(remoteUtil.getBitVectorForGroups(groups)))) { - missionService.createMission(mud.getMissionName(), mud.getMissionCreatorUid(), remoteUtil.bitVectorToString(remoteUtil.getBitVectorForGroups(groups)), mud.getMissionDescription(), mud.getMissionChatRoom(), mud.getMissionTool(), null, null, null); + missionService.createMission(mud.getMissionName(), mud.getMissionCreatorUid(), remoteUtil.bitVectorToString(remoteUtil.getBitVectorForGroups(groups)), mud.getMissionDescription(), mud.getMissionChatRoom(), null, null, null, null, mud.getMissionTool(), null, null, null, null); } if (logger.isDebugEnabled()) { diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/federation/MissionActionROLConverter.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/federation/MissionActionROLConverter.java index b38dc75a..daa39b28 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/federation/MissionActionROLConverter.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/federation/MissionActionROLConverter.java @@ -95,7 +95,10 @@ public ROL deleteMissionContentToROL(String missionName, String hash, String uid } - public ROL getInsertResourceROL(Metadata metadata, byte[] content) throws IOException { + /* + * generate save file ROL with content attached + */ + public ROL getInsertResourceROL(Metadata metadata, byte[] content) throws IOException { Resource res = new Resource(metadata); String metadataJson = mapper.writeValueAsString(res); @@ -116,6 +119,25 @@ public ROL getInsertResourceROL(Metadata metadata, byte[] content) throws IOExce return rol.build(); } + + /* + * generate save file ROL with only metadata attached + */ + public ROL getInsertResourceROLNoContent(Metadata metadata) throws IOException { + + Resource res = new Resource(metadata); + String metadataJson = mapper.writeValueAsString(res); + + if (logger.isDebugEnabled()) { + logger.debug("intercepted enterprise sync insert resource (without content)" + metadataJson); + } + + Objects.requireNonNull(metadataJson, "resource metadata"); + + Builder rol = ROL.newBuilder().setProgram("create resource\n" + metadataJson + ";"); + + return rol.build(); + } public ROL getUpdateMetadataROL(String hash, String key, String value) throws IOException { diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/federation/MissionFederationManager.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/federation/MissionFederationManager.java index 2de51a5d..d35b77ff 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/federation/MissionFederationManager.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/federation/MissionFederationManager.java @@ -24,7 +24,7 @@ public interface MissionFederationManager { void deleteMissionContent(String missionName, String hash, String uid, String creatorUid, NavigableSet groups); void archiveMission(String missionName, String serverName, NavigableSet groups); - + void insertResource(Metadata metadata, byte[] content, NavigableSet groups); void updateMetadata(String hash, String key, String value, NavigableSet groups); @@ -32,6 +32,7 @@ public interface MissionFederationManager { void setParent(String missionName, String parentMissionName, NavigableSet groups); void clearParent(String missionName, NavigableSet groups); + } \ No newline at end of file diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/federation/MissionFederationManagerROL.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/federation/MissionFederationManagerROL.java index 68260b48..1321de66 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/federation/MissionFederationManagerROL.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/federation/MissionFederationManagerROL.java @@ -19,6 +19,7 @@ import com.bbn.marti.sync.DataPackageFileBlocker; import com.bbn.marti.sync.EnterpriseSyncService; import com.bbn.marti.sync.Metadata; +import com.bbn.marti.sync.Metadata.Field; import com.bbn.marti.sync.model.Mission; import com.bbn.marti.sync.model.MissionChange; import com.bbn.marti.sync.service.MissionService; @@ -269,11 +270,19 @@ public void clearParent(String missionName, NavigableSet groups) { } } } - + + /* + * save file with without payload + */ @Override public void insertResource(Metadata metadata, byte[] content, NavigableSet groups) { - + if (!(coreConfig.getRemoteConfiguration().getFederation().isAllowMissionFederation())) { + + if (logger.isDebugEnabled()) { + logger.debug("skipping federation of a file " + metadata); + } + return; } @@ -297,20 +306,22 @@ public void insertResource(Metadata metadata, byte[] content, NavigableSet { + + protected String name; + + protected CopHierarchyNode parent; + protected Set children = new ConcurrentSkipListSet(); + + public CopHierarchyNode(String name) { + this.name = name; + } + + public String getName() { + return name; + } + public void setName(String name) { + this.name = name; + } + + @JsonIgnore + public CopHierarchyNode getParent() { return parent; } + + public void setParent(CopHierarchyNode parent) { this.parent = parent; } + + public Set getChildren() { return children; } + + public void setChildren(Set children) { this.children = children; } + + public static String getPath(CopHierarchyNode node) { + if (node == null) { + return ""; + } + + return getPath(node.getParent()) + "/" + node.getName(); + } + + @Override + public int compareTo(CopHierarchyNode that) { + return getPath(this).compareTo(getPath(that)); + } + + public void addChild(CopHierarchyNode child) { + children.add(child); + child.setParent(this); + } +} \ No newline at end of file diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/model/DataFeedDao.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/model/DataFeedDao.java new file mode 100644 index 00000000..d5369cd5 --- /dev/null +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/model/DataFeedDao.java @@ -0,0 +1,201 @@ +package com.bbn.marti.sync.model; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import javax.persistence.Cacheable; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Transient; +import javax.xml.bind.annotation.XmlRootElement; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.google.common.collect.ComparisonChain; + +import tak.server.feeds.DataFeed.DataFeedType; + +@JsonInclude(Include.NON_NULL) +@Entity +@Table(name = "data_feed") +public class DataFeedDao implements Serializable, Comparable { + + private static final long serialVersionUID = -5463730874196814957L; + + protected Long id = 0L; + protected String uuid = ""; + protected String name = ""; + protected String auth = ""; + protected boolean authRequired; + protected String protocol; + protected Integer port; + protected String feedGroup; + protected String iface; + protected boolean archive; + protected boolean anongroup; + protected boolean archiveOnly; + protected boolean sync; + protected Integer coreVersion; + protected String coreVersion2TlsVersions; + protected int type; + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(unique = true, nullable = false, columnDefinition="long") + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + @Column(name = "uuid", unique = true, nullable = false, columnDefinition="string") + public String getUUID() { + return uuid; + } + + public void setUUID(String uuid) { + this.uuid = uuid; + } + + @Column(name = "name", unique = false, nullable = false, columnDefinition="string") + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @Column(name = "auth", unique = false, nullable = true, columnDefinition="string") + public String getAuth() { + return auth; + } + + public void setAuth(String auth) { + this.auth = auth; + } + + @Column(name = "type", unique = false, nullable = false, columnDefinition="integer") + public int getType() { + return type; + } + + public void setType(int type) { + this.type = type; + } + + @Column(name = "port", unique = false, nullable = true, columnDefinition="integer") + public Integer getPort() { + return port; + } + + public void setPort(Integer port) { + this.port = port; + } + + @Column(name = "auth_required", unique = false, nullable = true, columnDefinition="boolean") + public boolean getAuthRequired() { + return authRequired; + } + + public void setAuthRequired(boolean authRequired) { + this.authRequired = authRequired; + } + + @Column(name = "protocol", unique = false, nullable = true, columnDefinition="string") + public String getProtocol() { + return protocol; + } + + public void setProtocol(String protocol) { + this.protocol = protocol; + } + + @Column(name = "feed_group", unique = false, nullable = true, columnDefinition="string") + public String getFeedGroup() { + return feedGroup; + } + + public void setFeedGroup(String feedGroup) { + this.feedGroup = feedGroup; + } + + @Column(name = "iface", unique = false, nullable = true, columnDefinition="string") + public String getIface() { + return iface; + } + + public void setIface(String iface) { + this.iface = iface; + } + + @Column(name = "archive", unique = false, nullable = true, columnDefinition="boolean") + public boolean getArchive() { + return archive; + } + + public void setArchive(boolean archive) { + this.archive = archive; + } + + @Column(name = "anongroup", unique = false, nullable = true, columnDefinition="boolean") + public boolean getAnongroup() { + return anongroup; + } + + public void setAnongroup(boolean anongroup) { + this.anongroup = anongroup; + } + + @Column(name = "archive_only", unique = false, nullable = true, columnDefinition="boolean") + public boolean getArchiveOnly() { + return archiveOnly; + } + + public void setArchiveOnly(boolean archiveOnly) { + this.archiveOnly = archiveOnly; + } + + @Column(name = "sync", unique = false, nullable = true, columnDefinition="boolean") + public boolean isSync() { + return sync; + } + + public void setSync(boolean sync) { + this.sync = sync; + } + + @Column(name = "core_version", unique = false, nullable = true, columnDefinition="integer") + public Integer getCoreVersion() { + return coreVersion; + } + + public void setCoreVersion(Integer coreVersion) { + this.coreVersion = coreVersion; + } + + @Column(name = "core_version_tls_versions", unique = false, nullable = true, columnDefinition="string") + public String getCoreVersion2TlsVersions() { + return coreVersion2TlsVersions; + } + + public void setCoreVersion2TlsVersions(String coreVersion2TlsVersions) { + this.coreVersion2TlsVersions = coreVersion2TlsVersions; + } + + + @Override + public int compareTo(DataFeedDao that) { + return ComparisonChain.start().compare(this.getId(), that.getId()).result(); + } + +} diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/model/Mission.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/model/Mission.java index 04929568..c740c619 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/model/Mission.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/model/Mission.java @@ -1,5 +1,3 @@ - - package com.bbn.marti.sync.model; import java.io.Serializable; @@ -25,7 +23,6 @@ import javax.persistence.ManyToMany; import javax.persistence.ManyToOne; import javax.persistence.OneToMany; -import javax.persistence.OrderBy; import javax.persistence.Table; import javax.persistence.Transient; @@ -41,6 +38,8 @@ import com.google.common.base.Strings; import com.google.common.collect.ComparisonChain; +import com.bbn.marti.maplayer.model.MapLayer; + import tak.server.Constants; /* @@ -51,7 +50,7 @@ @Entity @Table(name = "mission") @Cacheable -@JsonIgnoreProperties({"hibernateLazyInitializer", "handler"}) +@JsonIgnoreProperties(value = {"hibernateLazyInitializer", "handler"}, ignoreUnknown = true) @JsonInclude(JsonInclude.Include.NON_NULL) public class Mission implements Serializable, Comparable { @@ -70,6 +69,11 @@ public class Mission implements Serializable, Comparable { protected String name; protected String description; protected String chatRoom; + protected String baseLayer; + protected String bbox; + protected String boundingPolygon; + protected String path; + protected String classification; protected String tool; @@ -82,7 +86,9 @@ public class Mission implements Serializable, Comparable { protected String creatorUid; protected Date createTime; - + + protected Date lastEdited; + protected List> uidAdds; protected List> resourceAdds; @@ -97,6 +103,14 @@ public class Mission implements Serializable, Comparable { protected Set externalData = new ConcurrentSkipListSet(); + protected Set feeds = new ConcurrentSkipListSet<>(); + + protected Set mapLayers = new ConcurrentSkipListSet(); + + protected Long pageCount; + protected Long missionCount; + protected Long pageSize; + protected String passwordHash; protected MissionRole defaultRole; protected MissionRole ownerRole; @@ -126,6 +140,45 @@ public Mission(@NotNull String name) { setName(name); } + public Mission(Mission other, Date lastEdited) { + id = other.id; + name = other.name; + description = other.description; + chatRoom = other.chatRoom; + baseLayer = other.baseLayer; + bbox = other.bbox; + boundingPolygon = other.boundingPolygon; + path = other.path; + classification = other.classification; + tool = other.tool; + contents = other.contents; + uids = other.uids; + keywords = other.keywords; + creatorUid = other.creatorUid; + createTime = other.createTime; + uidAdds = other.uidAdds; + resourceAdds = other.resourceAdds; + groupVector = other.groupVector; + groups = other.groups; + parent = other.parent; + children = other.children; + externalData = other.externalData; + feeds = other.feeds; + mapLayers = other.mapLayers; + pageCount = other.pageCount; + missionCount = other.missionCount; + pageSize = other.pageSize; + passwordHash = other.passwordHash; + defaultRole = other.defaultRole; + ownerRole = other.ownerRole; + token = other.token; + missionChanges = other.missionChanges; + logs = other.logs; + expiration = other.expiration; + + this.lastEdited = lastEdited; + } + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(name = "id", unique = true, nullable = false) @@ -165,6 +218,51 @@ public void setChatRoom(String chatRoom) { this.chatRoom = chatRoom; } + @Column(name = "base_layer", unique = false, nullable = true) + public String getBaseLayer() { + return baseLayer; + } + + public void setBaseLayer(String baseLayer) { + this.baseLayer = baseLayer; + } + + @Column(name = "bbox", unique = false, nullable = true) + public String getBbox() { + return bbox; + } + + public void setBbox(String bbox) { + this.bbox = bbox; + } + + @Column(name = "bounding_polygon", unique = false, nullable = true) + public String getBoundingPolygon() { + return boundingPolygon; + } + + public void setBoundingPolygon(String boundingPolygon) { + this.boundingPolygon = boundingPolygon; + } + + @Column(name = "path", unique = false, nullable = true) + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + @Column(name = "classification", unique = false, nullable = true) + public String getClassification() { + return classification; + } + + public void setClassification(String classification) { + this.classification = classification; + } + @Column(name = "tool", unique = false, nullable = true) public String getTool() { return tool; @@ -238,6 +336,16 @@ public void setCreateTime(Date createTime) { this.createTime = createTime; } + @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = Constants.COT_DATE_FORMAT_PAD_MILLIS) + @Column(name = "last_edited", insertable = false, updatable = false) + public Date getLastEdited() { + return lastEdited; + } + + public void setLastEdited(Date lastEdited) { + this.lastEdited = lastEdited; + } + @Transient @JsonProperty("uids") public List> getUidAdds() { @@ -296,6 +404,16 @@ public int compareTo(Mission that) { public void setExternalData(Set externalData) { this.externalData = externalData; } + @OneToMany(mappedBy="mission", fetch = FetchType.EAGER) + public Set getMapLayers() { return mapLayers; } + + public void setMapLayers(Set mapLayers) { this.mapLayers = mapLayers; } + + @OneToMany(mappedBy="mission", fetch = FetchType.EAGER) + public Set getFeeds() { return feeds; } + + public void setFeeds(Set feeds) { this.feeds = feeds; } + @JsonIgnore @Column(name = "password_hash", unique = false, nullable = true) public String getPasswordHash() { @@ -438,5 +556,7 @@ public void clear() { getUids().clear(); getUidAdds().clear(); getExternalData().clear(); + getMapLayers().clear(); + getFeeds().clear(); } } \ No newline at end of file diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/model/MissionChange.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/model/MissionChange.java index 2f75f0ae..bee2686c 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/model/MissionChange.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/model/MissionChange.java @@ -28,6 +28,7 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import com.bbn.marti.maplayer.model.MapLayer; import com.bbn.marti.remote.sync.MissionChangeType; import com.bbn.marti.sync.service.MissionService; import com.bbn.marti.util.xml.DateAdapter; @@ -77,6 +78,8 @@ public void setMissionService(MissionService missionService) { protected String externalDataTool; protected String externalDataToken; protected String externalDataNotes; + protected String missionFeedUid; + protected String mapLayerUid; protected Mission mission; protected String missionName; // mission name is denormalized here to track all event by mission name, even deletions (because if the mission id foreign key was joined on, the mission name would be lost when the mission record is deleted. @@ -376,6 +379,67 @@ public void setExternalDataNotes(String externalDataNotes) { this.externalDataNotes = externalDataNotes; } + @JsonIgnore + @XmlTransient + @Column(name = "mission_feed_uid") + public String getMissionFeedUid() { + return missionFeedUid; + } + + public void setMissionFeedUid(String missionFeedUid) { + this.missionFeedUid = missionFeedUid; + } + + @Transient + @JsonProperty("missionFeed") + @XmlElement(name = "missionFeed") + public MissionFeed getMissionFeed() { + if (missionFeedUid != null) { + MissionFeed missionFeed = missionService.getMissionFeed(missionFeedUid); + if (missionFeed == null) { + missionFeed = new MissionFeed(); + missionFeed.setUid(missionFeedUid); + } + + return missionFeed; + } + + return null; + } + + @JsonIgnore + @XmlTransient + @Column(name = "map_layer_uid") + public String getMapLayerUid() { + return mapLayerUid; + } + + public void setMapLayerUid(String mapLayerUid) { + this.mapLayerUid = mapLayerUid; + } + + + @Transient + @JsonProperty("mapLayer") + @XmlElement(name = "mapLayer") + public MapLayer getMapLayer() { + try { + if (mapLayerUid != null) { + MapLayer mapLayer = missionService.getMapLayer(mapLayerUid); + if (mapLayer == null) { + mapLayer = new MapLayer(); + mapLayer.setUid(mapLayerUid); + } + + return mapLayer; + } + } catch (Exception e) { + logger.error("exception in getMapLayer", e); + } + + return null; + } + public void setExternalData(ExternalMissionData externalMissionData) { this.tempExternalMissionData = externalMissionData; } @@ -403,7 +467,5 @@ public ExternalMissionData getExternalData() { return null; } -} - - +} \ No newline at end of file diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/model/MissionFeed.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/model/MissionFeed.java new file mode 100644 index 00000000..cc14b70c --- /dev/null +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/model/MissionFeed.java @@ -0,0 +1,172 @@ +package com.bbn.marti.sync.model; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ComparisonChain; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.persistence.*; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlTransient; +import java.io.Serializable; + +@Entity +@Table(name = "mission_feed") +@Cacheable +@JsonInclude(JsonInclude.Include.NON_NULL) +public class MissionFeed implements Serializable, Comparable { + + protected static final Logger logger = LoggerFactory.getLogger(MissionFeed.class); + + protected String uid; + protected String dataFeedUid; + protected String filterBbox; + protected String filterType; + protected String filterCallsign; + protected Mission mission; + + public MissionFeed() { + + } + + @Id + @Column(name = "uid", unique = true, nullable = false) + @JsonProperty("uid") + @XmlElement(name = "uid") + public String getUid() { + return uid; + } + public void setUid(String uid) { + this.uid = uid; + } + + @Column(name = "data_feed_uid", nullable = false, columnDefinition = "TEXT") + public String getDataFeedUid() { + return dataFeedUid; + } + public void setDataFeedUid(String dataFeedUid) { + this.dataFeedUid = dataFeedUid; + } + + @Column(name = "filter_bbox", nullable = false, columnDefinition = "TEXT") + public String getFilterBbox() { + return filterBbox; + } + public void setFilterBbox(String filterBbox) { + this.filterBbox = filterBbox; + } + + @Column(name = "filter_type", nullable = false, columnDefinition = "TEXT") + public String getFilterType() { + return filterType; + } + public void setFilterType(String filterType) { + this.filterType = filterType; + } + + @Column(name = "filter_callsign", nullable = false, columnDefinition = "TEXT") + public String getFilterCallsign() { + return filterCallsign; + } + public void setFilterCallsign(String filterCallsign) { + this.filterCallsign = filterCallsign; + } + + @JsonIgnore + @XmlTransient + @ManyToOne + @JoinColumn(name="mission_id") + public Mission getMission() { return mission; } + public void setMission(Mission mission) { this.mission = mission; } + + + @Override + public int compareTo(MissionFeed that) { + return ComparisonChain.start() + .compare(this.uid, that.uid) + .compare(this.dataFeedUid, that.dataFeedUid) + .compare(this.filterBbox, that.filterBbox) + .compare(this.filterType, that.filterType) + .compare(this.filterCallsign, that.filterCallsign) + .result(); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((uid == null) ? 0 : uid.hashCode()); + result = prime * result + ((dataFeedUid == null) ? 0 : dataFeedUid.hashCode()); + result = prime * result + ((filterBbox == null) ? 0 : filterBbox.hashCode()); + result = prime * result + ((filterType == null) ? 0 : filterType.hashCode()); + result = prime * result + ((filterCallsign == null) ? 0 : filterCallsign.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + MissionFeed other = (MissionFeed) obj; + + if (uid == null) { + if (other.uid != null) + return false; + } else if (!uid.equals(other.uid)) + return false; + + if (dataFeedUid == null) { + if (other.dataFeedUid != null) + return false; + } else if (!dataFeedUid.equals(other.dataFeedUid)) + return false; + + if (filterBbox == null) { + if (other.filterBbox != null) + return false; + } else if (!filterBbox.equals(other.filterBbox)) + return false; + + if (filterType == null) { + if (other.filterType != null) + return false; + } else if (!filterType.equals(other.filterType)) + return false; + + if (filterCallsign == null) { + if (other.filterCallsign != null) + return false; + } else if (!filterCallsign.equals(other.filterCallsign)) + return false; + + return true; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("MissionFeed [uid="); + builder.append(uid); + + builder.append(", dataFeedUid="); + builder.append(dataFeedUid); + + builder.append(", filterBbox="); + builder.append(filterBbox); + + builder.append(", filterType="); + builder.append(filterType); + + builder.append(", filterCallsign="); + builder.append(filterCallsign); + + builder.append("]"); + return builder.toString(); + } +} \ No newline at end of file diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/model/MissionInvitation.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/model/MissionInvitation.java index a34127eb..714be402 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/model/MissionInvitation.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/model/MissionInvitation.java @@ -23,7 +23,7 @@ public class MissionInvitation implements Serializable, Comparable { + + @Query(value = "select * from data_feed where uuid = :uuid", nativeQuery = true) + List getDataFeedByUUID(@Param("uuid") String uuid); + + @Query(value = "select * from data_feed where name = :name", nativeQuery = true) + List getDataFeedByName(@Param("name") String name); + + @Query(value = "select id, uuid, name, type, auth, port, auth_required, protocol, feed_group, iface, archive, anongroup, archive_only, " + + "core_version, core_version_tls_versions, sync from data_feed order by name", nativeQuery = true) + List getDataFeeds(); + + @Query(value = "insert into data_feed (uuid, name, type, auth, port, auth_required, protocol, feed_group, iface, archive, anongroup, " + + "archive_only, core_version, core_version_tls_versions, sync) values (:uuid, :name, :type, :auth, :port, :authRequired, :protocol, " + + ":feedGroup, :iface, :archive, :anongroup, :archiveOnly, :coreVersion, :coreVersion2TlsVersions, :sync) returning id", nativeQuery = true) + Long addDataFeed(@Param("uuid") String uuid, @Param("name") String name, @Param("type") int type, + @Param("auth") String auth, @Param("port") Integer port, @Param("authRequired") boolean authRequired, + @Param("protocol") String protocol, @Param("feedGroup") String feedGroup, @Param("iface") String iface, + @Param("archive") boolean archive, @Param("anongroup") boolean anongroup, + @Param("archiveOnly") boolean archiveOnly, @Param("coreVersion") int coreVersion, + @Param("coreVersion2TlsVersions") String coreVersion2TlsVersions, + @Param("sync") boolean sync); + + @Query(value = "update data_feed set name = :name, type = :type, auth = :auth, port = :port, auth_required = :authRequired, protocol = :protocol, " + + "feed_group = :feedGroup, iface = :iface, archive = :archive, anongroup = :anongroup, archive_only = :archiveOnly, core_version = :coreVersion, " + + "core_version_tls_versions = :coreVersion2TlsVersions, sync = :sync where uuid = :uuid returning id", nativeQuery = true) + Long updateDataFeed(@Param("uuid") String uuid, @Param("name") String name, @Param("type") int type, + @Param("auth") String auth, @Param("port") Integer port, @Param("authRequired") boolean authRequired, + @Param("protocol") String protocol, @Param("feedGroup") String feedGroup, @Param("iface") String iface, + @Param("archive") boolean archive, @Param("anongroup") boolean anongroup, + @Param("archiveOnly") boolean archiveOnly, @Param("coreVersion") int coreVersion, + @Param("coreVersion2TlsVersions") String coreVersion2TlsVersions, + @Param("sync") boolean sync); + + @Query(value = "update data_feed set name = :name, type = :type, archive = :archive, archive_only = :archiveOnly, sync = :sync where uuid = :uuid returning id", nativeQuery = true) + Long modifyDataFeed(@Param("uuid") String uuid, @Param("name") String name, @Param("type") int type, + @Param("archive") boolean archive, @Param("archiveOnly") boolean archiveOnly, @Param("sync") boolean sync); + + @Query(value = "delete from data_feed where name = :name returning id", nativeQuery = true) + Long deleteDataFeed(@Param("name") String name); + + @Query(value = "delete from data_feed where id = :id returning id", nativeQuery = true) + Long deleteDataFeedById(@Param("id") Long id); + + @Query(value = "select tag from data_feed_tag where data_feed_id = :id", nativeQuery = true) + List getDataFeedTagsById(@Param("id") Long id); + + @Query(value = "insert into data_feed_tag (data_feed_id, tag) values (:id, :tag) returning data_feed_id", nativeQuery = true) + Long addDataFeedTag(@Param("id") Long id, @Param("tag") String tag); + + @Query(value = "delete from data_feed_tag where data_feed_id = :data_feed_id returning data_feed_id", nativeQuery = true) + List removeAllDataFeedTagsById(@Param("data_feed_id") Long id); + + @Query(value = "select filter_group from data_feed_filter_group where data_feed_id = :id", nativeQuery = true) + List getDataFeedFilterGroupsById(@Param("id") Long id); + + @Query(value = "insert into data_feed_filter_group (data_feed_id, filter_group) values (:id, :filterGroup) returning data_feed_id", nativeQuery = true) + Long addDataFeedFilterGroup(@Param("id") Long id, @Param("filterGroup") String filterGroup); + + @Query(value = "delete from data_feed_filter_group where data_feed_id = :data_feed_id returning data_feed_id", nativeQuery = true) + List removeAllDataFeedFilterGroupsById(@Param("data_feed_id") Long id); + + @Query(value = "select distinct uid from cot_router inner join data_feed_cot on cot_router.id = cot_router_id inner join data_feed on data_feed.id = data_feed_id where stale > now() and data_feed.uuid = :data_feed_uuid", nativeQuery = true) + List getFeedEventUids(@Param("data_feed_uuid") String dataFeedUuid); +} diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/repository/FederationEventRepository.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/repository/FederationEventRepository.java index bec0b913..97555ff2 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/repository/FederationEventRepository.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/repository/FederationEventRepository.java @@ -1,5 +1,3 @@ - - package com.bbn.marti.sync.repository; import java.util.Date; diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/repository/MissionChangeRepository.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/repository/MissionChangeRepository.java index 063eee5f..2ec44664 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/repository/MissionChangeRepository.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/repository/MissionChangeRepository.java @@ -21,7 +21,7 @@ public interface MissionChangeRepository extends JpaRepository= m.create_time\n" + START_END_PRED + "group by mc.external_data_uid, mc.external_data_token, mc.external_data_name, mc.external_data_tool, mc.external_data_notes, mc.creatoruid\n"; static final String EXTERNAL_DATA_REMOVES = - "select max(mc.id) as id, 3 as change_type, null as hash, max(mc.ts) as ts, max(mc.servertime) as servertime, null as uid, mc.creatoruid as creatoruid, null as mission_id, :missionName as mission_name, 0 as mission_createTime, mc.external_data_uid as external_data_uid, mc.external_data_name as external_data_name, mc.external_data_tool as external_data_tool, mc.external_data_token as external_data_token, mc.external_data_notes as external_data_notes \n" + + "select max(mc.id) as id, 3 as change_type, null as hash, max(mc.ts) as ts, max(mc.servertime) as servertime, null as uid, mc.creatoruid as creatoruid, null as mission_id, :missionName as mission_name, 0 as mission_createTime, mc.external_data_uid as external_data_uid, mc.external_data_name as external_data_name, mc.external_data_tool as external_data_tool, mc.external_data_token as external_data_token, mc.external_data_notes as external_data_notes, null as mission_feed_uid, null as map_layer_uid \n" + "from mission_change mc \n" + "inner join mission m on mc.mission_id = m.id\n" + //"left join mission_external_data med on med.id = mc.external_data_uid and med.mission_id = m.id\n" + //this join needs two criteria in the case of multiple missions and the same resource / or same hash different resource row "where \n" + "mc.change_type = 3\n" + // add change type by hash + "and mc.external_data_uid is not null \n" + "and m.name = :missionName\n" + "and mc.ts >= m.create_time\n" + START_END_PRED + @@ -155,29 +157,42 @@ public interface MissionChangeRepository extends JpaRepository= m.create_time\n" + START_END_PRED; static final String EXTERNAL_DATA_REMOVES_FULL_HISTORY = - "select mc.id as id, 3 as change_type, null as hash, mc.ts as ts, mc.servertime as servertime, null as uid, mc.creatoruid as creatoruid, null as mission_id, :missionName as mission_name, 0 as mission_createTime, mc.external_data_uid as external_data_uid, mc.external_data_name as external_data_name, mc.external_data_tool as external_data_tool, mc.external_data_token as external_data_token, mc.external_data_notes as external_data_notes \n" + + "select mc.id as id, 3 as change_type, null as hash, mc.ts as ts, mc.servertime as servertime, null as uid, mc.creatoruid as creatoruid, null as mission_id, :missionName as mission_name, 0 as mission_createTime, mc.external_data_uid as external_data_uid, mc.external_data_name as external_data_name, mc.external_data_tool as external_data_tool, mc.external_data_token as external_data_token, mc.external_data_notes as external_data_notes, null as mission_feed_uid, null as map_layer_uid \n" + "from mission_change mc \n" + "inner join mission m on mc.mission_id = m.id\n" + //"left join mission_external_data med on med.id = mc.external_data_uid and med.mission_id = m.id\n" + //this join needs two criteria in the case of multiple missions and the same resource / or same hash different resource row "where \n" + "mc.change_type = 3\n" + // add change type by hash + "and mc.external_data_uid is not null \n" + "and m.name = :missionName\n" + "and mc.ts >= m.create_time\n" + START_END_PRED; - static final String MISSION_CHANGES = MISSION_CREATES_AND_DELETES + "union all " + HASH_ADDS + "union all " + HASH_REMOVES + "union all " + UID_ADDS + "union all " + UID_REMOVES + "union all " + EXTERNAL_DATA_ADDS + "union all " + EXTERNAL_DATA_REMOVES; - static final String MISSION_CHANGES_FULL_HISTORY = MISSION_CREATES_AND_DELETES + "union all " + HASH_ADDS_FULL_HISTORY + "union all " + HASH_REMOVES_FULL_HISTORY + "union all " + UID_ADDS_FULL_HISTORY + "union all " + UID_REMOVES_FULL_HISTORY + "union all " + EXTERNAL_DATA_ADDS_FULL_HISTORY + "union all " + EXTERNAL_DATA_REMOVES_FULL_HISTORY; + static final String MAP_LAYERS_AND_FEEDS = + "select mc.id as id, change_type, null as hash, ts, servertime, null as uid, mc.creatoruid as creatoruid, null as mission_id, :missionName as mission_name, 0 as mission_createTime, external_data_uid, external_data_name, external_data_tool, external_data_token, external_data_notes, mission_feed_uid, map_layer_uid \n" + + "from mission_change mc \n" + + "inner join mission m on mc.mission_id = m.id\n" + + "where \n" + + "(change_type = 2 or change_type = 3)\n" + // add change type by hash + "and m.name = :missionName\n" + + "and ts >= create_time\n" + + "and (mission_feed_uid is not null or map_layer_uid is not null)\n"+ + START_END_PRED; + + static final String MISSION_CHANGES = MISSION_CREATES_AND_DELETES + "union all " + HASH_ADDS + "union all " + HASH_REMOVES + "union all " + UID_ADDS + "union all " + UID_REMOVES + "union all " + EXTERNAL_DATA_ADDS + "union all " + EXTERNAL_DATA_REMOVES + "union all " + MAP_LAYERS_AND_FEEDS; + static final String MISSION_CHANGES_FULL_HISTORY = MISSION_CREATES_AND_DELETES + "union all " + HASH_ADDS_FULL_HISTORY + "union all " + HASH_REMOVES_FULL_HISTORY + "union all " + UID_ADDS_FULL_HISTORY + "union all " + UID_REMOVES_FULL_HISTORY + "union all " + EXTERNAL_DATA_ADDS_FULL_HISTORY + "union all " + EXTERNAL_DATA_REMOVES_FULL_HISTORY + "union all " + MAP_LAYERS_AND_FEEDS; // Execute all mission change queries back-to-back, and combine the results @Query(value = MISSION_CHANGES, nativeQuery = true) @@ -191,7 +206,7 @@ public interface MissionChangeRepository extends JpaRepository { + MissionFeed save(MissionFeed missionFeed); + + MissionFeed getByUid(String uid); + + @Query(value = "select uid, data_feed_uid, filter_bbox, filter_type, filter_callsign, null as mission_id " + + "from mission_feed where uid = :uid ", nativeQuery = true) + MissionFeed getByUidNoMission(@Param("uid") String uid); + + @Transactional + void deleteByUid(String uid); + + @Transactional + @Modifying + @Query(value = "delete from mission_feed where mission_id = :mission_id", nativeQuery = true) + void deleteAllByMissionId(@Param("mission_id") Long mission_id); +} + diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/repository/MissionRepository.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/repository/MissionRepository.java index 04015fbd..a1191f3b 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/repository/MissionRepository.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/repository/MissionRepository.java @@ -1,11 +1,10 @@ - - package com.bbn.marti.sync.repository; import java.util.Date; import java.util.List; import org.springframework.cache.annotation.Cacheable; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; @@ -16,12 +15,14 @@ import tak.server.cache.MissionCacheResolver; public interface MissionRepository extends JpaRepository { + + String missionAttributes = "select id, create_time, last_edited, name, creatoruid, groups, description, chatroom, base_layer, bbox, path, classification, tool, parent_mission_id, password_hash, default_role_id, expiration, bounding_polygon"; - @Query(value = "select id, create_time, name, creatoruid, groups, description, chatroom, tool, parent_mission_id, password_hash, default_role_id, expiration from mission where name = :name ", nativeQuery = true) + @Query(value = missionAttributes + " from mission where name = :name ", nativeQuery = true) Mission getByNameNoCache(@Param("name") String name); @Cacheable(cacheResolver = MissionCacheResolver.MISSION_CACHE_RESOLVER, key="{#root.args[0] + '-byName'}", sync = true) - @Query(value = "select id, create_time, name, creatoruid, groups, description, chatroom, tool, parent_mission_id, password_hash, default_role_id, expiration from mission where name = :name ", nativeQuery = true) + @Query(value = missionAttributes + " from mission where name = :name ", nativeQuery = true) Mission getByName(@Param("name") String name); Long findMissionIdByName(String name); @@ -76,26 +77,26 @@ public interface MissionRepository extends JpaRepository { @Query(value = "select uid from mission_uid mu inner join mission m on m.id = mu.mission_id where m.name = ?", nativeQuery = true) List getMissionUids(String missionName); - @Query(value = "select id, create_time, name, creatoruid, groups, description, chatroom, tool, parent_mission_id, password_hash, default_role_id, expiration from mission where" + + @Query(value = missionAttributes + " from mission where" + "((:passwordProtected = false and password_hash is null) or :passwordProtected = true)" + // only include password protected missions if asked to "and ((:defaultRole = false and (default_role_id is null or default_role_id = 2)) or :defaultRole = true) " + // return new missions with default role of MISSION_SUBSCRIBER to older clients "AND " + RemoteUtil.GROUP_CLAUSE + " order by id desc ", nativeQuery = true) List getAllMissions(@Param("passwordProtected") boolean passwordProtected, @Param("defaultRole") boolean defaultRole, @Param("groupVector") String groupVector); - @Query(value = "select id, create_time, name, creatoruid, groups, description, chatroom, tool, parent_mission_id, password_hash, default_role_id, expiration from mission where" + + @Query(value = missionAttributes + " from mission where" + "((:passwordProtected = false and password_hash is null) or :passwordProtected = true)" + // only include password protected missions if asked to "and ((:defaultRole = false and (default_role_id is null or default_role_id = 2)) or :defaultRole = true) " + // return new missions with default role of MISSION_SUBSCRIBER to older clients "and tool = :tool AND " + RemoteUtil.GROUP_CLAUSE + " order by id desc ", nativeQuery = true) List getAllMissionsByTool(@Param("passwordProtected") boolean passwordProtected, @Param("defaultRole") boolean defaultRole, @Param("tool") String tool, @Param("groupVector") String groupVector); - @Query(value = "select id, create_time, name, creatoruid, groups, description, chatroom, tool, parent_mission_id, password_hash, default_role_id, expiration from mission where" + + @Query(value = missionAttributes + " from mission where" + "((:passwordProtected = false and password_hash is null) or :passwordProtected = true)" + // only include password protected missions if asked to "and ((:defaultRole = false and (default_role_id is null or default_role_id = 2)) or :defaultRole = true) " + // return new missions with default role of MISSION_SUBSCRIBER to older clients "AND " + RemoteUtil.GROUP_CLAUSE + " and create_time < (now() - (:ttl * INTERVAL '1 second'))" + " order by id desc ", nativeQuery = true) List getAllMissionsByTtl(@Param("passwordProtected") boolean passwordProtected, @Param("defaultRole") boolean defaultRole, @Param("groupVector") String groupVector, @Param("ttl") Integer ttl); - @Query(value = "select id, create_time, name, creatoruid, groups, description, chatroom, tool, parent_mission_id, password_hash, default_role_id, expiration from mission where" + + @Query(value = missionAttributes + " from mission where" + "((:passwordProtected = false and password_hash is null) or :passwordProtected = true)" + // only include password protected missions if asked to "and ((:defaultRole = false and (default_role_id is null or default_role_id = 2)) or :defaultRole = true) " + // return new missions with default role of MISSION_SUBSCRIBER to older clients "and " + RemoteUtil.GROUP_CLAUSE + @@ -115,19 +116,22 @@ List getAllMissionsByTtl(@Param("passwordProtected") boolean passwordPr "and tool = :tool AND " + RemoteUtil.GROUP_CLAUSE + " order by id desc ", nativeQuery = true) List getMissionNamesByTool(@Param("passwordProtected") boolean passwordProtected, @Param("defaultRole") boolean defaultRole, @Param("tool") String tool, @Param("groupVector") String groupVector); - @Query(value = "insert into mission (create_time, name, creatoruid, groups, description, chatroom, tool, password_hash, expiration) values (:createTime, :name, :creatorUid, " - + RemoteUtil.GROUP_VECTOR + ", :description, :chatRoom, :tool, :passwordHash, :expiration) returning id", nativeQuery = true) + @Query(value = "insert into mission (create_time, name, creatoruid, groups, description, chatroom, base_layer, bbox, path, classification, tool, password_hash, expiration, bounding_polygon) values (:createTime, :name, :creatorUid, " + + RemoteUtil.GROUP_VECTOR + ", :description, :chatRoom, :baseLayer, :bbox, :path, :classification, :tool, :passwordHash, :expiration, :boundingPolygon) returning id", nativeQuery = true) Long create(@Param("createTime") Date createTime, @Param("name") String name, @Param("creatorUid") String creatorUid, @Param("groupVector") - String groupVector, @Param("description") String description, @Param("chatRoom") String chatRoom, @Param("tool") String tool, @Param("passwordHash") String passwordHash, @Param("expiration") Long expiration); + String groupVector, @Param("description") String description, @Param("chatRoom") String chatRoom, @Param("baseLayer") String baseLayer, @Param("bbox") String bbox, @Param("path") String path, @Param("classification") String classification, @Param("tool") String tool, @Param("passwordHash") String passwordHash, @Param("expiration") Long expiration, @Param("boundingPolygon") String boundingPolygon); - @Query(value = "insert into mission (create_time, name, creatoruid, groups, description, chatroom, tool, password_hash, default_role_id, expiration) values (:createTime, :name, :creatorUid, " - + RemoteUtil.GROUP_VECTOR + ", :description, :chatRoom, :tool, :passwordHash, :defaultRoleId, :expiration) returning id", nativeQuery = true) - Long create(@Param("createTime") Date createTime, @Param("name") String name, @Param("creatorUid") String creatorUid, @Param("groupVector") String groupVector, @Param("description") String description, @Param("chatRoom") String chatRoom, @Param("tool") String tool, @Param("passwordHash") String passwordHash, - @Param("defaultRoleId") Long defaultRoleId, @Param("expiration") Long expiration); + @Query(value = "insert into mission (create_time, name, creatoruid, groups, description, chatroom, base_layer, bbox, path, classification, tool, password_hash, default_role_id, expiration, bounding_polygon) values (:createTime, :name, :creatorUid, " + + RemoteUtil.GROUP_VECTOR + ", :description, :chatRoom, :baseLayer, :bbox, :path, :classification, :tool, :passwordHash, :defaultRoleId, :expiration, :boundingPolygon) returning id", nativeQuery = true) + Long create(@Param("createTime") Date createTime, @Param("name") String name, @Param("creatorUid") String creatorUid, @Param("groupVector") + String groupVector, @Param("description") String description, @Param("chatRoom") String chatRoom, @Param("baseLayer") String baseLayer, + @Param("bbox") String bbox, @Param("path") String path, @Param("classification") String classification, @Param("tool") String tool, + @Param("passwordHash") String passwordHash, @Param("defaultRoleId") Long defaultRoleId, @Param("expiration") Long expiration, @Param("boundingPolygon") String boundingPolygon); - @Query(value = "update mission set description = :description, chatroom = :chatRoom, expiration = :expiration where name = :name and" + RemoteUtil.GROUP_CLAUSE + " returning id", nativeQuery = true) + @Query(value = "update mission set description = :description, chatroom = :chatRoom, base_layer = :baseLayer, bbox = :bbox, path = :path, classification = :classification, expiration = :expiration, bounding_polygon = :boundingPolygon where name = :name and" + RemoteUtil.GROUP_CLAUSE + " returning id", nativeQuery = true) Long update(@Param("name") String name, @Param("groupVector") String groupVector, @Param("description") String description, - @Param("chatRoom") String chatRoom, @Param("expiration") Long expiration); + @Param("chatRoom") String chatRoom, @Param("baseLayer") String baseLayer, @Param("bbox") String bbox, @Param("path") String path, + @Param("classification") String classification, @Param("expiration") Long expiration, @Param("boundingPolygon") String boundingPolygon); @Query(value = "update mission set groups = cast(:groupVectorMission as bit(" + RemoteUtil.GROUPS_BIT_VECTOR_LEN + ")) where name = :name and" + RemoteUtil.GROUP_CLAUSE + " returning id", nativeQuery = true) Long updateGroups(@Param("name") String name, @Param("groupVector") String groupVector, @Param("groupVectorMission") String groupVectorMission); @@ -155,4 +159,23 @@ Long update(@Param("name") String name, @Param("groupVector") String groupVector @Query(value = "select name from mission, mission_resource where resource_hash = :resource_hash and id = mission_id", nativeQuery = true) List getMissionNamesContainingHash(@Param("resource_hash") String resource_hash); + + static final String COP_QUERY = "select m.id, m.create_time, max(mc.servertime) as last_edited, m.name, m.creatoruid, m.groups, m.description, m.chatroom, m.base_layer, " + + "m.bbox, m.bounding_polygon, m.path, m.classification, m.tool, m.parent_mission_id, m.password_hash, m.default_role_id, m.expiration " + + "from mission m inner join mission_change mc on mc.mission_id = m.id " + + "where m.tool = :tool and " + RemoteUtil.GROUP_CLAUSE + " and (:path is null or path = :path) group by m.id order by m.id desc"; + @Query(value = COP_QUERY, nativeQuery = true) + List getAllCopMissions(@Param("groupVector") String groupVector, @Param("path") String path, @Param("tool") String tool); + + @Query(value = COP_QUERY + " offset :offset rows fetch next :size rows only", nativeQuery = true) + List getAllCopMissionsWithPaging(@Param("groupVector") String groupVector, @Param("path") String path, @Param("tool") String tool, @Param("offset") Integer offset, @Param("size") Integer size); + + @Query(value = "select count(*) from public.mission where tool = :tool", nativeQuery= true) + Long getMissionCountByTool(@Param("tool") String tool); + + @Query(value = "SELECT * FROM mission m INNER JOIN (SELECT DISTINCT data_feed_uid, mission_id FROM mission_feed) f ON f.mission_id = m.id WHERE f.data_feed_uid = :data_feed_uid", nativeQuery = true) + List getMissionsForDataFeed(@Param("data_feed_uid") String dataFeedUuid); + + @Query(value = "select distinct path from mission where tool = :tool and" + RemoteUtil.GROUP_CLAUSE, nativeQuery = true) + List getMissionPathsByTool(@Param("tool") String tool, @Param("groupVector") String groupVector); } \ No newline at end of file diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/repository/MissionSubscriptionRepository.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/repository/MissionSubscriptionRepository.java index 32248a46..e920560e 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/repository/MissionSubscriptionRepository.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/repository/MissionSubscriptionRepository.java @@ -17,11 +17,12 @@ public interface MissionSubscriptionRepository extends JpaRepository findAllByMissionNameNoMission(@Param("missionName") String missionName); - @Query(value = "select ms.uid, null as token, null as mission_id, ms.client_uid, ms.create_time, ms.role_id from mission m " + + @Query(value = "select ms.uid, null as token, null as mission_id, ms.client_uid, ms.username, ms.create_time, ms.role_id from mission m " + "inner join mission_subscription ms on m.id = ms.mission_id where m.name = :missionName", nativeQuery = true) @Cacheable(Constants.MISSION_SUBSCRIPTION_CACHE) List findAllByMissionNameNoMissionNoToken(@Param("missionName") String missionName); - @Query(value = "select ms.uid, ms.token, m.id as mission_id, ms.client_uid, ms.create_time, ms.role_id from mission m " + + @Query(value = "select ms.uid, ms.token, m.id as mission_id, ms.client_uid, ms.username, ms.create_time, ms.role_id from mission m " + "inner join mission_subscription ms on m.id = ms.mission_id where m.tool = 'public' ", nativeQuery = true) @Cacheable(Constants.MISSION_SUBSCRIPTION_CACHE) List findAll(); diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/service/MissionService.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/service/MissionService.java index e61d375c..6e716914 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/service/MissionService.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/service/MissionService.java @@ -4,6 +4,7 @@ import javax.servlet.http.HttpServletRequest; import com.bbn.marti.config.GeospatialFilter; +import com.bbn.marti.maplayer.model.MapLayer; import com.bbn.marti.remote.groups.Group; import com.bbn.marti.remote.sync.MissionContent; import com.bbn.marti.sync.model.*; @@ -23,7 +24,7 @@ */ public interface MissionService { - Mission createMission(String name, String creatorUid, String groupVector, String description, String chatRoom, String tool, String passwordHash, MissionRole defaultRole, Long expiration); + Mission createMission(String name, String creatorUid, String groupVector, String description, String chatRoom, String baseLayer, String bbox, String path, String classification, String tool, String passwordHash, MissionRole defaultRole, Long expiration, String boundingPolygon); Mission getMission(String missionName, boolean hydrateDetails); @@ -35,6 +36,10 @@ public interface MissionService { List getAllMissions(boolean passwordProtected, boolean defaultRole, String tool, NavigableSet groups); + List getAllCopsMissions(String tool, NavigableSet groups, String path, Integer page, Integer size); + + Long getMissionCount(String tool); + Mission deleteMission(String name, String creatorUid, String groupVector, boolean deepDelete); CotElement getLatestCotElement(String uid, String groupVector, Date end, ResultSetExtractor resultSetExtractor); @@ -65,13 +70,13 @@ public interface MissionService { MissionSubscription missionSubscribe(String missionName, String clientUid, String groupVector); - MissionSubscription missionSubscribe(String missionName, Long missionId, String clientUid, MissionRole role, String groupVector); + MissionSubscription missionSubscribe(String missionName, Long missionId, String clientUid, String username, MissionRole role, String groupVector); - void missionUnsubscribe(String missionName, String uid, String groupVector); + void missionUnsubscribe(String missionName, String uid, String username, String groupVector, boolean disconnectOnly); Mission addMissionContent(String missionName, MissionContent content, String creatorUid, String groupVector); - Mission addMissionContentAtTime(String missionName, MissionContent content, String creatorUid, String groupVector, Date date); + Mission addMissionContentAtTime(String missionName, MissionContent content, String creatorUid, String groupVector, Date date, String xmlContentForNotification); boolean addMissionPackage(String missionName, byte[] missionPackage, String creatorUid, NavigableSet groups, List conflicts); @@ -138,9 +143,15 @@ ExternalMissionData hydrate(String externalDataUid, String externalDataName, Str boolean validateRoleAssignment(Mission mission, HttpServletRequest request, MissionRole attemptAssign); - void setRole(Long missionId, String clientUid, Long roleId); + void setRoleByClientUid(Long missionId, String clientUid, Long roleId); + + void setRoleByUsername(Long missionId, String username, Long roleId); + + void setRoleByClientUidOrUsername(Long missionId, String clientUid, String username, Long roleId); - boolean setRole(Mission mission, String clientUid, MissionRole role); + boolean setRole(Mission mission, String clientUid, String username, MissionRole role); + + void setSubscriptionUsername(Long missionId, String clientUid, String username); boolean validatePermission(MissionPermission.Permission permission, HttpServletRequest request); @@ -178,4 +189,22 @@ ExternalMissionData hydrate(String externalDataUid, String externalDataName, Str Collection getLatestMissionCotWrappersForUids(String missionName, Set uids, String groupVector); List getCachedResourcesByHash(String missionName, String hash); + + MissionFeed getMissionFeed(String missionFeedUid); + + MissionFeed addFeedToMission(String missionName, String creatorUid, Mission mission, String dataFeedUid, String filterBbox, String filterType, String filterCallsign); + + void removeFeedFromMission(String missionName, String creatorUid, Mission mission, String missionFeedUid); + + MapLayer getMapLayer(String mapLayerUid); + + MapLayer addMapLayerToMission(String missionName, String creatorUid, Mission mission, MapLayer mapLayer); + + MapLayer updateMapLayer(String missionName, String creatorUid, Mission mission, MapLayer mapLayer); + + void removeMapLayerFromMission(String missionName, String creatorUid, Mission mission, String mapLayerUid); + + List getMissionsForDataFeed(String feed_uid); + + void sendLatestFeedEvents(Mission mission, MissionFeed missionFeed, String clientUid, String groupVector); } \ No newline at end of file diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/service/MissionServiceDefaultImpl.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/service/MissionServiceDefaultImpl.java index ff1a35f0..d30af2ff 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/service/MissionServiceDefaultImpl.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/sync/service/MissionServiceDefaultImpl.java @@ -1,16 +1,12 @@ package com.bbn.marti.sync.service; -import java.io.BufferedInputStream; -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.security.PrivateKey; import java.sql.Array; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; -import java.text.DateFormat; import java.text.SimpleDateFormat; import java.time.Duration; import java.util.*; @@ -18,9 +14,6 @@ import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; import javax.persistence.EntityNotFoundException; import javax.servlet.http.HttpServletRequest; @@ -28,8 +21,10 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.commons.lang3.StringUtils; import org.dom4j.DocumentException; import org.jetbrains.annotations.NotNull; +import org.locationtech.jts.geom.Polygon; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -58,6 +53,8 @@ import com.bbn.marti.dao.kml.JDBCCachingKMLDao; import com.bbn.marti.jwt.JwtUtils; import com.bbn.marti.logging.AuditLogUtil; +import com.bbn.marti.maplayer.MapLayerService; +import com.bbn.marti.maplayer.model.MapLayer; import com.bbn.marti.remote.CoreConfig; import com.bbn.marti.remote.RemoteSubscription; import com.bbn.marti.remote.SubmissionInterface; @@ -83,7 +80,7 @@ import com.bbn.marti.sync.model.*; import com.bbn.marti.sync.model.Mission.MissionAdd; import com.bbn.marti.sync.model.Mission.MissionAddDetails; -import com.bbn.marti.sync.model.MissionPermission.Permission; +import com.bbn.marti.sync.repository.DataFeedRepository; import com.bbn.marti.sync.repository.ExternalMissionDataRepository; import com.bbn.marti.sync.repository.LogEntryRepository; import com.bbn.marti.sync.repository.MissionChangeRepository; @@ -92,7 +89,9 @@ import com.bbn.marti.sync.repository.MissionRoleRepository; import com.bbn.marti.sync.repository.MissionSubscriptionRepository; import com.bbn.marti.sync.repository.ResourceRepository; +import com.bbn.marti.sync.repository.MissionFeedRepository; import com.bbn.marti.util.CommonUtil; +import com.bbn.marti.util.GeomUtils; import com.bbn.marti.util.TimeUtils; import com.bbn.marti.util.missionpackage.ContentType; import com.bbn.marti.util.missionpackage.MissionPackage; @@ -159,6 +158,12 @@ public class MissionServiceDefaultImpl implements MissionService { private final MissionSubscriptionRepository missionSubscriptionRepository; + private final MissionFeedRepository missionFeedRepository; + + private final MapLayerService mapLayerService; + + private final DataFeedRepository dataFeedRepository; + private final GroupManager groupManager; private final CommonUtil commonUtil; @@ -196,6 +201,9 @@ public MissionServiceDefaultImpl( ExternalMissionDataRepository externalMissionDataRepository, MissionInvitationRepository missionInvitationRepository, MissionSubscriptionRepository missionSubscriptionRepository, + MissionFeedRepository missionFeedRepository, + DataFeedRepository dataFeedRepository, + MapLayerService mapLayerService, GroupManager groupManager, CacheManager cacheManager, CommonUtil commonUtil, @@ -218,6 +226,9 @@ public MissionServiceDefaultImpl( this.externalMissionDataRepository = externalMissionDataRepository; this.missionInvitationRepository = missionInvitationRepository; this.missionSubscriptionRepository = missionSubscriptionRepository; + this.missionFeedRepository = missionFeedRepository; + this.dataFeedRepository = dataFeedRepository; + this.mapLayerService = mapLayerService; this.groupManager = groupManager; this.commonUtil = commonUtil; this.missionRoleRepository = missionRoleRepository; @@ -676,7 +687,7 @@ public Mission hydrate(Mission mission, boolean hydrateDetails) { resourceAdds.add(resourceAdd); } } - + Metrics.timer("method-exec-timer-hydrateMission-processChanges", "takserver", "MissionService").record(Duration.ofMillis(System.currentTimeMillis() - startProcessChanges)); @@ -1011,7 +1022,7 @@ public void missionInvite(Mission mission, MissionInvitation missionInvitation) } break; } - case user: { + case username: { RemoteSubscription subscription = subscriptionManager .getSubscriptionByUsersDisplayName(missionInvitation.getInvitee()); if (subscription != null) { @@ -1040,7 +1051,7 @@ public void missionInvite(Mission mission, MissionInvitation missionInvitation) if (contactUids == null || contactUids.length == 0) { logger.error("Unable to determine contactUids for invite! " - + missionInvitation.getType() + ", " + missionInvitation.getInvitee()); + + StringUtils.normalizeSpace(missionInvitation.getType()) + ", " + StringUtils.normalizeSpace(missionInvitation.getInvitee())); } else { subscriptionManager.sendMissionInvite(mission.getName(), contactUids, missionInvitation.getCreatorUid(), mission.getTool(), missionInvitation.getToken(), roleXml); @@ -1097,7 +1108,7 @@ public Set getAllMissionInvitationsForClient(String clientUid User user = subscription.getUser(); if (user.getConnectionType() == ConnectionType.CORE) { missionInvitations.addAll(missionInvitationRepository.findAllMissionInvitationsByInviteeIgnoreCaseAndType( - user.getDisplayName(), MissionInvitation.Type.user.name(), groupVector)); + user.getDisplayName(), MissionInvitation.Type.username.name(), groupVector)); } // add invitations for user's groups @@ -1120,11 +1131,21 @@ public Set getAllMissionInvitationsForClient(String clientUid public MissionSubscription missionSubscribe(String missionName, String clientUid, String groupVector) { Mission mission = getMissionService().getMission(missionName, groupVector); MissionRole defaultRole = getDefaultRole(mission); - return missionSubscribe(missionName, mission.getId(), clientUid, defaultRole, groupVector); + + String username = ""; + try { + username = SecurityContextHolder.getContext().getAuthentication().getName(); + } catch (Exception e) { + if (logger.isDebugEnabled()) { + logger.debug("exception getting username", e); + } + } + + return missionSubscribe(missionName, mission.getId(), clientUid, username, defaultRole, groupVector); } @Override - public MissionSubscription missionSubscribe(String missionName, Long missionId, String clientUid, MissionRole role, String groupVector) { + public MissionSubscription missionSubscribe(String missionName, Long missionId, String clientUid, String username, MissionRole role, String groupVector) { try { missionName = trimName(missionName); @@ -1132,8 +1153,8 @@ public MissionSubscription missionSubscribe(String missionName, Long missionId, MissionSubscription missionSubscription = null; - missionSubscription = missionSubscriptionRepository.findByMissionNameAndClientUidNoMission( - missionName, clientUid); + missionSubscription = missionSubscriptionRepository.findByMissionNameAndClientUidAndUsernameNoMission( + missionName, clientUid, username); if (missionSubscription == null) { @@ -1145,16 +1166,19 @@ public MissionSubscription missionSubscribe(String missionName, Long missionId, logger.debug("creating subscription with uid : " + subscriptionUid); } - missionSubscription = new MissionSubscription(subscriptionUid, token, null, clientUid, new Date(), role); - - missionSubscriptionRepository.subscribe(missionId, clientUid, new Date(), subscriptionUid, token, role.getId()); + missionSubscription = new MissionSubscription(subscriptionUid, token, null, clientUid, username, new Date(), role); - } else if (missionSubscription.getRole() != null && missionSubscription.getRole().compareTo(role) != 0) { + missionSubscriptionRepository.subscribe(missionId, clientUid, new Date(), subscriptionUid, token, role.getId(), username); - logger.error("updating subscription role"); - missionSubscription.setRole(role); + } else { + if (missionSubscription.getRole() != null && missionSubscription.getRole().compareTo(role) != 0) { + if (logger.isDebugEnabled()) { + logger.debug("updating subscription role"); + } + missionSubscription.setRole(role); - getMissionService().setRole(missionId, clientUid, role.getId()); + getMissionService().setRoleByClientUidOrUsername(missionId, clientUid, null, role.getId()); + } } return missionSubscription; @@ -1172,9 +1196,9 @@ public MissionSubscription missionSubscribe(String missionName, Long missionId, @Override @Transactional - public void missionUnsubscribe(String missionName, String uid, String groupVector) { + public void missionUnsubscribe(String missionName, String uid, String username, String groupVector, boolean disconnectOnly) { try { - subscriptionManagerProxy.getSubscriptionManagerForClientUid(uid).missionUnsubscribe(missionName, uid); + subscriptionManagerProxy.getSubscriptionManagerForClientUid(uid).missionUnsubscribe(missionName, uid, username, disconnectOnly); } catch (Exception e) { throw new TakException(e); } @@ -1192,7 +1216,7 @@ public List> getAllMissionSubscriptions() { private AtomicInteger addCount = new AtomicInteger(); @Override - public Mission addMissionContentAtTime(String missionName, MissionContent content, String creatorUid, String groupVector, Date date) { + public Mission addMissionContentAtTime(String missionName, MissionContent content, String creatorUid, String groupVector, Date date, String xmlContentForNotification) { if (logger.isDebugEnabled()) { logger.debug("addMissionContent " + content + " missionName: " + missionName + " creatorUid: " + creatorUid); @@ -1322,7 +1346,7 @@ public Mission addMissionContentAtTime(String missionName, MissionContent conten changeLogger.trace(" annoucing change " + changeXml); } - subscriptionManager.announceMissionChange(missionName, ChangeType.CONTENT, creatorUid, mission.getTool(), changeXml); + subscriptionManager.announceMissionChange(missionName, ChangeType.CONTENT, creatorUid, mission.getTool(), changeXml, xmlContentForNotification); if (logger.isDebugEnabled()) { logger.debug("mission change announced"); @@ -1348,7 +1372,7 @@ public Mission addMissionContentAtTime(String missionName, MissionContent conten @Override public Mission addMissionContent(String missionName, MissionContent content, String creatorUid, String groupVector) { - return addMissionContentAtTime(missionName, content, creatorUid, groupVector, new Date()); + return addMissionContentAtTime(missionName, content, creatorUid, groupVector, new Date(), null); } public Metadata addToEnterpriseSync(byte[] contents, String name, String mimeType, List keywords, @@ -1733,7 +1757,7 @@ public boolean addMissionPackage(String missionName, byte[] missionPackage, Stri MissionContent content = new MissionContent(); content.getHashes().add(hash); addMissionContentAtTime( - missionName, content, creatorUid, groupVector, pendingChange.getTimestamp()); + missionName, content, creatorUid, groupVector, pendingChange.getTimestamp(), null); } else { String hash = pendingChange.getTempResource().getHash(); deleteMissionContentAtTime(missionName, hash, null, @@ -1790,7 +1814,7 @@ public boolean addMissionPackage(String missionName, byte[] missionPackage, Stri // add the new checklist to the checklist mission MissionContent content = new MissionContent(); content.getHashes().add(metadata.getHash()); - addMissionContentAtTime(missionName, content, creatorUid, groupVector, now); + addMissionContentAtTime(missionName, content, creatorUid, groupVector, now, null); } } } @@ -2107,7 +2131,8 @@ public Mission deleteMission(String name, String creatorUid, String groupVector, @Override @CacheEvict(cacheResolver = MissionCacheResolver.MISSION_CACHE_RESOLVER, allEntries = true) public Mission createMission(String name, String creatorUid, String groupVector, String description, String chatRoom, - String tool, String passwordHash, MissionRole defaultRole, Long expiration) { + String baseLayer, String bbox, String path, String classification, String tool, String passwordHash, + MissionRole defaultRole, Long expiration, String boundingPolygon) { if (logger.isDebugEnabled()) { logger.debug("create mission " + name + " " + tool + " " + creatorUid); @@ -2148,9 +2173,9 @@ public Mission createMission(String name, String creatorUid, String groupVector, expirationValue = expiration; } if (defaultRole == null) { - id = missionRepository.create(new Date(), trimName(name), creatorUid, groupVector, description, chatRoom, tool, passwordHash, expirationValue); + id = missionRepository.create(new Date(), trimName(name), creatorUid, groupVector, description, chatRoom, baseLayer, bbox, path, classification, tool, passwordHash, expirationValue, boundingPolygon); } else { - id = missionRepository.create(new Date(), trimName(name), creatorUid, groupVector, description, chatRoom, tool, passwordHash, defaultRole.getId(), expirationValue); + id = missionRepository.create(new Date(), trimName(name), creatorUid, groupVector, description, chatRoom, baseLayer, bbox, path, classification, tool, passwordHash, defaultRole.getId(), expirationValue, boundingPolygon); } mission.setId(id); @@ -2358,6 +2383,45 @@ public List getAllMissions(boolean passwordProtected, boolean defaultRo return missions; } + @Override + @Cacheable(value = Constants.ALL_COPS_MISSION_CACHE, keyGenerator = "allCopsMissionsCacheKeyGenerator", sync = true) + public List getAllCopsMissions(String tool, NavigableSet groups, String path, Integer offset, Integer size) { + + if (logger.isDebugEnabled()) { + logger.debug("mission service getAllMissions cache miss"); + } + + String groupVector = RemoteUtil.getInstance().bitVectorToString(RemoteUtil.getInstance().getBitVectorForGroups(groups)); + + List missions; + + if (offset != null && size != null) { + missions = missionRepository.getAllCopMissionsWithPaging(groupVector, path, tool, offset, size); + } else { + missions = missionRepository.getAllCopMissions(groupVector, path, tool); + } + + for (Mission mission : missions) { + if (mission.isPasswordProtected()) { + mission.clear(); + } + + mission.setGroups(RemoteUtil.getInstance().getGroupNamesForBitVectorString( + mission.getGroupVector(), groups)); + } + + return missions; + } + + public Long getMissionCount(String tool) { + + if (tool == null) { + return 0L; + } + + return missionRepository.getMissionCountByTool(tool); + } + @Override public boolean exists(String missionName, String groupVector) { Mission mission = getMissionService().getMissionByNameCheckGroups(missionName, groupVector); @@ -2672,100 +2736,100 @@ public MissionRole getRoleFromToken( // String authorization = request.getHeader("MissionAuthorization") != null ? request.getHeader("MissionAuthorization") : request.getHeader("Authorization"); - if (authorization == null) { - return null; - } + if (authorization == null) { + return null; + } - // see if the request is using bearer/token authentication - if (!authorization.startsWith("Bearer ")) { - logger.error("Bearer not found : " + authorization + " : " + request.getServletPath()); - return null; - } + // see if the request is using bearer/token authentication + if (!authorization.startsWith("Bearer ")) { + logger.error("Bearer not found : " + StringUtils.normalizeSpace(authorization) + " : " + StringUtils.normalizeSpace(request.getServletPath())); + return null; + } - // parse the token's claims - Claims claims = null; - String token = authorization.substring(7); - try { + // parse the token's claims + Claims claims = null; + String token = authorization.substring(7); + try { - if (coreConfig.getRemoteConfiguration().getSecurity().getTls() == null) { - logger.error("unable to find tls configuration!"); - } + if (coreConfig.getRemoteConfiguration().getSecurity().getTls() == null) { + logger.error("unable to find tls configuration!"); + } - claims = MissionTokenUtils.getInstance(JwtUtils.getInstance().getPrivateKey()).decodeMissionToken(token); + claims = MissionTokenUtils.getInstance(JwtUtils.getInstance().getPrivateKey()).decodeMissionToken(token); - } catch (Exception e) { - logger.error("decodeMissionToken failed! : " + authorization + " : " + request.getServletPath()); - return null; - } + } catch (Exception e) { + logger.error("decodeMissionToken failed! : " + authorization + " : " + request.getServletPath()); + return null; + } - MissionTokenUtils.TokenType tokenType = MissionTokenUtils.TokenType.valueOf(claims.getSubject()); + MissionTokenUtils.TokenType tokenType = MissionTokenUtils.TokenType.valueOf(claims.getSubject()); - // - // make sure we have the correct token type - // - if (!ArrayUtils.contains(validTokenTypes, tokenType)) { - String expected = ""; - for (MissionTokenUtils.TokenType type : validTokenTypes) { - expected += type.name() + " "; - } - logger.error("found unexpected token type, expected " + expected - + ", found " + tokenType.name() + ", " + request.getServletPath()); - return null; - } + // + // make sure we have the correct token type + // + if (!ArrayUtils.contains(validTokenTypes, tokenType)) { + String expected = ""; + for (MissionTokenUtils.TokenType type : validTokenTypes) { + expected += type.name() + " "; + } + logger.error("found unexpected token type, expected " + expected + + ", found " + tokenType.name() + ", " + request.getServletPath()); + return null; + } - String missionName = (String) claims.get(MissionTokenUtils.MISSION_NAME_CLAIM); - if (missionName == null) { - logger.error("found token without MISSION_NAME_CLAIM!"); - return null; - } + String missionName = (String) claims.get(MissionTokenUtils.MISSION_NAME_CLAIM); + if (missionName == null) { + logger.error("found token without MISSION_NAME_CLAIM!"); + return null; + } - if (missionName.compareTo(mission.getName()) != 0) { - logger.error("illegal attempt to re-use token for different mission! : " + request.getServletPath()); - return null; - } + if (missionName.compareTo(mission.getName()) != 0) { + logger.error("illegal attempt to re-use token for different mission! : " + request.getServletPath()); + return null; + } - if (tokenType == MissionTokenUtils.TokenType.SUBSCRIPTION) { + if (tokenType == MissionTokenUtils.TokenType.SUBSCRIPTION) { - String subscriptionUid = claims.get(tokenType.name(), String.class); - if (subscriptionUid == null) { - logger.error("missing subscription uid for subscription token! : " + request.getServletPath()); - return null; - } + String subscriptionUid = claims.get(tokenType.name(), String.class); + if (subscriptionUid == null) { + logger.error("missing subscription uid for subscription token! : " + request.getServletPath()); + return null; + } - MissionSubscription missionSubscription = missionSubscriptionRepository. - findByUidAndMissionNameNoMission(subscriptionUid, mission.getName()); - if (missionSubscription == null) { - logger.error("can't find subscription for token! : " + subscriptionUid - + ", " + request.getServletPath()); - return null; - } + MissionSubscription missionSubscription = missionSubscriptionRepository. + findByUidAndMissionNameNoMission(subscriptionUid, mission.getName()); + if (missionSubscription == null) { + logger.error("can't find subscription for token! : " + subscriptionUid + + ", " + request.getServletPath()); + return null; + } - return missionSubscription.getRole(); + return missionSubscription.getRole(); - } else if (tokenType == MissionTokenUtils.TokenType.INVITATION) { + } else if (tokenType == MissionTokenUtils.TokenType.INVITATION) { - MissionInvitation invitation = missionInvitationRepository.findByToken(token); - if (invitation == null) { - logger.error("can't find invitation for token! : " + request.getServletPath()); - return null; - } + MissionInvitation invitation = missionInvitationRepository.findByToken(token); + if (invitation == null) { + logger.error("can't find invitation for token! : " + request.getServletPath()); + return null; + } - if (invitation.getMissionName().compareToIgnoreCase(mission.getName()) != 0) { - logger.error("illegal attempt to re-use invitation token for different mission! : " - + request.getServletPath()); - return null; - } + if (invitation.getMissionName().compareToIgnoreCase(mission.getName()) != 0) { + logger.error("illegal attempt to re-use invitation token for different mission! : " + + request.getServletPath()); + return null; + } - return invitation.getRole(); + return invitation.getRole(); - } else if (tokenType == MissionTokenUtils.TokenType.ACCESS) { + } else if (tokenType == MissionTokenUtils.TokenType.ACCESS) { - return getDefaultRole(mission); + return getDefaultRole(mission); - } else { - logger.error("unknown token type : " + tokenType.name() + ", " + request.getServletPath()); - return null; - } + } else { + logger.error("unknown token type : " + tokenType.name() + ", " + request.getServletPath()); + return null; + } } catch (Exception e) { logger.error("exception in getRoleFromToken!", e); @@ -2855,7 +2919,7 @@ public boolean validateRoleAssignment(Mission mission, HttpServletRequest reques @Override @CacheEvict(value = Constants.MISSION_SUBSCRIPTION_CACHE, allEntries = true) - public void setRole(Long missionId, String clientUid, Long roleId) { + public void setRoleByClientUid(Long missionId, String clientUid, Long roleId) { MapSqlParameterSource namedParameters = new MapSqlParameterSource(); namedParameters.addValue("roleId", roleId); namedParameters.addValue("clientUid", clientUid); @@ -2865,9 +2929,31 @@ public void setRole(Long missionId, String clientUid, Long roleId) { } @Override - public boolean setRole(Mission mission, String clientUid, MissionRole role) { + @CacheEvict(value = Constants.MISSION_SUBSCRIPTION_CACHE, allEntries = true) + public void setRoleByUsername(Long missionId, String username, Long roleId) { + MapSqlParameterSource namedParameters = new MapSqlParameterSource(); + namedParameters.addValue("roleId", roleId); + namedParameters.addValue("username", username); + namedParameters.addValue("missionId", missionId); + String sql = "update mission_subscription set role_id = :roleId where username = :username and mission_id = :missionId"; + new NamedParameterJdbcTemplate(dataSource).update(sql, namedParameters); + } + + @Override + @CacheEvict(value = Constants.MISSION_SUBSCRIPTION_CACHE, allEntries = true) + public void setRoleByClientUidOrUsername(Long missionId, String clientUid, String username, Long roleId) { + if (!Strings.isNullOrEmpty(username)) { + getMissionService().setRoleByUsername(missionId, username, roleId); + } else { + getMissionService().setRoleByClientUid(missionId, clientUid, roleId); + } + } + + @Override + public boolean setRole(Mission mission, String clientUid, String username, MissionRole role) { try { - getMissionService().setRole(mission.getId(), clientUid, role.getId()); + + getMissionService().setRoleByClientUidOrUsername(mission.getId(), clientUid, username, role.getId()); String roleXml = commonUtil.toXml(role); @@ -2881,6 +2967,17 @@ public boolean setRole(Mission mission, String clientUid, MissionRole role) { } } + @Override + @CacheEvict(value = Constants.MISSION_SUBSCRIPTION_CACHE, allEntries = true) + public void setSubscriptionUsername(Long missionId, String clientUid, String username) { + MapSqlParameterSource namedParameters = new MapSqlParameterSource(); + namedParameters.addValue("missionId", missionId); + namedParameters.addValue("clientUid", clientUid); + namedParameters.addValue("username", username); + String sql = "update mission_subscription set username = :username where client_uid = :clientUid and mission_id = :missionId"; + new NamedParameterJdbcTemplate(dataSource).update(sql, namedParameters); + } + @Override public boolean validatePermission(MissionPermission.Permission permission, HttpServletRequest request) { try { @@ -2939,17 +3036,28 @@ public boolean inviteOrUpdate(Mission mission, List subscri try { for (MissionSubscription next : subscriptions) { - MissionSubscription existing = missionSubscriptionRepository. - findByMissionNameAndClientUidNoMission(mission.getName(), next.getClientUid()); + MissionSubscription existing = null; + + if (!Strings.isNullOrEmpty(next.getUsername())) { + existing = missionSubscriptionRepository. + findByMissionNameAndUsernameNoMission(mission.getName(), next.getUsername()); + } else if (!Strings.isNullOrEmpty(next.getClientUid())) { + existing = missionSubscriptionRepository. + findByMissionNameAndClientUidNoMission(mission.getName(), next.getClientUid()); + } MissionRole role = missionRoleRepository.findFirstByRole(next.getRole().getRole()); if (existing != null) { - setRole(mission, next.getClientUid(), role); - + setRole(mission, next.getClientUid(), next.getUsername(), role); } else { - missionInvite(mission.getName(), next.getClientUid(), - MissionInvitation.Type.clientUid, role, creatorUid, groupVector); + if (!Strings.isNullOrEmpty(next.getUsername())) { + missionInvite(mission.getName(), next.getUsername(), + MissionInvitation.Type.username, role, creatorUid, groupVector); + } else if (!Strings.isNullOrEmpty(next.getClientUid())) { + missionInvite(mission.getName(), next.getClientUid(), + MissionInvitation.Type.clientUid, role, creatorUid, groupVector); + } } } @@ -3087,5 +3195,205 @@ public List findLatestCachedMissionChangesForHashes(@Param("missi public List getCachedResourcesByHash(String missionName, String hash) { return resourceRepository.findByHash(hash); } -} + private MissionChanges saveFeedChangeAtTime(String missionFeedUid, String creatorUid, MissionChangeType missionChangeType, + Mission mission, Date date) { + + MissionChanges changes = new MissionChanges(); + MissionChange change = new MissionChange(missionChangeType, mission); + change.setMissionFeedUid(missionFeedUid); + changes.add(change); + + change.setTimestamp(date); + change.setCreatorUid(creatorUid); + + missionChangeRepository.saveAndFlush(change); + + return changes; + } + + @Override + public MissionFeed getMissionFeed(String missionFeedUid) { + return missionFeedRepository.getByUidNoMission(missionFeedUid); + } + + @Override + @CacheEvict(cacheResolver = MissionCacheResolver.MISSION_CACHE_RESOLVER, allEntries = true) + public MissionFeed addFeedToMission(String missionName, String creatorUid, Mission mission, String dataFeedUid, String filterBbox, String filterType, String filterCallsign) { + MissionFeed missionFeed = new MissionFeed(); + missionFeed.setUid(UUID.randomUUID().toString()); + missionFeed.setDataFeedUid(dataFeedUid); + missionFeed.setFilterBbox(filterBbox); + missionFeed.setFilterType(filterType); + missionFeed.setFilterCallsign(filterCallsign); + missionFeed.setMission(mission); + missionFeed = missionFeedRepository.save(missionFeed); + + // record the change + MissionChanges changes = saveFeedChangeAtTime( + missionFeed.getUid(), creatorUid, MissionChangeType.ADD_CONTENT, mission, new Date()); + + // notify users of the change + subscriptionManager.announceMissionChange(mission.getName(), + ChangeType.DATA_FEED, creatorUid, mission.getTool(), commonUtil.toXml(changes)); + + return missionFeed; + } + + @Override + @CacheEvict(cacheResolver = MissionCacheResolver.MISSION_CACHE_RESOLVER, allEntries = true) + public void removeFeedFromMission(String missionName, String creatorUid, Mission mission, String missionFeedUid) { + + missionFeedRepository.deleteByUid(missionFeedUid); + + // record the change + MissionChanges changes = saveFeedChangeAtTime( + missionFeedUid, creatorUid, MissionChangeType.REMOVE_CONTENT, mission, new Date()); + + // notify users of the change + subscriptionManager.announceMissionChange(mission.getName(), + ChangeType.DATA_FEED, creatorUid, mission.getTool(), commonUtil.toXml(changes)); + + } + + private MissionChanges saveMapLayerChangeAtTime(String mapLayerUid, String creatorUid, MissionChangeType missionChangeType, + Mission mission, Date date) { + + MissionChanges changes = new MissionChanges(); + MissionChange change = new MissionChange(missionChangeType, mission); + change.setMapLayerUid(mapLayerUid); + changes.add(change); + + change.setTimestamp(date); + change.setCreatorUid(creatorUid); + + missionChangeRepository.saveAndFlush(change); + + return changes; + } + + @Override + public MapLayer getMapLayer(String mapLayerUid) { + MapLayer mapLayer = null; + try { + mapLayer = mapLayerService.getMapLayerForUid(mapLayerUid); + } catch (Exception e) { + logger.error("exception in getMapLayer", e); + } + return mapLayer; + } + + @Override + @CacheEvict(cacheResolver = MissionCacheResolver.MISSION_CACHE_RESOLVER, allEntries = true) + public MapLayer addMapLayerToMission(String missionName, String creatorUid, Mission mission, MapLayer mapLayer) { + MapLayer newMapLayer = mapLayerService.createMapLayer(mapLayer); + + // record the change + MissionChanges changes = saveMapLayerChangeAtTime( + mapLayer.getUid(), creatorUid, MissionChangeType.ADD_CONTENT, mission, new Date()); + + // notify users of the change + subscriptionManager.announceMissionChange(mission.getName(), + ChangeType.MAP_LAYER, creatorUid, mission.getTool(), commonUtil.toXml(changes)); + + return newMapLayer; + } + + @Override + @CacheEvict(cacheResolver = MissionCacheResolver.MISSION_CACHE_RESOLVER, allEntries = true) + public MapLayer updateMapLayer(String missionName, String creatorUid, Mission mission, MapLayer mapLayer) { + return mapLayerService.updateMapLayer(mapLayer); + } + + @Override + @CacheEvict(cacheResolver = MissionCacheResolver.MISSION_CACHE_RESOLVER, allEntries = true) + public void removeMapLayerFromMission(String missionName, String creatorUid, Mission mission, String mapLayerUid) { + + // record the change + MissionChanges changes = saveMapLayerChangeAtTime( + mapLayerUid, creatorUid, MissionChangeType.REMOVE_CONTENT, mission, new Date()); + + // notify users of the change + subscriptionManager.announceMissionChange(mission.getName(), + ChangeType.MAP_LAYER, creatorUid, mission.getTool(), commonUtil.toXml(changes)); + + mapLayerService.deleteMapLayer(mapLayerUid); + } + + @Override + @Cacheable(value = Constants.ALL_MISSION_CACHE, key="{#root.methodName, #root.args[0]}", sync = true) + public List getMissionsForDataFeed(String feed_uid) { + return missionRepository.getMissionsForDataFeed(feed_uid); + } + + @Override + public void sendLatestFeedEvents(Mission mission, MissionFeed missionFeed, String clientUid, String groupVector) { + try { + List clientUidList = Collections.singletonList(clientUid); + NavigableSet groups = groupManager.groupVectorToGroupSet(mission.getGroupVector()); + + Polygon polygon = null; + GeospatialFilter.BoundingBox boundingBox = null; + // use polygon over bbox + if (!Strings.isNullOrEmpty(mission.getBoundingPolygon())) { + polygon = GeomUtils.postgisBoundingPolygonToPolygon(mission.getBoundingPolygon()); + } + // fallback to bbox + else { + boundingBox = GeomUtils.getBoundingBoxFromBboxString(mission.getBbox()); + } + + int sent = 0; + for (DataFeedDao datafeed : dataFeedRepository.getDataFeedByUUID(missionFeed.getDataFeedUid())) { + + if (!datafeed.isSync()) { + continue; + } + + if (logger.isDebugEnabled()) { + logger.debug("found datafeed to sync " + datafeed.getUUID()); + } + + // list of unique event ids contained in the feed + List feedEventUids = dataFeedRepository.getFeedEventUids(missionFeed.getDataFeedUid()); + + Collection feedEvents = cotCacheHelper.getLatestCotWrappersForUids( + new HashSet<>(feedEventUids), groupVector, false); + + if (logger.isDebugEnabled()) { + logger.debug("found events to sync : " + feedEvents.size()); + } + + // iterate over the events + for (CotCacheWrapper feedEvent : feedEvents) { + + if (polygon != null && !GeomUtils.polygonContainsCoordinate(polygon, + feedEvent.getCotElement().lat, feedEvent.getCotElement().lon)) { + continue; + } else if (boundingBox != null && !GeomUtils.bboxContainsCoordinate( + boundingBox, feedEvent.getCotElement().lat, feedEvent.getCotElement().lon)) { + continue; + } + + long now = new Date().getTime(); + long staleSeconds = 365 * 24 * 60 * 60; + feedEvent.getCotElement().setServertime(now); + feedEvent.getCotElement().setStaletime(now + (staleSeconds * 1000)); + + // send the event to the COP subscriber + submission.submitCot( + feedEvent.getCotElement().toCotXml(), clientUidList, + new ArrayList(), groups, false, true); + + sent++; + } + + if (logger.isDebugEnabled()) { + logger.debug("filtered events sent : " + sent); + } + } + } catch (Exception e) { + logger.error("exception in sendLatestFeedEvents", e); + } + } +} \ No newline at end of file diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/util/GeomUtils.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/util/GeomUtils.java new file mode 100644 index 00000000..281ad9b2 --- /dev/null +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/util/GeomUtils.java @@ -0,0 +1,86 @@ +package com.bbn.marti.util; + + +import java.util.Arrays; +import com.bbn.marti.config.GeospatialFilter; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.Polygon; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class GeomUtils { + private static final GeometryFactory gf = new GeometryFactory(); + + private static final Logger logger = LoggerFactory.getLogger(GeomUtils.class); + + // string format: lat long, lat long, lat lon, ... + public static Polygon postgisBoundingPolygonToPolygon(String polyString) { + Polygon polygon = null; + + try { + Coordinate[] polygonPoints = Arrays.stream(polyString.split(",")) + .map(p->p.split(" ")) + .map(latlon-> new double[] {Double.valueOf(latlon[0]), Double.valueOf(latlon[1])}) + .map(latlon-> new Coordinate(latlon[1], latlon[0])) // flip lat lon so that its x,y instead of y,x + .toArray(Coordinate[]::new); + + polygon = gf.createPolygon(polygonPoints); + } catch (Exception e) { + logger.error("Could not parse polygon string into a polygon", e); + } + + return polygon; + } + + public static boolean polygonContainsCoordinate(Polygon polygon, double lat, double lon) { + Coordinate coord = new Coordinate(lon,lat); // flip lat lon so that its x,y instead of y,x + + return polygon.contains(gf.createPoint(coord)); + } + + // compute bbox from string and cache it for instant lookup next time + public static GeospatialFilter.BoundingBox getBoundingBoxFromBboxString(String bbox) { + + String[] bboxArr = bbox.split(","); + + if (bboxArr.length != 4) return null; + + double maxLat, minLat, maxLong, minLong; + try { + maxLat = Double.valueOf(bboxArr[0]); + minLong = Double.valueOf(bboxArr[1]); + minLat = Double.valueOf(bboxArr[2]); + maxLong = Double.valueOf(bboxArr[3]); + } catch (Exception e) { + return null; + } + + GeospatialFilter.BoundingBox boundingBox = new GeospatialFilter.BoundingBox(); + boundingBox.setMaxLatitude(maxLat); + boundingBox.setMinLongitude(minLong); + boundingBox.setMinLatitude(minLat); + boundingBox.setMaxLongitude(maxLong); + + return boundingBox; + } + + public static boolean bboxContainsCoordinate(GeospatialFilter.BoundingBox boundingBox, double lat, double lon) { + + boolean validLongitude = false; + if (boundingBox.getMaxLongitude() > boundingBox.getMinLongitude()) { + validLongitude = lon >= boundingBox.getMinLongitude() && lon <= boundingBox.getMaxLongitude(); + } else { + validLongitude = lon >= boundingBox.getMinLongitude() || lon <= boundingBox.getMaxLongitude(); + } + + // return the cot event if found within one of the inputs filters + if (validLongitude && + lat >= boundingBox.getMinLatitude() && lat <= boundingBox.getMaxLatitude()) { + return true; + } + + return false; + } +} diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/util/RestrictedInputStream.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/util/RestrictedInputStream.java deleted file mode 100644 index 4d3e0583..00000000 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/util/RestrictedInputStream.java +++ /dev/null @@ -1,217 +0,0 @@ - - -package com.bbn.marti.util; - -import java.io.InputStream; -import java.io.IOException; - -/** -* An input stream wrapper that will throw an exception if more than maxBytes are read before the underlying input stream is depleted. -* Intended for verifying the size of input streams from the web into storage. -* -* This stream wrapper passes all calls made to the underlying stream, while keeping track of how many bytes have been read. -* If a user can read more than the constructor argument "maxBytes" from the contained stream before it is depleted, then -* an IllegalStateException will be thrown. -* -* @note mark and reset are supported, but only if markSupported returns true. Behavior of mark and reset if more than mark(bytes) -* are read is especially indeterminate. This stream should never wrap another stream where mark has already been called -* and reset() is expected to function properly when called through this wrapper. Neither should mark and reset calls -* be made directly on the underlying stream. -* -* @note If an exception is thrown on a read call, the contents of the destination buffer should be considered arbitrary and meaningless. -* -* -* Internally, this class maintains a count (current) of the number of bytes that have been read. After each read call, -* the number of bytes read (assuming -1 was not returned) is added to current. If current exceeds bound, then an exception -* is thrown. -* -* This class maintains a guard around each call with boolean flag, "valid". If close or shallowClose (which returns the contained -* input stream without closing it) are called, this flag is voided. In this state, all calls that expect a return argument -* throw an exception, and read calls return -1. -*/ -public class RestrictedInputStream extends InputStream { - private InputStream stream; - public final long bound; - private long current = 0L; - private boolean valid = true; - private long lastmark = -1; - - public RestrictedInputStream(InputStream in, long maxBytes) { - if (maxBytes < 0) throw new IllegalArgumentException("maxBytes must be nonnegative"); - else if (in == null) throw new IllegalArgumentException("Input stream must be nonnull"); - - stream = in; - bound = maxBytes; - } - - /** - * Returns whether the stream is valid and has "toRead" bytes left before bound. - */ - private boolean amValid(int toRead) { - return valid && (current + (long) toRead <= bound); - } - - private boolean amValid() { - return valid && (current <= bound); - } - - /** - * Returns whether the stream was valid at the time of the call, and then invalidates the stream if it was valid. - */ - private boolean amAndUnsetValid() { - boolean result = amValid(); - if (result) unsetValid(); - return result; - } - - private void unsetValid() { - valid = false; - } - - private void add(int read) { - this.add((long) read); - } - - /** - * adds the given number of bytes to the current counter, throwing an exception if the bound is exceeded in doing so. - */ - private void add(long read) { - current += read; - if (current > bound) throw new IllegalStateException("Read more than " + bound + " bytes."); - } - - /** - * Voids out this streams pointer to the input stream (hopefully encouraging garbage collection) - */ - private void dealloc() { - this.stream = null; - } - - @Override - public int available() throws IOException { - if (amValid()) { - return stream.available(); - } else return 0; - } - - /** - * calls close on the input stream, voids out the input stream pointer, and unsets this input streams validity. - */ - @Override - public void close() throws IOException { - if (amAndUnsetValid()) { - stream.close(); - this.dealloc(); - } - } - - /** - * If valid, returns a pointer to the underlying stream and unsents this input stream's validity. - */ - public InputStream shallowClose() { - if (amAndUnsetValid()) { - InputStream in = stream; - this.dealloc(); - return in; - } else return null; - } - - /** - * Calls mark on the underlying stream if it claims to support mark. Sets the lastmark argument to the current # of - * bytes read. Lastmark is restored into current when reset is called, assuming mark was called. - */ - @Override - public void mark(int readlimit) { - if (amValid() && markSupported()) { - stream.mark(readlimit); - lastmark = current; - } else throw new IllegalStateException(); - } - - @Override - public boolean markSupported() { - if (amValid()) { - return stream.markSupported(); - } else throw new IllegalStateException(); - } - - @Override - public int read() throws IOException { - // hey, nobody ever said this method would be efficient - byte[] single = new byte[1]; - int nread = this.read(single); - if (nread != -1) { - return single[0]; - } else return -1; - } - - @Override - public int read(byte[] buffer) throws IOException { - return this.read(buffer, 0, buffer.length); - } - - /** - * calls the same read call on the underlying stream, returning the number of bytes read. If bound was exceeded with the - * read, then an exception is thrown. - */ - @Override - public int read(byte[] buffer, int offset, int len) throws IOException { - // guards - we're using our own buffer internally to read, so the underlying stream won't catch it - int nread; - if (amValid() && - (nread = stream.read(buffer, offset, len)) != -1) - { - this.add(nread); - return nread; - } else return -1; - } - - /** - * Returns the number of bytes recorded as having been read so far by this stream. - */ - public long readCount() { - if (amValid()) { - return current; - } else throw new IllegalStateException(); - } - - /** - * Returns the number of bytes that a client can safely read without encountering an exception. - */ - public long bytesLeft() { - if (amValid()) { - return bound - current; - } else throw new IllegalStateException(); - } - - /** - * Calls reset on the underlying client, assuming that markSupported returns true, and mark was previously called on this stream. - */ - @Override - public void reset() throws IOException { - if (amValid() && markSupported() && lastmark != -1) { - if (lastmark != -1) { - stream.reset(); - } - } else throw new IllegalStateException("Operation unsupported, or mark was never called."); - } - - /** - * Calls skip on the underyling stream. The count returned is added to the current count of bytes read. - */ - @Override - public long skip(long toSkip) throws IOException { - if (amValid()) { - long skipped = stream.skip(toSkip); - if (skipped != -1) { - this.add(skipped); - } - return skipped; - } else throw new IllegalStateException(); - } - - @Override - public String toString() { - return stream.toString(); - } -} \ No newline at end of file diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/util/spring/RequestHolderFilterBean.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/util/spring/RequestHolderFilterBean.java index 46a63d50..0de43db5 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/util/spring/RequestHolderFilterBean.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/util/spring/RequestHolderFilterBean.java @@ -41,6 +41,7 @@ public class RequestHolderFilterBean extends GenericFilterBean { private CoreConfig config; private final String apiMissions = "/api/missions/"; + private final String copMissions = "/api/cops/"; @Override public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { @@ -57,10 +58,17 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo if (config.getRemoteConfiguration().getNetwork().isAllowAllOrigins()) { resp.setHeader("Access-Control-Allow-Origin", "*"); resp.setHeader("Access-Control-Allow-Headers", "*"); + resp.setHeader("Access-Control-Allow-Methods", "*"); } String path = req.getRequestURI(); - int missionStart = path.indexOf(apiMissions); + String apiPath = apiMissions; + int missionStart = path.indexOf(apiPath); + + if (missionStart == -1) { + missionStart = path.indexOf(copMissions); + apiPath = copMissions; + } if (logger.isDebugEnabled()) { @@ -72,7 +80,7 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo boolean missionCreate = false; - int missionEnd = path.indexOf("/", missionStart + apiMissions.length()); + int missionEnd = path.indexOf("/", missionStart + apiPath.length()); if (missionEnd == -1) { missionEnd = path.length(); @@ -82,7 +90,7 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo } } - String missionName = path.substring(missionStart + apiMissions.length(), missionEnd); + String missionName = path.substring(missionStart + apiPath.length(), missionEnd); if (missionName != null && !missionName.isEmpty()) { missionName = missionService.trimName(missionName); @@ -93,7 +101,8 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo // if (missionName.compareTo("all") != 0 && missionName.compareTo("logs") != 0 - && missionName.compareTo("invitations") != 0) { + && missionName.compareTo("invitations") != 0 + && missionName.compareTo("hierarchy") != 0) { // // get the mission diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/Feed.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/Feed.java index 9581a69e..58f33021 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/Feed.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/Feed.java @@ -13,6 +13,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.bbn.security.web.MartiValidator; +import java.net.MalformedURLException; +import java.net.URL; @XmlRootElement(name = "feed") @@ -25,36 +27,68 @@ public static enum Type { SENSOR_POINT } - private int id; - private String uuid; - private Type type; - - private boolean active; - private String alias; - - private String address; - private String macAddress; - private String port; - private String roverPort; - private String ignoreEmbeddedKLV; - private String path; - private String protocol; - private String source; - private String networkTimeout; - private String bufferTime; - private String rtspReliable; - private String thumbnail; - private String classification; + protected int id; + protected String uuid; + protected Type type; + + protected boolean active; + protected String alias; - private String latitude; - private String longitude; - private String fov; - private String heading; - private String range; + protected String address; + protected String macAddress; + protected String port; + protected String roverPort; + protected String ignoreEmbeddedKLV; + protected String path; + protected String protocol; + protected String source; + protected String networkTimeout; + protected String bufferTime; + protected String rtspReliable; + protected String thumbnail; + protected String classification; + + protected String latitude; + protected String longitude; + protected String fov; + protected String heading; + protected String range; public Feed() { } - + + public Feed(FeedV2 feedV2) { + this.uuid = feedV2.uuid; + this.active = feedV2.active; + this.alias = feedV2.alias; + + try { + URL url = new URL(feedV2.getUrl()); + this.protocol = url.getProtocol(); + this.address = url.getHost(); + this.port = String.valueOf(url.getPort()); + this.path = url.getPath(); + } catch (MalformedURLException e) { + logger.error("exception parsing feedv2 url!!", e); + } + + this.macAddress = feedV2.macAddress; + this.roverPort = feedV2.roverPort; + this.ignoreEmbeddedKLV = feedV2.ignoreEmbeddedKLV; + this.source = feedV2.source; + this.networkTimeout = feedV2.networkTimeout; + this.bufferTime = feedV2.bufferTime; + this.rtspReliable = feedV2.rtspReliable; + this.thumbnail = feedV2.thumbnail; + this.classification = feedV2.classification; + this.latitude = feedV2.latitude; + this.longitude = feedV2.longitude; + this.fov = feedV2.fov; + this.heading = feedV2.heading; + this.range = feedV2.range; + } + + public boolean validate(Validator validator) { try { diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/FeedV2.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/FeedV2.java new file mode 100644 index 00000000..dbe52558 --- /dev/null +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/FeedV2.java @@ -0,0 +1,311 @@ +/* + * + * The Feed class stores metadata about video feeds + * + */ + +package com.bbn.marti.video; + +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import org.owasp.esapi.*; +import org.owasp.esapi.errors.ValidationException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.bbn.security.web.MartiValidator; + + +@XmlRootElement(name = "feed") +public class FeedV2 { + + protected static final Logger logger = LoggerFactory.getLogger(Feed.class); + + protected int id; + protected String uuid; + protected boolean active; + protected String alias; + protected String url; + protected Integer order; + + protected String macAddress; + protected String roverPort; + protected String ignoreEmbeddedKLV; + protected String source; + protected String networkTimeout; + protected String bufferTime; + protected String rtspReliable; + protected String thumbnail; + protected String classification; + + protected String latitude; + protected String longitude; + protected String fov; + protected String heading; + protected String range; + + protected Integer width; + protected Integer height; + protected Integer bitrate; + + public FeedV2() { } + + public FeedV2(Feed feedV1) { + this.uuid = feedV1.uuid; + this.active = feedV1.active; + this.alias = feedV1.alias; + + this.url = ""; + + if (!feedV1.protocol.equals("raw")) { + this.url += feedV1.protocol + "://"; + } + + this.url += feedV1.address; + + if (feedV1.port != null && !feedV1.port.equals("-1")) { + this.url += ":" + feedV1.port; + } + + if (feedV1.path != null && feedV1.path.length() > 0) { + if (!feedV1.path.startsWith("/")) { + this.url += "/"; + } + + this.url += feedV1.path; + } + + this.macAddress = feedV1.macAddress; + this.roverPort = feedV1.roverPort; + this.ignoreEmbeddedKLV = feedV1.ignoreEmbeddedKLV; + this.source = feedV1.source; + this.networkTimeout = feedV1.networkTimeout; + this.bufferTime = feedV1.bufferTime; + this.rtspReliable = feedV1.rtspReliable; + this.thumbnail = feedV1.thumbnail; + this.classification = feedV1.classification; + this.latitude = feedV1.latitude; + this.longitude = feedV1.longitude; + this.fov = feedV1.fov; + this.heading = feedV1.heading; + this.range = feedV1.range; + } + + public boolean validate(Validator validator) { + + try { + if (getAlias() == null || getUrl() == null) { + return false; + } + + validator.getValidInput("feed", getUuid(), + MartiValidator.Regex.MartiSafeString.name(), MartiValidator.DEFAULT_STRING_CHARS, true); + validator.getValidInput("feed", getAlias(), + MartiValidator.Regex.MartiSafeString.name(), MartiValidator.SHORT_STRING_CHARS, true); + validator.getValidInput("feed", getLatitude(), + MartiValidator.Regex.Coordinates.name(), MartiValidator.SHORT_STRING_CHARS, true); + validator.getValidInput("feed", getLongitude(), + MartiValidator.Regex.Coordinates.name(), MartiValidator.SHORT_STRING_CHARS, true); + validator.getValidInput("feed", getFov(), + MartiValidator.Regex.Double.name(), MartiValidator.SHORT_STRING_CHARS, true); + validator.getValidInput("feed", getHeading(), + MartiValidator.Regex.Double.name(), MartiValidator.SHORT_STRING_CHARS, true); + validator.getValidInput("feed", getRange(), + MartiValidator.Regex.Double.name(), MartiValidator.SHORT_STRING_CHARS, true); + + } catch (ValidationException e) { + e.printStackTrace(); + logger.error("Exception!", e); + return false; + } + + return true; + } + + @XmlTransient + @JsonIgnore + public int getId() { + return id; + } + public void setId(int id) { + this.id = id; + } + + @XmlAttribute(name = "active") + public boolean getActive() { + return active; + } + public void setActive(boolean active) { + this.active = active; + } + + @XmlAttribute(name = "uid") + public String getUuid() { + return uuid; + } + public void setUuid(String uuid) { + this.uuid = uuid; + } + + @XmlAttribute(name = "alias") + public String getAlias() { + return alias; + } + public void setAlias(String alias) { + this.alias = alias; + } + + @XmlAttribute(name = "url") + public String getUrl() { + return url; + } + public void setUrl(String url) { + this.url = url; + } + + @XmlAttribute(name = "order") + public Integer getOrder() { + return order; + } + public void setOrder(Integer order) { + this.order = order; + } + + @XmlAttribute(name = "preferredMacAddress") + public String getMacAddress() { + return macAddress; + } + public void setMacAddress(String macAddress) { + this.macAddress = macAddress; + } + + @XmlAttribute(name = "roverPort") + public String getRoverPort() { + return roverPort; + } + public void setRoverPort(String roverPort) { + this.roverPort = roverPort; + } + + @XmlAttribute(name = "ignoreEmbeddedKLV") + public String getIgnoreEmbeddedKLV() { + return ignoreEmbeddedKLV; + } + public void setIgnoreEmbeddedKLV(String ignoreEmbeddedKLV) { + this.ignoreEmbeddedKLV = ignoreEmbeddedKLV; + } + + @XmlAttribute(name = "source") + public String getSource() { + return source; + } + public void setSource(String source) { + this.source = source; + } + + @XmlAttribute(name = "timeout") + public String getNetworkTimeout() { + return networkTimeout; + } + public void setNetworkTimeout(String networkTimeout) { + this.networkTimeout = networkTimeout; + } + + @XmlAttribute(name = "buffer") + public String getBufferTime() { + return bufferTime; + } + public void setBufferTime(String bufferTime) { + this.bufferTime= bufferTime; + } + + @XmlAttribute(name = "rtspReliable") + public String getRtspReliable() { + return rtspReliable; + } + public void setRtspReliable(String rtspReliable) { + this.rtspReliable = rtspReliable; + } + + @XmlAttribute(name = "thumbnail") + public String getThumbnail() { + return thumbnail; + } + public void setThumbnail(String thumbnail) { + this.thumbnail = thumbnail; + } + + @XmlAttribute(name = "classification") + public String getClassification() { + return classification; + } + public void setClassification(String classification) { + this.classification = classification; + } + + @XmlAttribute(name = "latitude") + public String getLatitude() { + return latitude; + } + public void setLatitude(String latitude) { + this.latitude = latitude; + } + + @XmlAttribute(name = "longitude") + public String getLongitude() { + return longitude; + } + public void setLongitude(String longitude) { + this.longitude = longitude; + } + + @XmlAttribute(name = "fov") + public String getFov() { + return fov; + } + public void setFov(String fov) { + this.fov = fov; + } + + @XmlAttribute(name = "heading") + public String getHeading() { + return heading; + } + public void setHeading(String heading) { + this.heading = heading; + } + + @XmlAttribute(name = "range") + public String getRange() { + return range; + } + public void setRange(String range) { + this.range = range; + } + + @XmlAttribute(name = "width") + public Integer getWidth() { + return width; + } + public void setWidth(Integer width) { + this.width = width; + } + + @XmlAttribute(name = "height") + public Integer getHeight() { + return height; + } + public void setHeight(Integer height) { + this.height = height; + } + + @XmlAttribute(name = "bitrate") + public Integer getBitrate() { + return bitrate; + } + public void setBitrate(Integer bitrate) { + this.bitrate = bitrate; + } +} \ No newline at end of file diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoCollections.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoCollections.java new file mode 100644 index 00000000..f7075a3f --- /dev/null +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoCollections.java @@ -0,0 +1,29 @@ +/* + * + * The VideoConnections class stores a collection of Feeds. + * + */ + + +package com.bbn.marti.video; + +import java.util.ArrayList; +import java.util.List; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + + +@XmlRootElement(name = "videoCollections") +public class VideoCollections { + + public VideoCollections() { + this.videoConnections = new ArrayList(); + } + + @XmlElement(name = "videoCollection") + private List videoConnections = null; + + public List getVideoConnections() { + return videoConnections; + } +} diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnection.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnection.java new file mode 100644 index 00000000..917de120 --- /dev/null +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnection.java @@ -0,0 +1,131 @@ +package com.bbn.marti.video; + + +import com.fasterxml.jackson.annotation.JsonIgnore; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import javax.persistence.*; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; + + +@Entity +@Table(name = "video_connections_v2") +@Cacheable +@XmlRootElement(name = "videoCollection") +public class VideoConnection implements Serializable, Comparable { + + public VideoConnection() { + this.feeds = new ArrayList(); + } + + private Long id; + private String uid; + private boolean active; + private String alias; + private String thumbnail; + private String classification; + private String xml; + private String groupVector; + + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id", unique = true, nullable = false) + @XmlTransient + @JsonIgnore + public Long getId() { + return id; + } + public void setId(Long id) { + this.id = id; + } + + + @Column(name = "uid", unique = false, nullable = false, columnDefinition="VARCHAR") + @XmlAttribute(name = "uid") + public String getUuid() { + return uid; + } + public void setUuid(String uuid) { + this.uid = uuid; + } + + + @Column(name = "active", unique = false, nullable = false, columnDefinition="boolean") + @XmlAttribute(name = "active") + public boolean getActive() { + return active; + } + public void setActive(boolean active) { + this.active = active; + } + + + @Column(name = "alias", unique = false, nullable = false, columnDefinition="VARCHAR") + @XmlAttribute(name = "alias") + public String getAlias() { + return alias; + } + public void setAlias(String alias) { + this.alias = alias; + } + + + @Column(name = "thumbnail", unique = false, nullable = false, columnDefinition="VARCHAR") + @XmlAttribute(name = "thumbnail") + public String getThumbnail() { + return thumbnail; + } + public void setThumbnail(String thumbnail) { + this.thumbnail = thumbnail; + } + + + @Column(name = "classification", unique = false, nullable = false, columnDefinition="VARCHAR") + @XmlAttribute(name = "classification") + public String getClassification() { + return classification; + } + public void setClassification(String classification) { + this.classification = classification; + } + + @Column(name = "xml", columnDefinition = "VARCHAR") + @XmlTransient + @JsonIgnore + public String getXml() { + return xml; + } + public void setXml(String xml) { + this.xml = xml; + } + + @Column(name = "groups", columnDefinition = "bit varying") + @XmlTransient + @JsonIgnore + public String getGroupVector() { + return groupVector; + } + public void setGroupVector(String groupVector) { + this.groupVector = groupVector; + } + + + private List feeds = null; + @Transient + @XmlElement(name = "feed") + public List getFeeds() { + return feeds; + } + + + @Override + public int compareTo(VideoConnection videoConnection) { + return 0; + } +} \ No newline at end of file diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnectionManager.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnectionManager.java index 6e0d9b29..743c1d57 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnectionManager.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnectionManager.java @@ -24,6 +24,8 @@ import com.bbn.marti.EsapiServlet; import com.bbn.marti.remote.util.SecureXmlParser; +import com.bbn.marti.util.CommonUtil; + public class VideoConnectionManager extends EsapiServlet { @@ -31,11 +33,24 @@ public class VideoConnectionManager extends EsapiServlet { @Autowired private VideoManagerService videoManagerService; + + @Autowired + protected CommonUtil martiUtil; protected static final Logger logger = LoggerFactory.getLogger(VideoConnectionManager.class); private boolean addVideoConnections(HttpServletRequest request) throws JAXBException, IOException{ + String groupVector = null; + + try { + // Get group vector for the user associated with this session + groupVector = martiUtil.getGroupBitVector(request); + log.finer("groups bit vector: " + groupVector); + } catch (Exception e) { + log.fine("exception getting group membership for current web user " + e.getMessage()); + } + //String xml = org.apache.commons.io.IOUtils.toString(request.getInputStream()); org.w3c.dom.Document doc = SecureXmlParser.makeDocument(request.getInputStream()); if(doc != null) { @@ -49,7 +64,7 @@ private boolean addVideoConnections(HttpServletRequest request) throws JAXBExcep return false; } - if (!videoManagerService.addFeed(feed, validator)) { + if (!videoManagerService.addFeed(feed, groupVector, validator)) { return false; } } @@ -62,14 +77,24 @@ private boolean addVideoConnections(HttpServletRequest request) throws JAXBExcep } private boolean setActive(HttpServletRequest request) { - + + String groupVector = null; + + try { + // Get group vector for the user associated with this session + groupVector = martiUtil.getGroupBitVector(request); + log.finer("groups bit vector: " + groupVector); + } catch (Exception e) { + log.fine("exception getting group membership for current web user " + e.getMessage()); + } + String feedId = request.getParameter("id"); String active = request.getParameter("active"); - Feed feed = videoManagerService.getFeed(Integer.parseInt(feedId)); + Feed feed = videoManagerService.getFeed(Integer.parseInt(feedId), groupVector); feed.setActive(Boolean.parseBoolean(active)); - return videoManagerService.updateFeed(feed, validator); + return videoManagerService.updateFeed(feed, groupVector, validator); } /* @@ -115,11 +140,21 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) try { initAuditLog(request); + + String groupVector = null; + + try { + // Get group vector for the user associated with this session + groupVector = martiUtil.getGroupBitVector(request); + log.finer("groups bit vector: " + groupVector); + } catch (Exception e) { + log.fine("exception getting group membership for current web user " + e.getMessage()); + } JAXBContext jaxbContext = JAXBContext.newInstance(VideoConnections.class, Feed.class); Marshaller jaxbMarshaller = jaxbContext.createMarshaller(); - VideoConnections videoConnections = videoManagerService.getVideoConnections(false); + VideoConnections videoConnections = videoManagerService.getVideoConnections(false, true, groupVector); StringWriter sw = new StringWriter(); jaxbMarshaller.marshal(videoConnections, sw); @@ -141,9 +176,19 @@ protected void doDelete(HttpServletRequest request, HttpServletResponse response throws ServletException, IOException { try { initAuditLog(request); - + + String groupVector = null; + + try { + // Get group vector for the user associated with this session + groupVector = martiUtil.getGroupBitVector(request); + log.finer("groups bit vector: " + groupVector); + } catch (Exception e) { + log.fine("exception getting group membership for current web user " + e.getMessage()); + } + String id = request.getParameter("id"); - videoManagerService.deleteFeed(id); + videoManagerService.deleteFeed(id, groupVector); } catch (Exception e) { e.printStackTrace(); logger.error("Exception!", e); diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnectionManagerV2.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnectionManagerV2.java new file mode 100644 index 00000000..3d0ec5b4 --- /dev/null +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnectionManagerV2.java @@ -0,0 +1,94 @@ +package com.bbn.marti.video; + +import com.bbn.marti.network.BaseRestController; +import com.bbn.marti.remote.exception.TakException; +import com.bbn.marti.util.CommonUtil; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + + +@RestController +public class VideoConnectionManagerV2 extends BaseRestController { + + private static final Logger logger = LoggerFactory.getLogger(com.bbn.marti.video.VideoConnectionManagerV2.class); + + @Autowired + private HttpServletRequest request; + + @Autowired + private HttpServletResponse response; + + @Autowired + private VideoManagerService videoManagerService; + + @Autowired + private CommonUtil martiUtil; + + @RequestMapping(value = "/video", method = RequestMethod.GET) + @ResponseBody + public VideoCollections getVideoCollections(@RequestParam(value = "protocol", required = false) String protocol) { + try { + String groupVector = martiUtil.getGroupVectorBitString(request); + VideoCollections videoCollections = videoManagerService.getVideoCollections(protocol, true, groupVector); + return videoCollections; + } catch (Exception e) { + throw new TakException("exception in getVideoCollections", e); + } + } + + @RequestMapping(value = "/video", method = RequestMethod.POST) + public void createVideoConnection( + @RequestBody VideoCollections videoCollection) { + try { + String groupVector = martiUtil.getGroupVectorBitString(request); + videoManagerService.createVideoCollections(videoCollection, groupVector); + } catch (Exception e) { + throw new TakException("exception in createVideoConnection", e); + } + } + + @RequestMapping(value = "/video/{uid}", method = RequestMethod.GET) + @ResponseBody + public VideoConnection getVideoConnection(@PathVariable("uid") @NotNull String uid) { + try { + String groupVector = martiUtil.getGroupVectorBitString(request); + VideoConnection videoConnection = videoManagerService.getVideoConnection(uid, groupVector); + if (videoConnection == null) { + response.setStatus(HttpServletResponse.SC_NOT_FOUND); + } + return videoConnection; + } catch (Exception e) { + throw new TakException("exception in getVideoConnection", e); + } + } + + @RequestMapping(value = "/video/{uid}", method = RequestMethod.PUT) + public void updateVideoConnection( + @PathVariable("uid") @NotNull String uid, + @RequestBody VideoConnection videoConnection) { + try { + String groupVector = martiUtil.getGroupVectorBitString(request); + videoManagerService.updateVideoConnection(videoConnection, groupVector); + } catch (Exception e) { + throw new TakException("exception in updateVideoConnection", e); + } + } + + @RequestMapping(value = "/video/{uid}", method = RequestMethod.DELETE) + public void deleteVideoConnection( + @PathVariable("uid") @NotNull String uid) { + try { + String groupVector = martiUtil.getGroupVectorBitString(request); + videoManagerService.deleteVideoConnection(uid, groupVector); + } catch (Exception e) { + throw new TakException("exception in deleteVideoConnection", e); + } + } +} + diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnectionRepository.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnectionRepository.java new file mode 100644 index 00000000..44244777 --- /dev/null +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnectionRepository.java @@ -0,0 +1,51 @@ +package com.bbn.marti.video; + +import com.bbn.marti.remote.util.RemoteUtil; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import tak.server.Constants; + +import java.util.List; + +public interface VideoConnectionRepository extends JpaRepository { + + @CacheEvict(value = Constants.VIDEO_CACHE, allEntries = true) + @Query(value = "insert into video_connections_v2 (uid, active, alias, thumbnail, classification, xml, groups) values " + + " (:uid, :active, :alias, :thumbnail, :classification, :xml, " + RemoteUtil.GROUP_VECTOR + " ) returning id", nativeQuery = true) + Long create(@Param("uid") String uid, + @Param("active") boolean active, + @Param("alias") String alias, + @Param("thumbnail") String thumbnail, + @Param("classification") String classification, + @Param("xml") String xml, + @Param("groupVector") String groupVector); + + @CacheEvict(value = Constants.VIDEO_CACHE, allEntries = true) + @Query(value = "update video_connections_v2 set uid = :uid, active = :active, alias = :alias, " + + " thumbnail = :thumbnail, classification = :classification, xml = :xml " + + " where uid = :uid and " + RemoteUtil.GROUP_CLAUSE + " returning id", nativeQuery = true) + Long update(@Param("uid") String uid, + @Param("active") boolean active, + @Param("alias") String alias, + @Param("thumbnail") String thumbnail, + @Param("classification") String classification, + @Param("xml") String xml, + @Param("groupVector") String groupVector); + + @CacheEvict(value = Constants.VIDEO_CACHE, allEntries = true) + @Query(value = "delete from video_connections_v2 where uid = :uid and " + RemoteUtil.GROUP_CLAUSE + " returning id", nativeQuery = true) + Long delete(@Param("uid") String uid, @Param("groupVector") String groupVector); + + @Cacheable(Constants.VIDEO_CACHE) + @Query(value = "select id, uid, active, alias, thumbnail, classification, xml, groups from video_connections_v2" + + " where " + RemoteUtil.GROUP_CLAUSE, nativeQuery = true) + List get(@Param("groupVector") String groupVector); + + @Cacheable(Constants.VIDEO_CACHE) + @Query(value = "select id, uid, active, alias, thumbnail, classification, xml, groups from video_connections_v2" + + " where uid = :uid and " + RemoteUtil.GROUP_CLAUSE, nativeQuery = true) + VideoConnection getByUid(@Param("uid") String uid, @Param("groupVector") String groupVector); +} \ No newline at end of file diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnectionSender.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnectionSender.java index dfc9d132..f23e4d7c 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnectionSender.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnectionSender.java @@ -100,23 +100,33 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) try { initAuditLog(request); - + + CommonUtil martiUtil = SpringContextBeanForApi.getSpringContext().getBean(CommonUtil.class); + + Objects.requireNonNull(martiUtil, "marti util bean"); + + String groupVector = null; + + try { + // Get group vector for the user associated with this session + groupVector = martiUtil.getGroupBitVector(request); + log.finer("groups bit vector: " + groupVector); + } catch (Exception e) { + log.fine("exception getting group membership for current web user " + e.getMessage()); + } + String[] contacts = getContacts(request); String[] feedIds = request.getParameter("feedId").split("\\|"); for (String feedId : feedIds) { - Feed feed = videoManagerService.getFeed(Integer.parseInt(feedId)); + Feed feed = videoManagerService.getFeed(Integer.parseInt(feedId), groupVector); String senderUid = UUID.randomUUID().toString(); for (String contact : contacts) { String cotMessage = getCotMessage(senderUid, contact, feed.getAddress(), feed.getAlias(), feed.getPort(), feed.getRoverPort(), feed.getRtspReliable(), feed.getIgnoreEmbeddedKLV(), feed.getPath(), feed.getProtocol(), feed.getNetworkTimeout(), feed.getBufferTime()); - - CommonUtil martiUtil = SpringContextBeanForApi.getSpringContext().getBean(CommonUtil.class); - - Objects.requireNonNull(martiUtil, "marti util bean"); - + submission.submitCot(cotMessage, martiUtil.getGroupsFromRequest(request)); } } diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnectionUploader.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnectionUploader.java index 1648fa2b..fd3489a3 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnectionUploader.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnectionUploader.java @@ -16,17 +16,20 @@ import org.springframework.beans.factory.annotation.Autowired; import com.bbn.marti.EsapiServlet; +import com.bbn.marti.util.CommonUtil; import com.bbn.marti.video.Feed.Type; -//@WebServlet("/vcu/*") public class VideoConnectionUploader extends EsapiServlet { - private static final long serialVersionUID = 2500827655081259445L; + private static final long serialVersionUID = 2500827655081259445L; @Autowired private VideoManagerService videoManagerService; + @Autowired + protected CommonUtil martiUtil; + protected static final Logger logger = LoggerFactory.getLogger(VideoConnectionManager.class); private String getParameter(HttpServletRequest request, String parameter) { @@ -50,13 +53,23 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) try { initAuditLog(request); - + + String groupVector = null; + + try { + // Get group vector for the user associated with this session + groupVector = martiUtil.getGroupBitVector(request); + log.finer("groups bit vector: " + groupVector); + } catch (Exception e) { + log.fine("exception getting group membership for current web user " + e.getMessage()); + } + Feed feed = null; String feedId = request.getParameter("feedId"); if (feedId == null || feedId.equals("null")) { feed = new Feed(); } else { - feed = videoManagerService.getFeed(Integer.parseInt(feedId)); + feed = videoManagerService.getFeed(Integer.parseInt(feedId), groupVector); } feed.setUuid(getParameter(request, "uuid")); @@ -85,8 +98,8 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) feed.setClassification(getParameter(request, "classification")); if (!feed.validate(validator) - || (feed.getId() == 0 && !videoManagerService.addFeed(feed, validator)) - || !videoManagerService.updateFeed(feed, validator)) + || (feed.getId() == 0 && !videoManagerService.addFeed(feed, groupVector, validator)) + || !videoManagerService.updateFeed(feed, groupVector, validator)) { response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); return; diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnections.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnections.java index 54308eaa..fb1bef9a 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnections.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoConnections.java @@ -9,18 +9,15 @@ import java.util.ArrayList; import java.util.List; - import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; -import com.bbn.marti.video.Feed; - @XmlRootElement(name = "videoConnections") public class VideoConnections { - public VideoConnections() { - this.feeds = new ArrayList(); + public VideoConnections() { + this.feeds = new ArrayList(); } @XmlElement(name = "feed") diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoManagerService.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoManagerService.java index 14c3b194..aa4c0693 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoManagerService.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/video/VideoManagerService.java @@ -11,6 +11,8 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Iterator; +import java.util.UUID; import javax.naming.NamingException; import javax.sql.DataSource; @@ -28,10 +30,12 @@ import com.bbn.marti.JDBCQueryAuditLogHelper; import com.bbn.marti.logging.AuditLogUtil; +import com.bbn.marti.remote.util.RemoteUtil; import com.bbn.marti.remote.util.SecureXmlParser; import com.bbn.marti.video.Feed.Type; import com.bbn.security.web.MartiValidator; + public class VideoManagerService { @Autowired @@ -40,27 +44,32 @@ public class VideoManagerService { @Autowired private DataSource ds; + @Autowired + private VideoConnectionRepository videoConnectionRepository; + protected static final Logger logger = LoggerFactory.getLogger(VideoManagerService.class); - public boolean addFeed(Feed feed, Validator validator) { + public boolean addFeed(Feed feed, String groupVector, Validator validator) { boolean status = false; try { - + JAXBContext jaxbContext = JAXBContext.newInstance(Feed.class); Marshaller jaxbMarshaller = jaxbContext.createMarshaller(); StringWriter sw = new StringWriter(); jaxbMarshaller.marshal(feed, sw); String xml = sw.toString(); - // - // is the feed (URL/alias) in the database already? - // + // + // is the feed (URL/alias) in the database already? + // try (Connection connection = ds.getConnection(); PreparedStatement query = wrapper.prepareStatement( "select * from video_connections where uuid = ? " + - " and deleted=false", connection)) { - query.setString(1, feed.getUuid()); + " and deleted=false and ( groups is null or " + RemoteUtil.getInstance().getGroupClause() + ")", + connection)) { + query.setString(1, feed.getUuid()); + query.setString(2, groupVector); logger.debug(query.toString()); ResultSet results = query.executeQuery(); boolean found = results.next(); @@ -91,8 +100,8 @@ public boolean addFeed(Feed feed, Validator validator) { try (PreparedStatement quer = wrapper.prepareStatement("insert into video_connections (" + "owner, type, alias, " + - "latitude, longitude, fov, heading, range, uuid, xml) " + - "values (?,?,?,?,?,?,?,?,?,?) ", connection)) { + "latitude, longitude, fov, heading, range, uuid, xml, groups) " + + "values (?,?,?,?,?,?,?,?,?,?,?" + RemoteUtil.getInstance().getGroupType() +") ", connection)) { quer.setString(1, AuditLogUtil.getUsername()); quer.setString(2, feed.getType() != null ? feed.getType().toString() : "VIDEO"); @@ -103,7 +112,8 @@ public boolean addFeed(Feed feed, Validator validator) { quer.setString(7, feed.getHeading()); quer.setString(8, feed.getRange()); quer.setString(9, feed.getUuid()); - quer.setString(10, xml); + quer.setString(10, xml); + quer.setString(11, groupVector); logger.debug(query.toString()); quer.executeUpdate(); @@ -112,7 +122,7 @@ public boolean addFeed(Feed feed, Validator validator) { status = true; } else { - status = updateFeed(feed, validator); + status = updateFeed(feed, groupVector, validator); } } @@ -127,14 +137,15 @@ public boolean addFeed(Feed feed, Validator validator) { return status; } - public boolean updateFeed(Feed feed, Validator validator) { + public boolean updateFeed(Feed feed, String groupVector, Validator validator) { boolean status = false; try (Connection connection = ds.getConnection(); PreparedStatement query = wrapper.prepareStatement("update video_connections set " + - "owner=?, type=?, alias=?, " + - "latitude=?, longitude=?, fov=?, heading=?, range=?, uuid=?, xml=? " + - "where uuid=?", connection)){ + " owner=?, type=?, alias=?, " + + " latitude=?, longitude=?, fov=?, heading=?, range=?, uuid=?, xml=?, groups=?" + RemoteUtil.getInstance().getGroupType() + + " where uuid=? and ( groups is null or " + RemoteUtil.getInstance().getGroupClause() + " )", + connection)){ JAXBContext jaxbContext = JAXBContext.newInstance(Feed.class); Marshaller jaxbMarshaller = jaxbContext.createMarshaller(); @@ -159,9 +170,11 @@ public boolean updateFeed(Feed feed, Validator validator) { query.setString(7, feed.getHeading()); query.setString(8, feed.getRange()); query.setString(9, feed.getUuid()); - query.setString(10, xml); - query.setString(11, feed.getUuid()); - + query.setString(10, xml); + query.setString(11, groupVector); + query.setString(12, feed.getUuid()); + query.setString(13, groupVector); + logger.debug(query.toString()); query.executeUpdate(); @@ -227,11 +240,15 @@ public Feed feedFromResultSet(ResultSet results) { return feed; } - public Feed getFeed(int id) { + public Feed getFeed(int id, String groupVector) { Feed feed = null; - try (Connection connection = ds.getConnection(); PreparedStatement query = wrapper.prepareStatement("select * from video_connections where id = ?", connection)) { + try (Connection connection = ds.getConnection(); PreparedStatement query = wrapper.prepareStatement( + "select * from video_connections where id = ? and " + + " ( groups is null or " + RemoteUtil.getInstance().getGroupClause() + " )" + , connection)) { query.setInt(1, id); + query.setString(2, groupVector); try (ResultSet results = query.executeQuery()) { if (results.next()) { @@ -246,20 +263,21 @@ public Feed getFeed(int id) { return feed; } - public VideoConnections getVideoConnections(boolean onlyActive) { + public VideoConnections getVideoConnections(boolean onlyActive, boolean includeV2, String groupVector) { VideoConnections videoConnections = null; try { - String sql = "select * from video_connections where deleted=false"; + String sql = "select * from video_connections where deleted=false and" + + " ( groups is null or " + RemoteUtil.getInstance().getGroupClause() + " )"; if (onlyActive) { sql += " and xml like '%true%'"; } try (Connection connection = ds.getConnection(); PreparedStatement p = wrapper.prepareStatement(sql, connection)) { + p.setString(1, groupVector); try (ResultSet results = wrapper.doQuery(p)) { - videoConnections = new VideoConnections(); while (results.next()) { Feed feed = feedFromResultSet(results); @@ -268,17 +286,30 @@ public VideoConnections getVideoConnections(boolean onlyActive) { } } +// if (includeV2) { +// VideoCollections videoCollections = getVideoCollections(null, false, groupVector); +// for (VideoConnection videoConnection : videoCollections.getVideoConnections()) { +// for (FeedV2 feedV2 : videoConnection.getFeeds()) { +// Feed feedV1 = new Feed(feedV2); +// videoConnections.getFeeds().add(feedV1); +// } +// } +// } + } catch (NamingException | SQLException e) { logger.error("Exception!", e); } return videoConnections; } - public void deleteFeed(String feedId) { + public void deleteFeed(String feedId, String groupVector) { try (Connection connection = ds.getConnection(); PreparedStatement query = wrapper.prepareStatement( - "update video_connections set deleted=true where id = ?", connection)) { - query.setInt(1, Integer.parseInt(feedId)); + "update video_connections set deleted=true where id = ? and " + + "( groups is null or " + RemoteUtil.getInstance().getGroupClause() + " )" + , connection)) { + query.setInt(1, Integer.parseInt(feedId)); + query.setString(2, groupVector); logger.debug(query.toString()); query.executeUpdate(); @@ -288,4 +319,145 @@ public void deleteFeed(String feedId) { logger.error("Exception!", e); } } + + public void createVideoCollections(VideoCollections videoCollections, String groupVector) { + try { + for (VideoConnection videoConnection : videoCollections.getVideoConnections()) { + VideoConnection existingConnection = null; + if (videoConnection.getUuid() != null && !videoConnection.getUuid().isEmpty()) { + existingConnection = videoConnectionRepository.getByUid(videoConnection.getUuid(), groupVector); + } else { + videoConnection.setUuid(UUID.randomUUID().toString()); + } + + if (existingConnection != null) { + updateVideoConnection(videoConnection, groupVector); + } else { + videoConnectionRepository.create( + videoConnection.getUuid(), + videoConnection.getActive(), + videoConnection.getAlias(), + videoConnection.getThumbnail(), + videoConnection.getClassification(), + videoConnectionToXml(videoConnection), + groupVector); + } + } + } catch (Exception e) { + logger.error("exception in createVideoCollections!", e); + } + } + + private void inflateFeedsFromXml(VideoConnection videoConnection) { + try { + JAXBContext jaxbContext = JAXBContext.newInstance(VideoConnection.class); + Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller(); + Document doc = SecureXmlParser.makeDocument(videoConnection.getXml()); + VideoConnection tmpVideoConnection = (VideoConnection) jaxbUnmarshaller.unmarshal(doc); + videoConnection.getFeeds().addAll(tmpVideoConnection.getFeeds()); + } catch (Exception e) { + logger.error("exception in getVideoConnection!", e); + } + } + + private String videoConnectionToXml(VideoConnection videoConnection) { + try { + JAXBContext jaxbContext = JAXBContext.newInstance(VideoConnection.class); + Marshaller jaxbMarshaller = jaxbContext.createMarshaller(); + StringWriter sw = new StringWriter(); + jaxbMarshaller.marshal(videoConnection, sw); + String xml = sw.toString(); + return xml; + } catch (Exception e) { + logger.error("exception in videoConnectionToXml!", e); + } + return null; + } + + public VideoConnection getVideoConnection(String uid, String groupVector) { + try { + VideoConnection videoConnection = videoConnectionRepository.getByUid(uid, groupVector); + if (videoConnection != null) { + inflateFeedsFromXml(videoConnection); + } + return videoConnection; + } catch (Exception e) { + logger.error("exception in getVideoConnection!", e); + } + return null; + } + + public VideoCollections getVideoCollections(String protocol, boolean includeV1, String groupVector) { + try { + VideoCollections videoCollections = new VideoCollections(); + videoCollections.getVideoConnections().addAll(videoConnectionRepository.get(groupVector)); + for (VideoConnection videoConnection : videoCollections.getVideoConnections()) { + inflateFeedsFromXml(videoConnection); + } + + if (includeV1) { + VideoConnections videoConnectionsV1 = getVideoConnections(false, false, groupVector); + Iterator feedV1Iterator = videoConnectionsV1.getFeeds().iterator(); + while (feedV1Iterator.hasNext()) { + Feed feedV1 = feedV1Iterator.next(); + + VideoConnection wrapper = new VideoConnection(); + wrapper.setActive(true); + wrapper.setUuid(feedV1.getUuid()); + wrapper.setAlias(feedV1.getAlias()); + wrapper.setThumbnail(feedV1.getThumbnail()); + wrapper.setClassification(feedV1.getClassification()); + + FeedV2 wrapped = new FeedV2(feedV1); + wrapper.getFeeds().add(wrapped); + videoCollections.getVideoConnections().add(wrapper); + } + } + + if (protocol != null) { + Iterator videoConnectionIterator = videoCollections.getVideoConnections().iterator(); + while (videoConnectionIterator.hasNext()) { + VideoConnection videoConnection = videoConnectionIterator.next(); + Iterator feedV2Iterator = videoConnection.getFeeds().iterator(); + while (feedV2Iterator.hasNext()) { + FeedV2 feedV2 = feedV2Iterator.next(); + if (!feedV2.getUrl().startsWith(protocol)) { + feedV2Iterator.remove(); + } + } + if (videoConnection.getFeeds().isEmpty()) { + videoConnectionIterator.remove(); + } + } + } + + return videoCollections; + } catch (Exception e) { + logger.error("exception in getVideoCollections!", e); + } + return null; + } + + public void updateVideoConnection(VideoConnection videoConnection, String groupVector) { + try { + videoConnectionRepository.update( + videoConnection.getUuid(), + videoConnection.getActive(), + videoConnection.getAlias(), + videoConnection.getThumbnail(), + videoConnection.getClassification(), + videoConnectionToXml(videoConnection), + groupVector); + } catch (Exception e) { + logger.error("exception in updateVideoConnection!", e); + } + } + + public void deleteVideoConnection(String uid, String groupVector) { + try { + videoConnectionRepository.delete(uid, groupVector); + } catch (Exception e) { + logger.error("exception in deleteVideoConnection!", e); + } + } } diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/xmpp/XmppAPI.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/xmpp/XmppAPI.java index b37a7f22..9fcae3e7 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/xmpp/XmppAPI.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/marti/xmpp/XmppAPI.java @@ -8,6 +8,8 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import io.swagger.models.auth.In; +import org.apache.commons.lang3.StringUtils; import org.jivesoftware.whack.ExternalComponentManager; import org.owasp.esapi.Validator; import org.owasp.esapi.errors.ValidationException; @@ -157,10 +159,19 @@ ResponseEntity getFile( return new ResponseEntity(HttpStatus.NOT_MODIFIED); } - response.setContentType(metadata.getFirst(Metadata.Field.MIMEType)); - response.setContentLength(metadata.getSize()); + String mimeType = metadata.getFirst(Metadata.Field.MIMEType); + mimeType = validator.getValidInput("MIME Type", mimeType, "MartiSafeString", MartiValidator.DEFAULT_STRING_CHARS, false); + response.setContentType(mimeType); + + String sizeStrVal = String.valueOf(metadata.getSize()); + sizeStrVal = validator.getValidInput("Content length", sizeStrVal, "NonNegativeInteger", MartiValidator.LONG_STRING_CHARS, true); + response.setContentLength(Integer.parseInt(sizeStrVal)); + response.setHeader( "Cache-Control", "max-age=31536000" ); - response.setHeader( "ETag", metadata.getHash()); + + String hash = metadata.getHash(); + hash = validator.getValidInput("Header Etag", hash, "Hexidecimal", MartiValidator.LONG_STRING_CHARS, false); + response.setHeader( "ETag", hash); response.getOutputStream().write(contents); return new ResponseEntity(HttpStatus.OK); @@ -218,7 +229,9 @@ ResponseEntity putFile( if (!slotContentType.equalsIgnoreCase(contentType)) { String message = "Content type in request " + contentType + " does not correspond with slot content type " + slotContentType; - logger.info(message); + message = validator.getValidInput(XMPP_TOOL, message, + MartiValidator.Regex.MartiSafeString.name(), MartiValidator.DEFAULT_STRING_CHARS, true); + logger.info(message); // log forging prevention return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(message); } } @@ -244,7 +257,9 @@ ResponseEntity putFile( // Metadata metadata = syncStore.insertResource(toStore, contents, martiUtil.getGroupBitVector(request)); - response.setHeader("Location", request.getRequestURL().toString()); + String url = request.getRequestURL().toString(); + url = validator.getValidInput("Location", url, "MartiSafeString", MartiValidator.DEFAULT_STRING_CHARS, true); + response.setHeader("Location", url); return new ResponseEntity(HttpStatus.CREATED); } catch (ValidationException e) { diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/security/web/MartiValidator.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/security/web/MartiValidator.java index a8a7b6eb..35af98a9 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/security/web/MartiValidator.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/security/web/MartiValidator.java @@ -29,7 +29,8 @@ import org.owasp.esapi.errors.ValidationException; import org.owasp.esapi.reference.DefaultValidator; import org.owasp.esapi.reference.validation.DateValidationRule; -import org.springframework.stereotype.Component; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Custom Validator for Marti, which closely resembles the ESAPI DefaultValidator. @@ -46,6 +47,8 @@ public class MartiValidator extends DefaultValidator { public static final int DEFAULT_STRING_CHARS = 255; public static final int SHORT_STRING_CHARS = 128; + private static final Logger logger = LoggerFactory.getLogger(com.bbn.security.web.MartiValidator.class); + /** * Members of this enum map to regexes defined in apache-tomcat/lib/validation.properties * @@ -98,6 +101,12 @@ public void assertValidHTTPRequestParameterSet(String context, Set optional) throws ValidationException, IntrusionException { + if (logger.isDebugEnabled()) { + logger.debug(" assertValidHTTPRequestParameterSet. context: " + context + " request: " + request + + " required: " + required + + " optional: " + optional); + } + // This implementation in inefficient but performance is not much a of a concern. Set given = request.getParameterMap().keySet(); @@ -171,6 +180,11 @@ public Date getValidDate(String context, String input, DateFormat format, boolea @Override public String getValidInput(String context, String input, String type, int maxLength, boolean allowNull) throws ValidationException { String toValidate = (input == null) ? input : input.trim(); + + if (logger.isDebugEnabled()) { + logger.debug("getValidInput context " + context + " input: " + input + " type: " + type + " maxLength: " + maxLength + " allowNull: " + allowNull); + } + return super.getValidInput(context, toValidate, type, maxLength, allowNull); } @@ -181,6 +195,11 @@ public String getValidInput(String context, String input, String type, int maxLe public String getValidInput(String context, String input, String type, int maxLength, boolean allowNull, boolean canonicalize) throws ValidationException { String toValidate = (input == null) ? input : input.trim(); + + if (logger.isDebugEnabled()) { + logger.debug("getValidInput context " + context + " input: " + input + " type: " + type + " maxLength: " + maxLength + " allowNull: " + allowNull + " canonicalize: " + canonicalize); + } + return super.getValidInput(context, toValidate, type, maxLength, allowNull, canonicalize); } @@ -191,6 +210,12 @@ public String getValidInput(String context, String input, String type, int maxLe public String getValidInput(String context, String input, String type, int maxLength, boolean allowNull, ValidationErrorList errorList) { String toValidate = (input == null) ? input : input.trim(); + + + if (logger.isDebugEnabled()) { + logger.debug("getValidInput context " + context + " input: " + input + " type: " + type + " maxLength: " + maxLength + " allowNull: " + allowNull + " validationErrorList: " + errorList); + } + return super.getValidInput(context, toValidate, type, maxLength, allowNull, errorList); } @@ -200,11 +225,24 @@ public String getValidInput(String context, String input, String type, int maxLe @Override public String getValidInput(String context, String input, String type, int maxLength, boolean allowNull, boolean canonicalize, ValidationErrorList errorList) { + + if (logger.isDebugEnabled()) { + logger.debug("getValidInput context " + context + " input: " + input + " type: " + type + " maxLength: " + maxLength + " allowNull: " + allowNull + " canonicalize: " + canonicalize + " validationErrorList: " + errorList); + } + String toValidate = (input == null) ? input : input.trim(); return super.getValidInput(context, toValidate, type, maxLength, allowNull, canonicalize, errorList); } - - + @Override + public byte[] getValidFileContent(String context, byte[] input, int maxBytes, boolean allowNull) throws ValidationException, IntrusionException { + + if (logger.isDebugEnabled()) { + logger.debug("getValidFileContent: " + context, input, maxBytes, allowNull); + } + + return super.getValidFileContent(context, input, maxBytes, allowNull); + + } } diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/tak/tls/CertManager.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/tak/tls/CertManager.java index 496c9984..dc6f3315 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/tak/tls/CertManager.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/tak/tls/CertManager.java @@ -88,16 +88,12 @@ public X509Certificate generateCert(CertAndKeyGen certGen, String name, long val } public static X509Certificate signCertificate(PKCS10 request, CertKey issuerCertficate, CERTTYPE type, - long validityNotBeforeOffsetMinutes, long validity, + Date notBefore, Date notAfter, String signatureAlg, boolean addChannelsExtUsage, String ocspResponder) throws Exception { - PublicKey pub = request.getSubjectPublicKeyInfo(); X509CertInfo info = new X509CertInfo(); - long now = new Date().getTime(); - long validMs = validity * 1000 * 60 * 60 * 24; - long validityNotBeforeOffsetMS = validityNotBeforeOffsetMinutes * 60 * 1000; info.set(X509CertInfo.VALIDITY, new CertificateValidity( - new Date(now - validityNotBeforeOffsetMS), new Date(now + validMs))); + notBefore, notAfter)); info.set(X509CertInfo.SERIAL_NUMBER, new CertificateSerialNumber( new java.util.Random().nextInt() & 0x7fffffff)); info.set(X509CertInfo.VERSION, diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/tak/tls/CertManagerApi.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/tak/tls/CertManagerApi.java index 3db5ee21..1b22c362 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/tak/tls/CertManagerApi.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/tak/tls/CertManagerApi.java @@ -2,6 +2,9 @@ package com.bbn.tak.tls; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + import java.io.ByteArrayOutputStream; import java.io.StringWriter; import java.security.KeyStore; @@ -24,6 +27,7 @@ import org.springframework.context.annotation.Profile; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.RequestBody; @@ -319,6 +323,46 @@ ResponseEntity signClientCert( } } + @RequestMapping(value = "/tls/signClient/v2", method = RequestMethod.POST) + ResponseEntity signClientCertV2( + @RequestParam(value = "clientUid", defaultValue = "") String clientUid, + @RequestParam(value = "version", required = false) String version, + @RequestBody String base64CSR) + throws Exception { + + try { + TakCert cert = certManagerService.signClient(clientUid, version != null, base64CSR); + if (cert == null) { + throw new TakException("signClient returned null!"); + } + + takCertRepository.save(cert); + + ObjectMapper mapper = new ObjectMapper(); + ObjectNode rootNode = mapper.createObjectNode(); + + String clientCertPem = Util.certToPEM(cert.getX509Certificate(), false); + rootNode.put("signedCert", clientCertPem); + + int ndx = 0; + for (X509Certificate ca : cert.getX509CertificateChain()) { + String caPem = Util.certToPEM(ca, false); + rootNode.put("ca" + ndx++, caPem); + } + + String json = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(rootNode); + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + return new ResponseEntity(json, HttpStatus.OK); + + } catch (Exception e) { + logger.error("Exception in signClientCertV2!", e); + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + return null; + } + } + protected String getHttpUser() { return SecurityContextHolder.getContext().getAuthentication().getName(); } diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/tak/tls/Service/CertManagerService.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/tak/tls/Service/CertManagerService.java index cab94439..4a34197d 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/tak/tls/Service/CertManagerService.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/tak/tls/Service/CertManagerService.java @@ -9,6 +9,8 @@ import javax.naming.ldap.LdapName; import javax.naming.ldap.Rdn; +import io.jsonwebtoken.Claims; +import io.jsonwebtoken.SignatureAlgorithm; import org.apache.commons.codec.binary.Base64; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,8 +18,10 @@ import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.context.event.EventListener; import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.security.oauth2.provider.authentication.OAuth2AuthenticationDetails; import com.bbn.marti.config.*; +import com.bbn.marti.jwt.JwtUtils; import com.bbn.marti.remote.CoreConfig; import com.bbn.marti.remote.SubscriptionManagerLite; import com.bbn.marti.remote.exception.TakException; @@ -28,6 +32,7 @@ import sun.security.pkcs10.PKCS10; + /** * Created on 5/14/2018. */ @@ -141,6 +146,16 @@ public TakCert signClient(String clientUid, boolean addChannelsExtUsage, String X509Certificate[] caChain; X509Certificate signedCert = null; + // is the user authenticating to the enrollment endpoint via OAuth? + String token = null; + if (SecurityContextHolder.getContext().getAuthentication().getDetails() + instanceof OAuth2AuthenticationDetails) { + + // save the token with the certificate + token = ((OAuth2AuthenticationDetails)SecurityContextHolder.getContext(). + getAuthentication().getDetails()).getTokenValue(); + } + if (certificateSigning.getCA() == CAType.TAK_SERVER) { TAKServerCAConfig takServerCAConfig = certificateSigning.getTAKServerCAConfig(); @@ -159,8 +174,24 @@ public TakCert signClient(String clientUid, boolean addChannelsExtUsage, String responderUrl = tls.getResponderUrl(); } + long now = new Date().getTime(); + long validMs = validityDays * 1000 * 60 * 60 * 24; + long validityNotBeforeOffsetMS = validityNotBeforeOffsetMinutes * 60 * 1000; + Date notBefore = new Date(now - validityNotBeforeOffsetMS); + Date notAfter = new Date(now + validMs); + + // is the user authenticating to the enrollment endpoint via OAuth? + if (token != null) { + // set the certificate expiration to match the token expiration + Claims claims = JwtUtils.getInstance().parseClaims(token, SignatureAlgorithm.RS256); + Integer exp = (Integer)claims.get("exp"); + if (exp != null) { + notAfter = new Date(exp.longValue() * 1000); + } + } + signedCert = certManager.signCertificate( - csr, signingCert, CertManager.CERTTYPE.CLIENT, validityNotBeforeOffsetMinutes, validityDays, + csr, signingCert, CertManager.CERTTYPE.CLIENT, notBefore, notAfter, takServerCAConfig.getSignatureAlg(), addChannelsExtUsage, responderUrl); } else if (certificateSigning.getCA() == CAType.MICROSOFT_CA) { @@ -202,7 +233,7 @@ public TakCert signClient(String clientUid, boolean addChannelsExtUsage, String String username = usernameExtractor.extractUsername(signedCert); return new TakCert(signedCert, caChain, - getHttpUser(), username, new Date(), clientUid); + getHttpUser(), username, new Date(), clientUid, token); } catch (Exception e) { logger.error("exception in signClient!", e); diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/tak/tls/TakCert.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/tak/tls/TakCert.java index 51ab3507..5353ee9a 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/tak/tls/TakCert.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/tak/tls/TakCert.java @@ -39,13 +39,14 @@ public class TakCert implements Serializable, Comparable { private Date expirationDate; private Date effectiveDate; private Date revocationDate; + public String token; public TakCert(){ } public TakCert(X509Certificate cert, X509Certificate[] caChain, - String creator, String userDn, Date issuanceDate, String clientUid) + String creator, String userDn, Date issuanceDate, String clientUid, String token) throws CertificateEncodingException, NoSuchAlgorithmException { this.certificate = cert; this.subjectDn = cert.getSubjectX500Principal().getName(); @@ -57,6 +58,7 @@ public TakCert(X509Certificate cert, X509Certificate[] caChain, this.hash = RemoteUtil.getInstance().getCertSHA256Fingerprint(cert); this.clientUid = clientUid; this.caChain = caChain; + this.token = token; } @Id @@ -188,6 +190,14 @@ public void setRevocationDate(Date revocationDate) { this.revocationDate = revocationDate; } + @Column(name = "token", unique = false, columnDefinition="VARCHAR") + public String getToken() { + return token; + } + + public void setToken(String token) { + this.token = token; + } @Override public int compareTo(TakCert takCert) { diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/tak/tls/repository/TakCertRepository.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/tak/tls/repository/TakCertRepository.java index de0156cc..537d7db7 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/tak/tls/repository/TakCertRepository.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/tak/tls/repository/TakCertRepository.java @@ -29,7 +29,6 @@ public interface TakCertRepository extends JpaRepository { @Cacheable(Constants.CERTIFICATE_CACHE) TakCert findOneByHash(String hash); - @Cacheable(Constants.CERTIFICATE_CACHE) List findAllByUserDn(String userDn); @Cacheable(Constants.CERTIFICATE_CACHE) diff --git a/src/takserver-core/takserver-war/src/main/java/com/bbn/user/registration/service/UserRegistrationService.java b/src/takserver-core/takserver-war/src/main/java/com/bbn/user/registration/service/UserRegistrationService.java index 08a9b0cf..d11de8cd 100644 --- a/src/takserver-core/takserver-war/src/main/java/com/bbn/user/registration/service/UserRegistrationService.java +++ b/src/takserver-core/takserver-war/src/main/java/com/bbn/user/registration/service/UserRegistrationService.java @@ -1,12 +1,6 @@ package com.bbn.user.registration.service; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Properties; -import java.util.Random; -import java.util.Set; -import java.util.UUID; +import java.util.*; import java.util.stream.Collectors; import javax.mail.internet.MimeMessage; @@ -209,7 +203,16 @@ public boolean activateUser( } else if (privateGroup != null) { String privateGroupName = !Strings.isNullOrEmpty(privateGroup.getGroup()) ? privateGroup.getGroup() : privateGroup.getDomain(); - groupNames = new String[] { privateGroupName }; + + List groups = new ArrayList<>(); + groups.add(privateGroupName); + if (privateGroup.getGroups() != null) { + for (String groupz : privateGroup.getGroups()) { + groups.add(groupz); + } + } + groupNames = groups.toArray(new String[0]); + } else { groupNames = new String[] { "__ANON__" }; } diff --git a/src/takserver-core/takserver-war/src/main/java/tak/server/cache/ActiveGroupCacheHelper.java b/src/takserver-core/takserver-war/src/main/java/tak/server/cache/ActiveGroupCacheHelper.java index 06927624..03f478d3 100644 --- a/src/takserver-core/takserver-war/src/main/java/tak/server/cache/ActiveGroupCacheHelper.java +++ b/src/takserver-core/takserver-war/src/main/java/tak/server/cache/ActiveGroupCacheHelper.java @@ -63,7 +63,7 @@ public void setActiveGroupsForUser(String username, List groups) { } // add the active groups to the cache - getActiveGroupsCache().put(username, groups); + activeGroupsCache.put(username, groups); // save the active groups to the database saveActiveGroupsForUser(username, groups); diff --git a/src/takserver-core/takserver-war/src/main/java/tak/server/cache/ContactCacheHelper.java b/src/takserver-core/takserver-war/src/main/java/tak/server/cache/ContactCacheHelper.java index b43d6d58..f46e5210 100644 --- a/src/takserver-core/takserver-war/src/main/java/tak/server/cache/ContactCacheHelper.java +++ b/src/takserver-core/takserver-war/src/main/java/tak/server/cache/ContactCacheHelper.java @@ -3,10 +3,10 @@ import java.util.List; import java.util.concurrent.TimeUnit; +import javax.annotation.PostConstruct; + import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.event.ContextRefreshedEvent; -import org.springframework.context.event.EventListener; import com.bbn.marti.remote.ClientEndpoint; import com.bbn.marti.remote.CoreConfig; @@ -24,7 +24,7 @@ public class ContactCacheHelper { @Autowired private CoreConfig config; - @EventListener({ContextRefreshedEvent.class}) + @PostConstruct public void init() { if (log.isDebugEnabled()) { @@ -54,7 +54,7 @@ public Cache> getContactsCache() { return contactCache; } - public String getKeyGetCachedClientEndpointData(boolean connected, boolean recent) { - return "getCachedClientEndpointData_" + connected + "_" + recent; + public String getKeyGetCachedClientEndpointData(boolean connected, boolean recent, long secAgo) { + return "getCachedClientEndpointData_" + connected + "_" + recent + "_" + secAgo; } -} +} \ No newline at end of file diff --git a/src/takserver-core/takserver-war/src/main/java/tak/server/cache/MissionCacheResolver.java b/src/takserver-core/takserver-war/src/main/java/tak/server/cache/MissionCacheResolver.java index 3b0390d3..e5df1134 100644 --- a/src/takserver-core/takserver-war/src/main/java/tak/server/cache/MissionCacheResolver.java +++ b/src/takserver-core/takserver-war/src/main/java/tak/server/cache/MissionCacheResolver.java @@ -35,6 +35,7 @@ public Collection resolveCaches(CacheOperationInvocationContext if ((CacheOperation)context.getOperation() instanceof CacheEvictOperation) { caches.add(cacheManager.getCache(Constants.ALL_MISSION_CACHE)); + caches.add(cacheManager.getCache(Constants.ALL_COPS_MISSION_CACHE)); // caches.add(cacheManager.getCache(cacheName + MissionChangeCacheResolver.SUFFIX)); } diff --git a/src/takserver-core/takserver-war/src/main/java/tak/server/cache/PluginDatafeedCacheHelper.java b/src/takserver-core/takserver-war/src/main/java/tak/server/cache/PluginDatafeedCacheHelper.java new file mode 100644 index 00000000..41e10272 --- /dev/null +++ b/src/takserver-core/takserver-war/src/main/java/tak/server/cache/PluginDatafeedCacheHelper.java @@ -0,0 +1,88 @@ +package tak.server.cache; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.event.ContextRefreshedEvent; +import org.springframework.context.event.EventListener; + +import com.bbn.marti.remote.CoreConfig; +import com.bbn.marti.remote.exception.TakException; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; + +import tak.server.plugins.PluginDataFeed; + +public class PluginDatafeedCacheHelper { + + private static final Logger log = LoggerFactory.getLogger(PluginDatafeedCacheHelper.class); + + private Cache> pluginDatafeedCache; + + public static final String ALL_PLUGIN_DATAFEED_KEY = "ALL_PLUGIN_DATAFEED_KEY"; + + @Autowired + private CoreConfig config; + + private boolean init = false; + + @EventListener({ContextRefreshedEvent.class}) + public void init() { + + if (log.isDebugEnabled()) { + log.debug("in init"); + } + + pluginDatafeedCache = Caffeine.newBuilder() + .expireAfterWrite(config.getCachedConfiguration().getBuffer().getQueue().getPluginDatafeedCacheSeconds(), TimeUnit.SECONDS) + .build(); + + log.info("Done initializing PluginDatafeedCacheHelper with buffer cache value: {} seconds", String.valueOf(config.getCachedConfiguration().getBuffer().getQueue().getPluginDatafeedCacheSeconds())); + + init = true; + } + + public void clearCache() { + + try { + + if (pluginDatafeedCache != null) { + pluginDatafeedCache.invalidateAll(); + } + } catch (Exception e) { + throw new TakException(e); + } + } + + public Cache> getPluginDatafeedCache() { + if (!init) { + throw new TakException("plugin data feed cache not initialized"); + } + + return pluginDatafeedCache; + } + + public List getPluginDatafeed(String feedUuid) { + return getPluginDatafeedCache().getIfPresent(feedUuid); + } + + public void cachePluginDatafeed(String feedUuid, List pluginDatafeed) { + getPluginDatafeedCache().put(feedUuid, pluginDatafeed); + } + + public List getAllPluginDatafeeds() { + return getPluginDatafeedCache().getIfPresent(ALL_PLUGIN_DATAFEED_KEY); + } + + public void cacheAllPluginDatafeeds(List allPluginDatafeeds) { + getPluginDatafeedCache().put(ALL_PLUGIN_DATAFEED_KEY, allPluginDatafeeds); + } + + public void invalidate(String key) { + getPluginDatafeedCache().invalidate(key); + } + +} diff --git a/src/takserver-core/takserver-war/src/main/java/tak/server/plugins/PluginDataApi.java b/src/takserver-core/takserver-war/src/main/java/tak/server/plugins/PluginDataApi.java new file mode 100644 index 00000000..76b3d93b --- /dev/null +++ b/src/takserver-core/takserver-war/src/main/java/tak/server/plugins/PluginDataApi.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2013-2015 Raytheon BBN Technologies. Licensed to US Government with unlimited rights. + */ + +package tak.server.plugins; + +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import com.bbn.marti.cot.search.model.ApiResponse; +import com.bbn.marti.network.BaseRestController; +import com.bbn.marti.remote.exception.TakException; +import com.bbn.marti.sync.api.MissionApi.ValidatedBy; +import com.bbn.marti.util.spring.RequestHolderBean; +import com.google.common.base.Charsets; + +import tak.server.Constants; +import tak.server.PluginManager; + +/* + * + * REST API for plugin data + * + * base path is /Marti/api + * + */ +@RestController +public class PluginDataApi extends BaseRestController { + + private static final Logger logger = LoggerFactory.getLogger(PluginDataApi.class); + + @Autowired(required = false) + private PluginManager pluginManager; + + @Autowired + private RequestHolderBean requestHolderBean; + + /* + * Submit generic to a plugin for processing. Data must be UTF-8 text. + */ + @RequestMapping(value = "/plugins/{name:.+}/submit", method = RequestMethod.PUT) + public ResponseEntity> submitToPluginUTF8( + @PathVariable("name") @ValidatedBy("MartiSafeString") @NotNull String pluginClassName, + @RequestParam(value = "scope", required = false) String scope, + @RequestBody(required = true) byte[] requestBodyBytes) { + + // TODO: handle IllegalArgumentException better + try { + if (requestBodyBytes == null) { + throw new IllegalArgumentException("null data submitted to plugin - ignoring."); + } + + String contentType = requestHolderBean.getRequest().getHeader("content-type"); + + String requestBodyString = new String(requestBodyBytes, Charsets.UTF_8); + + logger.info("submit " + pluginClassName); + logger.info("body string: " + requestBodyString); + logger.info("content type: " + contentType); + + try { + pluginManager.submitDataToPlugin(pluginClassName, scope, requestBodyString, contentType); + } catch (Exception e) { + throw new TakException("error accesing PluginManager process - is it running?", e); + } + + } catch (Exception e) { + + + logger.error("exception submitting plugin data", e); + } + + return new ResponseEntity>(new ApiResponse(Constants.API_VERSION, PluginInfo.class.getName(), "data submitted to " + pluginClassName), HttpStatus.OK); + } + +} \ No newline at end of file diff --git a/src/takserver-core/takserver-war/src/main/java/tak/server/plugins/PluginManagerApi.java b/src/takserver-core/takserver-war/src/main/java/tak/server/plugins/PluginManagerApi.java index 675d2226..a08e3320 100644 --- a/src/takserver-core/takserver-war/src/main/java/tak/server/plugins/PluginManagerApi.java +++ b/src/takserver-core/takserver-war/src/main/java/tak/server/plugins/PluginManagerApi.java @@ -64,8 +64,8 @@ public ResponseEntity>> getAllPluginInfo() { return result; } - @RequestMapping(value = "/plugins/info/all/status", method = RequestMethod.POST) - public ResponseEntity> changeAllPluginStatus(@RequestParam("status") boolean status) { + @RequestMapping(value = "/plugins/info/all/started", method = RequestMethod.POST) + public ResponseEntity> changeAllPluginStartedStatus(@RequestParam("status") boolean status) { ResponseEntity> result = null; try { if (status) { @@ -86,8 +86,8 @@ public ResponseEntity> changeAllPluginStatus(@RequestParam( return result; } - @RequestMapping(value = "/plugins/info/status", method = RequestMethod.POST) - public ResponseEntity> changePluginStatus(@RequestParam("name") String name, @RequestParam("status") boolean status) { + @RequestMapping(value = "/plugins/info/started", method = RequestMethod.POST) + public ResponseEntity> changePluginStartedStatus(@RequestParam("name") String name, @RequestParam("status") boolean status) { ResponseEntity> result = null; try { if (status) { @@ -107,5 +107,43 @@ public ResponseEntity> changePluginStatus(@RequestParam("na return result; } + + @RequestMapping(value = "/plugins/info/enabled", method = RequestMethod.POST) + public ResponseEntity> changePluginEnabledSetting(@RequestParam("name") String name, @RequestParam("status") boolean isEnabled) { + ResponseEntity> result = null; + try { + pluginManager.setPluginEnabled(name, isEnabled); + + result = new ResponseEntity>(new ApiResponse(Constants.API_VERSION, Boolean.class.getName(), isEnabled), HttpStatus.OK); + } catch (Exception e) { + logger.debug("Exception changing plugin status.", e); + } + + if (result == null) { + //This would be an error condition + result = new ResponseEntity>(new ApiResponse(Constants.API_VERSION, Boolean.class.getName(), isEnabled), HttpStatus.INTERNAL_SERVER_ERROR); + } + + return result; + } + + @RequestMapping(value = "/plugins/info/archive", method = RequestMethod.POST) + public ResponseEntity> changePluginArchiveSetting(@RequestParam("name") String name, @RequestParam("archiveEnabled") boolean archiveEnabled) { + ResponseEntity> result = null; + try { + pluginManager.setPluginArchive(name, archiveEnabled); + + result = new ResponseEntity>(new ApiResponse(Constants.API_VERSION, Boolean.class.getName(), archiveEnabled), HttpStatus.OK); + } catch (Exception e) { + logger.debug("Exception changing plugin status.", e); + } + + if (result == null) { + //This would be an error condition + result = new ResponseEntity>(new ApiResponse(Constants.API_VERSION, Boolean.class.getName(), archiveEnabled), HttpStatus.INTERNAL_SERVER_ERROR); + } + + return result; + } } \ No newline at end of file diff --git a/src/takserver-core/takserver-war/src/main/java/tak/server/system/ApiDependencyProxy.java b/src/takserver-core/takserver-war/src/main/java/tak/server/system/ApiDependencyProxy.java index 68332d77..1d6c3988 100644 --- a/src/takserver-core/takserver-war/src/main/java/tak/server/system/ApiDependencyProxy.java +++ b/src/takserver-core/takserver-war/src/main/java/tak/server/system/ApiDependencyProxy.java @@ -8,6 +8,7 @@ import com.bbn.marti.sync.EnterpriseSyncService; import com.bbn.marti.sync.repository.MissionRoleRepository; +import tak.server.PluginManager; import tak.server.cache.CoTCacheHelper; public class ApiDependencyProxy implements ApplicationContextAware { @@ -93,5 +94,6 @@ public CoTCacheHelper cotCacheHelper() { } return cotCacheHelper; - } + } + } diff --git a/src/takserver-core/takserver-war/src/main/java/tak/server/util/ExecutorSource.java b/src/takserver-core/takserver-war/src/main/java/tak/server/util/ExecutorSource.java new file mode 100644 index 00000000..6a9cc17b --- /dev/null +++ b/src/takserver-core/takserver-war/src/main/java/tak/server/util/ExecutorSource.java @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2013-2015 Raytheon BBN Technologies. Licensed to US Government with unlimited rights. + */ + +package tak.server.util; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import com.bbn.marti.config.Buffer.Queue; +import com.bbn.marti.config.Configuration; +import com.bbn.marti.remote.CoreConfig; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +public class ExecutorSource { + + private final Configuration config; + + private final Queue queue; + + private final int POOL_SIZE_INITIAL = 1; + + private final int NUM_AVAIL_CORES = Runtime.getRuntime().availableProcessors(); + + private final int DEFAULT_POOL_MAX = NUM_AVAIL_CORES; + + private final int POOL_SIZE_MAX; + + public final int EXEC_QUEUE_SIZE; + + public final boolean IS_LOW_CORE; // forceLowConcurrency option in config can be used to set low concurrency mode + + public final ExecutorService missionRepositoryProcessor; + + public ExecutorSource(CoreConfig coreConfig) { + + config = coreConfig.getRemoteConfiguration(); + + queue = config.getBuffer().getQueue(); + + POOL_SIZE_MAX = DEFAULT_POOL_MAX < queue.getDefaultMaxPoolSize() ? queue.getDefaultMaxPoolSize() : DEFAULT_POOL_MAX; + + EXEC_QUEUE_SIZE = config.getBuffer().getQueue().getDefaultExecQueueSize() * NUM_AVAIL_CORES; + + IS_LOW_CORE = NUM_AVAIL_CORES < 4 || config.isForceLowConcurrency(); // forceLowConcurrency option in config can be used to set low concurrency mode + + missionRepositoryProcessor = newExecutorService("MissionRepository", POOL_SIZE_INITIAL, POOL_SIZE_MAX * 2); + + } + + private ExecutorService newExecutorService(String name, int initialPoolSize, int maxPoolSize) { + + return newExecutorService(name, initialPoolSize, maxPoolSize, EXEC_QUEUE_SIZE); + } + + private static ExecutorService newExecutorService(String name, int initialPoolSize, int maxPoolSize, int queueSize) { + + ThreadFactory threadFactory = + new ThreadFactoryBuilder() + .setNameFormat(name + "-%1$d") + .setUncaughtExceptionHandler(new TakServerExceptionHandler()) + .build(); + + BlockingQueue workQueue = new LinkedBlockingQueue<>(queueSize); + return new TakServerThreadPoolExecutor(initialPoolSize, maxPoolSize, 60L, TimeUnit.SECONDS, workQueue, threadFactory); + } + + @SuppressWarnings("unused") + private static ScheduledExecutorService newScheduledExecutor(String name, int size) { + + ThreadFactory threadFactory = + new ThreadFactoryBuilder() + .setNameFormat(name + "-%1$d") + .setUncaughtExceptionHandler(new TakServerExceptionHandler()) + .build(); + + return new ScheduledThreadPoolExecutor(size, threadFactory); + } + + private ThreadPoolTaskExecutor websocketExecutor() { + + ThreadFactory threadFactory = + new ThreadFactoryBuilder() + .setNameFormat("takserver-socket-%1$d") + .setUncaughtExceptionHandler(new TakServerExceptionHandler()) + .build(); + + ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor(); + taskExecutor.setCorePoolSize(POOL_SIZE_INITIAL); + taskExecutor.setMaxPoolSize(POOL_SIZE_MAX); + taskExecutor.setQueueCapacity(EXEC_QUEUE_SIZE); + taskExecutor.setAllowCoreThreadTimeOut(true); + taskExecutor.setKeepAliveSeconds(120); + taskExecutor.setThreadFactory(threadFactory); + + return taskExecutor; + } + + private static class TakServerThreadPoolExecutor extends ThreadPoolExecutor { + + TakServerThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, BlockingQueue workQueue, ThreadFactory threadFactory) { + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory); + } + + Logger logger = LoggerFactory.getLogger(TakServerThreadPoolExecutor.class); + + public TakServerThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, BlockingQueue workQueue) { + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue); + } + + @Override + public void afterExecute(Runnable r, Throwable t) { + super.afterExecute(r, t); + // If submit() method is called instead of execute() + if (t == null && r instanceof Future) { + try { + ((Future) r).get(); + } catch (CancellationException e) { + t = e; + } catch (ExecutionException e) { + t = e.getCause(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + if (t != null) { + // Exception occurred + + logger.error("Uncaught exception ", t); + } + // can perform cleanup actions here + } + } + + private static class TakServerExceptionHandler implements Thread.UncaughtExceptionHandler { + + Logger logger = LoggerFactory.getLogger(TakServerExceptionHandler.class); + + @Override + public void uncaughtException(Thread thread, Throwable t) { + logger.error("Uncaught exception", t); + } + } +} + diff --git a/src/takserver-fig-core/src/main/proto/message.proto b/src/takserver-fig-core/src/main/proto/message.proto index 6e78b1eb..9ccada30 100644 --- a/src/takserver-fig-core/src/main/proto/message.proto +++ b/src/takserver-fig-core/src/main/proto/message.proto @@ -19,6 +19,12 @@ message Message { repeated string destCallsigns = 6; repeated string provenance = 7; + + bool archive = 8; + + string feedUuid = 9; + + string connectionId = 10; } diff --git a/src/takserver-package/launcher/build.gradle b/src/takserver-package/launcher/build.gradle index 2875f72f..7b6eb54f 100644 --- a/src/takserver-package/launcher/build.gradle +++ b/src/takserver-package/launcher/build.gradle @@ -71,8 +71,12 @@ if [ ! -e /opt/tak/conf/retention/retention-service.yml ]; then cp /opt/tak/retention/retention-service.yml /opt/tak/conf/retention/ fi -if [ ! -e /opt/tak/conf/retention/misison-archiving-config.yml ]; then - cp /opt/tak/retention/misison-archiving-config.yml /opt/tak/conf/retention/ +if [ -f /opt/tak/conf/retention/misison-archiving-config.yml ]; then + rm -rf /opt/tak/conf/retention/misison-archiving-config.yml +fi + +if [ ! -e /opt/tak/conf/retention/mission-archiving-config.yml ]; then + cp /opt/tak/retention/mission-archiving-config.yml /opt/tak/conf/retention/ fi if [ ! -d "/opt/tak/mission-archive" ]; then diff --git a/src/takserver-package/takserver/build.gradle b/src/takserver-package/takserver/build.gradle index 70878dbb..ecafc310 100644 --- a/src/takserver-package/takserver/build.gradle +++ b/src/takserver-package/takserver/build.gradle @@ -73,8 +73,12 @@ if [ ! -e /opt/tak/conf/retention/retention-service.yml ]; then cp /opt/tak/retention/retention-service.yml /opt/tak/conf/retention/ fi -if [ ! -e /opt/tak/conf/retention/misison-archiving-config.yml ]; then - cp /opt/tak/retention/misison-archiving-config.yml /opt/tak/conf/retention/ +if [ -f /opt/tak/conf/retention/misison-archiving-config.yml ]; then + rm -rf /opt/tak/conf/retention/misison-archiving-config.yml +fi + +if [ ! -e /opt/tak/conf/retention/mission-archiving-config.yml ]; then + cp /opt/tak/retention/mission-archiving-config.yml /opt/tak/conf/retention/ fi if [ ! -d "/opt/tak/mission-archive" ]; then diff --git a/src/takserver-plugin-manager/build.gradle b/src/takserver-plugin-manager/build.gradle index 12ee13cf..e13c2332 100644 --- a/src/takserver-plugin-manager/build.gradle +++ b/src/takserver-plugin-manager/build.gradle @@ -40,6 +40,8 @@ bootJar { dependencies { + compile group: 'xerces', name: 'xercesImpl', version: xerces_version + compile group: 'org.slf4j', name: 'slf4j-api', version: slf4j_version compile group: 'org.slf4j', name: 'log4j-over-slf4j', version: slf4j_version compile group: 'org.apache.logging.log4j', name: 'log4j-api', version: log4j_api_version diff --git a/src/takserver-plugin-manager/src/main/java/tak/server/plugins/PluginStarter.java b/src/takserver-plugin-manager/src/main/java/tak/server/plugins/PluginStarter.java index 304f51a1..5b347d7f 100644 --- a/src/takserver-plugin-manager/src/main/java/tak/server/plugins/PluginStarter.java +++ b/src/takserver-plugin-manager/src/main/java/tak/server/plugins/PluginStarter.java @@ -14,7 +14,6 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.ignite.Ignite; -import org.apache.ignite.IgniteAtomicLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -22,7 +21,7 @@ import org.springframework.context.event.EventListener; import org.springframework.context.support.GenericApplicationContext; -import com.google.common.base.Strings; +import com.bbn.marti.remote.ServerInfo; import atakmap.commoncommo.protobuf.v1.MessageOuterClass.Message; import io.nats.client.Connection; @@ -31,12 +30,8 @@ import tak.server.CommonConstants; import tak.server.messaging.Messenger; - public class PluginStarter { - private final String natsURL; - private final String natsClusterId; - private Connection natsConnection; private Dispatcher dispatcher; @@ -63,26 +58,29 @@ public class PluginStarter { private final ForkJoinPool interceptorSendPool = newForkJoinPool("plugin-interceptor-send-worker"); private final ForkJoinPool interceptorProcessPool = newForkJoinPool("plugin-interceptor-process-worker"); + + private final ServerInfo serverInfo; + private final PluginApi pluginApi; - public PluginStarter(String natsURL, String natsClusterId) { + public PluginStarter(ServerInfo serverInfo, PluginApi pluginApi) { starterPool = Executors.newWorkStealingPool(Runtime.getRuntime().availableProcessors() * 2); - this.natsURL = natsURL; - this.natsClusterId = natsClusterId; + this.serverInfo = serverInfo; + this.pluginApi = pluginApi; } @EventListener(PluginsLoadedEvent.class) private void init() { try { - initSenderPlugins(); - initReceiverPlugins(); - initSenderReceiverPlugins(); - initInterceptorPlugins(); + startSenderPlugins(); + startReceiverPlugins(); + startSenderReceiverPlugins(); + startInterceptorPlugins(); - if (Strings.isNullOrEmpty(natsClusterId)) { - initIgniteListener(); - } else { + if (serverInfo.isCluster()) { initNatsListener(); + } else { + initIgniteListener(); } } catch (Exception e) { logger.error("error inititalizing plugins", e); @@ -98,13 +96,13 @@ private void submitBytesToReceivers(byte[] rawMessage) { Message message = Message.parseFrom(rawMessage); receiverPlugins.forEach((name, receiver) -> { - if (receiver.getPluginInfo().isEnabled()) { + if (receiver.getPluginInfo().isStarted()) { receiver.onMessage(message); } }); senderReceiverPlugins.forEach((name, senderReceiver) -> { - if (senderReceiver.getPluginInfo().isEnabled()) { + if (senderReceiver.getPluginInfo().isStarted()) { senderReceiver.onMessage(message); } }); @@ -139,7 +137,8 @@ private void submitBytesToReceivers(byte[] rawMessage) { Message.Builder mb = processedMessage.toBuilder(); // add default provenance to guard against loops - mb.addProvenance("PluginManager"); + mb.addProvenance(tak.server.Constants.PLUGIN_MANAGER_PROVENANCE); + mb.addProvenance(tak.server.Constants.PLUGIN_INTERCEPTOR_PROVENANCE); fpm.send(mb.build()); } catch (InterruptedException | ExecutionException e) { @@ -160,13 +159,14 @@ private void submitBytesToReceivers(byte[] rawMessage) { private void initNatsListener() { try { - natsConnection = Nats.connect(natsURL); + natsConnection = Nats.connect(serverInfo.getNatsURL()); // message dispatcher dispatcher = natsConnection.createDispatcher(); dispatcher.subscribe(CommonConstants.CLUSTER_PLUGIN_SUBSCRIBE_TOPIC, CommonConstants.CLUSTER_PLUGIN_SUBSCRIBE_GROUP, m -> - submitBytesToReceivers((byte[]) (byte[]) m.getData())); + submitBytesToReceivers((byte[]) (byte[]) m.getData()) + ); } catch (Exception e) { logger.error("exception connecting to NATS server to receive messages", e); } @@ -196,81 +196,65 @@ private void initIgniteListener() { }); } - private void initSenderReceiverPlugins() { + private void startSenderReceiverPlugins() { senderReceiverPlugins = context.getBeansOfType(MessageSenderReceiver.class); - AtomicInteger senderReceiverCount = new AtomicInteger(); - senderReceiverPlugins.forEach((name, senderReceiver) -> { starterPool.execute(() -> { - senderReceiver.start(); - - if (logger.isDebugEnabled()) { - logger.debug("started sender-receiver plugin " + name + " " + senderReceiver.getClass().getName()); + + if (senderReceiver.getPluginInfo().isEnabled()) { + senderReceiver.internalStart(); + logger.info("started senderReceiver plugin named {}, class {}", name, senderReceiver.getClass().getName()); } - senderReceiverCount.incrementAndGet(); }); }); } - private void initReceiverPlugins() { + private void startReceiverPlugins() { receiverPlugins = context.getBeansOfType(MessageReceiver.class); - AtomicInteger receiverCount = new AtomicInteger(); - receiverPlugins.forEach((name, receiver) -> { starterPool.execute(() -> { - receiver.start(); - - if (logger.isDebugEnabled()) { - logger.debug("started receiver plugin " + name + " " + receiver.getClass().getName()); + + if (receiver.getPluginInfo().isEnabled()) { + receiver.internalStart(); + logger.info("started receiver plugin named {}, class {}", name, receiver.getClass().getName()); } - receiverCount.incrementAndGet(); }); }); } - private void initSenderPlugins() { + private void startSenderPlugins() { senderPlugins = context.getBeansOfType(MessageSender.class); - AtomicInteger senderCount = new AtomicInteger(); - senderPlugins.forEach((name, sender) -> { starterPool.execute(() -> { - sender.start(); - - if (logger.isDebugEnabled()) { - logger.debug("started sender plugin " + name + " " + sender.getClass().getName()); + + if (sender.getPluginInfo().isEnabled()) { + sender.internalStart(); + logger.info("started sender plugin named {}, class {}", name, sender.getClass().getName()); } - senderCount.incrementAndGet(); }); }); } - private void initInterceptorPlugins() { + private void startInterceptorPlugins() { interceptorPlugins = context.getBeansOfType(MessageInterceptor.class); - - AtomicInteger interceptorCount = new AtomicInteger(); // track any registration of interceptor plugins so that the messaging process submission service can behave accordingly - if (!interceptorPlugins.isEmpty()) { - ignite.atomicLong(PluginManagerConstants.INTERCEPTOR_REGISTRATION_KEY, 1, true); - } else { - ignite.atomicLong(PluginManagerConstants.INTERCEPTOR_REGISTRATION_KEY, 0, true); - } + pluginApi.addInterceptorPluginsActive(interceptorPlugins.size()); interceptorPlugins.forEach((name, interceptor) -> { starterPool.execute(() -> { - interceptor.start(); - - if (logger.isDebugEnabled()) { - logger.debug("started interceptor plugin " + name + " " + interceptor.getClass().getName()); + + if (interceptor.getPluginInfo().isEnabled()) { + interceptor.internalStart(); + logger.info("started interceptor plugin named {}, class {}", name, interceptor.getClass().getName()); } - interceptorCount.incrementAndGet(); }); }); } diff --git a/src/takserver-plugin-manager/src/main/java/tak/server/plugins/manager/loader/PluginLoader.java b/src/takserver-plugin-manager/src/main/java/tak/server/plugins/manager/loader/PluginLoader.java index 9ac9a9cd..9af41916 100644 --- a/src/takserver-plugin-manager/src/main/java/tak/server/plugins/manager/loader/PluginLoader.java +++ b/src/takserver-plugin-manager/src/main/java/tak/server/plugins/manager/loader/PluginLoader.java @@ -1,5 +1,11 @@ package tak.server.plugins.manager.loader; +import java.io.IOException; +import java.net.URL; +import java.net.URLDecoder; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; import java.util.UUID; import org.slf4j.Logger; @@ -13,6 +19,7 @@ import org.springframework.context.support.GenericApplicationContext; import org.springframework.core.type.filter.AnnotationTypeFilter; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; import tak.server.PluginRegistry; @@ -21,9 +28,11 @@ import tak.server.plugins.MessageSender; import tak.server.plugins.MessageSenderReceiver; import tak.server.plugins.MessageSenderReceiverBase; +import tak.server.plugins.PluginBase; import tak.server.plugins.PluginInfo; import tak.server.plugins.PluginsLoadedEvent; import tak.server.plugins.TakServerPlugin; +import tak.server.plugins.TakServerPluginVersion; public class PluginLoader { @@ -79,17 +88,93 @@ private void init() { if (Strings.isNullOrEmpty(name)) { name = bd.getBeanClassName(); } - + UUID id = UUID.randomUUID(); pluginInfo.setName(name); pluginInfo.setDescription(description); pluginInfo.setClassName(clazz.getName()); pluginInfo.setId(id); // TODO: unique id strategy - based on fully qualified class name? - + + PluginSystemConfiguration pluginSytemConfiguration = new PluginSystemConfiguration(clazz); + + // get the plugin's isEnabled property from the Plugin Configuration file if it exists + Boolean isEnabled = true; + if (pluginSytemConfiguration.containsProperty(PluginSystemConfiguration.PLUGIN_ENABLED_PROPERTY)) { + logger.debug("Plugin configuration contain {}", PluginSystemConfiguration.PLUGIN_ENABLED_PROPERTY); + isEnabled = (Boolean)pluginSytemConfiguration.getProperty(PluginSystemConfiguration.PLUGIN_ENABLED_PROPERTY); + }else { + logger.debug("Plugin configuration does NOT contain {}", PluginSystemConfiguration.PLUGIN_ENABLED_PROPERTY); + } + pluginInfo.setEnabled(isEnabled); + logger.info("Set isEnabled for plugin {} to {}", clazz, isEnabled); + pluginInfo.setStarted(false); // Not yet started + + // get the archiveEnabled property from the Plugin Configuration file if it exists + Boolean archiveEnabled = true; + if (pluginSytemConfiguration.containsProperty(PluginSystemConfiguration.ARCHIVE_ENABLED_PROPERTY)) { + logger.debug("Plugin configuration contain {}", PluginSystemConfiguration.ARCHIVE_ENABLED_PROPERTY); + archiveEnabled = (Boolean)pluginSytemConfiguration.getProperty(PluginSystemConfiguration.ARCHIVE_ENABLED_PROPERTY); + }else { + logger.debug("Plugin configuration does NOT contain {}", PluginSystemConfiguration.ARCHIVE_ENABLED_PROPERTY); + } + pluginInfo.setArchiveEnabled(archiveEnabled); + logger.info("Set archiveEnabled for plugin {} to {}", clazz, archiveEnabled); + // instantiate plugin Object pluginInstance = clazz.newInstance(); - + + // set version from file. this will be overridden if version annotations are found + Integer major = null; + Integer minor = null; + Integer patch = 0; + String hash = null; + String tag = null; + if (pluginInstance instanceof PluginBase) { + try { + String path = pluginInstance.getClass().asSubclass(pluginInstance.getClass()).getProtectionDomain() + .getCodeSource().getLocation().getPath(); + String decodedPath = URLDecoder.decode(path, "UTF-8"); + URL url = loadResources("ver.json", decodedPath); + + Map result = new ObjectMapper().readValue(url.openStream(), HashMap.class); + + if (result.get("major") != null) { + major = (Integer) result.get("major"); + } + if (result.get("minor") != null) { + minor = (Integer) result.get("minor"); + } + if (result.get("patch") != null) { + patch = (Integer) result.get("patch"); + } + if (result.get("hash") != null) { + hash = (String) result.get("hash"); + } + if (result.get("branch") != null) { + tag = (String) result.get("branch"); + } + } catch (Exception e) { + logger.error("Could not load version file. Consider upgrding the plugin: " + name, e); + } + } + + TakServerPluginVersion pluginVersionAnnotation = clazz.getAnnotation(TakServerPluginVersion.class); + if (pluginVersionAnnotation != null) { + if (pluginVersionAnnotation.major() != -1) major = pluginVersionAnnotation.major(); + if (pluginVersionAnnotation.minor() != -1) minor = pluginVersionAnnotation.minor(); + if (pluginVersionAnnotation.patch() != -1) patch = pluginVersionAnnotation.patch(); + if (!"".equals(pluginVersionAnnotation.commitHash())) hash = pluginVersionAnnotation.commitHash(); + if (!"".equals(pluginVersionAnnotation.tag())) tag = pluginVersionAnnotation.tag(); + } + + if (major != null && minor != null && patch != null && hash != null) { + pluginInfo.setVersion(major + "." + minor + "." + patch + "." + hash); + } + if (tag != null) { + pluginInfo.setTag(tag); + } + if (pluginInstance instanceof MessageSenderReceiverBase) { MessageSenderReceiver senderReceiverInstance = (MessageSenderReceiver) pluginInstance; @@ -98,11 +183,11 @@ private void init() { context.registerBean(id.toString(), MessageSenderReceiver.class, () -> senderReceiverInstance); pluginInfo.setSender(true); pluginInfo.setReceiver(true); - pluginInfo.setEnabled(true); - logger.info("registered sender-receiver plugin " + senderReceiverInstance + " name: " + name + " description: " + description); senderReceiverInstance.setPluginInfo(pluginInfo); + logger.info("Registered sender-receiver plugin instance: {}, name: {}", senderReceiverInstance, name); + } else if (pluginInstance instanceof MessageSender) { MessageSender senderPluginInstance = (MessageSender) pluginInstance; @@ -113,11 +198,10 @@ private void init() { context.registerBean(id.toString(), MessageSender.class, () -> senderPluginInstance); pluginInfo.setSender(true); - pluginInfo.setEnabled(true); senderPluginInstance.setPluginInfo(pluginInfo); - logger.info("registered sender plugin " + senderPluginInstance + " name: " + name + " description" + description); + logger.info("Registered sender plugin instance: {}, name: {}", senderPluginInstance, name); } else if (pluginInstance instanceof MessageReceiver) { @@ -129,11 +213,10 @@ private void init() { context.registerBean(id.toString(), MessageReceiver.class, () -> receiverPluginInstance); pluginInfo.setReceiver(true); - pluginInfo.setEnabled(true); receiverPluginInstance.setPluginInfo(pluginInfo); - logger.info("registered receiver plugin " + receiverPluginInstance); + logger.info("Registered receiver plugin instance: {}, name: {}", receiverPluginInstance, name); } else if (pluginInstance instanceof MessageInterceptor) { @@ -145,11 +228,10 @@ private void init() { context.registerBean(id.toString(), MessageInterceptor.class, () -> interceptorPluginInstance); pluginInfo.setInterceptor(true); - pluginInfo.setEnabled(true); interceptorPluginInstance.setPluginInfo(pluginInfo); - logger.info("registered interceptor plugin " + interceptorPluginInstance); + logger.info("Registered interceptor plugin instance: {}, name: {}", interceptorPluginInstance, name); } else { logger.error("Skipping invalid plugin type " + pluginInstance.getClass().getName()); @@ -171,4 +253,15 @@ private void init() { // get spring to set properties now applicationEventPublisher.publishEvent(new PluginsLoadedEvent(this, "plugins loaded")); } + + public static URL loadResources(String name, String path) throws IOException { + final Enumeration systemResources = PluginBase.class.getClassLoader().getResources(name); + while (systemResources.hasMoreElements()) { + URL url = systemResources.nextElement(); + if (url.getPath().toLowerCase().contains(path.toLowerCase())) { + return url; + } + } + return null; + } } diff --git a/src/takserver-plugin-manager/src/main/java/tak/server/plugins/manager/loader/PluginSystemConfiguration.java b/src/takserver-plugin-manager/src/main/java/tak/server/plugins/manager/loader/PluginSystemConfiguration.java new file mode 100644 index 00000000..fed6966c --- /dev/null +++ b/src/takserver-plugin-manager/src/main/java/tak/server/plugins/manager/loader/PluginSystemConfiguration.java @@ -0,0 +1,171 @@ +package tak.server.plugins.manager.loader; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.Writer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.Yaml; + +public class PluginSystemConfiguration { + private static final Logger logger = LoggerFactory.getLogger(PluginSystemConfiguration.class); + + private Map obj; + + static final String PLUGIN_CONFIG_BASE = "conf/plugins/"; + + static final String[] RESERVED_KEYWORDS = { "server", "tak", "system" }; + public static final String ARCHIVE_ENABLED_PROPERTY = "system.archive"; + public static final String PLUGIN_ENABLED_PROPERTY = "system.enable"; + + public PluginSystemConfiguration() { + obj = new HashMap(); + } + + public PluginSystemConfiguration(Class clazz) { + String configFileName = PLUGIN_CONFIG_BASE + clazz.getName() + ".yaml"; + File f = new File(configFileName); + if (!f.exists()) { + f.getParentFile().mkdirs(); + try { + f.createNewFile(); + } catch (IOException e) { + logger.error(e.getMessage()); + } + } + try (InputStream inputStream = new FileInputStream(f);){ + Yaml yaml = new Yaml(); + obj = yaml.load(inputStream); + } catch (Exception e) { + logger.error(e.getMessage()); + } + if (obj == null) { + if (logger.isDebugEnabled()) { + logger.debug("The config was empty. Create empty properties map."); + } + obj = new HashMap(); + } + } + + public Object getProperty(String key) { + List propChain = Arrays.asList(key.split("\\.")); + Object cur = obj; + for (String prop : propChain) { + if (cur instanceof Map) { + cur = ((Map) cur).get(prop); + } else { + logger.error("no such property: " + key); + return null; + } + } + return cur; + } + + public boolean containsProperty(String property) { + String[] propChain = property.split("\\."); + Object cur = obj; + for (String prop : propChain) { + if (cur instanceof Map) { + cur = ((Map) cur).get(prop); + } else { + return false; + } + } + if (cur != null) { + return true; + } else { + return false; + } + } + + public List getProperties() { + List properties = new ArrayList(); + for (String prop : obj.keySet()) { + properties.add(prop); + if (obj.get(prop) instanceof Map) { + properties.addAll(this.getProperties(prop, (Map)obj.get(prop))); + } + } + return properties; + } + + private List getProperties(String baseProp, Map map) { + List props = new ArrayList(); + for (String prop : map.keySet()) { + String subProp = baseProp + "." + prop; + props.add(subProp); + if (map.get(prop) instanceof Map) { + props.addAll(this.getProperties(subProp, (Map) map.get(prop))); + } + } + return props; + } + + public void setProperty(String key, Object value) throws Exception{ + + logger.info("Calling setProperty: key {}, value {}", key, value); + + List propChain = Arrays.asList(key.split("\\.")); + Object cur = obj; + for (int i=0 ; i < propChain.size()-1 ; i++) { + String prop = propChain.get(i); + if (cur instanceof Map) { + if (((Map) cur).containsKey(prop)) { + cur = ((Map) cur).get(prop); + }else { + ((Map) cur).put(prop, new HashMap<>()); + cur = ((Map) cur).get(prop); + } + } else { + throw new Exception("Key creates conflicts with existing configuration"); + } + } + String lastKey = propChain.get(propChain.size()-1); + if (cur instanceof Map) { + ((Map) cur).put(lastKey,value); + }else { + throw new Exception("Key creates conflicts with existing configuration"); + } + } + + public void save(Class clazz) throws IOException { + + synchronized (this) { + String configFileName = PLUGIN_CONFIG_BASE + clazz.getName() + ".yaml"; + + logger.info("Saving plugin configuration to {}", configFileName); + + File f = new File(configFileName); + if (!f.exists()) { + f.getParentFile().mkdirs(); + try { + f.createNewFile(); + } catch (IOException e) { + logger.error(e.getMessage()); + } + } + if (obj == null) { + if (logger.isDebugEnabled()) { + logger.debug("Nothing to save in plugin configuration"); + } + return; + } + + try(Writer writer = new FileWriter(f);){ + Yaml yaml = new Yaml(); + yaml.dump(obj, writer); + } + } + + } + +} diff --git a/src/takserver-plugin-manager/src/main/java/tak/server/plugins/service/DistributedClusterPluginManager.java b/src/takserver-plugin-manager/src/main/java/tak/server/plugins/service/DistributedClusterPluginManager.java index 06876319..6f505e6c 100644 --- a/src/takserver-plugin-manager/src/main/java/tak/server/plugins/service/DistributedClusterPluginManager.java +++ b/src/takserver-plugin-manager/src/main/java/tak/server/plugins/service/DistributedClusterPluginManager.java @@ -25,7 +25,7 @@ public class DistributedClusterPluginManager extends DistributedPluginManager { private static final long serialVersionUID = 753070297094974770L; private static final Logger logger = LoggerFactory.getLogger(DistributedClusterPluginManager.class); - private IgniteCache pluginCache; + private IgniteCache pluginStartedCache; private ContinuousQuery continuousPluginQuery = new ContinuousQuery<>(); @EventListener({ PluginsStartedEvent.class }) @@ -33,19 +33,19 @@ private void initCache() { // initialize plugins only if they arent in the cache getAllPlugins() .stream() - .forEach(plugin -> getPluginCache().putIfAbsent(plugin.getPluginInfo().getName(), new Boolean(true))); + .forEach(plugin -> getPluginStartedCache().putIfAbsent(plugin.getPluginInfo().getName(), new Boolean(plugin.getPluginInfo().isStarted()))); // pull the most up to date plugin status from cache and set them getAllPlugins() .stream() .forEach(plugin -> { - Boolean status = getPluginCache().get(plugin.getPluginInfo().getName()); - plugin.getPluginInfo().setEnabled(status.booleanValue()); + Boolean statusInCache = getPluginStartedCache().get(plugin.getPluginInfo().getName()); + plugin.getPluginInfo().setStarted(statusInCache.booleanValue()); - if (!status.booleanValue()) { + if (!statusInCache.booleanValue()) { plugin.internalStop(); // the stop failed (likely due to incompatibility - so re-cache as running) - if (plugin.getPluginInfo().isEnabled()) { + if (plugin.getPluginInfo().isStarted()) { updateCache(plugin.getPluginInfo().getName(), true); } } @@ -59,14 +59,14 @@ private void initCache() { // plugin matches .filter(plugin -> plugin.getPluginInfo().getName().equals(e.getKey())) // make sure we are only setting a new status if its not already set - .filter(plugin -> plugin.getPluginInfo().isEnabled() != e.getValue().booleanValue()) + .filter(plugin -> plugin.getPluginInfo().isStarted() != e.getValue().booleanValue()) .forEach(plugin -> { if (e.getValue().booleanValue()) { plugin.internalStart(); } else { plugin.internalStop(); // the stop failed (likely due to incompatibility - so re-cache as running) - if (plugin.getPluginInfo().isEnabled()) { + if (plugin.getPluginInfo().isStarted()) { updateCache(plugin.getPluginInfo().getName(), true); } } @@ -74,7 +74,7 @@ private void initCache() { } }); - getPluginCache().query(continuousPluginQuery); + getPluginStartedCache().query(continuousPluginQuery); } @Override @@ -127,20 +127,20 @@ public void stopPluginByName(String name) { updateCache(name, false); } - private IgniteCache getPluginCache() { + private IgniteCache getPluginStartedCache() { - if (pluginCache == null) { + if (pluginStartedCache == null) { CacheConfiguration cfg = new CacheConfiguration(); cfg.setName(CommonConstants.PLUGIN_CACHE); cfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); - pluginCache = PluginManagerDependencyInjectionProxy.getInstance().ignite().getOrCreateCache(cfg); + pluginStartedCache = PluginManagerDependencyInjectionProxy.getInstance().ignite().getOrCreateCache(cfg); } - return pluginCache; + return pluginStartedCache; } private void updateCache(String name, boolean status) { - getPluginCache().put(name, status); + getPluginStartedCache().put(name, status); } } diff --git a/src/takserver-plugin-manager/src/main/java/tak/server/plugins/service/DistributedPluginManager.java b/src/takserver-plugin-manager/src/main/java/tak/server/plugins/service/DistributedPluginManager.java index 83279d67..8cbc8d4a 100644 --- a/src/takserver-plugin-manager/src/main/java/tak/server/plugins/service/DistributedPluginManager.java +++ b/src/takserver-plugin-manager/src/main/java/tak/server/plugins/service/DistributedPluginManager.java @@ -1,15 +1,20 @@ package tak.server.plugins.service; import java.util.Collection; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.ignite.services.Service; import org.apache.ignite.services.ServiceContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Strings; + import tak.server.PluginManager; +import tak.server.plugins.MessageInterceptorBase; import tak.server.plugins.PluginInfo; import tak.server.plugins.PluginLifecycle; +import tak.server.plugins.manager.loader.PluginSystemConfiguration; import tak.server.plugins.util.PluginManagerDependencyInjectionProxy; public class DistributedPluginManager implements PluginManager, Service { @@ -47,16 +52,27 @@ public Collection getAllPluginInfo() { public void startAllPlugins() { getAllPlugins() .stream() - .filter(plugin -> !plugin.getPluginInfo().isEnabled()) - .forEach(plugin -> plugin.internalStart()); + .filter(plugin -> !plugin.getPluginInfo().isStarted()) + .forEach(plugin -> { + plugin.internalStart(); + if (plugin instanceof MessageInterceptorBase) + PluginManagerDependencyInjectionProxy.getInstance().pluginApi().addInterceptorPluginsActive(1); + + }); } @Override public void stopAllPlugins() { getAllPlugins() .stream() - .filter(plugin -> plugin.getPluginInfo().isEnabled()) - .forEach(plugin -> plugin.internalStop()); + .filter(plugin -> plugin.getPluginInfo().isStarted()) + .forEach(plugin -> { + plugin.internalStop(); + if (plugin instanceof MessageInterceptorBase) + PluginManagerDependencyInjectionProxy.getInstance().pluginApi().addInterceptorPluginsActive(-1); + + }); + } @Override @@ -64,8 +80,13 @@ public void startPluginByName(String name) { getAllPlugins() .stream() .filter(plugin -> plugin.getPluginInfo().getName().equals(name)) - .filter(plugin -> !plugin.getPluginInfo().isEnabled()) - .forEach(plugin -> plugin.internalStart()); + .filter(plugin -> !plugin.getPluginInfo().isStarted()) + .forEach(plugin -> { + plugin.internalStart(); + if (plugin instanceof MessageInterceptorBase) + PluginManagerDependencyInjectionProxy.getInstance().pluginApi().addInterceptorPluginsActive(1); + + }); } @Override @@ -73,11 +94,108 @@ public void stopPluginByName(String name) { getAllPlugins() .stream() .filter(plugin -> plugin.getPluginInfo().getName().equals(name)) - .filter(plugin -> plugin.getPluginInfo().isEnabled()) - .forEach(plugin -> plugin.internalStop()); + .filter(plugin -> plugin.getPluginInfo().isStarted()) + .forEach(plugin -> { + plugin.internalStop(); + if (plugin instanceof MessageInterceptorBase) + PluginManagerDependencyInjectionProxy.getInstance().pluginApi().addInterceptorPluginsActive(-1); + + }); + } + + @Override + public void setPluginEnabled(String name, boolean isPluginEnabled) { + + getAllPlugins() + .stream() + .filter(plugin -> plugin.getPluginInfo().getName().equals(name)) + .filter(plugin -> plugin.getPluginInfo().isEnabled() != isPluginEnabled) + .forEach(plugin -> { + PluginInfo pluginInfo = plugin.getPluginInfo(); + pluginInfo.setEnabled(isPluginEnabled); + + persistPluginEnabledPropertyInPluginConfigurationFile(pluginInfo, isPluginEnabled); + }); + } + + @Override + public void setPluginArchive(String name, boolean isArchiveEnabled) { + getAllPlugins() + .stream() + .filter(plugin -> plugin.getPluginInfo().getName().equals(name)) + .filter(plugin -> plugin.getPluginInfo().isArchiveEnabled() != isArchiveEnabled) + .forEach(plugin -> { + PluginInfo pluginInfo = plugin.getPluginInfo(); + pluginInfo.setArchiveEnabled(isArchiveEnabled); + + persistPluginArchiveEnabledPropertyInPluginConfigurationFile(pluginInfo, isArchiveEnabled); + + }); + } protected Collection getAllPlugins() { return PluginManagerDependencyInjectionProxy.getInstance().pluginStarter().getAllPlugins(); } + + @Override + public void submitDataToPlugin(String pluginClassName, String scope, String data, String contentType) { + + if (Strings.isNullOrEmpty(pluginClassName)) { + throw new IllegalArgumentException("plugin class name is empty"); + } + + if (data == null) { + throw new IllegalArgumentException("null data can't be submitted to plugin"); + } + + if (Strings.isNullOrEmpty(contentType)) { + throw new IllegalArgumentException("content type must be specified"); + } + + AtomicInteger pluginDataSubmitCounter = new AtomicInteger(); + + // submit the data to any plugin matching the class name + getAllPlugins() + .stream() + .filter(plugin -> plugin.getPluginInfo().getClassName().equals(pluginClassName)) + .forEach(plugin -> { + plugin.onSubmitData(scope, data, contentType); + pluginDataSubmitCounter.incrementAndGet(); + }); + + if (pluginDataSubmitCounter.get() == 0) { + throw new IllegalArgumentException("no plugin with class name " + pluginClassName + " is currently installed."); + + } + } + + private synchronized void persistPluginEnabledPropertyInPluginConfigurationFile(PluginInfo pluginInfo, boolean isEnabled) { + + try { + // persist the configuration + Class clazz = Class.forName(pluginInfo.getClassName()); + PluginSystemConfiguration pluginSytemConfiguration = new PluginSystemConfiguration(clazz); + pluginSytemConfiguration.setProperty(PluginSystemConfiguration.PLUGIN_ENABLED_PROPERTY, isEnabled); + pluginSytemConfiguration.save(clazz); + } catch(Exception e) { + logger.error("Error in persisting plugin configuration", e); + } + + } + + private synchronized void persistPluginArchiveEnabledPropertyInPluginConfigurationFile(PluginInfo pluginInfo, boolean isArchiveEnabled) { + + try { + // persist the configuration + Class clazz = Class.forName(pluginInfo.getClassName()); + PluginSystemConfiguration pluginSytemConfiguration = new PluginSystemConfiguration(clazz); + pluginSytemConfiguration.setProperty(PluginSystemConfiguration.ARCHIVE_ENABLED_PROPERTY, isArchiveEnabled); + pluginSytemConfiguration.save(clazz); + } catch(Exception e) { + logger.error("Error in persisting plugin configuration", e); + } + + } + } diff --git a/src/takserver-plugin-manager/src/main/java/tak/server/plugins/service/PluginService.java b/src/takserver-plugin-manager/src/main/java/tak/server/plugins/service/PluginService.java index b9b129c3..92cc5081 100644 --- a/src/takserver-plugin-manager/src/main/java/tak/server/plugins/service/PluginService.java +++ b/src/takserver-plugin-manager/src/main/java/tak/server/plugins/service/PluginService.java @@ -3,9 +3,17 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.ForkJoinPool.ForkJoinWorkerThreadFactory; +import java.util.concurrent.ForkJoinWorkerThread; import org.apache.ignite.Ignite; import org.apache.ignite.Ignition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -21,8 +29,12 @@ import tak.server.PluginRegistry; import tak.server.ignite.IgniteConfigurationHolder; import tak.server.messaging.Messenger; +import tak.server.plugins.PluginApi; +import tak.server.plugins.PluginDataFeedApi; import tak.server.plugins.PluginManagerConstants; +import tak.server.plugins.PluginSelfStopApi; import tak.server.plugins.PluginStarter; +import tak.server.plugins.SystemInfoApi; import tak.server.plugins.manager.loader.PluginLoader; import tak.server.plugins.messaging.MessageConverter; import tak.server.plugins.messaging.PluginClusterMessenger; @@ -34,6 +46,8 @@ public class PluginService implements CommandLineRunner { private static Ignite ignite = null; + private static final Logger logger = LoggerFactory.getLogger(PluginService.class); + public static void main(String[] args) { SpringApplication application = new SpringApplication(PluginService.class); @@ -56,7 +70,7 @@ public static void main(String[] args) { if (ignite == null) { System.exit(1); } - + // start sping boot app application.run(args); } @@ -65,18 +79,14 @@ public static void main(String[] args) { public void run(String... args) throws Exception { } @Bean - @Profile("!" + Constants.CLUSTER_PROFILE_NAME) - PluginStarter pluginIntializer(Ignite ignite) { - return new PluginStarter("", ""); + ServerInfo serverInfo(Ignite ignite) { + return ignite.services(ignite.cluster().forAttribute(Constants.TAK_PROFILE_KEY, Constants.MESSAGING_PROFILE_NAME)) + .serviceProxy(Constants.DISTRIBUTED_SERVER_INFO, ServerInfo.class, false); } @Bean - @Profile(Constants.CLUSTER_PROFILE_NAME) - PluginStarter pluginClusterIntializer(Ignite ignite) { - ServerInfo serverInfo = ignite.services(ignite.cluster().forAttribute(Constants.TAK_PROFILE_KEY, Constants.MESSAGING_PROFILE_NAME)) - .serviceProxy(Constants.DISTRIBUTED_SERVER_INFO, ServerInfo.class, false); - - return new PluginStarter(serverInfo.getNatsURL(), serverInfo.getNatsClusterId()); + PluginStarter pluginIntializer(Ignite ignite, PluginDataFeedApi pdfApi, ServerInfo serverInfo, PluginApi pluginApi, PluginSelfStopApi pluginSelfStopApi) { + return new PluginStarter(serverInfo, pluginApi); } @Bean @@ -117,6 +127,7 @@ Ignite ignite() { @Bean @Profile("!" + Constants.CLUSTER_PROFILE_NAME) public PluginManager pluginManager(Ignite ignite) { + DistributedPluginManager dpm = new DistributedPluginManager(); ignite.services(ClusterGroupDefinition.getPluginManagerClusterDeploymentGroup(ignite)).deployNodeSingleton(Constants.DISTRIBUTED_PLUGIN_MANAGER, dpm); @@ -126,14 +137,74 @@ public PluginManager pluginManager(Ignite ignite) { @Bean @Profile(Constants.CLUSTER_PROFILE_NAME) public PluginManager pluginClusterManager(Ignite ignite) { + DistributedPluginManager dpm = new DistributedClusterPluginManager(); ignite.services(ClusterGroupDefinition.getPluginManagerClusterDeploymentGroup(ignite)).deployNodeSingleton(Constants.DISTRIBUTED_PLUGIN_MANAGER, dpm); return ignite.services(ClusterGroupDefinition.getPluginManagerClusterDeploymentGroup(ignite)).serviceProxy(Constants.DISTRIBUTED_PLUGIN_MANAGER, PluginManager.class, false); } + + + @Bean + public SystemInfoApi systemInfoApi(Ignite ignite) { + return ignite.services(ClusterGroupDefinition.getMessagingClusterDeploymentGroup(ignite)).serviceProxy(Constants.DISTRIBUTED_SYSTEM_INFO_API, SystemInfoApi.class, false); + } + @Bean public PluginManagerDependencyInjectionProxy pmdip() { return new PluginManagerDependencyInjectionProxy(); } + + private boolean accessApi(PluginDataFeedApi api) { + api.getAllPluginDataFeeds(); + return true; + } + + private CompletableFuture canAccessApi(final PluginDataFeedApi api) { + + try { + return CompletableFuture.completedFuture(accessApi(api)); + } catch (Exception e) { + try { + Thread.sleep(250L); + } catch (InterruptedException e1) { + logger.error("interruped sleep", e1); + } + return canAccessApi(api); + } + } + + @Bean + public PluginDataFeedApi pluginDataFeedApi(Ignite ignite) { + + + final PluginDataFeedApi api = ignite.services(ClusterGroupDefinition.getMessagingClusterDeploymentGroup(ignite)).serviceProxy(Constants.DISTRIBUTED_PLUGIN_DATA_FEED_API, PluginDataFeedApi.class, false); + + boolean isApiAvailable = false; + + // block and wait for PluginDataFeedApi to become available in messaging process + try { + isApiAvailable = canAccessApi(api).get(); + } catch (InterruptedException | ExecutionException e) { + logger.error("interrupted checking api availablity", e); + } + + logger.info("data feed api available: " + isApiAvailable); + + return api; + } + + @Bean + public PluginApi pluginApi(Ignite ignite) { + return ignite.services(ClusterGroupDefinition.getMessagingClusterDeploymentGroup(ignite)).serviceProxy(Constants.DISTRIBUTED_PLUGIN_API, PluginApi.class, false); + } + + @Bean + public PluginSelfStopApi pluginSelfStopApi(Ignite ignite) { + + final PluginSelfStopApi api = ignite.services(ClusterGroupDefinition.getMessagingClusterDeploymentGroup(ignite)).serviceProxy(Constants.DISTRIBUTED_PLUGIN_SELF_STOP_API, PluginSelfStopApi.class, false); + + return api; + } } diff --git a/src/takserver-plugin-manager/src/main/java/tak/server/plugins/util/PluginManagerDependencyInjectionProxy.java b/src/takserver-plugin-manager/src/main/java/tak/server/plugins/util/PluginManagerDependencyInjectionProxy.java index 8c7a7730..1ee40bb8 100644 --- a/src/takserver-plugin-manager/src/main/java/tak/server/plugins/util/PluginManagerDependencyInjectionProxy.java +++ b/src/takserver-plugin-manager/src/main/java/tak/server/plugins/util/PluginManagerDependencyInjectionProxy.java @@ -7,7 +7,10 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; +import com.bbn.marti.remote.ServerInfo; + import tak.server.PluginRegistry; +import tak.server.plugins.PluginApi; import tak.server.plugins.PluginStarter; /* @@ -83,4 +86,18 @@ public PluginStarter pluginStarter() { return pluginStarter; } + + private PluginApi pluginApi = null; + + public PluginApi pluginApi() { + if (pluginApi == null) { + synchronized (this) { + if (pluginApi == null) { + pluginApi = springContext.getBean(PluginApi.class); + } + } + } + + return pluginApi; + } } diff --git a/src/takserver-plugins/build.gradle b/src/takserver-plugins/build.gradle index 56a94c55..5ad46241 100644 --- a/src/takserver-plugins/build.gradle +++ b/src/takserver-plugins/build.gradle @@ -19,6 +19,8 @@ publishing { } dependencies { + compile group: 'xerces', name: 'xercesImpl', version: xerces_version + compile group: 'org.apache.logging.log4j', name: 'log4j-api', version: log4j_api_version compile group: 'org.apache.logging.log4j', name: 'log4j-to-slf4j', version: log4j_api_version compile group: 'org.slf4j', name: 'slf4j-api', version: slf4j_version diff --git a/src/takserver-plugins/src/main/java/com/bbn/marti/remote/util/DateUtil.java b/src/takserver-plugins/src/main/java/com/bbn/marti/remote/util/DateUtil.java index 04dcd24e..ab30382a 100644 --- a/src/takserver-plugins/src/main/java/com/bbn/marti/remote/util/DateUtil.java +++ b/src/takserver-plugins/src/main/java/com/bbn/marti/remote/util/DateUtil.java @@ -55,12 +55,4 @@ public static final String toCotTime(long millisSinceEpochUtc) { public static final String toCotTimeMillis(long millisSinceEpochUtc) { return cotDateParserMillis().print(millisSinceEpochUtc); } - - public static final void main(String[] args) { - System.out.println(DateUtil.toCotTime(System.currentTimeMillis())); - String test = "2016-04-26T14:53:09.213Z"; - System.out.println("Normal out: " + DateUtil.millisFromCotTimeStr(test)); - System.out.println("UTC out: " + iso().withZoneUTC().parseDateTime(test).getMillis()); - } - } diff --git a/src/takserver-plugins/src/main/java/tak/server/Constants.java b/src/takserver-plugins/src/main/java/tak/server/Constants.java index 307b5721..20a6b8be 100644 --- a/src/takserver-plugins/src/main/java/tak/server/Constants.java +++ b/src/takserver-plugins/src/main/java/tak/server/Constants.java @@ -97,6 +97,7 @@ public class Constants { // distributed cache / messaging names (ignite) public static final String ALL_MISSION_CACHE = "allMissionCache"; + public static final String ALL_COPS_MISSION_CACHE = "allCopsMissionCache"; public static final String MISSION_ROLE_CACHE = "missionRoleCache"; public static final String MISSION_SUBSCRIPTION_CACHE = "missionSubscriptionCache"; public static final String ENTERPRISE_SYNC_CACHE_NAME = "enterprise-sync-cache"; @@ -114,6 +115,7 @@ public class Constants { public static final String LATEST_COT_CACHE = "latest-cot-cache"; public static final String CLIENT_MSG_TS_CACHE = "client-msg-ts-cache"; public static final String CERTIFICATE_CACHE = "certificate-cache"; + public static final String VIDEO_CACHE = "video-cache"; // distributed message topics (ignite) public static final String SUBMISSION_TOPIC_BASE = "submission-topic-"; @@ -146,20 +148,22 @@ public class Constants { public static final String DEFAULT_FLOWTAG_TEXT = "marti"; public static final String USER_KEY = "user"; public static final String GROUPS_KEY = "groups"; + public static final String GROUPS_BIT_VECTOR_KEY = "groups.bit.vector"; public static final String NOFEDV2_KEY = "nofedv2"; public static final String REPEATER_KEY = "repeater"; public static final String TOPICS_KEY = "topics"; public static final String OFFLINE_CHANGE_TIME_KEY = "offlineChangeTime"; public static final String CLIENT_UID_KEY = "clientUid"; public static final String DO_NOT_BROKER_KEY = "brokering.needed"; - public static final String ARCHIVE_EVENT_KEY = "respository.archive"; + public static final String ARCHIVE_EVENT_KEY = "respository.archive"; // value is a Boolean public static final String SUBSCRIBER_HITS_KEY = "SUBSCRIBER_HITS_KEY"; public static final String CONNECTION_ID_KEY = "connection.id"; public static final String COT_MESSENGER_TOPIC_KEY = "tak.messenger.topic"; public static final String PROCESSED_COUNT = "processed.count"; - public static final String MESSAGING_ARCHIVER = "messaging.archiver"; + public static final String MESSAGING_ARCHIVER = "messaging.archiver"; // value is a String public static final String PLUGIN_PROVENANCE = "plugin.provenance"; public static final String STORE_FORWARD_KEY = "storeforward"; + public static final String DATA_FEED_KEY = "data.feed"; // Grid Service Names public static final String DISTRIBUTED_FEDERATION_MANAGER = "distributed-federation-manager"; @@ -181,6 +185,10 @@ public class Constants { public static final String DISTRIBUTED_RETENTION_POLICY_CONFIGURATION = "distributed-retention-policy-configuration"; public static final String DISTRIBUTED_MISSION_ARCHIVE_MANAGER = "distributed-mission-archive-manager"; public static final String DISTRIBUTED_QOS_MANAGER = "distributed-qos-manager"; + public static final String DISTRIBUTED_SYSTEM_INFO_API = "distributed-system-info-api"; + public static final String DISTRIBUTED_PLUGIN_DATA_FEED_API = "distributed-plugin-data-feed-api"; + public static final String DISTRIBUTED_PLUGIN_API = "plugin-api"; + public static final String DISTRIBUTED_PLUGIN_SELF_STOP_API = "distributed-plugin-self-stop-api"; // Bean Names public static final String DISTRIBUTED_COT_MESSENGER = "cotMessenger"; @@ -235,7 +243,9 @@ public class Constants { public static final String METRIC_CLIENT_DISCONNECT = "client.disconnect"; public static final String METRIC_REPOSITORY_QUEUE_FULL_SKIP = "message.repository.q.skip"; - - + + // Provenance keys used in Message + public static final String PLUGIN_MANAGER_PROVENANCE = "PluginManager"; + public static final String PLUGIN_INTERCEPTOR_PROVENANCE = "PluginInterceptor"; } \ No newline at end of file diff --git a/src/takserver-plugins/src/main/java/tak/server/plugins/Interceptor.java b/src/takserver-plugins/src/main/java/tak/server/plugins/Interceptor.java index 4eda6cb0..f41d2712 100644 --- a/src/takserver-plugins/src/main/java/tak/server/plugins/Interceptor.java +++ b/src/takserver-plugins/src/main/java/tak/server/plugins/Interceptor.java @@ -22,4 +22,6 @@ public interface Interceptor extends PluginLifecycle { * @see atakmap.commoncommo.protobuf.v1.Message */ void send(T message); + + void send(T message, String feedUuid); } diff --git a/src/takserver-plugins/src/main/java/tak/server/plugins/MessageInterceptorBase.java b/src/takserver-plugins/src/main/java/tak/server/plugins/MessageInterceptorBase.java index 600bb9ee..7879c3b2 100644 --- a/src/takserver-plugins/src/main/java/tak/server/plugins/MessageInterceptorBase.java +++ b/src/takserver-plugins/src/main/java/tak/server/plugins/MessageInterceptorBase.java @@ -1,21 +1,27 @@ package tak.server.plugins; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.antlr.v4.parse.GrammarTreeVisitor.locals_return; import org.springframework.beans.factory.annotation.Autowired; import atakmap.commoncommo.protobuf.v1.MessageOuterClass.Message; +import tak.server.Constants; import tak.server.messaging.Messenger; import tak.server.plugins.messaging.MessageConverter; public abstract class MessageInterceptorBase extends PluginBase implements MessageInterceptor { -// private static final Logger logger = LoggerFactory.getLogger(MessageReceiverBase.class); + private static final Logger logger = LoggerFactory.getLogger(MessageReceiverBase.class); @Autowired private Messenger messenger; @Autowired private MessageConverter converter; - - private PluginInfo pluginInfo; + + @Autowired + private PluginDataFeedApi pluginDataFeedApi; MessageInterceptorBase() throws ReservedConfigurationException { super(); @@ -25,6 +31,10 @@ protected MessageConverter getConverter() { return converter; } + protected PluginDataFeedApi getPluginDataFeedApi() { + return pluginDataFeedApi; + } + @Override public Message intercept(Message message) { return message; @@ -32,31 +42,12 @@ public Message intercept(Message message) { @Override public void send(Message message) { - if (pluginInfo.isEnabled()) { - // send the message to TAK Server - messenger.send(message); - } - } - - @Override - public final void internalStart() { - start(); - pluginInfo.setEnabled(true); - } - - @Override - public final void internalStop() { - stop(); - pluginInfo.setEnabled(false); + logger.info("send is NO-OP for interceptors. use the intercept method."); } - + @Override - public final PluginInfo getPluginInfo() { - return pluginInfo; + public void send(Message message, String feedUuid) { + logger.info("send is NO-OP for interceptors. use the intercept method."); } - @Override - public final void setPluginInfo(PluginInfo pluginInfo) { - this.pluginInfo = pluginInfo; - } } \ No newline at end of file diff --git a/src/takserver-plugins/src/main/java/tak/server/plugins/MessageReceiverBase.java b/src/takserver-plugins/src/main/java/tak/server/plugins/MessageReceiverBase.java index 6d81f0d5..1f702b72 100644 --- a/src/takserver-plugins/src/main/java/tak/server/plugins/MessageReceiverBase.java +++ b/src/takserver-plugins/src/main/java/tak/server/plugins/MessageReceiverBase.java @@ -6,36 +6,11 @@ import atakmap.commoncommo.protobuf.v1.MessageOuterClass.Message; public abstract class MessageReceiverBase extends PluginBase implements MessageReceiver { + private static final Logger logger = LoggerFactory.getLogger(MessageReceiverBase.class); - private PluginInfo pluginInfo; public MessageReceiverBase() throws ReservedConfigurationException { super(); } - @Override - public final void internalStart() { - start(); - pluginInfo.setEnabled(true); - } - - @Override - public final void internalStop() { - try { - stop(); - pluginInfo.setEnabled(false); - } catch (AbstractMethodError e) { - logger.info("Recompile plugin " + getPluginInfo().getName() + " with TAK Server Plugin SDK version 4.2 or higher to support the stop method."); - } - } - - @Override - public final PluginInfo getPluginInfo() { - return pluginInfo; - } - - @Override - public final void setPluginInfo(PluginInfo pluginInfo) { - this.pluginInfo = pluginInfo; - } } diff --git a/src/takserver-plugins/src/main/java/tak/server/plugins/MessageSenderBase.java b/src/takserver-plugins/src/main/java/tak/server/plugins/MessageSenderBase.java index ddcabd89..6968cdde 100644 --- a/src/takserver-plugins/src/main/java/tak/server/plugins/MessageSenderBase.java +++ b/src/takserver-plugins/src/main/java/tak/server/plugins/MessageSenderBase.java @@ -5,6 +5,7 @@ import org.springframework.beans.factory.annotation.Autowired; import atakmap.commoncommo.protobuf.v1.MessageOuterClass.Message; +import tak.server.Constants; import tak.server.messaging.Messenger; import tak.server.plugins.messaging.MessageConverter; @@ -17,43 +18,46 @@ public abstract class MessageSenderBase extends PluginBase implements MessageSen @Autowired private MessageConverter converter; - private PluginInfo pluginInfo; - + @Autowired + private PluginDataFeedApi pluginDataFeedApi; + protected MessageConverter getConverter() { return converter; } + + protected PluginDataFeedApi getPluginDataFeedApi() { + return pluginDataFeedApi; + } @Override public void send(Message message) { + if (pluginInfo.isEnabled()) { + + Message.Builder mb = message.toBuilder(); + mb.setArchive(pluginInfo.isArchiveEnabled()); + mb.addProvenance(Constants.PLUGIN_MANAGER_PROVENANCE); + message = mb.build(); + // send the message to TAK Server messenger.send(message); } } - - @Override - public final void internalStart() { - start(); - pluginInfo.setEnabled(true); - } - - @Override - public final void internalStop() { - try { - stop(); - pluginInfo.setEnabled(false); - } catch (AbstractMethodError e) { - logger.info("Recompile plugin " + getPluginInfo().getName() + " with TAK Server Plugin SDK version 4.2 or higher to support the stop method."); - } - } @Override - public final PluginInfo getPluginInfo() { - return pluginInfo; + public void send(Message message, String feedUuid) { + + if (pluginInfo.isEnabled()) { + + Message.Builder mb = message.toBuilder(); + mb.setArchive(pluginInfo.isArchiveEnabled()); + mb.setFeedUuid(feedUuid); + mb.addProvenance(Constants.PLUGIN_MANAGER_PROVENANCE); + message = mb.build(); + + // send the message to TAK Server + messenger.send(message); + } } - @Override - public final void setPluginInfo(PluginInfo pluginInfo) { - this.pluginInfo = pluginInfo; - } } diff --git a/src/takserver-plugins/src/main/java/tak/server/plugins/MessageSenderReceiverBase.java b/src/takserver-plugins/src/main/java/tak/server/plugins/MessageSenderReceiverBase.java index b9c683d6..5d5aa948 100644 --- a/src/takserver-plugins/src/main/java/tak/server/plugins/MessageSenderReceiverBase.java +++ b/src/takserver-plugins/src/main/java/tak/server/plugins/MessageSenderReceiverBase.java @@ -5,6 +5,7 @@ import org.springframework.beans.factory.annotation.Autowired; import atakmap.commoncommo.protobuf.v1.MessageOuterClass.Message; +import tak.server.Constants; import tak.server.messaging.Messenger; import tak.server.plugins.messaging.MessageConverter; @@ -16,8 +17,9 @@ public abstract class MessageSenderReceiverBase extends PluginBase implements Me @Autowired private MessageConverter converter; - - private PluginInfo pluginInfo; + + @Autowired + private PluginDataFeedApi pluginDataFeedApi; MessageSenderReceiverBase() throws ReservedConfigurationException { super(); @@ -26,38 +28,40 @@ public abstract class MessageSenderReceiverBase extends PluginBase implements Me protected MessageConverter getConverter() { return converter; } + + protected PluginDataFeedApi getPluginDataFeedApi() { + return pluginDataFeedApi; + } @Override public void send(Message message) { + if (pluginInfo.isEnabled()) { + + Message.Builder mb = message.toBuilder(); + mb.setArchive(pluginInfo.isArchiveEnabled()); + mb.addProvenance(Constants.PLUGIN_MANAGER_PROVENANCE); + message = mb.build(); + // send the message to TAK Server messenger.send(message); } } - + @Override - public final void internalStart() { - start(); - pluginInfo.setEnabled(true); - } - - @Override - public final void internalStop() { - try { - stop(); - pluginInfo.setEnabled(false); - } catch (AbstractMethodError e) { - logger.info("Recompile plugin " + getPluginInfo().getName() + " with TAK Server Plugin SDK version 4.2 or higher to support the stop method."); + public void send(Message message, String feedUuid) { + + if (pluginInfo.isEnabled()) { + + Message.Builder mb = message.toBuilder(); + mb.setArchive(pluginInfo.isArchiveEnabled()); + mb.setFeedUuid(feedUuid); + mb.addProvenance(Constants.PLUGIN_MANAGER_PROVENANCE); + message = mb.build(); + + // send the message to TAK Server + messenger.send(message); } } - @Override - public final PluginInfo getPluginInfo() { - return pluginInfo; - } - - @Override - public final void setPluginInfo(PluginInfo pluginInfo) { - this.pluginInfo = pluginInfo; - } } \ No newline at end of file diff --git a/src/takserver-plugins/src/main/java/tak/server/plugins/PluginApi.java b/src/takserver-plugins/src/main/java/tak/server/plugins/PluginApi.java new file mode 100644 index 00000000..1260115e --- /dev/null +++ b/src/takserver-plugins/src/main/java/tak/server/plugins/PluginApi.java @@ -0,0 +1,9 @@ +package tak.server.plugins; + +/* + */ +public interface PluginApi { + + void addInterceptorPluginsActive(int n); + +} diff --git a/src/takserver-plugins/src/main/java/tak/server/plugins/PluginBase.java b/src/takserver-plugins/src/main/java/tak/server/plugins/PluginBase.java index 6e159ac6..6a71535a 100644 --- a/src/takserver-plugins/src/main/java/tak/server/plugins/PluginBase.java +++ b/src/takserver-plugins/src/main/java/tak/server/plugins/PluginBase.java @@ -1,11 +1,62 @@ package tak.server.plugins; -public abstract class PluginBase { +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +public abstract class PluginBase implements PluginLifecycle{ + + private static final Logger logger = LoggerFactory.getLogger(PluginBase.class); protected PluginConfiguration config; + + protected PluginInfo pluginInfo; + + @Autowired + protected SystemInfoApi systemInfoApi; + + @Autowired + private PluginApi pluginApi; + + @Autowired + private PluginSelfStopApi pluginSelfStopApi; public PluginBase() throws ReservedConfigurationException { config = new PluginConfiguration(this.getClass()); } + @Override + public final void internalStart() { + start(); + pluginInfo.setStarted(true); + } + + @Override + public final void internalStop() { + try { + stop(); + pluginInfo.setStarted(false); + } catch (AbstractMethodError e) { + logger.info("Recompile plugin " + getPluginInfo().getName() + " with TAK Server Plugin SDK version 4.2 or higher to support the stop method."); + } + } + + @Override + public final PluginInfo getPluginInfo() { + return pluginInfo; + } + + @Override + public final void setPluginInfo(PluginInfo pluginInfo) { + this.pluginInfo = pluginInfo; + } + + @Override + public final void selfStop() { + + logger.info("Calling selfStop for plugin {}", pluginInfo.getName()); + pluginSelfStopApi.pluginSelfStop(pluginInfo.getName()); + logger.info("Done calling selfStop for plugin {}", pluginInfo.getName()); + + } } diff --git a/src/takserver-plugins/src/main/java/tak/server/plugins/PluginConfiguration.java b/src/takserver-plugins/src/main/java/tak/server/plugins/PluginConfiguration.java index b173c9ac..bd444aaf 100644 --- a/src/takserver-plugins/src/main/java/tak/server/plugins/PluginConfiguration.java +++ b/src/takserver-plugins/src/main/java/tak/server/plugins/PluginConfiguration.java @@ -22,13 +22,13 @@ public class PluginConfiguration { static final String PLUGIN_CONFIG_BASE = "conf/plugins/"; - static final String[] RESERVED_KEYWORDS = { "server", "tak" }; + static final String[] RESERVED_KEYWORDS = { "server", "tak", "system" }; public PluginConfiguration() { obj = new HashMap(); } - public PluginConfiguration(Class clazz) throws ReservedConfigurationException { + public PluginConfiguration(Class clazz) { String configFileName = PLUGIN_CONFIG_BASE + clazz.getName() + ".yaml"; File f = new File(configFileName); if (!f.exists()) { @@ -39,11 +39,10 @@ public PluginConfiguration(Class clazz) throws ReservedConfigurationException { logger.error(e.getMessage()); } } - try { - InputStream inputStream = new FileInputStream(f); + try (InputStream inputStream = new FileInputStream(f);){ Yaml yaml = new Yaml(); obj = yaml.load(inputStream); - } catch (FileNotFoundException e) { + } catch (Exception e) { logger.error(e.getMessage()); } if (obj == null) { @@ -52,12 +51,9 @@ public PluginConfiguration(Class clazz) throws ReservedConfigurationException { } obj = new HashMap(); } else { - // check for reserved keywords - for (String keyword : RESERVED_KEYWORDS) { - if (obj.containsKey(keyword)) { - throw new ReservedConfigurationException(keyword); - } - } + // remove reserved keywords + obj.keySet().removeAll(Arrays.asList(RESERVED_KEYWORDS)); + } } diff --git a/src/takserver-plugins/src/main/java/tak/server/plugins/PluginDataApi.java b/src/takserver-plugins/src/main/java/tak/server/plugins/PluginDataApi.java new file mode 100644 index 00000000..4847634b --- /dev/null +++ b/src/takserver-plugins/src/main/java/tak/server/plugins/PluginDataApi.java @@ -0,0 +1,21 @@ +package tak.server.plugins; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + */ +public interface PluginDataApi { + + static final Logger logger = LoggerFactory.getLogger(PluginDataApi.class); + + /* + * Submit data to a plugin for processing by the plugin. Implementation of this function is provided by the plugin. + * + */ + default void onSubmitData(String scope, String data, String contentType) { + + logger.info("submitDataToPlugin method not implemented in plugin class " + getClass().getName() + " must be implemented in order to submit data."); + } + +} diff --git a/src/takserver-plugins/src/main/java/tak/server/plugins/PluginDataFeed.java b/src/takserver-plugins/src/main/java/tak/server/plugins/PluginDataFeed.java new file mode 100644 index 00000000..f117d4c3 --- /dev/null +++ b/src/takserver-plugins/src/main/java/tak/server/plugins/PluginDataFeed.java @@ -0,0 +1,76 @@ +package tak.server.plugins; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + + +public class PluginDataFeed implements Serializable{ + + private static final long serialVersionUID = 1336926585144753864L; + + private String uuid; + + private String name; + + private List tags = new ArrayList<>(); + + private boolean archive; + + private boolean sync; + + public PluginDataFeed(String uuid, String name, List tags, boolean archive, boolean sync) { + this.uuid = uuid; + this.name = name; + this.tags.addAll(tags); + this.archive = archive; + this.sync = sync; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public List getTags() { + return tags; + } + + public void setTags(List tags) { + this.tags = tags; + } + + public boolean isArchive() { + return archive; + } + + public void setArchive(boolean archive) { + this.archive = archive; + } + + public boolean isSync() { + return sync; + } + + public void setSync(boolean sync) { + this.sync = sync; + } + + @Override + public String toString() { + return "PluginDataFeed [uuid=" + uuid + ", name=" + name + ", tags=" + tags + ", archive=" + archive + ", sync=" + + sync + "]"; + } + +} diff --git a/src/takserver-plugins/src/main/java/tak/server/plugins/PluginDataFeedApi.java b/src/takserver-plugins/src/main/java/tak/server/plugins/PluginDataFeedApi.java new file mode 100644 index 00000000..689cce55 --- /dev/null +++ b/src/takserver-plugins/src/main/java/tak/server/plugins/PluginDataFeedApi.java @@ -0,0 +1,13 @@ +package tak.server.plugins; + +import java.util.Collection; +import java.util.List; + +public interface PluginDataFeedApi { + + PluginDataFeed create(String uuid, String name, List tags, boolean archive, boolean sync); + PluginDataFeed create(String uuid, String name, List tags); + void delete(String uuid); + Collection getAllPluginDataFeeds(); + +} diff --git a/src/takserver-plugins/src/main/java/tak/server/plugins/PluginInfo.java b/src/takserver-plugins/src/main/java/tak/server/plugins/PluginInfo.java index 2b1b594e..3531241b 100644 --- a/src/takserver-plugins/src/main/java/tak/server/plugins/PluginInfo.java +++ b/src/takserver-plugins/src/main/java/tak/server/plugins/PluginInfo.java @@ -12,12 +12,16 @@ public class PluginInfo { private String name; private String description; private String className; + private String version; + private String tag; private UUID id; boolean isSender; boolean isReceiver; boolean isInterceptor; private boolean isEnabled; + private boolean isStarted; private String exceptionMessage; + private boolean archiveEnabled; public String getName() { return name; @@ -67,12 +71,37 @@ public boolean isEnabled() { public void setEnabled(boolean isEnabled) { this.isEnabled = isEnabled; } + public boolean isStarted() { + return isStarted; + } + public void setStarted(boolean isStarted) { + this.isStarted = isStarted; + } public boolean isInterceptor() { return isInterceptor; } public void setInterceptor(boolean isInterceptor) { this.isInterceptor = isInterceptor; } + public boolean isArchiveEnabled() { + return archiveEnabled; + } + public void setArchiveEnabled(boolean archiveEnabled) { + this.archiveEnabled = archiveEnabled; + } + public String getVersion() { + return version; + } + public void setVersion(String version) { + this.version = version; + } + public String getTag() { + return tag; + } + public void setTag(String tag) { + this.tag = tag; + } + @Override public int hashCode() { final int prime = 31; diff --git a/src/takserver-plugins/src/main/java/tak/server/plugins/PluginLifecycle.java b/src/takserver-plugins/src/main/java/tak/server/plugins/PluginLifecycle.java index 931dd5dc..925a30af 100644 --- a/src/takserver-plugins/src/main/java/tak/server/plugins/PluginLifecycle.java +++ b/src/takserver-plugins/src/main/java/tak/server/plugins/PluginLifecycle.java @@ -1,6 +1,6 @@ package tak.server.plugins; -public interface PluginLifecycle { +public interface PluginLifecycle extends PluginDataApi { void start(); void stop(); @@ -10,4 +10,7 @@ public interface PluginLifecycle { PluginInfo getPluginInfo(); void setPluginInfo(PluginInfo pluginInfo); + + void selfStop(); + } diff --git a/src/takserver-plugins/src/main/java/tak/server/plugins/PluginSelfStopApi.java b/src/takserver-plugins/src/main/java/tak/server/plugins/PluginSelfStopApi.java new file mode 100644 index 00000000..272721a8 --- /dev/null +++ b/src/takserver-plugins/src/main/java/tak/server/plugins/PluginSelfStopApi.java @@ -0,0 +1,7 @@ +package tak.server.plugins; + +public interface PluginSelfStopApi { + + void pluginSelfStop(String pluginName); + +} diff --git a/src/takserver-plugins/src/main/java/tak/server/plugins/Sender.java b/src/takserver-plugins/src/main/java/tak/server/plugins/Sender.java index f1c87c2c..f6a75830 100644 --- a/src/takserver-plugins/src/main/java/tak/server/plugins/Sender.java +++ b/src/takserver-plugins/src/main/java/tak/server/plugins/Sender.java @@ -12,4 +12,6 @@ public interface Sender extends PluginLifecycle { * @see atakmap.commoncommo.protobuf.v1.Message */ void send(T message); + + void send(T message, String feedUuid); } diff --git a/src/takserver-plugins/src/main/java/tak/server/plugins/SystemInfoApi.java b/src/takserver-plugins/src/main/java/tak/server/plugins/SystemInfoApi.java new file mode 100644 index 00000000..16b5475f --- /dev/null +++ b/src/takserver-plugins/src/main/java/tak/server/plugins/SystemInfoApi.java @@ -0,0 +1,9 @@ +package tak.server.plugins; + +/** + */ +public interface SystemInfoApi { + + String getTAKServerUrl(); + +} diff --git a/src/takserver-plugins/src/main/java/tak/server/plugins/TakServerPluginVersion.java b/src/takserver-plugins/src/main/java/tak/server/plugins/TakServerPluginVersion.java new file mode 100644 index 00000000..80b896b6 --- /dev/null +++ b/src/takserver-plugins/src/main/java/tak/server/plugins/TakServerPluginVersion.java @@ -0,0 +1,16 @@ +package tak.server.plugins; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE}) +public @interface TakServerPluginVersion { + int major() default -1; + int minor() default -1; + int patch() default -1; + String commitHash() default ""; + String tag() default ""; +} \ No newline at end of file diff --git a/src/takserver-plugins/src/main/java/tak/server/plugins/messaging/PluginMessenger.java b/src/takserver-plugins/src/main/java/tak/server/plugins/messaging/PluginMessenger.java index 99a26a99..51e64d74 100644 --- a/src/takserver-plugins/src/main/java/tak/server/plugins/messaging/PluginMessenger.java +++ b/src/takserver-plugins/src/main/java/tak/server/plugins/messaging/PluginMessenger.java @@ -50,7 +50,7 @@ public void send(Message message) { // if callsigns or uids are specified, set them in the message if (message.getDestCallsignsCount() != 0 || message.getDestClientUidsCount() != 0) { Message.Builder mb = message.toBuilder(); - + Detail.Builder db = mb.getPayloadBuilder().getCotEventBuilder().getDetailBuilder(); try { diff --git a/src/takserver-retention/conf/retention/misison-archiving-config.yml b/src/takserver-retention/conf/retention/mission-archiving-config.yml similarity index 100% rename from src/takserver-retention/conf/retention/misison-archiving-config.yml rename to src/takserver-retention/conf/retention/mission-archiving-config.yml diff --git a/src/takserver-retention/mission-archive/mission-store.yml b/src/takserver-retention/mission-archive/mission-store.yml index bc215a74..340aca08 100644 --- a/src/takserver-retention/mission-archive/mission-store.yml +++ b/src/takserver-retention/mission-archive/mission-store.yml @@ -1,2 +1,2 @@ --- -missonArchiveStoreEntries: +missionArchiveStoreEntries: diff --git a/src/takserver-retention/src/main/java/tak/server/retention/config/MissionArchiveStoreConfig.java b/src/takserver-retention/src/main/java/tak/server/retention/config/MissionArchiveStoreConfig.java index 4df99686..d15180bf 100644 --- a/src/takserver-retention/src/main/java/tak/server/retention/config/MissionArchiveStoreConfig.java +++ b/src/takserver-retention/src/main/java/tak/server/retention/config/MissionArchiveStoreConfig.java @@ -8,28 +8,26 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; -import tak.server.retention.service.MissionArchiveHelper; - @Configuration @ConfigurationProperties @PropertySource(name="mission-store", factory=YamlPropertySourceFactory.class, value="file:mission-archive/mission-store.yml") public class MissionArchiveStoreConfig implements Serializable { - private List missonArchiveStoreEntries = new ArrayList<>(); + private List missionArchiveStoreEntries = new ArrayList<>(); - public synchronized List getMissonArchiveStoreEntries() { - return missonArchiveStoreEntries; + public synchronized List getMissionArchiveStoreEntries() { + return missionArchiveStoreEntries; } - public synchronized void setMissonArchiveStoreEntries(List missonArchiveStoreEntries) { - this.missonArchiveStoreEntries = missonArchiveStoreEntries; + public synchronized void setMissionArchiveStoreEntries(List missionArchiveStoreEntries) { + this.missionArchiveStoreEntries = missionArchiveStoreEntries; } - public synchronized void addMissionEntry(MissonArchiveStoreEntry missonArchiveStoreEntry) { - missonArchiveStoreEntries.add(missonArchiveStoreEntry); + public synchronized void addMissionEntry(MissionArchiveStoreEntry missionArchiveStoreEntry) { + missionArchiveStoreEntries.add(missionArchiveStoreEntry); } - public static class MissonArchiveStoreEntry { + public static class MissionArchiveStoreEntry { private String missionName; private String createTimeMs; private String archiveTimeMs; @@ -75,7 +73,7 @@ public void setId(int id) { } @Override public String toString() { - return "MissonArchiveStoreEntry [missionName=" + missionName + ", createTimeMs=" + createTimeMs + return "MissionArchiveStoreEntry [missionName=" + missionName + ", createTimeMs=" + createTimeMs + ", archiveTimeMs=" + archiveTimeMs + ", createTime=" + createTime + ", archiveTime=" + archiveTime + ", id=" + id + "]"; } @@ -83,6 +81,6 @@ public String toString() { @Override public String toString() { - return "MissionArchiveStoreConfig [missonArchiveStoreEntries=" + missonArchiveStoreEntries + "]"; + return "MissionArchiveStoreConfig [missionArchiveStoreEntries=" + missionArchiveStoreEntries + "]"; } } diff --git a/src/takserver-retention/src/main/java/tak/server/retention/config/MissionArchivingCronConfig.java b/src/takserver-retention/src/main/java/tak/server/retention/config/MissionArchivingCronConfig.java index e3656a05..2414b493 100644 --- a/src/takserver-retention/src/main/java/tak/server/retention/config/MissionArchivingCronConfig.java +++ b/src/takserver-retention/src/main/java/tak/server/retention/config/MissionArchivingCronConfig.java @@ -9,7 +9,7 @@ @Configuration @ConfigurationProperties -@PropertySource(name="mission-archive-cron-sched", factory=YamlPropertySourceFactory.class, value="file:conf/retention/misison-archiving-config.yml") +@PropertySource(name="mission-archive-cron-sched", factory=YamlPropertySourceFactory.class, value="file:conf/retention/mission-archiving-config.yml") public class MissionArchivingCronConfig implements Serializable { private String missionCronExpression = "-"; diff --git a/src/takserver-retention/src/main/java/tak/server/retention/service/DistributedMissionArchiveManager.java b/src/takserver-retention/src/main/java/tak/server/retention/service/DistributedMissionArchiveManager.java index 54392493..f390d750 100644 --- a/src/takserver-retention/src/main/java/tak/server/retention/service/DistributedMissionArchiveManager.java +++ b/src/takserver-retention/src/main/java/tak/server/retention/service/DistributedMissionArchiveManager.java @@ -21,7 +21,7 @@ public class DistributedMissionArchiveManager implements MissionArchiveManager, Service { - private static final String MISSION_ARCHIVE_CONFIG = "conf/mission-archive/misison-archiving-config.yml"; + private static final String MISSION_ARCHIVE_CONFIG = "conf/retention/mission-archiving-config.yml"; @Autowired MissionArchiveHelper missionArchiveHelper; diff --git a/src/takserver-retention/src/main/java/tak/server/retention/service/MissionArchiveHelper.java b/src/takserver-retention/src/main/java/tak/server/retention/service/MissionArchiveHelper.java index 8a359e6f..3c57f96d 100644 --- a/src/takserver-retention/src/main/java/tak/server/retention/service/MissionArchiveHelper.java +++ b/src/takserver-retention/src/main/java/tak/server/retention/service/MissionArchiveHelper.java @@ -36,7 +36,7 @@ import tak.server.cot.XmlContainer; import tak.server.retention.config.MissionArchiveStoreConfig; -import tak.server.retention.config.MissionArchiveStoreConfig.MissonArchiveStoreEntry; +import tak.server.retention.config.MissionArchiveStoreConfig.MissionArchiveStoreEntry; public class MissionArchiveHelper { @@ -46,16 +46,16 @@ public class MissionArchiveHelper { private static final String MISSION_STORE_FILE = ARCHIVE_DIR + "mission-store.yml"; @Autowired - private MissionArchiveStoreConfig missionArchiveStore; + MissionArchiveStoreConfig missionArchiveStore; @Autowired RetentionQueryService retentionQueryService; public void removeExpiredArchiveEntries(double ttlDays) { - List entriesToRemove = new ArrayList<>(); - List entriesToKeep = new ArrayList<>(); + List entriesToRemove = new ArrayList<>(); + List entriesToKeep = new ArrayList<>(); - missionArchiveStore.getMissonArchiveStoreEntries().forEach(missionArchiveStoreEntry-> { + missionArchiveStore.getMissionArchiveStoreEntries().forEach(missionArchiveStoreEntry-> { double msInArchive = new Date().getTime() - Double.valueOf(missionArchiveStoreEntry.getArchiveTimeMs()); double daysInArchive = msInArchive / 1000 / 60 / 60 / 24; if (daysInArchive > ttlDays) { @@ -67,14 +67,14 @@ public void removeExpiredArchiveEntries(double ttlDays) { if (entriesToRemove.size() == 0) return; - missionArchiveStore.setMissonArchiveStoreEntries(entriesToKeep); + missionArchiveStore.setMissionArchiveStoreEntries(entriesToKeep); try { ObjectMapper mapper = new ObjectMapper(new YAMLFactory().enable(YAMLGenerator.Feature.MINIMIZE_QUOTES)); File file = new File(MISSION_STORE_FILE); MissionArchiveStoreConfig newMissionArchiveStoreConfig = new MissionArchiveStoreConfig(); - newMissionArchiveStoreConfig.setMissonArchiveStoreEntries(entriesToKeep); + newMissionArchiveStoreConfig.setMissionArchiveStoreEntries(entriesToKeep); mapper.writeValue(file, newMissionArchiveStoreConfig); logger.info("Removed Expired Mission Archive Entries " + Arrays.toString(entriesToRemove.toArray())); @@ -95,7 +95,7 @@ public void removeExpiredArchiveEntries(double ttlDays) { public String getMissionStoreJson() { try { MissionArchiveStoreConfig newMissionArchiveStoreConfig = new MissionArchiveStoreConfig(); - newMissionArchiveStoreConfig.setMissonArchiveStoreEntries(missionArchiveStore.getMissonArchiveStoreEntries()); + newMissionArchiveStoreConfig.setMissionArchiveStoreEntries(missionArchiveStore.getMissionArchiveStoreEntries()); ObjectMapper jsonWriter = new ObjectMapper(); return jsonWriter.writeValueAsString(newMissionArchiveStoreConfig); @@ -132,13 +132,13 @@ public synchronized void archiveMissionAndDelete(Map mission) { public String restoreMissionFromArchive(int id) { try { - Optional matchingEntryOp = missionArchiveStore.getMissonArchiveStoreEntries().stream().filter(m->m.getId() == id).findFirst(); + Optional matchingEntryOp = missionArchiveStore.getMissionArchiveStoreEntries().stream().filter(m->m.getId() == id).findFirst(); if (!matchingEntryOp.isPresent()) { return "Mission to restore not found in mission index"; } - MissonArchiveStoreEntry matchingEntry = matchingEntryOp.get(); + MissionArchiveStoreEntry matchingEntry = matchingEntryOp.get(); String filename = ARCHIVE_DIR + matchingEntry.getCreateTime() + "_" + matchingEntry.getMissionName() + ".zip"; Map files = new HashMap<>(); @@ -217,9 +217,9 @@ public String restoreMissionFromArchive(int id) { File file = new File(MISSION_STORE_FILE); // remove the mission from the mission store index - missionArchiveStore.getMissonArchiveStoreEntries().remove(matchingEntry); + missionArchiveStore.getMissionArchiveStoreEntries().remove(matchingEntry); MissionArchiveStoreConfig newMissionArchiveStoreConfig = new MissionArchiveStoreConfig(); - newMissionArchiveStoreConfig.setMissonArchiveStoreEntries(missionArchiveStore.getMissonArchiveStoreEntries()); + newMissionArchiveStoreConfig.setMissionArchiveStoreEntries(missionArchiveStore.getMissionArchiveStoreEntries()); mapper.writeValue(file, newMissionArchiveStoreConfig); // lastly, remove the zip archive file @@ -251,17 +251,17 @@ private void writeMissionToArchive(String filename, String missionName, Timestam try { File file = new File(MISSION_STORE_FILE); - MissonArchiveStoreEntry missonArchiveStoreEntry = new MissonArchiveStoreEntry(); - missonArchiveStoreEntry.setMissionName(missionName); - missonArchiveStoreEntry.setCreateTimeMs(String.valueOf(createTime.getTime())); - missonArchiveStoreEntry.setArchiveTimeMs(String.valueOf(archiveTime.getTime())); - missonArchiveStoreEntry.setCreateTime(createTime.toString().replace(":", "-")); - missonArchiveStoreEntry.setArchiveTime(archiveTime.toString().replace(":", "-")); - missonArchiveStoreEntry.setId(filename.hashCode()); - missionArchiveStore.addMissionEntry(missonArchiveStoreEntry); + MissionArchiveStoreEntry missionArchiveStoreEntry = new MissionArchiveStoreEntry(); + missionArchiveStoreEntry.setMissionName(missionName); + missionArchiveStoreEntry.setCreateTimeMs(String.valueOf(createTime.getTime())); + missionArchiveStoreEntry.setArchiveTimeMs(String.valueOf(archiveTime.getTime())); + missionArchiveStoreEntry.setCreateTime(createTime.toString().replace(":", "-")); + missionArchiveStoreEntry.setArchiveTime(archiveTime.toString().replace(":", "-")); + missionArchiveStoreEntry.setId(filename.hashCode()); + missionArchiveStore.addMissionEntry(missionArchiveStoreEntry); MissionArchiveStoreConfig newMissionArchiveStoreConfig = new MissionArchiveStoreConfig(); - newMissionArchiveStoreConfig.setMissonArchiveStoreEntries(missionArchiveStore.getMissonArchiveStoreEntries()); + newMissionArchiveStoreConfig.setMissionArchiveStoreEntries(missionArchiveStore.getMissionArchiveStoreEntries()); mapper.writeValue(file, newMissionArchiveStoreConfig); } catch (Exception e) { diff --git a/src/takserver-schemamanager/src/main/resources/db/migration/V12__mission_api_tables.sql b/src/takserver-schemamanager/src/main/resources/db/migration/V12__mission_api_tables.sql index b817ec07..ab679292 100644 --- a/src/takserver-schemamanager/src/main/resources/db/migration/V12__mission_api_tables.sql +++ b/src/takserver-schemamanager/src/main/resources/db/migration/V12__mission_api_tables.sql @@ -3,6 +3,7 @@ -- -- Migration script to TAK Server schema version 12 -- Adds support for Mission API +-- This script is a copy of mission_schema_20160224.sql -- -- drop table if exists mission cascade; diff --git a/src/takserver-schemamanager/src/main/resources/db/migration/V45__video_connections_add_groups.sql b/src/takserver-schemamanager/src/main/resources/db/migration/V45__video_connections_add_groups.sql new file mode 100644 index 00000000..469b83c3 --- /dev/null +++ b/src/takserver-schemamanager/src/main/resources/db/migration/V45__video_connections_add_groups.sql @@ -0,0 +1 @@ +alter table video_connections add column groups bit varying; \ No newline at end of file diff --git a/src/takserver-schemamanager/src/main/resources/db/migration/V46__create_video_connections_table_v2.sql b/src/takserver-schemamanager/src/main/resources/db/migration/V46__create_video_connections_table_v2.sql new file mode 100644 index 00000000..6da07238 --- /dev/null +++ b/src/takserver-schemamanager/src/main/resources/db/migration/V46__create_video_connections_table_v2.sql @@ -0,0 +1,31 @@ +-- +-- Copyright (c) 2016 Raytheon BBN Technologies. Licensed to US Government with unlimited rights. +-- + +-- +-- Migration script to schema version 11 +-- Creates the video_connections table. +-- Schema version 11 was originally committedd on 12 February 2016 + +CREATE TABLE video_connections_v2 ( + id integer NOT NULL, + uid text, + active boolean DEFAULT true, + alias text, + thumbnail text, + classification text, + xml text, + groups bit varying + ); + +DROP SEQUENCE IF EXISTS video_connection_id_seq_v2 ; +CREATE SEQUENCE video_connection_id_seq_v2 + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE video_connection_id_seq_v2 OWNED BY video_connections_v2.id; +ALTER TABLE video_connections_v2 ALTER COLUMN id SET DEFAULT nextval('video_connection_id_seq_v2'::regclass); +ALTER TABLE ONLY video_connections_v2 ADD CONSTRAINT video_connection_pkey_v2 PRIMARY KEY (id); \ No newline at end of file diff --git a/src/takserver-schemamanager/src/main/resources/db/migration/V47__add_cop_attributes_to_mission.sql b/src/takserver-schemamanager/src/main/resources/db/migration/V47__add_cop_attributes_to_mission.sql new file mode 100644 index 00000000..53469428 --- /dev/null +++ b/src/takserver-schemamanager/src/main/resources/db/migration/V47__add_cop_attributes_to_mission.sql @@ -0,0 +1,19 @@ +-- Table: public.mission_feed + +DROP TABLE IF EXISTS public.mission_feed; + +CREATE TABLE public.mission_feed +( + uid character varying(255), + data_feed_uid text, + filter_bbox text, + filter_type text, + filter_callsign text, + mission_id bigint +) +WITH ( + OIDS=FALSE +); + +alter table mission add column base_layer text; +alter table mission add column bbox text; \ No newline at end of file diff --git a/src/takserver-schemamanager/src/main/resources/db/migration/V48__data_sync_add_feed_permission.sql b/src/takserver-schemamanager/src/main/resources/db/migration/V48__data_sync_add_feed_permission.sql new file mode 100644 index 00000000..d30df656 --- /dev/null +++ b/src/takserver-schemamanager/src/main/resources/db/migration/V48__data_sync_add_feed_permission.sql @@ -0,0 +1,7 @@ + +-- MISSION_MANAGE_FEEDS +insert into permission (permission) values (6); + +-- MISSION_OWNER -> MISSION_MANAGE_FEEDS +insert into role_permission (role_id, permission_id) values (1,7); + diff --git a/src/takserver-schemamanager/src/main/resources/db/migration/V49__certificate_add_token.sql b/src/takserver-schemamanager/src/main/resources/db/migration/V49__certificate_add_token.sql new file mode 100644 index 00000000..0167c419 --- /dev/null +++ b/src/takserver-schemamanager/src/main/resources/db/migration/V49__certificate_add_token.sql @@ -0,0 +1 @@ +alter table certificate add column token character varying; diff --git a/src/takserver-schemamanager/src/main/resources/db/migration/V50__cop_hierarchy_and_classification.sql b/src/takserver-schemamanager/src/main/resources/db/migration/V50__cop_hierarchy_and_classification.sql new file mode 100644 index 00000000..b68a8d35 --- /dev/null +++ b/src/takserver-schemamanager/src/main/resources/db/migration/V50__cop_hierarchy_and_classification.sql @@ -0,0 +1,3 @@ + +alter table mission add column path text; +alter table mission add column classification text; \ No newline at end of file diff --git a/src/takserver-schemamanager/src/main/resources/db/migration/V51__data_feed_cot.sql b/src/takserver-schemamanager/src/main/resources/db/migration/V51__data_feed_cot.sql new file mode 100644 index 00000000..1c93b900 --- /dev/null +++ b/src/takserver-schemamanager/src/main/resources/db/migration/V51__data_feed_cot.sql @@ -0,0 +1,64 @@ +DROP TABLE IF EXISTS data_feed_type_pl; +create table data_feed_type_pl ( + id serial primary key, + feed_type text unique not null +); +insert into data_feed_type_pl (feed_type) values ('Streaming'); +insert into data_feed_type_pl (feed_type) values ('API'); +insert into data_feed_type_pl (feed_type) values ('Plugin'); + +DROP TABLE IF EXISTS data_feed; +CREATE TABLE data_feed ( + id bigint NOT NULL, + uuid character varying(255) UNIQUE NOT NULL, + name character varying(255) NOT NULL, + type bigint NOT NULL, + auth character varying(255), + port bigint, + auth_required boolean DEFAULT false, + protocol character varying(255), + feed_group character varying(255), + iface character varying(255), + archive boolean DEFAULT true, + anongroup boolean DEFAULT false, + sync boolean DEFAULT false, + archive_only boolean DEFAULT false, + core_version bigint, + core_version_tls_versions character varying(255) +); + +CREATE SEQUENCE data_feed_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE data_feed_id_seq OWNED BY data_feed.id; +ALTER TABLE ONLY data_feed ALTER COLUMN id SET DEFAULT nextval('data_feed_id_seq'::regclass); +ALTER TABLE ONLY data_feed ADD CONSTRAINT data_feed_pkey PRIMARY KEY (id); + +DROP TABLE IF EXISTS data_feed_cot; +CREATE TABLE data_feed_cot ( + cot_router_id int NOT NULL, + data_feed_id int NOT NULL +); + +CREATE INDEX data_feed_cot_cot_router_id_idx ON data_feed_cot(cot_router_id); +CREATE INDEX data_feed_cot_data_feed_id_idx ON data_feed_cot(data_feed_id); + +CREATE TABLE data_feed_tag ( + data_feed_id bigint NOT NULL, + tag character varying(255) NOT NULL +); + +ALTER TABLE ONLY data_feed_tag ADD CONSTRAINT data_feed_tag_pkey PRIMARY KEY (data_feed_id, tag); +ALTER TABLE ONLY data_feed_tag ADD CONSTRAINT data_feed_tag_data_feed_id_fk FOREIGN KEY (data_feed_id) REFERENCES data_feed(id); + +CREATE TABLE data_feed_filter_group ( + data_feed_id bigint NOT NULL, + filter_group character varying(255) NOT NULL +); + +ALTER TABLE ONLY data_feed_filter_group ADD CONSTRAINT data_feed_filter_group_pkey PRIMARY KEY (data_feed_id, filter_group); +ALTER TABLE ONLY data_feed_filter_group ADD CONSTRAINT data_feed_filter_group_data_feed_id_fk FOREIGN KEY (data_feed_id) REFERENCES data_feed(id); diff --git a/src/takserver-schemamanager/src/main/resources/db/migration/V52__add_username_to_mission_subscription.sql b/src/takserver-schemamanager/src/main/resources/db/migration/V52__add_username_to_mission_subscription.sql new file mode 100644 index 00000000..f42255cf --- /dev/null +++ b/src/takserver-schemamanager/src/main/resources/db/migration/V52__add_username_to_mission_subscription.sql @@ -0,0 +1,5 @@ + +alter table mission_subscription add column username text; +alter table client_endpoint add column username text; + +create unique index client_endpoint_idx2 on client_endpoint(callsign, uid, username); \ No newline at end of file diff --git a/src/takserver-schemamanager/src/main/resources/db/migration/V53__add_map_layers_to_mission.sql b/src/takserver-schemamanager/src/main/resources/db/migration/V53__add_map_layers_to_mission.sql new file mode 100644 index 00000000..6c8d0839 --- /dev/null +++ b/src/takserver-schemamanager/src/main/resources/db/migration/V53__add_map_layers_to_mission.sql @@ -0,0 +1,10 @@ +alter table maplayer add column min_zoom integer; +alter table maplayer add column max_zoom integer; +alter table maplayer add column tile_type character varying(255); +alter table maplayer add column server_parts character varying(255); +alter table maplayer add column background_color character varying(255); +alter table maplayer add column tile_update character varying(255); +alter table maplayer add column ignore_errors boolean default false; +alter table maplayer add column invert_y_coordinate boolean default false; + +alter table maplayer add column mission_id bigint; diff --git a/src/takserver-schemamanager/src/main/resources/db/migration/V54__add_group_compare_function.sql b/src/takserver-schemamanager/src/main/resources/db/migration/V54__add_group_compare_function.sql new file mode 100644 index 00000000..61868a16 --- /dev/null +++ b/src/takserver-schemamanager/src/main/resources/db/migration/V54__add_group_compare_function.sql @@ -0,0 +1,8 @@ +CREATE OR REPLACE FUNCTION bitwiseAndGroups(groupVector character varying, groups bit varying) RETURNS boolean + LANGUAGE plpgsql IMMUTABLE + AS $$ +BEGIN + return groupVector::bit(32768) & + lpad(groups::character varying, 32768, '0')::bit(32768)::bit varying <> + 0::bit(32768)::bit varying; +END;$$ RETURNS NULL ON NULL INPUT; \ No newline at end of file diff --git a/src/takserver-schemamanager/src/main/resources/db/migration/V55__add_feeds_and_maplayers_to_changes.sql b/src/takserver-schemamanager/src/main/resources/db/migration/V55__add_feeds_and_maplayers_to_changes.sql new file mode 100644 index 00000000..08067927 --- /dev/null +++ b/src/takserver-schemamanager/src/main/resources/db/migration/V55__add_feeds_and_maplayers_to_changes.sql @@ -0,0 +1,2 @@ +ALTER TABLE public.mission_change ADD COLUMN mission_feed_uid character varying(255); +ALTER TABLE public.mission_change ADD COLUMN map_layer_uid character varying(255); \ No newline at end of file diff --git a/src/takserver-schemamanager/src/main/resources/db/migration/V56__add_last_edited_to_mission.sql b/src/takserver-schemamanager/src/main/resources/db/migration/V56__add_last_edited_to_mission.sql new file mode 100644 index 00000000..155d4313 --- /dev/null +++ b/src/takserver-schemamanager/src/main/resources/db/migration/V56__add_last_edited_to_mission.sql @@ -0,0 +1 @@ +ALTER TABLE public.mission ADD COLUMN last_edited timestamp(3) with time zone; \ No newline at end of file diff --git a/src/takserver-schemamanager/src/main/resources/db/migration/V57__add_attributes_to_maplayer.sql b/src/takserver-schemamanager/src/main/resources/db/migration/V57__add_attributes_to_maplayer.sql new file mode 100644 index 00000000..1d26f582 --- /dev/null +++ b/src/takserver-schemamanager/src/main/resources/db/migration/V57__add_attributes_to_maplayer.sql @@ -0,0 +1,9 @@ +alter table maplayer add column north numeric; +alter table maplayer add column south numeric; +alter table maplayer add column east numeric; +alter table maplayer add column west numeric; +alter table maplayer add column additional_parameters character varying(255); +alter table maplayer add column coordinate_system character varying(255); +alter table maplayer add column version character varying(255); +alter table maplayer add column layers integer; +alter table maplayer add column opacity integer; diff --git a/src/takserver-schemamanager/src/main/resources/db/migration/V58__drop_mission_subscription_pkey.sql b/src/takserver-schemamanager/src/main/resources/db/migration/V58__drop_mission_subscription_pkey.sql new file mode 100644 index 00000000..ba8deb9b --- /dev/null +++ b/src/takserver-schemamanager/src/main/resources/db/migration/V58__drop_mission_subscription_pkey.sql @@ -0,0 +1 @@ +ALTER TABLE ONLY mission_subscription DROP CONSTRAINT mission_subscription_pkey; \ No newline at end of file diff --git a/src/takserver-schemamanager/src/main/resources/db/migration/V59__mission_polygon.sql b/src/takserver-schemamanager/src/main/resources/db/migration/V59__mission_polygon.sql new file mode 100644 index 00000000..70f71806 --- /dev/null +++ b/src/takserver-schemamanager/src/main/resources/db/migration/V59__mission_polygon.sql @@ -0,0 +1 @@ +alter table mission add column bounding_polygon text; \ No newline at end of file diff --git a/src/takserver-schemamanager/src/main/resources/db/migration/V60__change_layers_to_string.sql b/src/takserver-schemamanager/src/main/resources/db/migration/V60__change_layers_to_string.sql new file mode 100644 index 00000000..d4dd45c0 --- /dev/null +++ b/src/takserver-schemamanager/src/main/resources/db/migration/V60__change_layers_to_string.sql @@ -0,0 +1,2 @@ +alter table maplayer drop column layers; +alter table maplayer add column layers character varying(255); \ No newline at end of file diff --git a/src/takserver-schemamanager/src/main/resources/db/migration/V61__add_content_to_error_logs.sql b/src/takserver-schemamanager/src/main/resources/db/migration/V61__add_content_to_error_logs.sql new file mode 100644 index 00000000..eebe8b97 --- /dev/null +++ b/src/takserver-schemamanager/src/main/resources/db/migration/V61__add_content_to_error_logs.sql @@ -0,0 +1 @@ +alter table error_logs add column contents bytea; diff --git a/src/takserver-schemamanager/src/main/resources/db/migration/V62__change_feed_group_to_bit.sql b/src/takserver-schemamanager/src/main/resources/db/migration/V62__change_feed_group_to_bit.sql new file mode 100644 index 00000000..2cc3d025 --- /dev/null +++ b/src/takserver-schemamanager/src/main/resources/db/migration/V62__change_feed_group_to_bit.sql @@ -0,0 +1,2 @@ +alter table data_feed add column groups bit varying; +update data_feed set groups = rpad('', 32768, '1')::bit(32768)::bit varying where groups is null or groups = ''; \ No newline at end of file diff --git a/src/takserver-schemamanager/src/main/resources/db/migration/V63__cleanup_client_endpoint_index.sql b/src/takserver-schemamanager/src/main/resources/db/migration/V63__cleanup_client_endpoint_index.sql new file mode 100644 index 00000000..58359c47 --- /dev/null +++ b/src/takserver-schemamanager/src/main/resources/db/migration/V63__cleanup_client_endpoint_index.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS client_endpoint_idx1; \ No newline at end of file diff --git a/src/takserver-takcl-core/docs/Development.md b/src/takserver-takcl-core/docs/Development.md index e97f1594..b467a832 100644 --- a/src/takserver-takcl-core/docs/Development.md +++ b/src/takserver-takcl-core/docs/Development.md @@ -6,7 +6,6 @@ The purpose of TAKCL, or the TAK Command Line, is two fold: 2. Provide easy command line access to the methods and libraries created for orchestrating said tests. - ## Architecture ### Overview, Packages, and Source Sets @@ -20,36 +19,35 @@ Takcl is broken down into six source sets within takserver-takcl-core/src: * test - test files The tests themselves are stored under the corresponding project: - * takserver-core/src/integrationTest - -The UserManager also uses TAKCL behind the scenes, and is located in takserver-usermanager +* [takserver-core/src/integrationTest](../../takserver-core/src/integrationTest) +The UserManager also uses TAKCL behind the scenes, and is located in [takserver-usermanager](../../takserver-usermanager) -The architecture can be broken down into two major components: TACKL and the Test Framework: +The architecture can be broken down into two major components: TACKL, and the Test Framework: TAKCL contains the following major components: - * AppModules - These expose core server interaction functionality utilized to orchestrate tests in a way that also allows - utilization of methods from the command line and are contained in the package com.bbn.marti.takcl.AppModules - - The abstract classes for these are located in the core source set in the package com.bbn.marti.takcl.AppModules - - The implementation of takcl.jar is located in the exe source set - - The rest of the implementation is located in the main source set + * **AppModules** - These implement core server interaction functionality and expose them for CLI usage. + - They are located in the package _com.bbn.marti.takcl.AppModules_ + - The abstract classes for these are [located](../src/core/java/com/bbn/marti/takcl/AppModules) in the core source set + - The implementation of takcl.jar is [located](../src/exe/java/com/bbn/marti/takcl/AppModules/) in the exe source set + - The rest of the implementations are [located](../src/core/java/com/bbn/marti/takcl/AppModules) in the main source set. * CLI - This allows AppModules to be utilized from the command line with little effort. - - The classes to utilize this are located in the com.bbn.marti.takcl.cli package in the core source set - - They are used by the UserManager and the exe source set + - The classes to utilize this are [located](../src/core/java/com/bbn/marti/takcl/cli) in the core source set + - They are currently used by the [UserManager](../../takserver-usermanager) and [takcl.jar](../src/exe) * Client - This wraps functionality for simulating a client that can be controlled and monitored by the framework. - - These are located in the com.bbn.marti.takcl.connectivity package in the main source set + - These are [located](../src/main/java/com/bbn/marti/takcl/connectivity) in the main source set - This package also contains the portion used to bring up and tear down server instances The Test Framework contains the following components: * Data - A set of structures used to represent objects and relations to TAKServer - - They are located in the com.bbn.marti.test.shared.data package in the core source set + - They are [located](../src/core/java/com/bbn/marti/test/shared) in the core source set - Although restrictive, they allow fast development of tests by using predefined servers, users, and connections - * Engines - A set of Execution, Validation, And State Management abstractions to simplify interactions with the server. - - They exist in the com.bbn.marti.test.shared.engines package of the main source set + * Engines - A set of Execution, Validation, And State Management abstractions to simplify interactions with the server + - They are [located](../src/main/java/com/bbn/marti/test/shared/engines) in the main source set * Tests - The actual test files. Some of these are placed within the integrationTest source directories of TAKServer components. - - These are located in the integrationTest source set in takserver-core. Some tests exist in the main project, but they are largely old, incomplete, or redundant and not exeucted. - + - These are located in the [integrationTest](../../takserver-core/src/integrationTest/java) source set of + takserver-core. Some tests exist in the main project, but they are largely old, incomplete, or redundant and not exeucted. ### CLI @@ -70,7 +68,6 @@ The Test Framework contains the following components: * [AppModuleInterface.java](../src/core/java/com/bbn/marti/takcl/AppModules/generic/AppModuleInterface.java) * [ServerAppModuleInterface.java](../src/core/java/com/bbn/marti/takcl/AppModules/generic/ServerAppModuleInterface.java) - An example method signature is as follows: @@ -93,17 +90,63 @@ The advancec CLI abstraction allows significantly more complex command line anno ### Test Structure -Each piece of higher level functionality is captured in the [EngineInterface.java](../src/main/java/com/bbn/marti/test/shared/engines/EngineInterface.java). This is then implemented in [TestEngine.java](../src/main/java/com/bbn/marti/test/shared/engines/TestEngine.java), which serves one simple purpose: To execute the following classes, in order, which also implement the EngineInterface: - - [ActionEngine](../src/main/java/com/bbn/marti/test/shared/engines/ActionEngine.java) +Each piece of higher level functionality is captured in the +[EngineInterface.java](../src/main/java/com/bbn/marti/test/shared/engines/EngineInterface.java). This is then +implemented in [TestEngine.java](../src/main/java/com/bbn/marti/test/shared/engines/TestEngine.java), which serves one +simple purpose: To execute the following classes, in order, which also implement the EngineInterface: + - [ActionEngine](../src/main/java/com/bbn/marti/test/shared/engines/action/ActionEngine.java) * Executes the action against the server - [VerificationEngine](../src/main/java/com/bbn/marti/test/shared/engines/verification/VerificationEngine.java) * Verifies the expected behavior has impacted the rest of the system as expected - [StateEngine](../src/main/java/com/bbn/marti/test/shared/engines/state/StateEngine.java) * Updates the current state for the next TestEngine call -To keep things organized modification of engine data and operations relating to each phase should be kept within the engine at all costs. It has proven to be difficult to manage the integrity of the environment when elements creep outside of their designated area. +To keep things organized modification of engine data and operations relating to each phase should be kept within the +engine at all costs. It has proven to be difficult to manage the integrity of the environment when elements creep +outside of their designated area. Each engine has its own data structure where necessary, that is typically read by +other engines for data necessary for a complete TestEngine cycle. The tests themselves start, stop and execute the +features within the TestEngine, isolated from the processes within the engine. The engine components + +#### ActionEngine + +The action engine is where all the "action" happens, and where actual interactions with the server are performed. It +also has the following related classes: +The ActionEngine is also where the timeouts used are defined. There is a _setSleepMultiplier_ method to override most +of the timeout values (useful for federation tests), and some individual methods for timeouts that tend to vary +individually. Since only the ActionEngine should be modifying its own data, it is the responsibility of each overridden +EngineInterface method to call `data.engineIterationDataClear();` at the start of each method clear the state +information from the previous run. + +**ActionClient** - The events executed by the superclass should be relatively self-explanatory. The state data has been +documented. It is vital that any new state data is included in the _hasChanged_ and _clearIterationData_ methods of the +ActionClient, as this data is only intended to stick around for a single TestEngine cycle. + +**ActionEngineData** - This class encapsulates data that is consistent through the lifetime of the server and moderates +access to the ActionClients by the StateEngine and VerificationEngine. + +**ActionTimeoutManager** - This class contains the timeout data used to dictate the test flow. + +#### VerificationEngine + +The verification engine contains a similar structure to the TestEngine, but has significantly more logic contained +within the individual overridden methods. Since only the ActionEngine should be modifying its own data, it is the +responsibility of each overridden EngineInterface method to call `data.engineIterationDataClear();` at the end of each +method to clear the state information from the next run. It has the following related classes: -The tests themselves start, stop and execute the features within the test engine, isolated from the processes within the engine. +**VerificationData** This class wraps the verification data management for each test execution + +**UserTrafficValidator** - Used to ingest, validate, and display the proper or inproper flow of CoT traffic for a +user. It is created with the expectations and compared against the actual behavior via the _validateExpectations_ call. +It also contains a _FAILURE_DELAY_TIME_. The purpose of this is that if a message isn't received, the testing will wait +a significantly longer time than the default to see if one does come in. It was previously used to indicate resource +issues where there was simply too much load on the server to process the message in a reasonable amount of time vs the +server simply not sending the message. + +#### StateEngine + +The StateEngine is used to record the final state after an event defined in the EngineInterface has been executed and +validated. It is probably the most complex Engine component, and contains a number of classes to break down the state +data in a reasonable manner. The **EnvironmentState** class serves as the data for this class. ### Test Execution @@ -115,6 +158,26 @@ The first thing that should be done when developing a test is to encapsulate ind At this point, a test can be made based on the functionality and combined with other functionality. Once a test class has been created, it should be added to the TestRunnerModule, and then added to the gitlab-ci.yaml file. +## Test User formatting +The test users are formatted as follows: + +`___[_optional_differentiator]` + +Groups are 0, 1, 2, 3, or t/f, where t and f indicate access to the \__ANON__ group. + +A and B may be used to allow functionally identical users or inputs. + +Websockets and cert authentication (which is currently only supported on websockets) are indicated by authwssuser. + +Where: + * server is s0/s1/s2/s3 indicating the server + * input indicates the input that is being used + * input groups indicates the groups associated with the input + * authentication indicates the authentication user being used (anonuser means no authentication) + * authentication groups indicates the groups associated with the user + * resultant groups indicates the resultant groups + * _optional differentiator is used to differentiate between functionally identical users + ## Test Tags Test tagging has been added to encapsulate higher level environment conditions and configurations that should be diff --git a/src/takserver-takcl-core/src/core/java/com/bbn/marti/takcl/TAKCLProfilingLogging.java b/src/takserver-takcl-core/src/core/java/com/bbn/marti/takcl/TAKCLProfilingLogging.java index 16512c40..de7c3c56 100644 --- a/src/takserver-takcl-core/src/core/java/com/bbn/marti/takcl/TAKCLProfilingLogging.java +++ b/src/takserver-takcl-core/src/core/java/com/bbn/marti/takcl/TAKCLProfilingLogging.java @@ -137,21 +137,27 @@ public static LogActivity getSend(AbstractUser user) { switch (protocol) { case INPUT_TCP: + case DATAFEED_TCP: assert (protocol.canSend()); return sendTcp; case INPUT_UDP: + case DATAFEED_UDP: assert (protocol.canSend()); return sendUdp; case INPUT_MCAST: + case DATAFEED_MCAST: assert (protocol.canSend()); return sendMcast; case INPUT_STCP: + case DATAFEED_STCP: assert (protocol.canSend()); return sendStcp; case INPUT_TLS: + case DATAFEED_TLS: assert (protocol.canSend()); return sendTls; case INPUT_SSL: + case DATAFEED_SSL: assert (protocol.canSend()); return sendSsl; default: @@ -165,12 +171,15 @@ public static LogActivity getReceive(AbstractUser user) { switch (protocol) { case INPUT_STCP: + case DATAFEED_STCP: assert (protocol.canListen()); return receiveStcp; case INPUT_SSL: + case DATAFEED_SSL: assert (protocol.canListen()); return receiveSsl; case INPUT_TLS: + case DATAFEED_TLS: assert (protocol.canListen()); return receiveTls; case SUBSCRIPTION_TCP: diff --git a/src/takserver-takcl-core/src/core/java/com/bbn/marti/test/shared/data/connections/AbstractConnection.java b/src/takserver-takcl-core/src/core/java/com/bbn/marti/test/shared/data/connections/AbstractConnection.java index 1413d57d..3baae7e7 100644 --- a/src/takserver-takcl-core/src/core/java/com/bbn/marti/test/shared/data/connections/AbstractConnection.java +++ b/src/takserver-takcl-core/src/core/java/com/bbn/marti/test/shared/data/connections/AbstractConnection.java @@ -1,18 +1,20 @@ package com.bbn.marti.test.shared.data.connections; +import java.util.Comparator; +import java.util.Set; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + import com.bbn.marti.config.AuthType; -import com.bbn.marti.config.Network; +import com.bbn.marti.config.DataFeed; +import com.bbn.marti.config.Input; import com.bbn.marti.config.Subscription; import com.bbn.marti.test.shared.data.GroupSetProfiles; import com.bbn.marti.test.shared.data.protocols.ProtocolProfiles; import com.bbn.marti.test.shared.data.servers.AbstractServerProfile; import com.bbn.marti.test.shared.data.users.AbstractUser; import com.bbn.marti.test.shared.data.users.UserFilter; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.util.Comparator; -import java.util.Set; /** * Created on 3/4/16. @@ -55,13 +57,13 @@ public static boolean getAnonAccess(@NotNull AuthType authType, @NotNull GroupSe /** * Generates a {@link com.bbn.marti.config.Network.Input} input for this connection. This connection must be a proper input type or this will throw an exception! * - * @return A usable + * @return A usable Input */ - public final Network.Input getConfigInput() { + public final Input getConfigInput() { if (getConnectionType() != ProtocolProfiles.ConnectionType.INPUT) { throw new RuntimeException(("Cannot generate an input for for connection '" + getConsistentUniqueReadableIdentifier() + "' because it is not an input type!")); } - Network.Input input = new Network.Input(); + Input input = new Input(); input.setName(this.getConsistentUniqueReadableIdentifier()); input.setProtocol(getProtocol().getValue()); input.setPort(getPort()); @@ -78,6 +80,35 @@ public final Network.Input getConfigInput() { return input; } + + + /** + * Generates a {@link com.bbn.marti.config.Network.DataFeed} datafeed for this connection. This connection must be a proper datafeed type or this will throw an exception! + * + * @return A usable DataFeed + */ + public final DataFeed getConfigDataFeed() { + if (getConnectionType() != ProtocolProfiles.ConnectionType.DATAFEED) { + throw new RuntimeException(("Cannot generate a datafeed for connection '" + getConsistentUniqueReadableIdentifier() + "' because it is not an datafeed type!")); + } + DataFeed dataFeed = new DataFeed(); + dataFeed.setName(this.getConsistentUniqueReadableIdentifier()); + dataFeed.setProtocol(getProtocol().getValue()); + dataFeed.setPort(getPort()); + dataFeed.setAuth(getAuthType()); + dataFeed.setGroup(getMCastGroup()); + dataFeed.setAnongroup(getRawAnonAccessFlag()); + dataFeed.setType(getType()); + Integer networkVersion = getProtocol().getCoreNetworkVersion(); + if (networkVersion != null) { + dataFeed.setCoreVersion(networkVersion); + } + if (getGroupSet().groupSet != null) { + dataFeed.getFiltergroup().addAll(getGroupSet().groupSet); + } + + return dataFeed; + } /** @@ -251,6 +282,12 @@ public final String toString() { */ public abstract Boolean getRawAnonAccessFlag(); + /** + * Gets the value of the data feed type (could be null/unset) + * + * @return The value of the data feed type, if any + */ + public abstract String getType(); /** * Gets the users associated with the connection diff --git a/src/takserver-takcl-core/src/core/java/com/bbn/marti/test/shared/data/connections/BaseConnections.java b/src/takserver-takcl-core/src/core/java/com/bbn/marti/test/shared/data/connections/BaseConnections.java index c3eee8b4..d1118802 100644 --- a/src/takserver-takcl-core/src/core/java/com/bbn/marti/test/shared/data/connections/BaseConnections.java +++ b/src/takserver-takcl-core/src/core/java/com/bbn/marti/test/shared/data/connections/BaseConnections.java @@ -42,8 +42,34 @@ public enum BaseConnections { subudp(ProtocolProfiles.SUBSCRIPTION_UDP, 17717, AuthType.ANONYMOUS, null, GroupSetProfiles.Set_None), substcp(ProtocolProfiles.SUBSCRIPTION_STCP, 17718, AuthType.ANONYMOUS, null, GroupSetProfiles.Set_None), submcast(ProtocolProfiles.SUBSCRIPTION_MCAST, 17719, AuthType.ANONYMOUS, null, GroupSetProfiles.Set_None, "239.2.3.1"), - stcp3(ProtocolProfiles.INPUT_STCP, 17720, AuthType.ANONYMOUS, null, GroupSetProfiles.Set_3); - + stcp3(ProtocolProfiles.INPUT_STCP, 17720, AuthType.ANONYMOUS, null, GroupSetProfiles.Set_3), + + // Data feed connections + stcp01_data(ProtocolProfiles.DATAFEED_STCP, 18742, AuthType.ANONYMOUS, null, "Streaming", GroupSetProfiles.Set_01), + stcp12_data(ProtocolProfiles.DATAFEED_STCP, 18743, AuthType.ANONYMOUS, null, "Streaming", GroupSetProfiles.Set_12), + stcp0_data(ProtocolProfiles.DATAFEED_STCP, 17723, AuthType.ANONYMOUS, null, "Streaming", GroupSetProfiles.Set_0), + authstcp_data(ProtocolProfiles.DATAFEED_STCP, 18746, AuthType.FILE, null, "Streaming", GroupSetProfiles.Set_None), + stcp01t_data(ProtocolProfiles.DATAFEED_STCP, 17725, AuthType.ANONYMOUS, true, "Streaming", GroupSetProfiles.Set_01), + stcp2f_data(ProtocolProfiles.DATAFEED_STCP, 17726, AuthType.ANONYMOUS, false, "Streaming", GroupSetProfiles.Set_2), + tcp12_data(ProtocolProfiles.DATAFEED_TCP, 18749, AuthType.ANONYMOUS, null, "Streaming", GroupSetProfiles.Set_12), + tcp01t_data(ProtocolProfiles.DATAFEED_TCP, 18747, AuthType.ANONYMOUS, true, "Streaming", GroupSetProfiles.Set_01), + tcp2f_data(ProtocolProfiles.DATAFEED_TCP, 18748, AuthType.ANONYMOUS, false, "Streaming", GroupSetProfiles.Set_2), + udp01_data(ProtocolProfiles.DATAFEED_UDP, 18752, AuthType.ANONYMOUS, null, "Streaming", GroupSetProfiles.Set_01), + udp12t_data(ProtocolProfiles.DATAFEED_UDP, 18753, AuthType.ANONYMOUS, true, "Streaming", GroupSetProfiles.Set_12), + udp3f_data(ProtocolProfiles.DATAFEED_UDP, 18754, AuthType.ANONYMOUS, false, "Streaming", GroupSetProfiles.Set_3), + stcp_data(ProtocolProfiles.DATAFEED_STCP, 8088, AuthType.ANONYMOUS, null, "Streaming", GroupSetProfiles.Set_None), + stcp3_data(ProtocolProfiles.DATAFEED_STCP, 18720, AuthType.ANONYMOUS, null, "Streaming", GroupSetProfiles.Set_3), + mcast_data(ProtocolProfiles.DATAFEED_MCAST, 18734, AuthType.ANONYMOUS, null, "Streaming", GroupSetProfiles.Set_None, "239.2.3.1"), + saproxy_data(ProtocolProfiles.DATAFEED_MCAST, 6969, AuthType.ANONYMOUS, null, "Streaming", GroupSetProfiles.Set_None, "239.2.3.1"), + authssl_data(ProtocolProfiles.DATAFEED_TLS, 8090, AuthType.FILE, null, "Streaming", GroupSetProfiles.Set_None), + mcast01_data(ProtocolProfiles.DATAFEED_MCAST, 18755, AuthType.ANONYMOUS, null, "Streaming", GroupSetProfiles.Set_01, "239.2.3.1"), + udp_data(ProtocolProfiles.DATAFEED_UDP, 8087, AuthType.ANONYMOUS, null, "Streaming", GroupSetProfiles.Set_None), + ssl_data(ProtocolProfiles.DATAFEED_SSL, 8089, AuthType.ANONYMOUS, null, "Streaming", GroupSetProfiles.Set_None), + mcast12t_data(ProtocolProfiles.DATAFEED_MCAST, 18756, AuthType.ANONYMOUS, true, "Streaming", GroupSetProfiles.Set_12, "239.2.3.1"), + tls_data(ProtocolProfiles.DATAFEED_TLS, 8089, AuthType.ANONYMOUS, null, "Streaming", GroupSetProfiles.Set_None), + authtls_data(ProtocolProfiles.DATAFEED_TLS, 8090, AuthType.FILE, null, "Streaming", GroupSetProfiles.Set_None), + mcast3f_data(ProtocolProfiles.DATAFEED_MCAST, 18715, AuthType.ANONYMOUS, false, "Streaming", GroupSetProfiles.Set_3, "239.2.3.1"), + tcp_data(ProtocolProfiles.DATAFEED_TCP, 8087, AuthType.ANONYMOUS, null, "Streaming", GroupSetProfiles.Set_None); BaseConnections(@NotNull ProtocolProfiles protocol, @NotNull int port, @NotNull AuthType authType, @Nullable Boolean isAnon, @NotNull GroupSetProfiles groupSet, @NotNull String mcastGroup) { this.protocol = protocol; @@ -52,6 +78,7 @@ public enum BaseConnections { this.isAnon = isAnon; this.groupSet = groupSet; this.mcastGroup = mcastGroup; + this.type = null; } BaseConnections(@NotNull ProtocolProfiles protocol, @NotNull int port, @NotNull AuthType authType, @Nullable Boolean isAnon, @NotNull GroupSetProfiles groupSet) { @@ -61,6 +88,27 @@ public enum BaseConnections { this.isAnon = isAnon; this.groupSet = groupSet; this.mcastGroup = null; + this.type = null; + } + + BaseConnections(@NotNull ProtocolProfiles protocol, @NotNull int port, @NotNull AuthType authType, @Nullable Boolean isAnon, @NotNull String type, @NotNull GroupSetProfiles groupSet, @NotNull String mcastGroup) { + this.protocol = protocol; + this.port = port; + this.authType = authType; + this.isAnon = isAnon; + this.groupSet = groupSet; + this.mcastGroup = mcastGroup; + this.type = type; + } + + BaseConnections(@NotNull ProtocolProfiles protocol, @NotNull int port, @NotNull AuthType authType, @Nullable Boolean isAnon, @NotNull String type, @NotNull GroupSetProfiles groupSet) { + this.protocol = protocol; + this.port = port; + this.authType = authType; + this.isAnon = isAnon; + this.groupSet = groupSet; + this.mcastGroup = null; + this.type = type; } public static BaseConnections getByProtocolProfile(ProtocolProfilesInterface protocolProfile) { @@ -84,6 +132,9 @@ public static BaseConnections getByProtocolProfile(ProtocolProfilesInterface pro @Nullable private final String mcastGroup; + + @Nullable + private final String type; @NotNull public ProtocolProfiles getProtocol() { @@ -114,6 +165,11 @@ public GroupSetProfiles getGroupSet() { public String getMcastGroup() { return mcastGroup; } + + @Nullable + public String getType() { + return type; + } public boolean requiresAuthentication() { return AbstractConnection.requiresAuthentication(authType); diff --git a/src/takserver-takcl-core/src/core/java/com/bbn/marti/test/shared/data/connections/MutableConnection.java b/src/takserver-takcl-core/src/core/java/com/bbn/marti/test/shared/data/connections/MutableConnection.java index 2c2bdc54..1818dc42 100644 --- a/src/takserver-takcl-core/src/core/java/com/bbn/marti/test/shared/data/connections/MutableConnection.java +++ b/src/takserver-takcl-core/src/core/java/com/bbn/marti/test/shared/data/connections/MutableConnection.java @@ -76,6 +76,9 @@ public MutableConnection create() { @Nullable private final String mcastGroup; + @Nullable + private final String type; + @NotNull private final AbstractServerProfile serverProfile; @@ -128,6 +131,11 @@ public String getMCastGroup() { return mcastGroup; } + @Override + public String getType() { + return type; + } + @Override public String getConsistentUniqueReadableIdentifier() { return consistentUniqueReadableIdentifier; @@ -161,6 +169,7 @@ public MutableConnection(@NotNull ImmutableConnections immutableConnection, @Not this.originalRawAnonAccess = immutableConnection.getRawAnonAccessFlag(); this.rawAnonAccess = this.originalRawAnonAccess; this.mcastGroup = immutableConnection.getMCastGroup(); + this.type = immutableConnection.getType(); } public MutableConnection(@NotNull String identifier, @NotNull BaseConnections connectionModel, int port, @NotNull AbstractServerProfile serverProfile) { @@ -174,6 +183,7 @@ public MutableConnection(@NotNull String identifier, @NotNull BaseConnections co this.originalRawAnonAccess = connectionModel.getRawAnon(); this.rawAnonAccess = this.originalRawAnonAccess; this.mcastGroup = connectionModel.getMcastGroup(); + this.type = connectionModel.getType(); } public void removeFromGroup(@NotNull GroupProfiles group) { diff --git a/src/takserver-takcl-core/src/core/java/com/bbn/marti/test/shared/data/generated/ImmutableConnections.java b/src/takserver-takcl-core/src/core/java/com/bbn/marti/test/shared/data/generated/ImmutableConnections.java index 88e92089..1cf84fee 100644 --- a/src/takserver-takcl-core/src/core/java/com/bbn/marti/test/shared/data/generated/ImmutableConnections.java +++ b/src/takserver-takcl-core/src/core/java/com/bbn/marti/test/shared/data/generated/ImmutableConnections.java @@ -157,6 +157,39 @@ public class ImmutableConnections extends AbstractConnection { public static final ImmutableConnections sCLI_authstcpA = new ImmutableConnections("sCLI_authstcpA", ImmutableServerProfiles.SERVER_CLI, BaseConnections.authstcp, 17878, "sCLI_authstcpA_authuser012_012f", "sCLI_authstcpA_authuser_f", "sCLI_authstcpA_authuser01_01f", "NV_sCLI_authstcpA_authuser12_012f", "NV_sCLI_authstcpA_authuser01_01f", "NV_sCLI_authstcpA_authuser012_012f", "sCLI_authstcpA_authuser12_012f", "NV_sCLI_authstcpA_authuser3_3f", "NV_sCLI_authstcpA_authuser2_2f", "sCLI_authstcpA_authuser3_3f", "sCLI_authstcpA_authuser0_0f", "sCLI_authstcpA_authuser2_2f", "NV_sCLI_authstcpA_authuser0_0f", "NV_sCLI_authstcpA_authuser_f"); public static final ImmutableConnections s1_substcp = new ImmutableConnections("s1_substcp", ImmutableServerProfiles.SERVER_1, BaseConnections.substcp, 17793, "NV_s1_substcp_anonuser_t_B", "NV_s1_substcp_anonuser_t_A", "s1_substcp_anonuser_t", "s1_substcp_anonuser_t_A", "NV_s1_substcp_anonuser_t", "s1_substcp_anonuser_t_B"); public static final ImmutableConnections s2_saproxy = new ImmutableConnections("s2_saproxy", ImmutableServerProfiles.SERVER_2, BaseConnections.saproxy, 17801, "s2_saproxy_anonuser_t", "NV_s2_saproxy_anonuser_t", "s2_saproxy_anonuser_t_B", "s2_saproxy_anonuser_t_A", "NV_s2_saproxy_anonuser_t_A", "NV_s2_saproxy_anonuser_t_B"); + + // Data feed connections + public static final ImmutableConnections s0_stcp12_data = new ImmutableConnections("s0_stcp12_data", ImmutableServerProfiles.SERVER_0, BaseConnections.stcp12_data, 18743, "NV_s0_stcp12_anonuser_12f_B", "NV_s0_stcp12_anonuser_12f_A", "NV_s0_stcp12_anonuser_12f", "s0_stcp12_anonuser_12f", "s0_stcp12_anonuser_12f_A", "s0_stcp12_anonuser_12f_B"); + public static final ImmutableConnections s0_udp12t_data = new ImmutableConnections("s0_udp12t_data", ImmutableServerProfiles.SERVER_0, BaseConnections.udp12t_data, 18753, "s0_udp12t_anonuser_12t", "NV_s0_udp12t_anonuser_12t_A", "NV_s0_udp12t_anonuser_12t", "s0_udp12t_anonuser_12t_A", "s0_udp12t_anonuser_12t_B", "NV_s0_udp12t_anonuser_12t_B"); + public static final ImmutableConnections s0_stcp_data = new ImmutableConnections("s0_stcp_data", ImmutableServerProfiles.SERVER_0, BaseConnections.stcp_data, 18732, "NV_s0_stcp_anonuser_t_B", "NV_s0_stcp_anonuser_t_A", "s0_stcp_anonuser_t_A", "s0_stcp_anonuser_t_B", "s0_stcp_anonuser_t", "NV_s0_stcp_anonuser_t"); + public static final ImmutableConnections s0_tcp01t_data = new ImmutableConnections("s0_tcp01t_data", ImmutableServerProfiles.SERVER_0, BaseConnections.tcp01t_data, 18750, "s0_tcp01t_anonuser_01t_B", "s0_tcp01t_anonuser_01t_A", "NV_s0_tcp01t_anonuser_01t_A", "NV_s0_tcp01t_anonuser_01t_B", "s0_tcp01t_anonuser_01t", "NV_s0_tcp01t_anonuser_01t"); + public static final ImmutableConnections s0_stcp3_data = new ImmutableConnections("s0_stcp3_data", ImmutableServerProfiles.SERVER_0, BaseConnections.stcp3_data, 18762, "NV_s0_stcp3_anonuser_3f", "s0_stcp3_anonuser_3f", "NV_s0_stcp3_anonuser_3f_A", "s0_stcp3_anonuser_3f_B", "NV_s0_stcp3_anonuser_3f_B", "s0_stcp3_anonuser_3f_A"); + public static final ImmutableConnections s0_authstcpA_data = new ImmutableConnections("s0_authstcpA_data", ImmutableServerProfiles.SERVER_0, BaseConnections.authstcp_data, 18758, "NV_s0_authstcpA_authuser012_012f", "NV_s0_authstcpA_authuser01_01f", "s0_authstcpA_authuser01_01f", "NV_s0_authstcpA_authuser0_0f", "NV_s0_authstcpA_authuser_f", "NV_s0_authstcpA_authuser12_012f", "NV_s0_authstcpA_authuser2_2f", "s0_authstcpA_authuser_f", "s0_authstcpA_authuser12_012f", "s0_authstcpA_authuser012_012f", "NV_s0_authstcpA_authuser3_3f", "s0_authstcpA_authuser0_0f", "s0_authstcpA_authuser3_3f", "s0_authstcpA_authuser2_2f"); + public static final ImmutableConnections s0_authstcp_data = new ImmutableConnections("s0_authstcp_data", ImmutableServerProfiles.SERVER_0, BaseConnections.authstcp_data, 18746, "NV_s0_authstcp_authuser01_01f", "NV_s0_authstcp_authuser12_012f", "NV_s0_authstcp_authuser_f", "s0_authstcp_authuser_f", "s0_authstcp_authuser012_012f", "s0_authstcp_authuser01_01f", "NV_s0_authstcp_authuser0_0f", "NV_s0_authstcp_authuser2_2f", "NV_s0_authstcp_authuser3_3f", "NV_s0_authstcp_authuser012_012f", "s0_authstcp_authuser12_012f", "s0_authstcp_authuser0_0f", "s0_authstcp_authuser3_3f", "s0_authstcp_authuser2_2f"); + public static final ImmutableConnections s0_udp3f_data = new ImmutableConnections("s0_udp3f_data", ImmutableServerProfiles.SERVER_0, BaseConnections.udp3f_data, 18754, "NV_s0_udp3f_anonuser_3f_B", "s0_udp3f_anonuser_3f_A", "s0_udp3f_anonuser_3f_B", "NV_s0_udp3f_anonuser_3f", "NV_s0_udp3f_anonuser_3f_A", "s0_udp3f_anonuser_3f"); + public static final ImmutableConnections s0_udp01_data = new ImmutableConnections("s0_udp01_data", ImmutableServerProfiles.SERVER_0, BaseConnections.udp01_data, 18752, "s0_udp01_anonuser_01f", "s0_udp01_anonuser_01f_B", "s0_udp01_anonuser_01f_A", "NV_s0_udp01_anonuser_01f", "NV_s0_udp01_anonuser_01f_A", "NV_s0_udp01_anonuser_01f_B"); + public static final ImmutableConnections s0_stcp01_data = new ImmutableConnections("s0_stcp01_data", ImmutableServerProfiles.SERVER_0, BaseConnections.stcp01_data, 18742, "s0_stcp01_anonuser_01f_A", "s0_stcp01_anonuser_01f_B", "s0_stcp01_anonuser_01f", "NV_s0_stcp01_anonuser_01f", "NV_s0_stcp01_anonuser_01f_B", "NV_s0_stcp01_anonuser_01f_A"); + public static final ImmutableConnections s0_mcast_data = new ImmutableConnections("s0_mcast_data", ImmutableServerProfiles.SERVER_0, BaseConnections.mcast_data, 18734, "s0_mcast_anonuser_t_A", "NV_s0_mcast_anonuser_t", "s0_mcast_anonuser_t_B", "NV_s0_mcast_anonuser_t_B", "NV_s0_mcast_anonuser_t_A", "s0_mcast_anonuser_t"); + public static final ImmutableConnections s0_stcp2f_data = new ImmutableConnections("s0_stcp2f_data", ImmutableServerProfiles.SERVER_0, BaseConnections.stcp2f_data, 18748, "NV_s0_stcp2f_anonuser_2f_B", "s0_stcp2f_anonuser_2f", "NV_s0_stcp2f_anonuser_2f_A", "s0_stcp2f_anonuser_2f_A", "s0_stcp2f_anonuser_2f_B", "NV_s0_stcp2f_anonuser_2f"); + public static final ImmutableConnections s0_stcp01t_data = new ImmutableConnections("s0_stcp01t_data", ImmutableServerProfiles.SERVER_0, BaseConnections.stcp01t_data, 18747, "NV_s0_stcp01t_anonuser_01t_A", "NV_s0_stcp01t_anonuser_01t_B", "NV_s0_stcp01t_anonuser_01t", "s0_stcp01t_anonuser_01t", "s0_stcp01t_anonuser_01t_A", "s0_stcp01t_anonuser_01t_B"); + public static final ImmutableConnections s0_tcp12_data = new ImmutableConnections("s0_tcp12_data", ImmutableServerProfiles.SERVER_0, BaseConnections.tcp12_data, 18749, "s0_tcp12_anonuser_12f_B", "s0_tcp12_anonuser_12f_A", "s0_tcp12_anonuser_12f", "NV_s0_tcp12_anonuser_12f_A", "NV_s0_tcp12_anonuser_12f_B", "NV_s0_tcp12_anonuser_12f"); + public static final ImmutableConnections s0_saproxy_data = new ImmutableConnections("s0_saproxy_data", ImmutableServerProfiles.SERVER_0, BaseConnections.saproxy_data, 18735, "s0_saproxy_anonuser_t_A", "NV_s0_saproxy_anonuser_t_B", "NV_s0_saproxy_anonuser_t_A", "s0_saproxy_anonuser_t", "s0_saproxy_anonuser_t_B", "NV_s0_saproxy_anonuser_t"); + public static final ImmutableConnections s0_authsslA_data = new ImmutableConnections("s0_authsslA_data", ImmutableServerProfiles.SERVER_0, BaseConnections.authssl_data, 18740, "s0_authsslA_authuser3_3f", "NV_s0_authsslA_authuser_f", "NV_s0_authsslA_authuser01_01f", "s0_authsslA_authuser2_2f", "NV_s0_authsslA_authuser12_012f", "s0_authsslA_authuser01_01f", "s0_authsslA_authuser12_012f", "s0_authsslA_authuser0_0f", "s0_authsslA_authuser_f", "s0_authsslA_authuser012_012f", "NV_s0_authsslA_authuser2_2f", "NV_s0_authsslA_authuser012_012f", "NV_s0_authsslA_authuser3_3f", "NV_s0_authsslA_authuser0_0f"); + public static final ImmutableConnections s0_mcast01_data = new ImmutableConnections("s0_mcast01_data", ImmutableServerProfiles.SERVER_0, BaseConnections.mcast01_data, 18755, "NV_s0_mcast01_anonuser_01f", "NV_s0_mcast01_anonuser_01f_A", "NV_s0_mcast01_anonuser_01f_B", "s0_mcast01_anonuser_01f", "s0_mcast01_anonuser_01f_A", "s0_mcast01_anonuser_01f_B"); + public static final ImmutableConnections s0_udp_data = new ImmutableConnections("s0_udp_data", ImmutableServerProfiles.SERVER_0, BaseConnections.udp_data, 18731, "s0_udp_anonuser_t", "s0_udp_anonuser_t_A", "s0_udp_anonuser_t_B", "NV_s0_udp_anonuser_t_B", "NV_s0_udp_anonuser_t", "NV_s0_udp_anonuser_t_A"); + public static final ImmutableConnections s0_ssl_data = new ImmutableConnections("s0_ssl_data", ImmutableServerProfiles.SERVER_0, BaseConnections.ssl_data, 18737, "NV_s0_ssl_anonuser_t_A", "NV_s0_ssl_anonuser_t_B", "NV_s0_ssl_anonuser_t", "s0_ssl_anonuser_t", "s0_ssl_anonuser_t_B", "s0_ssl_anonuser_t_A"); + public static final ImmutableConnections s0_authssl_data = new ImmutableConnections("s0_authssl_data", ImmutableServerProfiles.SERVER_0, BaseConnections.authssl_data, 18739, "s0_authssl_authuser12_012f", "NV_s0_authssl_authuser012_012f", "s0_authssl_authuser3_3f", "s0_authssl_authuser_f", "NV_s0_authssl_authuser3_3f", "s0_authssl_authuser012_012f", "NV_s0_authssl_authuser01_01f", "NV_s0_authssl_authuser2_2f", "s0_authssl_authuser01_01f", "NV_s0_authssl_authuser0_0f", "NV_s0_authssl_authuser_f", "s0_authssl_authuser2_2f", "NV_s0_authssl_authuser12_012f", "s0_authssl_authuser0_0f"); + public static final ImmutableConnections s0_mcast12t_data = new ImmutableConnections("s0_mcast12t_data", ImmutableServerProfiles.SERVER_0, BaseConnections.mcast12t_data, 18756, "s0_mcast12t_anonuser_12t", "NV_s0_mcast12t_anonuser_12t_B", "NV_s0_mcast12t_anonuser_12t_A", "NV_s0_mcast12t_anonuser_12t", "s0_mcast12t_anonuser_12t_A", "s0_mcast12t_anonuser_12t_B"); + public static final ImmutableConnections s0_tls_data = new ImmutableConnections("s0_tls_data", ImmutableServerProfiles.SERVER_0, BaseConnections.tls_data, 18738, "NV_s0_tls_anonuser_t_B", "NV_s0_tls_anonuser_t_A", "NV_s0_tls_anonuser_t", "s0_tls_anonuser_t_B", "s0_tls_anonuser_t_A", "s0_tls_anonuser_t"); + public static final ImmutableConnections s0_authtls_data = new ImmutableConnections("s0_authtls_data", ImmutableServerProfiles.SERVER_0, BaseConnections.authtls_data, 18741, "s0_authtls_authuser12_012f", "NV_s0_authtls_authuser01_01f", "NV_s0_authtls_authuser_f", "s0_authtls_authuser01_01f", "s0_authtls_authuser0_0f", "s0_authtls_authuser3_3f", "s0_authtls_authuser2_2f", "NV_s0_authtls_authuser12_012f", "s0_authtls_authuser012_012f", "NV_s0_authtls_authuser2_2f", "NV_s0_authtls_authuser3_3f", "NV_s0_authtls_authuser012_012f", "s0_authtls_authuser_f", "NV_s0_authtls_authuser0_0f"); + public static final ImmutableConnections s0_stcpA_data = new ImmutableConnections("s0_stcpA_data", ImmutableServerProfiles.SERVER_0, BaseConnections.stcp_data, 18733, "s0_stcpA_anonuser_t", "s0_stcpA_anonuser_t_A", "s0_stcpA_anonuser_t_B", "NV_s0_stcpA_anonuser_t", "NV_s0_stcpA_anonuser_t_B", "NV_s0_stcpA_anonuser_t_A"); + public static final ImmutableConnections s0_mcast3f_data = new ImmutableConnections("s0_mcast3f_data", ImmutableServerProfiles.SERVER_0, BaseConnections.mcast3f_data, 18757, "NV_s0_mcast3f_anonuser_3f_B", "NV_s0_mcast3f_anonuser_3f_A", "NV_s0_mcast3f_anonuser_3f", "s0_mcast3f_anonuser_3f_A", "s0_mcast3f_anonuser_3f", "s0_mcast3f_anonuser_3f_B"); + public static final ImmutableConnections s0_tcp_data = new ImmutableConnections("s0_tcp_data", ImmutableServerProfiles.SERVER_0, BaseConnections.tcp_data, 18730, "s0_tcp_anonuser_t", "s0_tcp_anonuser_t_B", "s0_tcp_anonuser_t_A", "NV_s0_tcp_anonuser_t_B", "NV_s0_tcp_anonuser_t_A", "NV_s0_tcp_anonuser_t"); + public static final ImmutableConnections s0_saproxyA_data = new ImmutableConnections("s0_saproxyA_data", ImmutableServerProfiles.SERVER_0, BaseConnections.saproxy_data, 18736, "s0_saproxyA_anonuser_t", "s0_saproxyA_anonuser_t_B", "s0_saproxyA_anonuser_t_A", "NV_s0_saproxyA_anonuser_t", "NV_s0_saproxyA_anonuser_t_B", "NV_s0_saproxyA_anonuser_t_A"); + public static final ImmutableConnections s0_stcp0_data = new ImmutableConnections("s0_stcp0_data", ImmutableServerProfiles.SERVER_0, BaseConnections.stcp0_data, 18744, "s0_stcp0_anonuser_0f", "NV_s0_stcp0_anonuser_0f_B", "NV_s0_stcp0_anonuser_0f_A", "s0_stcp0_anonuser_0f_B", "s0_stcp0_anonuser_0f_A", "NV_s0_stcp0_anonuser_0f"); + public static final ImmutableConnections s0_tcp2f_data = new ImmutableConnections("s0_tcp2f_data", ImmutableServerProfiles.SERVER_0, BaseConnections.tcp2f_data, 18751, "NV_s0_tcp2f_anonuser_2f", "NV_s0_tcp2f_anonuser_2f_A", "NV_s0_tcp2f_anonuser_2f_B", "s0_tcp2f_anonuser_2f", "s0_tcp2f_anonuser_2f_A", "s0_tcp2f_anonuser_2f_B"); + + ////////////////////////// BD6A4745-76B4-42DD-AE35-5C2251DD6301 // End Generated Users ////////////////////////// @@ -184,6 +217,8 @@ public class ImmutableConnections extends AbstractConnection { private final BaseConnections connectionModel; @Nullable private final String mcastGroup; + @Nullable + private final String type; private String[] genUsersStrings; public BaseConnections getConnectionModel() { @@ -199,6 +234,7 @@ private ImmutableConnections(@NotNull String identifier, @NotNull ImmutableServe this.groupSet = modelInput.getGroupSet(); this.isAnon = modelInput.getRawAnon(); this.mcastGroup = modelInput.getMcastGroup(); + this.type = modelInput.getType(); this.genUsersStrings = genUserNameList; this.connectionModel = modelInput; @@ -298,6 +334,11 @@ public String getDynamicName() { return consistentUniqueReadableIdentifier; } + @Override + public String getType() { + return type; + } + @Override public AbstractServerProfile getServer() { return server; diff --git a/src/takserver-takcl-core/src/core/java/com/bbn/marti/test/shared/data/protocols/ProtocolProfiles.java b/src/takserver-takcl-core/src/core/java/com/bbn/marti/test/shared/data/protocols/ProtocolProfiles.java index 27d2307c..2bcc9957 100644 --- a/src/takserver-takcl-core/src/core/java/com/bbn/marti/test/shared/data/protocols/ProtocolProfiles.java +++ b/src/takserver-takcl-core/src/core/java/com/bbn/marti/test/shared/data/protocols/ProtocolProfiles.java @@ -23,7 +23,13 @@ public enum ProtocolProfiles implements ProtocolProfilesInterface, Comparator getInputs() { + public List getInputs() { return configuration.getNetwork().getInput(); } + + public List getDataFeeds() { + return configuration.getNetwork().getDatafeed(); + } public void saveChanges() { try { @@ -161,10 +176,10 @@ public boolean isFileAuthEnabled() { return (configuration.getAuth().getFile() != null); } - public void addInput(final Network.Input input) { - List inputList = configuration.getNetwork().getInput(); + public void addInput(final Input input) { + List inputList = configuration.getNetwork().getInput(); - for (Network.Input loopInput : inputList) { + for (Input loopInput : inputList) { if (loopInput.getName().equals(input.getName())) { inputList.remove(loopInput); break; @@ -173,16 +188,29 @@ public void addInput(final Network.Input input) { inputList.add(input); saveChanges(); } + + public void addDataFeed(final DataFeed dataFeed) { + List dataFeedList = configuration.getNetwork().getDatafeed(); + + for (DataFeed loopFeed : dataFeedList) { + if (loopFeed.getName().equals(dataFeed.getName())) { + dataFeedList.remove(loopFeed); + break; + } + } + dataFeedList.add(dataFeed); + saveChanges(); + } public void addConnectionIfNecessary(@NotNull AbstractConnection connection) { ProtocolProfiles.ConnectionType connectionType = connection.getConnectionType(); if (connectionType == ProtocolProfiles.ConnectionType.INPUT) { Set inputNameSet = new HashSet<>(); - for (Network.Input loopinput : getInputs()) { + for (Input loopinput : getInputs()) { inputNameSet.add(loopinput.getName()); } - Network.Input input = connection.getConfigInput(); + Input input = connection.getConfigInput(); if (!inputNameSet.contains(input.getName())) { addInput(input); if (connection.getProtocol().isTLS()) { @@ -208,6 +236,22 @@ public void addConnectionIfNecessary(@NotNull AbstractConnection connection) { } } + } else if (connectionType == ProtocolProfiles.ConnectionType.DATAFEED){ + Set feedNameSet = new HashSet<>(); + for (DataFeed loopFeed : getDataFeeds()) { + feedNameSet.add(loopFeed.getName()); + } + DataFeed dataFeed = connection.getConfigDataFeed(); + if (!feedNameSet.contains(dataFeed.getName())) { + addDataFeed(dataFeed); + if (connection.getProtocol().isTLS()) { + setSSLSecuritySettings(); + } + } + + if (connection.getAuthType() == AuthType.FILE) { + fileAuth(true); + } } else { throw new RuntimeException("Invalid ConnectionType of type '" + connection.getConsistentUniqueReadableIdentifier() + "'!"); } @@ -238,9 +282,9 @@ public List getStaticSubscriptions() { */ @Command public void removeInput(final String inputName) { - List inputList = configuration.getNetwork().getInput(); + List inputList = configuration.getNetwork().getInput(); - for (Network.Input input : inputList) { + for (Input input : inputList) { if (input.getName().equals(inputName)) { inputList.remove(input); break; diff --git a/src/takserver-takcl-core/src/main/java/com/bbn/marti/takcl/AppModules/OnlineInputModule.java b/src/takserver-takcl-core/src/main/java/com/bbn/marti/takcl/AppModules/OnlineInputModule.java index 7e833d05..afc418d3 100644 --- a/src/takserver-takcl-core/src/main/java/com/bbn/marti/takcl/AppModules/OnlineInputModule.java +++ b/src/takserver-takcl-core/src/main/java/com/bbn/marti/takcl/AppModules/OnlineInputModule.java @@ -1,11 +1,16 @@ package com.bbn.marti.takcl.AppModules; -import com.bbn.marti.config.Network; +import java.util.Collection; + +import org.jetbrains.annotations.NotNull; + +import com.bbn.marti.config.Input; import com.bbn.marti.remote.InputMetric; import com.bbn.marti.remote.groups.ConnectionModifyResult; import com.bbn.marti.remote.groups.NetworkInputAddResult; -import com.bbn.marti.takcl.AppModules.generic.ServerAppModuleInterface; +import com.bbn.marti.remote.service.InputManager; import com.bbn.marti.takcl.TakclIgniteHelper; +import com.bbn.marti.takcl.AppModules.generic.ServerAppModuleInterface; import com.bbn.marti.takcl.cli.simple.Command; import com.bbn.marti.takcl.config.common.TakclRunMode; import com.bbn.marti.test.shared.data.connections.AbstractConnection; @@ -14,10 +19,6 @@ import com.bbn.marti.test.shared.data.servers.AbstractServerProfile; import com.bbn.marti.test.shared.data.servers.ImmutableServerProfiles; import com.bbn.marti.test.shared.data.templates.ImmutableConnectionsTemplate_471E9257; -import org.jetbrains.annotations.NotNull; -import com.bbn.marti.remote.service.InputManager; - -import java.util.Collection; /** * Used to modify the server's inputs through an online remote interface. Will not work if there is no server to connect to. @@ -75,7 +76,7 @@ public void halt() { */ @Command(description = "Adds a network input with the specified getConsistentUniqueReadableIdentifier, protocol, and port to the server.") public NetworkInputAddResult add(@NotNull ProtocolProfiles protocol, @NotNull String port, @NotNull String name) { - Network.Input input = new Network.Input(); + Input input = new Input(); input.setName(name); input.setProtocol(protocol.getValue()); input.setPort(Integer.parseInt(port)); @@ -87,7 +88,7 @@ public NetworkInputAddResult add(@NotNull ProtocolProfiles protocol, @NotNull St return inputManager.createInput(input); } - public NetworkInputAddResult add(Network.Input input) { + public NetworkInputAddResult add(Input input) { return inputManager.createInput(input); } @@ -118,6 +119,10 @@ public void add(BaseConnections input) { public void add(AbstractConnection networkInput) { inputManager.createInput(networkInput.getConfigInput()); } + + public void addDataFeed(AbstractConnection networkInput) { + inputManager.createDataFeed(networkInput.getConfigDataFeed()); + } /** * Removes a predefined input from the server @@ -152,9 +157,20 @@ public void listPredefinedInputs() { public void remove(String name) { inputManager.deleteInput(name); } + + /** + * Removes the dataFeed with the specified getConsistentUniqueReadableIdentifier + * + * @param name The getConsistentUniqueReadableIdentifier of the data feed to remove + * @command + */ + @Command(description = "Removes the data feed with the specified getConsistentUniqueReadableIdentifier.") + public void removeDataFeed(String name) { + inputManager.deleteDataFeed(name); + } public Collection getInputMetricList() { - return inputManager.getInputMetrics(); + return inputManager.getInputMetrics(false); } /** @@ -169,7 +185,7 @@ public String getList() { String printString = ""; for (InputMetric inputMetric : inputMetrics) { - Network.Input input = inputMetric.getInput(); + Input input = inputMetric.getInput(); String groupString = null; for (String group : input.getFiltergroup()) { @@ -196,8 +212,8 @@ public String getList() { // } // public ConnectionModifyResult addInputToGroup(@NotNull String inputIdentifier, @NotNull String groupName) { - Collection inputMetrics = inputManager.getInputMetrics(); - Network.Input modInput = null; + Collection inputMetrics = inputManager.getInputMetrics(false); + Input modInput = null; for (InputMetric im : inputMetrics) { if (im.getInput().getName().equals(inputIdentifier)) { @@ -214,8 +230,8 @@ public ConnectionModifyResult addInputToGroup(@NotNull String inputIdentifier, @ } public ConnectionModifyResult removeInputFromGroup(@NotNull String inputIdentifier, @NotNull String groupName) { - Collection inputMetrics = inputManager.getInputMetrics(); - Network.Input modInput = null; + Collection inputMetrics = inputManager.getInputMetrics(false); + Input modInput = null; for (InputMetric im : inputMetrics) { if (im.getInput().getName().equals(inputIdentifier)) { diff --git a/src/takserver-takcl-core/src/main/java/com/bbn/marti/takcl/SSLHelper.java b/src/takserver-takcl-core/src/main/java/com/bbn/marti/takcl/SSLHelper.java index de7896ba..20948893 100644 --- a/src/takserver-takcl-core/src/main/java/com/bbn/marti/takcl/SSLHelper.java +++ b/src/takserver-takcl-core/src/main/java/com/bbn/marti/takcl/SSLHelper.java @@ -326,6 +326,7 @@ private static void produceCerts(@NotNull String certType, @NotNull Path makeCer File cwd = makeCertPath.getParent().toFile(); ProcessBuilder pb = new ProcessBuilder(cmd); pb.directory(cwd); + pb.inheritIO(); pb.environment().put("COUNTRY", "US"); pb.environment().put("STATE", "MA"); pb.environment().put("CITY", "Cambridge"); diff --git a/src/takserver-takcl-core/src/main/java/com/bbn/marti/takcl/connectivity/RunnableLocalServer.java b/src/takserver-takcl-core/src/main/java/com/bbn/marti/takcl/connectivity/RunnableLocalServer.java index 20f99ed8..ca47d416 100644 --- a/src/takserver-takcl-core/src/main/java/com/bbn/marti/takcl/connectivity/RunnableLocalServer.java +++ b/src/takserver-takcl-core/src/main/java/com/bbn/marti/takcl/connectivity/RunnableLocalServer.java @@ -1,20 +1,25 @@ package com.bbn.marti.takcl.connectivity; -import com.bbn.marti.config.Network; -import com.bbn.marti.takcl.TAKCLCore; -import com.bbn.marti.takcl.TestExceptions; -import com.bbn.marti.test.shared.data.protocols.ProtocolProfiles; -import com.bbn.marti.test.shared.data.servers.AbstractServerProfile; -import com.bbn.marti.test.shared.data.servers.ImmutableServerProfiles; -import org.jetbrains.annotations.Nullable; - import java.io.Console; import java.io.File; import java.io.IOException; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import org.jetbrains.annotations.Nullable; + +import com.bbn.marti.config.Input; +import com.bbn.marti.config.Network; +import com.bbn.marti.takcl.TAKCLCore; +import com.bbn.marti.test.shared.data.protocols.ProtocolProfiles; +import com.bbn.marti.test.shared.data.servers.AbstractServerProfile; +import com.bbn.marti.test.shared.data.servers.ImmutableServerProfiles; public class RunnableLocalServer extends AbstractRunnableServer { @@ -136,15 +141,15 @@ protected void innerStopServer() { protected void innerConfigureServer(@Nullable String sessionIdentifier, boolean enableRemoteDebug) { // Removing default inputs from servers other than SERVER_0 since they will cause bind conflicts if (!serverIdentifier.getConsistentUniqueReadableIdentifier().equals(ImmutableServerProfiles.SERVER_0.getConsistentUniqueReadableIdentifier())) { - List inputList = new LinkedList<>(this.getOfflineConfigModule().getInputs()); - for (Network.Input input : inputList) { + List inputList = new LinkedList<>(this.getOfflineConfigModule().getInputs()); + for (Input input : inputList) { if ((input.getPort() == 8088 && input.getName().equals("streamtcp")) || (input.getPort() == 8087 && (input.getName().equals("stdudp") || input.getName().equals("stdtcp")))) { this.getOfflineConfigModule().removeInput(input.getName()); } } } else { - for (Network.Input input : this.getOfflineConfigModule().getInputs()) { + for (Input input : this.getOfflineConfigModule().getInputs()) { Integer networkVersion = ProtocolProfiles.getInputByValue(input.getProtocol()).getCoreNetworkVersion(); if (networkVersion != null) { input.setCoreVersion(networkVersion); @@ -165,15 +170,6 @@ protected void innerDeployServer(@Nullable String sessionIdentifier, long startT Map additionalEnvironmentVariables = new HashMap<>(); - -// try { -// ProcessBuilder psProcessBuilder = new ProcessBuilder("ps", "-aux"); -// initFilepaths(sessionIdentifier, startTimeMs, "psaux", psProcessBuilder); -// psProcessBuilder.start().waitFor(); -// } catch (IOException | InterruptedException e) { -// throw new RuntimeException(e); -// } - List profiles; if (TAKCLCore.useMonolithProfile) { profiles = new ArrayList<>(1); diff --git a/src/takserver-takcl-core/src/main/java/com/bbn/marti/takcl/connectivity/implementations/SendingClient.java b/src/takserver-takcl-core/src/main/java/com/bbn/marti/takcl/connectivity/implementations/SendingClient.java index 102688c7..11029d08 100644 --- a/src/takserver-takcl-core/src/main/java/com/bbn/marti/takcl/connectivity/implementations/SendingClient.java +++ b/src/takserver-takcl-core/src/main/java/com/bbn/marti/takcl/connectivity/implementations/SendingClient.java @@ -84,6 +84,7 @@ private boolean sendMessage(@NotNull Document doc, boolean omitXmlDeclaration) { switch (user.getConnection().getProtocol()) { case INPUT_TCP: + case DATAFEED_TCP: sendTCPMessage(user.getServer().getUrl(), user.getConnection().getPort(), xmlData); // dl.begin("stateChangeListener.onMessageSent"); stateChangeListener.onMessageSent(xmlData); @@ -91,6 +92,7 @@ private boolean sendMessage(@NotNull Document doc, boolean omitXmlDeclaration) { break; case INPUT_UDP: + case DATAFEED_UDP: try { sendUDPMessage(user.getServer().getUrl(), user.getConnection().getPort(), udpPort, xmlData); // dl.begin("stateChangeListener.onMessageSent"); @@ -102,6 +104,7 @@ private boolean sendMessage(@NotNull Document doc, boolean omitXmlDeclaration) { break; case INPUT_MCAST: + case DATAFEED_MCAST: try { // dl.begin("Construct DatagramPacket"); DatagramPacket data = new DatagramPacket(xmlData.getBytes(), xmlData.length(), diff --git a/src/takserver-takcl-core/src/main/java/com/bbn/marti/takcl/connectivity/missions/MissionModels.java b/src/takserver-takcl-core/src/main/java/com/bbn/marti/takcl/connectivity/missions/MissionModels.java index 72240913..994726e1 100644 --- a/src/takserver-takcl-core/src/main/java/com/bbn/marti/takcl/connectivity/missions/MissionModels.java +++ b/src/takserver-takcl-core/src/main/java/com/bbn/marti/takcl/connectivity/missions/MissionModels.java @@ -729,7 +729,8 @@ public enum MissionUserPermission { MISSION_DELETE, // Can read, write, and delete mission data MISSION_SET_ROLE, // Can set user roles MISSION_SET_PASSWORD, // Can set the mission password - MISSION_UPDATE_GROUPS // Can update the mission groups + MISSION_UPDATE_GROUPS, // Can update the mission groups + MISSION_MANAGE_FEEDS // TODO: Add tests? } // Server Admin: Default owner role for everything. diff --git a/src/takserver-takcl-core/src/main/java/com/bbn/marti/test/shared/engines/ActionEngine.java b/src/takserver-takcl-core/src/main/java/com/bbn/marti/test/shared/engines/ActionEngine.java index 4ca6ab07..71de3610 100644 --- a/src/takserver-takcl-core/src/main/java/com/bbn/marti/test/shared/engines/ActionEngine.java +++ b/src/takserver-takcl-core/src/main/java/com/bbn/marti/test/shared/engines/ActionEngine.java @@ -410,6 +410,17 @@ public void onlineRemoveInputAndVerify(@NotNull AbstractConnection input) { System.out.println("Removing input \"" + input.getConsistentUniqueReadableIdentifier() + "\"(" + tTot + "s)."); sleep(SLEEP_ADDREMOVE_INPUT); } + + @Override + public void onlineRemoveDataFeedAndVerify(@NotNull AbstractConnection dataFeed) { + data.engineIterationDataClear(); + double t0 = System.currentTimeMillis(); + serverManager.getServerInstance(dataFeed.getServer()).getOnlineInputModule().removeDataFeed(dataFeed.getConsistentUniqueReadableIdentifier()); + double t1 = System.currentTimeMillis(); + double tTot = (t1 - t0) / 1000; + System.out.println("Removing data feed \"" + dataFeed.getConsistentUniqueReadableIdentifier() + "\"(" + tTot + "s)."); + sleep(SLEEP_ADDREMOVE_INPUT); + } @Override public void attemptSendFromUserAndVerify(@NotNull AbstractUser sendingUser, @NotNull AbstractUser... targetUsers) { @@ -539,6 +550,23 @@ public void onlineAddInput(@NotNull AbstractConnection input) { sleep(SLEEP_TIME_GROUP_MANIPULATION); } } + + @Override + public void onlineAddDataFeed(@NotNull AbstractConnection dataFeed) { + data.engineIterationDataClear(); + double t0 = System.currentTimeMillis(); + serverManager.getServerInstance(dataFeed.getServer()).getOnlineInputModule().addDataFeed(dataFeed); + double t1 = System.currentTimeMillis(); + double tTot = (t1 - t0) / 1000; + serverManager.getServerInstance(dataFeed.getServer()).getOnlineInputModule().addDataFeed(dataFeed); + System.out.println("Added input \"" + dataFeed.getConsistentUniqueReadableIdentifier() + "\"(" + tTot + "s)."); + sleep(SLEEP_ADDREMOVE_INPUT); + + for (GroupProfiles group : dataFeed.getGroupSet().getGroups()) { + serverManager.getServerInstance(dataFeed.getServer()).getOnlineInputModule().addInputToGroup(dataFeed.getConsistentUniqueReadableIdentifier(), group.name()); + sleep(SLEEP_TIME_GROUP_MANIPULATION); + } + } @Override public void startServer(@NotNull AbstractServerProfile server, @NotNull String sessionIdentifier) { @@ -599,6 +627,15 @@ public void offlineAddSubscriptionFromInputToServer(@NotNull AbstractConnection serverManager.getServerInstance(serverProvidingSubscription).getOfflineConfigModule().addStaticSubscription(targetInput.generateMatchingStaticSubscription()); System.out.println("Static subsciption added to server '" + serverProvidingSubscription + "' for input '" + targetInput + "'."); } + + @Override + public void offlineAddSubscriptionFromDataFeedToServer(@NotNull AbstractConnection targetDataFeed, @NotNull AbstractServerProfile serverProvidingSubscription) { + data.engineIterationDataClear(); + AbstractServerProfile targetServer = targetDataFeed.getServer(); + serverManager.getServerInstance(targetServer).getOfflineConfigModule().addDataFeed(targetDataFeed.getConfigDataFeed()); + serverManager.getServerInstance(serverProvidingSubscription).getOfflineConfigModule().addStaticSubscription(targetDataFeed.generateMatchingStaticSubscription()); + System.out.println("Static subsciption added to server '" + serverProvidingSubscription + "' for data feed '" + targetDataFeed + "'."); + } @Override public void offlineFederateServers(boolean useV1Federation, boolean useV2Federation, @NotNull AbstractServerProfile... serversToFederate) { diff --git a/src/takserver-takcl-core/src/main/java/com/bbn/marti/test/shared/engines/EngineInterface.java b/src/takserver-takcl-core/src/main/java/com/bbn/marti/test/shared/engines/EngineInterface.java index d35a55c4..0a6261e2 100644 --- a/src/takserver-takcl-core/src/main/java/com/bbn/marti/test/shared/engines/EngineInterface.java +++ b/src/takserver-takcl-core/src/main/java/com/bbn/marti/test/shared/engines/EngineInterface.java @@ -27,6 +27,8 @@ public interface EngineInterface { void disconnectClientAndVerify(@NotNull AbstractUser disconnectingUser); void onlineRemoveInputAndVerify(@NotNull AbstractConnection input); + + void onlineRemoveDataFeedAndVerify(@NotNull AbstractConnection dataFeed); void attemptSendFromUserAndVerify(@NotNull AbstractUser sendingUser, @NotNull AbstractUser... targetUsers); @@ -41,6 +43,8 @@ public interface EngineInterface { void authenticateAndVerifyClient(@NotNull AbstractUser users); void onlineAddInput(@NotNull AbstractConnection input); + + void onlineAddDataFeed(@NotNull AbstractConnection dataFeed); void startServer(@NotNull AbstractServerProfile server, @NotNull String sessionIdentifier); @@ -56,6 +60,8 @@ public interface EngineInterface { void offlineAddSubscriptionFromInputToServer(@NotNull AbstractConnection targetInput, @NotNull AbstractServerProfile serverProvidingSubscription); + void offlineAddSubscriptionFromDataFeedToServer(AbstractConnection targetDataFeed, AbstractServerProfile serverProvidingSubscription); + void offlineFederateServers(boolean useV1Federation, boolean useV2Federation, @NotNull AbstractServerProfile... serversToFederate); void offlineAddOutboundFederateConnection(boolean useV2Federation, @NotNull AbstractServerProfile sourceServer, @NotNull AbstractServerProfile targetServer); diff --git a/src/takserver-takcl-core/src/main/java/com/bbn/marti/test/shared/engines/TestEngine.java b/src/takserver-takcl-core/src/main/java/com/bbn/marti/test/shared/engines/TestEngine.java index d5e6c7b0..25e113cd 100644 --- a/src/takserver-takcl-core/src/main/java/com/bbn/marti/test/shared/engines/TestEngine.java +++ b/src/takserver-takcl-core/src/main/java/com/bbn/marti/test/shared/engines/TestEngine.java @@ -125,6 +125,16 @@ public void offlineAddSubscriptionFromInputToServer(@NotNull AbstractConnection stateEngine.offlineAddSubscriptionFromInputToServer(targetInput, serverProvidingSubscription); } } + + @Override + public void offlineAddSubscriptionFromDataFeedToServer(@NotNull AbstractConnection targetInput, @NotNull AbstractServerProfile serverProvidingSubscription) { + if (!TAKCLCore.useRunningServer) { + TestLogger.executeEngineCommand("offlineAddSubscriptionFromInputToServer"); + actionEngine.offlineAddSubscriptionFromDataFeedToServer(targetInput, serverProvidingSubscription); + verificationEngine.offlineAddSubscriptionFromDataFeedToServer(targetInput, serverProvidingSubscription); + stateEngine.offlineAddSubscriptionFromDataFeedToServer(targetInput, serverProvidingSubscription); + } + } @Override public void offlineFederateServers(boolean useV1Federation, boolean useV2Federation, @NotNull AbstractServerProfile... serversToFederate) { @@ -204,7 +214,14 @@ public synchronized void onlineRemoveInputAndVerify(@NotNull AbstractConnection verificationEngine.onlineRemoveInputAndVerify(input); stateEngine.onlineRemoveInputAndVerify(input); } - + + @Override + public synchronized void onlineRemoveDataFeedAndVerify(@NotNull AbstractConnection dataFeed) { + TestLogger.executeEngineCommand("onlineRemoveDataFeedAndVerify"); + actionEngine.onlineRemoveDataFeedAndVerify(dataFeed); + verificationEngine.onlineRemoveDataFeedAndVerify(dataFeed); + stateEngine.onlineRemoveDataFeedAndVerify(dataFeed); + } @Override public synchronized void onlineAddInput(@NotNull AbstractConnection input) { @@ -214,6 +231,14 @@ public synchronized void onlineAddInput(@NotNull AbstractConnection input) { stateEngine.onlineAddInput(input); } + @Override + public synchronized void onlineAddDataFeed(@NotNull AbstractConnection input) { + TestLogger.executeEngineCommand("onlineAddDataFeed"); + actionEngine.onlineAddDataFeed(input); + verificationEngine.onlineAddDataFeed(input); + stateEngine.onlineAddDataFeed(input); + } + @Override /** diff --git a/src/takserver-takcl-core/src/main/java/com/bbn/marti/test/shared/engines/state/StateEngine.java b/src/takserver-takcl-core/src/main/java/com/bbn/marti/test/shared/engines/state/StateEngine.java index 51ec7f93..ca073fdd 100644 --- a/src/takserver-takcl-core/src/main/java/com/bbn/marti/test/shared/engines/state/StateEngine.java +++ b/src/takserver-takcl-core/src/main/java/com/bbn/marti/test/shared/engines/state/StateEngine.java @@ -75,6 +75,14 @@ public void onlineRemoveInputAndVerify(@NotNull AbstractConnection input) { // data.clearTestIterationData(); data.updateState(); } + + @Override + public void onlineRemoveDataFeedAndVerify(@NotNull AbstractConnection dataFeed) { + // TODO: Validate validate validate!! Check. Maybe add a thread for delayed checking? + data.getState(dataFeed).setInactiveInDeployment(); +// data.clearTestIterationData(); + data.updateState(); + } @Override public void attemptSendFromUserAndVerify(@NotNull AbstractUser sendingUser, @NotNull AbstractUser... targetUsers) { @@ -127,6 +135,13 @@ public void onlineAddInput(@NotNull AbstractConnection input) { data.getState(input).setActiveInDeployment(); data.updateState(); } + + @Override + public void onlineAddDataFeed(@NotNull AbstractConnection dataFeed) { + // TODO: Validate validate validate!! Check + data.getState(dataFeed).setActiveInDeployment(); + data.updateState(); + } @Override public void startServer(@NotNull AbstractServerProfile server, @NotNull String sessionIdentifier) { @@ -176,6 +191,12 @@ public void offlineAddSubscriptionFromInputToServer(@NotNull AbstractConnection data.getState(serverProvidingSubscription).addConnectionSubscriptionTarget(targetInput); data.updateState(); } + + @Override + public void offlineAddSubscriptionFromDataFeedToServer(@NotNull AbstractConnection targetInput, @NotNull AbstractServerProfile serverProvidingSubscription) { + data.getState(serverProvidingSubscription).addConnectionSubscriptionTarget(targetInput); + data.updateState(); + } @Override public void offlineFederateServers(boolean useV1Federation, boolean useV2Federation, @NotNull AbstractServerProfile... serversToFederate) { diff --git a/src/takserver-takcl-core/src/main/java/com/bbn/marti/test/shared/engines/verification/VerificationEngine.java b/src/takserver-takcl-core/src/main/java/com/bbn/marti/test/shared/engines/verification/VerificationEngine.java index 82c31a06..05e80b2e 100644 --- a/src/takserver-takcl-core/src/main/java/com/bbn/marti/test/shared/engines/verification/VerificationEngine.java +++ b/src/takserver-takcl-core/src/main/java/com/bbn/marti/test/shared/engines/verification/VerificationEngine.java @@ -249,6 +249,11 @@ public void onlineRemoveInputAndVerify(@NotNull AbstractConnection input) { data.validateAllUserExpectations("InputRemoved"); data.engineIterationDataClear(); } + + @Override + public void onlineRemoveDataFeedAndVerify(@NotNull AbstractConnection dataFeed) { + onlineRemoveInputAndVerify(dataFeed); + } @Override public void attemptSendFromUserAndVerify(@NotNull AbstractUser sendingUser, @NotNull AbstractUser... targetUsers) { @@ -312,13 +317,22 @@ public void authenticateAndVerifyClient(@NotNull AbstractUser user) { @Override public void offlineAddSubscriptionFromInputToServer(@NotNull AbstractConnection targetInput, @NotNull AbstractServerProfile serverProvidingSubscription) { data.engineIterationDataClear(); - + } + + @Override + public void offlineAddSubscriptionFromDataFeedToServer(@NotNull AbstractConnection targetInput, @NotNull AbstractServerProfile serverProvidingSubscription) { + data.engineIterationDataClear(); } @Override public void onlineAddInput(@NotNull AbstractConnection input) { data.engineIterationDataClear(); } + + @Override + public void onlineAddDataFeed(@NotNull AbstractConnection dataFeed) { + data.engineIterationDataClear(); + } @Override public void startServer(@NotNull AbstractServerProfile server, @NotNull String sessionIdentifier) { diff --git a/src/testing/grpc-streaming-cot-client/src/main/java/grpc/GrpcClientInputTest.java b/src/testing/grpc-streaming-cot-client/src/main/java/grpc/GrpcClientInputTest.java index 21391d2c..a94cc78c 100644 --- a/src/testing/grpc-streaming-cot-client/src/main/java/grpc/GrpcClientInputTest.java +++ b/src/testing/grpc-streaming-cot-client/src/main/java/grpc/GrpcClientInputTest.java @@ -2,6 +2,7 @@ import static io.grpc.MethodDescriptor.generateFullMethodName; +import java.util.Date; import java.io.FileInputStream; import java.security.KeyStore; import java.util.concurrent.BlockingQueue; @@ -20,7 +21,16 @@ import com.google.common.base.Strings; import atakmap.commoncommo.protobuf.v1.Takmessage; +import atakmap.commoncommo.protobuf.v1.ContactOuterClass.Contact; +import atakmap.commoncommo.protobuf.v1.Cotevent.CotEvent; +import atakmap.commoncommo.protobuf.v1.DetailOuterClass.Detail; +import atakmap.commoncommo.protobuf.v1.GroupOuterClass.Group; +import atakmap.commoncommo.protobuf.v1.Precisionlocation.PrecisionLocation; +import atakmap.commoncommo.protobuf.v1.StatusOuterClass.Status; import atakmap.commoncommo.protobuf.v1.Takmessage.TakMessage; +import atakmap.commoncommo.protobuf.v1.Takmessage.TakMessage.Builder; +import atakmap.commoncommo.protobuf.v1.TakvOuterClass.Takv; +import atakmap.commoncommo.protobuf.v1.TrackOuterClass.Track; import io.grpc.ClientCall; import io.grpc.ManagedChannel; import io.grpc.Metadata; @@ -61,14 +71,14 @@ public class GrpcClientInputTest { // TODO EDIT THESE TO FIT YOUR ENV private final String host = "localhost"; - private final int port = 30001; + private final int port = 8079; private final String[] tlsVersions = { "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" }; // path to user cert to use for x509 auth: - private final String keystoreFile = ""; - private final String keystorePassword = ""; + private final String keystoreFile = "admin.jks" ; + private final String keystorePassword = "atakatak"; // path to trust store: - private final String truststoreFile = ""; - private final String truststorePassword = ""; + private final String truststoreFile = "truststore-root.jks"; + private final String truststorePassword = "atakatak"; // Bounded Executor pool for grpc input server and channel builders private final BlockingQueue workQueue = new BlockingArrayQueue<>(0, 1, 1000); @@ -105,8 +115,8 @@ public void testGrpc() throws Exception { openSendStream(); openReceiveStream(); - while (true) { - } + // keep the process running + while (true) {} } public void openSendStream() { @@ -130,6 +140,8 @@ public void onMessage(ClientSubscription message) { @Override public void onReady() { + // send in an SA once the connection is good + clientCall.sendMessage(cot2protoBuf()); } }, new Metadata()); @@ -145,7 +157,8 @@ public void openReceiveStream() { @Override public void onNext(Takmessage.TakMessage value) { // ECHO - clientCall.sendMessage(value); + // clientCall.sendMessage(value); + System.out.println(value); } @Override @@ -201,4 +214,51 @@ private SslContext buildClientSslContext() throws Exception { .trustManager(trustMgrFactory) .build(); } + + public TakMessage cot2protoBuf() { + try { + CotEvent.Builder cotEventBuilder = CotEvent.newBuilder(); + cotEventBuilder.setType("a-f-G-U-C-I"); + cotEventBuilder.setUid("GLIDER-909"); + cotEventBuilder.setHow("h-g-i-g-o"); + cotEventBuilder.setSendTime(new Date().getTime()); + cotEventBuilder.setStartTime(new Date().getTime()); + cotEventBuilder.setStaleTime(new Date().getTime()); + + cotEventBuilder.setLat(41.644598); + cotEventBuilder.setLon(-70.610919); + cotEventBuilder.setHae(9999999.0); + cotEventBuilder.setCe(9999999.0); + cotEventBuilder.setLe(9999999.0); + + Contact.Builder contactBuilder = Contact.newBuilder(); + contactBuilder.setCallsign("glider909"); + contactBuilder.setEndpoint("*:-1:stcp"); + Contact contact = contactBuilder.build(); + Detail.Builder detailBuilder = Detail.newBuilder(); + detailBuilder.setContact(contact); + + Group.Builder groupBuilder = Group.newBuilder(); + groupBuilder.setName("Magenta"); + groupBuilder.setRole("Team Member"); + + Group group = groupBuilder.build(); + detailBuilder.setGroup(group); + + + Detail detail = detailBuilder.build(); + cotEventBuilder.setDetail(detail); + + CotEvent cotEvent = cotEventBuilder.build(); + + TakMessage.Builder takMessageBuilder = TakMessage.newBuilder(); + takMessageBuilder.setCotEvent(cotEvent); + TakMessage takMessage = takMessageBuilder.build(); + + return takMessage; + } catch (Exception e) { + System.out.println("exception in cot2protoBuf!" + e); + return null; + } + } } diff --git a/src/testing/grpc-streaming-cot-client/src/main/proto/cotevent.proto b/src/testing/grpc-streaming-cot-client/src/main/proto/cotevent.proto index 2d04d513..536ef6c8 100644 --- a/src/testing/grpc-streaming-cot-client/src/main/proto/cotevent.proto +++ b/src/testing/grpc-streaming-cot-client/src/main/proto/cotevent.proto @@ -39,5 +39,9 @@ message CotEvent { // This is optional - if omitted, then the cot message // had no data under Detail detail = 15; + + uint64 submissionTime = 16; + uint64 creationTime = 17; + } diff --git a/src/testing/grpc-streaming-cot-client/src/main/proto/fig.proto b/src/testing/grpc-streaming-cot-client/src/main/proto/fig.proto index 18f7f331..2e102983 100644 --- a/src/testing/grpc-streaming-cot-client/src/main/proto/fig.proto +++ b/src/testing/grpc-streaming-cot-client/src/main/proto/fig.proto @@ -35,7 +35,7 @@ message FederatedEvent { } message GeoEvent { - int64 sendTime = 1; + int64 sendTime = 1; int64 startTime = 2; int64 staleTime = 3; double lat = 4; // WSG-84 @@ -58,7 +58,7 @@ message GeoEvent { double course = 21; BinaryBlob binary = 22; //image repeated string ptpUids = 23; - repeated string ptpCallsigns = 24; // ugh...shouldn't need this + repeated string ptpCallsigns = 24; } message BinaryBlob { @@ -89,6 +89,7 @@ message Identity { message Subscription { Identity identity = 1; string filter = 2; + TakServerVersion version = 3; } message ClientHealth { @@ -113,9 +114,18 @@ message ServerHealth { message ROL { string program = 1; repeated BinaryBlob payload = 2; + repeated string federateGroups = 3; } message FederateGroups { ServerHealth streamUpdate = 1; repeated string federateGroups = 2; } + +message TakServerVersion { + int64 major = 1; // major version number + int64 minor = 2; // minor version number + int64 patch = 3; // patch version number + string branch = 4; // branch name + string variant = 5; // variant name +} diff --git a/src/testing/grpc-streaming-cot-client/src/main/proto/message.proto b/src/testing/grpc-streaming-cot-client/src/main/proto/message.proto index 432def3f..9ccada30 100644 --- a/src/testing/grpc-streaming-cot-client/src/main/proto/message.proto +++ b/src/testing/grpc-streaming-cot-client/src/main/proto/message.proto @@ -8,10 +8,23 @@ package atakmap.commoncommo.protobuf.v1; message Message { TakMessage payload = 1; - string source = 2; + string source = 2; + + string clientId = 3; - repeated string groups = 3; + repeated string groups = 4; + + repeated string destClientUids = 5; + + repeated string destCallsigns = 6; + + repeated string provenance = 7; + + bool archive = 8; + + string feedUuid = 9; + + string connectionId = 10; - string clientId = 4; } diff --git a/src/testing/load_test/base_config.yml b/src/testing/load_test/base_config.yml index e73d0fdb..23271e70 100644 --- a/src/testing/load_test/base_config.yml +++ b/src/testing/load_test/base_config.yml @@ -7,7 +7,7 @@ connection: udp: 8087 authentication: - cert: "./certs/my_cert.p12" + cert: "./certs/ser_cert.p12" password: "atakatak" # if no password, use empty string diff --git a/src/testing/load_test/base_config_plugin_datafeed_load_test.yml b/src/testing/load_test/base_config_plugin_datafeed_load_test.yml new file mode 100644 index 00000000..cc96bf69 --- /dev/null +++ b/src/testing/load_test/base_config_plugin_datafeed_load_test.yml @@ -0,0 +1,45 @@ +# Command: python3 tak_tester.py --config --test-pytak +# Command: python3 tak_tester.py --config --test-pytak-proto + +connection: + host: "172.31.83.222" + tls: 8089 + https: 8443 + udp: 8087 + +authentication: + cert: "/home/centos/user.p12" + password: "atakatak" # if no password, use empty string + + +PyTAK: + self_sa_delta: 3600 # seconds + offset: 1.0 # time between starting clients + clients: 400 + + websocket_path: "takproto/1" + + missions: + random: False + subscribe: + - mission_datafeed_load_test + send_mission_cot: False + mission_write_interval: 20 # seconds + +Missions: + creatorUid: PyTAK-1 + group: __ANON__ + + missions: + - mission_datafeed_load_test: + description: "Load test datafeed!" + #datafeed_uuids: + # - "feedUUID_loadtest_1" + # - "feedUUID_loadtest_2" + # - "feedUUID_loadtest_3" + datafeed_uuid_pattern: + pattern: feedUUID_loadtest_[i] + start_i: 1 + end_i: 50 + + diff --git a/src/testing/load_test/mission_api.py b/src/testing/load_test/mission_api.py index 52741d1b..2416d13e 100644 --- a/src/testing/load_test/mission_api.py +++ b/src/testing/load_test/mission_api.py @@ -128,6 +128,20 @@ def get_mission_cot(self, mission): if response.status_code != 200: print(("ERROR: ", response.text)) return + + def get_client_endpoints(self): + + cep_url = self.base_url + "api/clientEndPoints" + + #print("client endpoints url: " + cep_url) + + response = self.get(cep_url) + + #print(("clientEndPoints response: ", response.text)) + + if response.status_code != 200: + print(("ERROR: ", response.text)) + return def create_mission(self, mission_name, creator_uid=None, @@ -218,6 +232,13 @@ def unsubscribe_from_mission(self, mission_name, subscriber_uid=None): response = self.delete(unsubscribe_url, params=params) return response + def add_datafeed_to_mission(self, mission_name, datafeed_uuid, creator_uid): + my_url = self.base_mission_api + mission_name + "/feed" + params = {'creatorUid': creator_uid, 'dataFeedUid': datafeed_uuid} + + response = self.post(my_url, params=params) + return response + ########### End Mission Information/Subscription ##################### ############## Enterprise Sync ############################################# @@ -445,6 +466,29 @@ def initialize_takserver(self, config_dict): for f in attrs.get("files"): self.add_mission_content(name, content_hash=self.file_hash_map.get(f)) + # Add datafeed to mission: + if "datafeed_uuids" in attrs: + print("Adding datafeeds to mission " + name) + for datafeed_uuid in attrs.get("datafeed_uuids"): + self.add_datafeed_to_mission(name, datafeed_uuid = datafeed_uuid, creator_uid=attrs.get("creatorUid", creatorUid)) + print("Added datafeed " + datafeed_uuid + " to mission " + name) + print("Done adding datafeeds to mission " + name) + print() + if "datafeed_uuid_pattern" in attrs: + pattern = attrs.get("datafeed_uuid_pattern").get("pattern") + start_i = attrs.get("datafeed_uuid_pattern").get("start_i") + end_i = attrs.get("datafeed_uuid_pattern").get("end_i") + if (pattern is None or start_i is None or end_i is None): + print("ERROR: missing params in datafeed_uuid_pattern") + exit(1) + print("Adding datafeeds with pattern " + pattern + " to mission " + name) + for i in range(int(start_i), int(end_i)+1): + datafeed_uuid = pattern.replace("[i]", str(i)) + self.add_datafeed_to_mission(name, datafeed_uuid = datafeed_uuid, creator_uid=attrs.get("creatorUid", creatorUid)) + print("Added datafeed " + datafeed_uuid + " to mission " + name) + print("Done adding datafeeds with pattern " + pattern + " to mission" + name) + + def cleanup(self): print("cleaning up initialization data on server") self.delete_all_test_files() @@ -562,6 +606,7 @@ def get_and_subscribe_to_mission(self, mission_name): c['data']['downloaded'] = True def make_requests(self): + if self.do_write: if len(self.all_missions) == 0: self.get_all_missions() @@ -671,13 +716,3 @@ def subscribe_to_mission(self, mission_name, subscriber_uid=None): return True -# if __name__ == "__main__": -# -# cert = "certs/takserver.p12" -# password = "atakatak" -# host = "3.236.246.198" -# port = 8443 -# m = MissionApiSession(host, port, certfile=cert, password=password, uid="load-test-12") -# -# pprint(m.search_files()) -# m.delete_all_test_files() diff --git a/src/testing/load_test/pyTakStreamingCot.py b/src/testing/load_test/pyTakStreamingCot.py index 11c9e892..9a6e619a 100644 --- a/src/testing/load_test/pyTakStreamingCot.py +++ b/src/testing/load_test/pyTakStreamingCot.py @@ -95,15 +95,17 @@ def __init__(self, async def read_handler(self, reader): while True: data = await reader.readuntil(b'') + self.logger.debug("Client receives an event message") if self.mission_config.get("react_to_change_message", False): await self.read_socket.send(data) - # connection_data = self.data_dict[self.uid] - # connection_data['read'] += 1 - # connection_data['bytes'] += len(data) - # self.data_dict[self.uid] = connection_data - + connection_data = self.data_dict[self.uid] + connection_data['read'] += 1 + #connection_data['bytes'] += len(data) + self.data_dict[self.uid] = connection_data + async def send_self_sa(self, writer): + self.logger.debug("Client sends self_sa message") self_sa_message = CotMessage(uid=self.uid, lat=str(self.location[0]), lon=str(self.location[1])) self_sa_message.add_callsign_detail(group_name="Red", platform="PyTAKStreamingCot") writer.write(self_sa_message.to_string()) @@ -111,6 +113,7 @@ async def send_self_sa(self, writer): self.last_sa_write = time.time() async def send_mission_cot(self, writer): + self.logger.debug("Client sending mission_cot") for mission in self.data_sync_sess.missions_subscribed: if self.mission_cot_prob < random.uniform(0, 1): continue @@ -220,6 +223,10 @@ async def connect_socket(self): for i in range(POOL_SIZE): self.read_pool.apply_async(read_thread_zmq, args=(port, data_sync_port)) await asyncio.sleep(1) + + + # get list of recent client connect / disconnect events + self.data_sync_sess.get_client_endpoints() read_task = asyncio.ensure_future(self.read_handler(reader)) write_task = asyncio.ensure_future(self.write_handler(writer)) @@ -284,7 +291,7 @@ def read_thread_zmq(port: int, data_sync_port): class PyTAKStreamingCotProcess(multiprocessing.Process): def __init__(self, address=None, uid=None, cert=None, password=None, self_sa_delta=5.0, - mission_config=None, data_dict=None): + mission_config=None, data_dict=None, debug=None): multiprocessing.Process.__init__(self) self.address = address self.uid = uid @@ -293,6 +300,7 @@ def __init__(self, address=None, uid=None, cert=None, password=None, self_sa_del self.self_sa_delta = self_sa_delta self.mission_config = mission_config self.data_dict = data_dict + self.debug = debug self.agent = None def run(self): @@ -303,7 +311,8 @@ def run(self): password=self.password, self_sa_delta=self.self_sa_delta, mission_config=self.mission_config, - data_dict=self.data_dict) + data_dict=self.data_dict, + debug=self.debug) if self.uid is None: self.uid = self.agent.uid asyncio.get_event_loop().run_until_complete(self.agent.connect_socket()) diff --git a/src/testing/load_test/pyTakStreamingProto.py b/src/testing/load_test/pyTakStreamingProto.py index e60ddcb9..16700bf4 100644 --- a/src/testing/load_test/pyTakStreamingProto.py +++ b/src/testing/load_test/pyTakStreamingProto.py @@ -111,7 +111,6 @@ async def negotiate_protocol(self, reader, writer): self.logger.debug(data) - msg = CotMessage(msg=data) if msg.server_protocol_version_support() == '1': response = CotMessage(uid=msg.uid) @@ -133,38 +132,43 @@ async def read_handler(self, reader): else: data = await reader.read(reader._limit) while True: - if self.mission_config.get("react_to_change_message", False): - if leftovers is None: - fullbuffer = data - else: - fullbuffer = leftovers + data - leftovers = None - while len(fullbuffer) > 0: - if msg_size == 0: # get the message size - first_byte = fullbuffer[0].to_bytes(1, byteorder="big") - if first_byte != b'\xbf': - raise Exception("Magic byte is wrong: " + str(first_byte)) - msg_size = get_msg_size(fullbuffer[1:]) - if not msg_size: - leftovers = fullbuffer - break - if len(fullbuffer) < msg_size: + + if leftovers is None: + fullbuffer = data + else: + fullbuffer = leftovers + data + leftovers = None + while len(fullbuffer) > 0: + if msg_size == 0: # get the message size + first_byte = fullbuffer[0].to_bytes(1, byteorder="big") + if first_byte != b'\xbf': + raise Exception("Magic byte is wrong: " + str(first_byte)) + msg_size = get_msg_size(fullbuffer[1:]) + if not msg_size: leftovers = fullbuffer break + if len(fullbuffer) < msg_size: + leftovers = fullbuffer + break - next_msg = fullbuffer[:msg_size] - fullbuffer = fullbuffer[msg_size:] + next_msg = fullbuffer[:msg_size] + fullbuffer = fullbuffer[msg_size:] + + if self.mission_config.get("react_to_change_message", False): await self.read_socket.send(next_msg) - leftovers = None - msg_size = 0 + + connection_data = self.data_dict[self.uid] + connection_data['read'] += 1 + #connection_data['bytes'] += len(data) + self.data_dict[self.uid] = connection_data + + leftovers = None + msg_size = 0 + data = await reader.read(reader._limit) if data == b'': await asyncio.sleep(0.0001) - #connection_data = self.data_dict[self.uid] - #connection_data['read'] += count - #connection_data['bytes'] += len(data) - #self.data_dict[self.uid] = connection_data async def send_self_sa(self, writer): self_sa_message = CotProtoMessage(uid=self.uid, lat=str(self.location[0]), lon=str(self.location[1])) @@ -249,8 +253,6 @@ async def write_handler(self, writer): await self.send_self_sa(writer) elif write_action == "mission_cot": await self.send_mission_cot(writer) - # elif write_action == "data_sync": - # await asyncio.get_event_loop().run_in_executor(self.pool, self.data_sync_sess.make_requests) await asyncio.get_event_loop().run_in_executor(self.pool, partial(data_sync_thread.join, timeout=0.1)) if not data_sync_thread.is_alive(): raise RuntimeError("There was a mission api exception") @@ -289,6 +291,9 @@ async def connect_socket(self): self.read_pool.apply_async(read_thread_zmq, args=(port, data_sync_port)) await asyncio.sleep(1) + # get list of recent client connect / disconnect events + self.data_sync_sess.get_client_endpoints() + read_task = asyncio.ensure_future(self.read_handler(reader)) write_task = asyncio.ensure_future(self.write_handler(writer)) diff --git a/src/testing/load_test/pyTakWebsocket.py b/src/testing/load_test/pyTakWebsocket.py index 6f6cbfae..5840b24d 100644 --- a/src/testing/load_test/pyTakWebsocket.py +++ b/src/testing/load_test/pyTakWebsocket.py @@ -166,6 +166,9 @@ async def connect_websocket(self): for i in range(POOL_SIZE): self.read_pool.apply_async(read_thread_zmq, args=(port, data_sync_port)) await asyncio.sleep(1) + + # get list of recent client connect / disconnect events + self.data_sync_sess.get_client_endpoints() read_task = asyncio.ensure_future(self.read_handler(ws)) write_task = asyncio.ensure_future(self.write_handler(ws)) diff --git a/src/testing/load_test/requirements.txt b/src/testing/load_test/requirements.txt index 20cda8b0..26fa76a5 100644 --- a/src/testing/load_test/requirements.txt +++ b/src/testing/load_test/requirements.txt @@ -1,8 +1,10 @@ -lxml==4.2.5 +lxml==4.6.3 protobuf==3.11.3 pyOpenSSL==19.1.0 PyYAML==5.1 -requests==2.19.1 -urllib3==1.23 +requests==2.26 +urllib3==1.26 websockets -pyzmq~=19.0.2 \ No newline at end of file +pyzmq~=19.0.2 +boto3==1.18 + diff --git a/src/testing/load_test/tak_tester.py b/src/testing/load_test/tak_tester.py index 99dc6491..82fbda39 100755 --- a/src/testing/load_test/tak_tester.py +++ b/src/testing/load_test/tak_tester.py @@ -35,7 +35,7 @@ def test_mission_api(host, port, cert, password, verbose=True): logger.info("subcribing to a mission?...") files = {'file': open('Mission-API.pdf', 'rb')} - response = sess.post(enterprise_sync + "upload", params={"name": "test_file.pdf", "creatorUid": "test_User"}, files=files, headers={"Content-Type": "application/pdf"}) + response = sess.post(enterprise_sync + "upload", params={"name": "test_file.pdf", "creatorUid": "test"}, files=files, headers={"Content-Type": "application/pdf"}) logger.info(pformat(response)) @@ -160,6 +160,7 @@ def test_websocket_proto(host, port, websocket_path, cert, password, for p in procs: p.join() exit(0) + logger.info("Done with setting up " + str(clients) + " clients") while True: try: @@ -246,7 +247,8 @@ def test_streaming_cot(host, port, cert, password, sequential_uids=True, mission_config=None, mission_port=None, - mission_api_config=None): + mission_api_config=None, + debug=False): if sequential_uids: uids = ["PyTAK-%04d" % i for i in range(clients)] else: @@ -286,7 +288,8 @@ def test_streaming_cot(host, port, cert, password, uid=uids[i], self_sa_delta=self_sa_delta, mission_config=mission_api_config, - data_dict=data_dict) + data_dict=data_dict, + debug = debug) procs.append(p) p.start() time.sleep(offset) @@ -514,7 +517,8 @@ def test_udp(host, port, interval=0.25, track_uid=None): sequential_uids=args.sequential_uids, mission_config=config.get("Missions"), mission_port=https, - mission_api_config=mission_api_config) + mission_api_config=mission_api_config, + debug = args.verbose) if args.test_pytak_proto: logger.info("") diff --git a/src/testing/load_test/utils.py b/src/testing/load_test/utils.py index 1cb51665..b183b12f 100644 --- a/src/testing/load_test/utils.py +++ b/src/testing/load_test/utils.py @@ -91,12 +91,12 @@ def print_data_dict(connection_data): total_clients = len(connection_data) connected_clients = 0 num_writes = 0 - #num_reads = 0 + num_reads = 0 #num_bytes = 0 for client_data in connection_data.values(): if client_data['connected']: connected_clients += 1 - #num_reads += client_data['read'] + num_reads += client_data['read'] num_writes += client_data['write'] #num_bytes += client_data['bytes'] # try: @@ -115,13 +115,16 @@ def print_data_dict(connection_data): # prev_data['bytes'] = num_bytes + diff_reads = num_reads - prev_data['read'] + prev_data['read'] = num_reads + diff_writes = num_writes - prev_data['write'] - prev_data['write'] = num_writes + prev_data['write'] = num_writes # ; num_reads/client: {reads:>10}; num_bytes: {bytes:>10}" \ - return "clients connected: {conn:>3} / {tot:>3}; num_writes: {writes:>10}" \ + return "clients connected: {conn:>3} / {tot:>3}; num_writes: {writes:>10}; num_reads: {reads:>10} " \ .format(conn=connected_clients, tot=total_clients, - writes=diff_writes)#, - #reads=diff_reads, + writes=diff_writes, + reads=diff_reads) #, #bytes=diff_bytes) \ No newline at end of file diff --git a/src/testing/netty-streaming-cot-client/README.md b/src/testing/netty-streaming-cot-client/README.md new file mode 100644 index 00000000..ed6d4509 --- /dev/null +++ b/src/testing/netty-streaming-cot-client/README.md @@ -0,0 +1,9 @@ +## Setup +You'll need a TAK Server with a grpc input similiar to the following: + + +## Build +./gradlew shadowjar + +## Run +java -jar build/libs/nettytestclient-1.0-uber.jar \ No newline at end of file diff --git a/src/testing/netty-streaming-cot-client/build.gradle b/src/testing/netty-streaming-cot-client/build.gradle new file mode 100644 index 00000000..884daa99 --- /dev/null +++ b/src/testing/netty-streaming-cot-client/build.gradle @@ -0,0 +1,62 @@ +apply plugin: 'java' +apply plugin: 'maven-publish' +apply plugin: 'idea' +apply plugin: 'eclipse' +apply plugin: 'application' + +mainClassName = 'netty.NettyClientInputTest' + +apply plugin: 'com.github.johnrengelman.shadow' + + +buildscript { + repositories { + mavenCentral() + maven { + url 'https://plugins.gradle.org/m2' + } + } + dependencies { + classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.8' + classpath 'com.github.jengelman.gradle.plugins:shadow:6.1.0' + } +} + +//apply plugin: 'com.github.johnrengelman.shadow' + +repositories { + mavenCentral() + mavenLocal() + jcenter() +} +dependencies { + + compile group: 'org.eclipse.jetty', name: 'jetty-util', version: '9.4.10.RC0' + + + compile 'io.netty:netty-tcnative-boringssl-static:' + boringssl_static_version + + compile group: 'io.netty', name: 'netty-all', version: netty_epoll_version + + compile group: 'io.netty', name: 'netty-transport-native-epoll', version: netty_epoll_version, classifier: 'linux-x86_64' + + implementation group: 'com.google.guava', name: 'guava', version: '11.0.2' + + + compile 'javax.annotation:javax.annotation-api:1.3.2' + + // for netty performance, put javassist on the classpath + compile group: 'org.javassist', name: 'javassist', version: javassist_version +} + + +shadowJar { + baseName = 'nettytestclient' + classifier = 'uber' + version = '1.0' +} + +clean { + delete 'bin' +} + diff --git a/src/testing/netty-streaming-cot-client/gradle.properties b/src/testing/netty-streaming-cot-client/gradle.properties new file mode 100644 index 00000000..491bb141 --- /dev/null +++ b/src/testing/netty-streaming-cot-client/gradle.properties @@ -0,0 +1,5 @@ +javassist_version = 3.24.1-GA +boringssl_static_version = 2.0.30.Final +netty_epoll_version = 4.1.58.Final + + diff --git a/src/testing/netty-streaming-cot-client/gradle/wrapper/gradle-wrapper.jar b/src/testing/netty-streaming-cot-client/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 00000000..87b738cb Binary files /dev/null and b/src/testing/netty-streaming-cot-client/gradle/wrapper/gradle-wrapper.jar differ diff --git a/src/testing/netty-streaming-cot-client/gradle/wrapper/gradle-wrapper.properties b/src/testing/netty-streaming-cot-client/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..a4f0001d --- /dev/null +++ b/src/testing/netty-streaming-cot-client/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-6.4.1-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/src/testing/netty-streaming-cot-client/gradlew b/src/testing/netty-streaming-cot-client/gradlew new file mode 100755 index 00000000..af6708ff --- /dev/null +++ b/src/testing/netty-streaming-cot-client/gradlew @@ -0,0 +1,172 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/src/testing/netty-streaming-cot-client/gradlew.bat b/src/testing/netty-streaming-cot-client/gradlew.bat new file mode 100644 index 00000000..0f8d5937 --- /dev/null +++ b/src/testing/netty-streaming-cot-client/gradlew.bat @@ -0,0 +1,84 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/src/testing/netty-streaming-cot-client/settings.gradle b/src/testing/netty-streaming-cot-client/settings.gradle new file mode 100644 index 00000000..41611a4a --- /dev/null +++ b/src/testing/netty-streaming-cot-client/settings.gradle @@ -0,0 +1,3 @@ +rootProject.name = 'grpc-streaming' + + diff --git a/src/testing/netty-streaming-cot-client/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslClientContext.java b/src/testing/netty-streaming-cot-client/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslClientContext.java new file mode 100644 index 00000000..f266cf24 --- /dev/null +++ b/src/testing/netty-streaming-cot-client/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslClientContext.java @@ -0,0 +1,350 @@ +/* + * Copyright 2016 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package io.netty.handler.ssl; + +import io.netty.internal.tcnative.CertificateCallback; +import io.netty.util.internal.SuppressJava6Requirement; +import io.netty.util.internal.SystemPropertyUtil; +import io.netty.util.internal.logging.InternalLogger; +import io.netty.util.internal.logging.InternalLoggerFactory; +import io.netty.internal.tcnative.SSL; +import io.netty.internal.tcnative.SSLContext; + +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Set; + +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLException; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509ExtendedTrustManager; +import javax.net.ssl.X509TrustManager; +import javax.security.auth.x500.X500Principal; + +/** + * A client-side {@link SslContext} which uses OpenSSL's SSL/TLS implementation. + *

    Instances of this class must be {@link #release() released} or else native memory will leak! + * + *

    Instances of this class must not be released before any {@link ReferenceCountedOpenSslEngine} + * which depends upon the instance of this class is released. Otherwise if any method of + * {@link ReferenceCountedOpenSslEngine} is called which uses this class's JNI resources the JVM may crash. + */ +public final class ReferenceCountedOpenSslClientContext extends ReferenceCountedOpenSslContext { + private static final InternalLogger logger = + InternalLoggerFactory.getInstance(ReferenceCountedOpenSslClientContext.class); + private static final Set SUPPORTED_KEY_TYPES = Collections.unmodifiableSet(new LinkedHashSet( + Arrays.asList(OpenSslKeyMaterialManager.KEY_TYPE_RSA, + OpenSslKeyMaterialManager.KEY_TYPE_DH_RSA, + OpenSslKeyMaterialManager.KEY_TYPE_EC, + OpenSslKeyMaterialManager.KEY_TYPE_EC_RSA, + OpenSslKeyMaterialManager.KEY_TYPE_EC_EC))); + private static final boolean ENABLE_SESSION_TICKET = + SystemPropertyUtil.getBoolean("jdk.tls.client.enableSessionTicketExtension", false); + private final OpenSslSessionContext sessionContext; + + ReferenceCountedOpenSslClientContext(X509Certificate[] trustCertCollection, TrustManagerFactory trustManagerFactory, + X509Certificate[] keyCertChain, PrivateKey key, String keyPassword, + KeyManagerFactory keyManagerFactory, Iterable ciphers, + CipherSuiteFilter cipherFilter, ApplicationProtocolConfig apn, + String[] protocols, long sessionCacheSize, long sessionTimeout, + boolean enableOcsp, String keyStore) throws SSLException { + super(ciphers, cipherFilter, apn, sessionCacheSize, sessionTimeout, SSL.SSL_MODE_CLIENT, keyCertChain, + ClientAuth.NONE, protocols, false, enableOcsp, true); + boolean success = false; + try { + sessionContext = newSessionContext(this, ctx, engineMap, trustCertCollection, trustManagerFactory, + keyCertChain, key, keyPassword, keyManagerFactory, keyStore); + if (ENABLE_SESSION_TICKET) { + sessionContext.setTicketKeys(); + } + success = true; + } finally { + if (!success) { + release(); + } + } + } + + @Override + public OpenSslSessionContext sessionContext() { + return sessionContext; + } + + static OpenSslSessionContext newSessionContext(ReferenceCountedOpenSslContext thiz, long ctx, + OpenSslEngineMap engineMap, + X509Certificate[] trustCertCollection, + TrustManagerFactory trustManagerFactory, + X509Certificate[] keyCertChain, PrivateKey key, + String keyPassword, KeyManagerFactory keyManagerFactory, + String keyStore) throws SSLException { + if (key == null && keyCertChain != null || key != null && keyCertChain == null) { + throw new IllegalArgumentException( + "Either both keyCertChain and key needs to be null or none of them"); + } + OpenSslKeyMaterialProvider keyMaterialProvider = null; + try { + try { + if (!OpenSsl.useKeyManagerFactory()) { + if (keyManagerFactory != null) { + throw new IllegalArgumentException( + "KeyManagerFactory not supported"); + } + if (keyCertChain != null/* && key != null*/) { + setKeyMaterial(ctx, keyCertChain, key, keyPassword); + } + } else { + // javadocs state that keyManagerFactory has precedent over keyCertChain + if (keyManagerFactory == null && keyCertChain != null) { + char[] keyPasswordChars = keyStorePassword(keyPassword); + KeyStore ks = buildKeyStore(keyCertChain, key, keyPasswordChars, keyStore); + if (ks.aliases().hasMoreElements()) { + keyManagerFactory = new OpenSslX509KeyManagerFactory(); + } else { + keyManagerFactory = new OpenSslCachingX509KeyManagerFactory( + KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm())); + } + keyManagerFactory.init(ks, keyPasswordChars); + keyMaterialProvider = providerFor(keyManagerFactory, keyPassword); + } else if (keyManagerFactory != null) { + keyMaterialProvider = providerFor(keyManagerFactory, keyPassword); + } + + if (keyMaterialProvider != null) { + OpenSslKeyMaterialManager materialManager = new OpenSslKeyMaterialManager(keyMaterialProvider); + SSLContext.setCertificateCallback(ctx, new OpenSslClientCertificateCallback( + engineMap, materialManager)); + } + } + } catch (Exception e) { + throw new SSLException("failed to set certificate and key", e); + } + + // On the client side we always need to use SSL_CVERIFY_OPTIONAL (which will translate to SSL_VERIFY_PEER) + // to ensure that when the TrustManager throws we will produce the correct alert back to the server. + // + // See: + // - https://www.openssl.org/docs/man1.0.2/man3/SSL_CTX_set_verify.html + // - https://github.com/netty/netty/issues/8942 + SSLContext.setVerify(ctx, SSL.SSL_CVERIFY_OPTIONAL, VERIFY_DEPTH); + + try { + if (trustCertCollection != null) { + trustManagerFactory = buildTrustManagerFactory(trustCertCollection, trustManagerFactory, keyStore); + } else if (trustManagerFactory == null) { + trustManagerFactory = TrustManagerFactory.getInstance( + TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init((KeyStore) null); + } + final X509TrustManager manager = chooseTrustManager(trustManagerFactory.getTrustManagers()); + + // IMPORTANT: The callbacks set for verification must be static to prevent memory leak as + // otherwise the context can never be collected. This is because the JNI code holds + // a global reference to the callbacks. + // + // See https://github.com/netty/netty/issues/5372 + + setVerifyCallback(ctx, engineMap, manager); + } catch (Exception e) { + if (keyMaterialProvider != null) { + keyMaterialProvider.destroy(); + } + throw new SSLException("unable to setup trustmanager", e); + } + OpenSslClientSessionContext context = new OpenSslClientSessionContext(thiz, keyMaterialProvider); + keyMaterialProvider = null; + return context; + } finally { + if (keyMaterialProvider != null) { + keyMaterialProvider.destroy(); + } + } + } + + @SuppressJava6Requirement(reason = "Guarded by java version check") + private static void setVerifyCallback(long ctx, OpenSslEngineMap engineMap, X509TrustManager manager) { + // Use this to prevent an error when running on java < 7 + if (useExtendedTrustManager(manager)) { + SSLContext.setCertVerifyCallback(ctx, + new ExtendedTrustManagerVerifyCallback(engineMap, (X509ExtendedTrustManager) manager)); + } else { + SSLContext.setCertVerifyCallback(ctx, new TrustManagerVerifyCallback(engineMap, manager)); + } + } + + // No cache is currently supported for client side mode. + static final class OpenSslClientSessionContext extends OpenSslSessionContext { + OpenSslClientSessionContext(ReferenceCountedOpenSslContext context, OpenSslKeyMaterialProvider provider) { + super(context, provider); + } + + @Override + public void setSessionTimeout(int seconds) { + if (seconds < 0) { + throw new IllegalArgumentException(); + } + } + + @Override + public int getSessionTimeout() { + return 0; + } + + @Override + public void setSessionCacheSize(int size) { + if (size < 0) { + throw new IllegalArgumentException(); + } + } + + @Override + public int getSessionCacheSize() { + return 0; + } + + @Override + public void setSessionCacheEnabled(boolean enabled) { + // ignored + } + + @Override + public boolean isSessionCacheEnabled() { + return false; + } + } + + private static final class TrustManagerVerifyCallback extends AbstractCertificateVerifier { + private final X509TrustManager manager; + + TrustManagerVerifyCallback(OpenSslEngineMap engineMap, X509TrustManager manager) { + super(engineMap); + this.manager = manager; + } + + @Override + void verify(ReferenceCountedOpenSslEngine engine, X509Certificate[] peerCerts, String auth) + throws Exception { + // doing this to disable hostname verfication that we don't want + manager.checkClientTrusted(peerCerts, auth); + //manager.checkServerTrusted(peerCerts, auth); + + } + } + + @SuppressJava6Requirement(reason = "Usage guarded by java version check") + private static final class ExtendedTrustManagerVerifyCallback extends AbstractCertificateVerifier { + private final X509ExtendedTrustManager manager; + + ExtendedTrustManagerVerifyCallback(OpenSslEngineMap engineMap, X509ExtendedTrustManager manager) { + super(engineMap); + this.manager = OpenSslTlsv13X509ExtendedTrustManager.wrap(manager); + } + + @Override + void verify(ReferenceCountedOpenSslEngine engine, X509Certificate[] peerCerts, String auth) + throws Exception { + //manager.checkServerTrusted(peerCerts, auth, engine); + // doing this to disable hostname verfication that we don't want + manager.checkClientTrusted(peerCerts, auth); + //manager.checkServerTrusted(peerCerts, auth); + + } + } + + private static final class OpenSslClientCertificateCallback implements CertificateCallback { + private final OpenSslEngineMap engineMap; + private final OpenSslKeyMaterialManager keyManagerHolder; + + OpenSslClientCertificateCallback(OpenSslEngineMap engineMap, OpenSslKeyMaterialManager keyManagerHolder) { + this.engineMap = engineMap; + this.keyManagerHolder = keyManagerHolder; + } + + @Override + public void handle(long ssl, byte[] keyTypeBytes, byte[][] asn1DerEncodedPrincipals) throws Exception { + final ReferenceCountedOpenSslEngine engine = engineMap.get(ssl); + // May be null if it was destroyed in the meantime. + if (engine == null) { + return; + } + try { + final Set keyTypesSet = supportedClientKeyTypes(keyTypeBytes); + final String[] keyTypes = keyTypesSet.toArray(new String[0]); + final X500Principal[] issuers; + if (asn1DerEncodedPrincipals == null) { + issuers = null; + } else { + issuers = new X500Principal[asn1DerEncodedPrincipals.length]; + for (int i = 0; i < asn1DerEncodedPrincipals.length; i++) { + issuers[i] = new X500Principal(asn1DerEncodedPrincipals[i]); + } + } + keyManagerHolder.setKeyMaterialClientSide(engine, keyTypes, issuers); + } catch (Throwable cause) { + logger.debug("request of key failed", cause); + engine.initHandshakeException(cause); + } + } + + /** + * Gets the supported key types for client certificates. + * + * @param clientCertificateTypes {@code ClientCertificateType} values provided by the server. + * See https://www.ietf.org/assignments/tls-parameters/tls-parameters.xml. + * @return supported key types that can be used in {@code X509KeyManager.chooseClientAlias} and + * {@code X509ExtendedKeyManager.chooseEngineClientAlias}. + */ + private static Set supportedClientKeyTypes(byte[] clientCertificateTypes) { + if (clientCertificateTypes == null) { + // Try all of the supported key types. + return SUPPORTED_KEY_TYPES; + } + Set result = new HashSet(clientCertificateTypes.length); + for (byte keyTypeCode : clientCertificateTypes) { + String keyType = clientKeyType(keyTypeCode); + if (keyType == null) { + // Unsupported client key type -- ignore + continue; + } + result.add(keyType); + } + return result; + } + + private static String clientKeyType(byte clientCertificateType) { + // See also http://www.ietf.org/assignments/tls-parameters/tls-parameters.xml + switch (clientCertificateType) { + case CertificateCallback.TLS_CT_RSA_SIGN: + return OpenSslKeyMaterialManager.KEY_TYPE_RSA; // RFC rsa_sign + case CertificateCallback.TLS_CT_RSA_FIXED_DH: + return OpenSslKeyMaterialManager.KEY_TYPE_DH_RSA; // RFC rsa_fixed_dh + case CertificateCallback.TLS_CT_ECDSA_SIGN: + return OpenSslKeyMaterialManager.KEY_TYPE_EC; // RFC ecdsa_sign + case CertificateCallback.TLS_CT_RSA_FIXED_ECDH: + return OpenSslKeyMaterialManager.KEY_TYPE_EC_RSA; // RFC rsa_fixed_ecdh + case CertificateCallback.TLS_CT_ECDSA_FIXED_ECDH: + return OpenSslKeyMaterialManager.KEY_TYPE_EC_EC; // RFC ecdsa_fixed_ecdh + default: + return null; + } + } + } +} diff --git a/src/testing/netty-streaming-cot-client/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslEngine.java b/src/testing/netty-streaming-cot-client/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslEngine.java new file mode 100644 index 00000000..e247470e --- /dev/null +++ b/src/testing/netty-streaming-cot-client/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslEngine.java @@ -0,0 +1,2439 @@ +/* + * Copyright 2016 The Netty Project + * + * The Netty Project licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package io.netty.handler.ssl; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufAllocator; +import io.netty.internal.tcnative.Buffer; +import io.netty.internal.tcnative.SSL; +import io.netty.util.AbstractReferenceCounted; +import io.netty.util.CharsetUtil; +import io.netty.util.ReferenceCounted; +import io.netty.util.ResourceLeakDetector; +import io.netty.util.ResourceLeakDetectorFactory; +import io.netty.util.ResourceLeakTracker; +import io.netty.util.internal.EmptyArrays; +import io.netty.util.internal.ObjectUtil; +import io.netty.util.internal.PlatformDependent; +import io.netty.util.internal.StringUtil; +import io.netty.util.internal.SuppressJava6Requirement; +import io.netty.util.internal.UnstableApi; +import io.netty.util.internal.logging.InternalLogger; +import io.netty.util.internal.logging.InternalLoggerFactory; + +import java.nio.ByteBuffer; +import java.nio.ReadOnlyBufferException; +import java.security.Principal; +import java.security.cert.Certificate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.locks.Lock; + +import javax.crypto.spec.SecretKeySpec; +import javax.net.ssl.SSLEngine; +import javax.net.ssl.SSLEngineResult; +import javax.net.ssl.SSLException; +import javax.net.ssl.SSLHandshakeException; +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLPeerUnverifiedException; +import javax.net.ssl.SSLSession; +import javax.net.ssl.SSLSessionBindingEvent; +import javax.net.ssl.SSLSessionBindingListener; +import javax.net.ssl.SSLSessionContext; +import javax.security.cert.X509Certificate; + +import com.google.common.primitives.Longs; + +import static io.netty.handler.ssl.OpenSsl.memoryAddress; +import static io.netty.handler.ssl.SslUtils.PROTOCOL_SSL_V2; +import static io.netty.handler.ssl.SslUtils.PROTOCOL_SSL_V2_HELLO; +import static io.netty.handler.ssl.SslUtils.PROTOCOL_SSL_V3; +import static io.netty.handler.ssl.SslUtils.PROTOCOL_TLS_V1; +import static io.netty.handler.ssl.SslUtils.PROTOCOL_TLS_V1_1; +import static io.netty.handler.ssl.SslUtils.PROTOCOL_TLS_V1_2; +import static io.netty.handler.ssl.SslUtils.PROTOCOL_TLS_V1_3; +import static io.netty.handler.ssl.SslUtils.SSL_RECORD_HEADER_LENGTH; +import static io.netty.util.internal.EmptyArrays.EMPTY_CERTIFICATES; +import static io.netty.util.internal.EmptyArrays.EMPTY_JAVAX_X509_CERTIFICATES; +import static io.netty.util.internal.ObjectUtil.checkNotNull; +import static java.lang.Integer.MAX_VALUE; +import static java.lang.Math.min; +import static javax.net.ssl.SSLEngineResult.HandshakeStatus.FINISHED; +import static javax.net.ssl.SSLEngineResult.HandshakeStatus.NEED_TASK; +import static javax.net.ssl.SSLEngineResult.HandshakeStatus.NEED_UNWRAP; +import static javax.net.ssl.SSLEngineResult.HandshakeStatus.NEED_WRAP; +import static javax.net.ssl.SSLEngineResult.HandshakeStatus.NOT_HANDSHAKING; +import static javax.net.ssl.SSLEngineResult.Status.BUFFER_OVERFLOW; +import static javax.net.ssl.SSLEngineResult.Status.BUFFER_UNDERFLOW; +import static javax.net.ssl.SSLEngineResult.Status.CLOSED; +import static javax.net.ssl.SSLEngineResult.Status.OK; + +/** + * Implements a {@link SSLEngine} using + * OpenSSL BIO abstractions. + *

    Instances of this class must be {@link #release() released} or else native memory will leak! + * + *

    Instances of this class must be released before the {@link ReferenceCountedOpenSslContext} + * the instance depends upon are released. Otherwise if any method of this class is called which uses the + * the {@link ReferenceCountedOpenSslContext} JNI resources the JVM may crash. + */ + +/** + * + * + * see handshakeFinished() for custom logic needed for setting the session id + * + */ +public class ReferenceCountedOpenSslEngine extends SSLEngine implements ReferenceCounted, ApplicationProtocolAccessor { + + private static final InternalLogger logger = InternalLoggerFactory.getInstance(ReferenceCountedOpenSslEngine.class); + + private static final ResourceLeakDetector leakDetector = + ResourceLeakDetectorFactory.instance().newResourceLeakDetector(ReferenceCountedOpenSslEngine.class); + private static final int OPENSSL_OP_NO_PROTOCOL_INDEX_SSLV2 = 0; + private static final int OPENSSL_OP_NO_PROTOCOL_INDEX_SSLV3 = 1; + private static final int OPENSSL_OP_NO_PROTOCOL_INDEX_TLSv1 = 2; + private static final int OPENSSL_OP_NO_PROTOCOL_INDEX_TLSv1_1 = 3; + private static final int OPENSSL_OP_NO_PROTOCOL_INDEX_TLSv1_2 = 4; + private static final int OPENSSL_OP_NO_PROTOCOL_INDEX_TLSv1_3 = 5; + private static final int[] OPENSSL_OP_NO_PROTOCOLS = { + SSL.SSL_OP_NO_SSLv2, + SSL.SSL_OP_NO_SSLv3, + SSL.SSL_OP_NO_TLSv1, + SSL.SSL_OP_NO_TLSv1_1, + SSL.SSL_OP_NO_TLSv1_2, + SSL.SSL_OP_NO_TLSv1_3 + }; + + /** + * Depends upon tcnative ... only use if tcnative is available! + */ + static final int MAX_PLAINTEXT_LENGTH = SSL.SSL_MAX_PLAINTEXT_LENGTH; + /** + * Depends upon tcnative ... only use if tcnative is available! + */ + private static final int MAX_RECORD_SIZE = SSL.SSL_MAX_RECORD_LENGTH; + + private static final SSLEngineResult NEED_UNWRAP_OK = new SSLEngineResult(OK, NEED_UNWRAP, 0, 0); + private static final SSLEngineResult NEED_UNWRAP_CLOSED = new SSLEngineResult(CLOSED, NEED_UNWRAP, 0, 0); + private static final SSLEngineResult NEED_WRAP_OK = new SSLEngineResult(OK, NEED_WRAP, 0, 0); + private static final SSLEngineResult NEED_WRAP_CLOSED = new SSLEngineResult(CLOSED, NEED_WRAP, 0, 0); + private static final SSLEngineResult CLOSED_NOT_HANDSHAKING = new SSLEngineResult(CLOSED, NOT_HANDSHAKING, 0, 0); + + // OpenSSL state + private long ssl; + private long networkBIO; + + private enum HandshakeState { + /** + * Not started yet. + */ + NOT_STARTED, + /** + * Started via unwrap/wrap. + */ + STARTED_IMPLICITLY, + /** + * Started via {@link #beginHandshake()}. + */ + STARTED_EXPLICITLY, + /** + * Handshake is finished. + */ + FINISHED + } + + private HandshakeState handshakeState = HandshakeState.NOT_STARTED; + private boolean receivedShutdown; + private volatile boolean destroyed; + private volatile String applicationProtocol; + private volatile boolean needTask; + + // Reference Counting + private final ResourceLeakTracker leak; + private final AbstractReferenceCounted refCnt = new AbstractReferenceCounted() { + @Override + public ReferenceCounted touch(Object hint) { + if (leak != null) { + leak.record(hint); + } + + return ReferenceCountedOpenSslEngine.this; + } + + @Override + protected void deallocate() { + shutdown(); + if (leak != null) { + boolean closed = leak.close(ReferenceCountedOpenSslEngine.this); + assert closed; + } + parentContext.release(); + } + }; + + private volatile ClientAuth clientAuth = ClientAuth.NONE; + private volatile Certificate[] localCertificateChain; + + // Updated once a new handshake is started and so the SSLSession reused. + private volatile long lastAccessed = -1; + + private String endPointIdentificationAlgorithm; + // Store as object as AlgorithmConstraints only exists since java 7. + private Object algorithmConstraints; + private List sniHostNames; + + // Mark as volatile as accessed by checkSniHostnameMatch(...) and also not specify the SNIMatcher type to allow us + // using it with java7. + private volatile Collection matchers; + + // SSL Engine status variables + private boolean isInboundDone; + private boolean outboundClosed; + + final boolean jdkCompatibilityMode; + private final boolean clientMode; + final ByteBufAllocator alloc; + private final OpenSslEngineMap engineMap; + private final OpenSslApplicationProtocolNegotiator apn; + private final ReferenceCountedOpenSslContext parentContext; + private final OpenSslSession session; + private final ByteBuffer[] singleSrcBuffer = new ByteBuffer[1]; + private final ByteBuffer[] singleDstBuffer = new ByteBuffer[1]; + private final boolean enableOcsp; + private int maxWrapOverhead; + private int maxWrapBufferSize; + private Throwable handshakeException; + + /** + * Create a new instance. + * @param context Reference count release responsibility is not transferred! The callee still owns this object. + * @param alloc The allocator to use. + * @param peerHost The peer host name. + * @param peerPort The peer port. + * @param jdkCompatibilityMode {@code true} to behave like described in + * https://docs.oracle.com/javase/7/docs/api/javax/net/ssl/SSLEngine.html. + * {@code false} allows for partial and/or multiple packets to be process in a single + * wrap or unwrap call. + * @param leakDetection {@code true} to enable leak detection of this object. + */ + ReferenceCountedOpenSslEngine(ReferenceCountedOpenSslContext context, final ByteBufAllocator alloc, String peerHost, + int peerPort, boolean jdkCompatibilityMode, boolean leakDetection) { + super(peerHost, peerPort); + OpenSsl.ensureAvailability(); + this.alloc = checkNotNull(alloc, "alloc"); + apn = (OpenSslApplicationProtocolNegotiator) context.applicationProtocolNegotiator(); + clientMode = context.isClient(); + if (PlatformDependent.javaVersion() >= 7) { + session = new ExtendedOpenSslSession(new DefaultOpenSslSession(context.sessionContext())) { + private String[] peerSupportedSignatureAlgorithms; + private List requestedServerNames; + + @Override + public List getRequestedServerNames() { + if (clientMode) { + return Java8SslUtils.getSniHostNames(sniHostNames); + } else { + synchronized (ReferenceCountedOpenSslEngine.this) { + if (requestedServerNames == null) { + if (isDestroyed()) { + requestedServerNames = Collections.emptyList(); + } else { + String name = SSL.getSniHostname(ssl); + if (name == null) { + requestedServerNames = Collections.emptyList(); + } else { + // Convert to bytes as we do not want to do any strict validation of the + // SNIHostName while creating it. + requestedServerNames = + Java8SslUtils.getSniHostName( + SSL.getSniHostname(ssl).getBytes(CharsetUtil.UTF_8)); + } + } + } + return requestedServerNames; + } + } + } + + @Override + public String[] getPeerSupportedSignatureAlgorithms() { + synchronized (ReferenceCountedOpenSslEngine.this) { + if (peerSupportedSignatureAlgorithms == null) { + if (isDestroyed()) { + peerSupportedSignatureAlgorithms = EmptyArrays.EMPTY_STRINGS; + } else { + String[] algs = SSL.getSigAlgs(ssl); + if (algs == null) { + peerSupportedSignatureAlgorithms = EmptyArrays.EMPTY_STRINGS; + } else { + Set algorithmList = new LinkedHashSet(algs.length); + for (String alg: algs) { + String converted = SignatureAlgorithmConverter.toJavaName(alg); + + if (converted != null) { + algorithmList.add(converted); + } + } + peerSupportedSignatureAlgorithms = algorithmList.toArray(new String[0]); + } + } + } + return peerSupportedSignatureAlgorithms.clone(); + } + } + + @Override + public List getStatusResponses() { + byte[] ocspResponse = null; + if (enableOcsp && clientMode) { + synchronized (ReferenceCountedOpenSslEngine.this) { + if (!isDestroyed()) { + ocspResponse = SSL.getOcspResponse(ssl); + } + } + } + return ocspResponse == null ? + Collections.emptyList() : Collections.singletonList(ocspResponse); + } + }; + } else { + session = new DefaultOpenSslSession(context.sessionContext()); + } + engineMap = context.engineMap; + enableOcsp = context.enableOcsp; + // context.keyCertChain will only be non-null if we do not use the KeyManagerFactory. In this case + // localCertificateChain will be set in setKeyMaterial(...). + localCertificateChain = context.keyCertChain; + + this.jdkCompatibilityMode = jdkCompatibilityMode; + Lock readerLock = context.ctxLock.readLock(); + readerLock.lock(); + final long finalSsl; + try { + finalSsl = SSL.newSSL(context.ctx, !context.isClient()); + } finally { + readerLock.unlock(); + } + synchronized (this) { + ssl = finalSsl; + try { + networkBIO = SSL.bioNewByteBuffer(ssl, context.getBioNonApplicationBufferSize()); + + // Set the client auth mode, this needs to be done via setClientAuth(...) method so we actually call the + // needed JNI methods. + setClientAuth(clientMode ? ClientAuth.NONE : context.clientAuth); + + if (context.protocols != null) { + setEnabledProtocols(context.protocols); + } + + // Use SNI if peerHost was specified and a valid hostname + // See https://github.com/netty/netty/issues/4746 + if (clientMode && SslUtils.isValidHostNameForSNI(peerHost)) { + SSL.setTlsExtHostName(ssl, peerHost); + sniHostNames = Collections.singletonList(peerHost); + } + + if (enableOcsp) { + SSL.enableOcsp(ssl); + } + + if (!jdkCompatibilityMode) { + SSL.setMode(ssl, SSL.getMode(ssl) | SSL.SSL_MODE_ENABLE_PARTIAL_WRITE); + } + + // setMode may impact the overhead. + calculateMaxWrapOverhead(); + } catch (Throwable cause) { + // Call shutdown so we are sure we correctly release all native memory and also guard against the + // case when shutdown() will be called by the finalizer again. + shutdown(); + + PlatformDependent.throwException(cause); + } + } + + // Now that everything looks good and we're going to successfully return the + // object so we need to retain a reference to the parent context. + parentContext = context; + parentContext.retain(); + + // Only create the leak after everything else was executed and so ensure we don't produce a false-positive for + // the ResourceLeakDetector. + leak = leakDetection ? leakDetector.track(this) : null; + } + + final synchronized String[] authMethods() { + if (isDestroyed()) { + return EmptyArrays.EMPTY_STRINGS; + } + return SSL.authenticationMethods(ssl); + } + + final boolean setKeyMaterial(OpenSslKeyMaterial keyMaterial) throws Exception { + synchronized (this) { + if (isDestroyed()) { + return false; + } + SSL.setKeyMaterial(ssl, keyMaterial.certificateChainAddress(), keyMaterial.privateKeyAddress()); + } + localCertificateChain = keyMaterial.certificateChain(); + return true; + } + + final synchronized SecretKeySpec masterKey() { + if (isDestroyed()) { + return null; + } + return new SecretKeySpec(SSL.getMasterKey(ssl), "AES"); + } + + /** + * Sets the OCSP response. + */ + @UnstableApi + public void setOcspResponse(byte[] response) { + if (!enableOcsp) { + throw new IllegalStateException("OCSP stapling is not enabled"); + } + + if (clientMode) { + throw new IllegalStateException("Not a server SSLEngine"); + } + + synchronized (this) { + if (!isDestroyed()) { + SSL.setOcspResponse(ssl, response); + } + } + } + + /** + * Returns the OCSP response or {@code null} if the server didn't provide a stapled OCSP response. + */ + @UnstableApi + public byte[] getOcspResponse() { + if (!enableOcsp) { + throw new IllegalStateException("OCSP stapling is not enabled"); + } + + if (!clientMode) { + throw new IllegalStateException("Not a client SSLEngine"); + } + + synchronized (this) { + if (isDestroyed()) { + return EmptyArrays.EMPTY_BYTES; + } + return SSL.getOcspResponse(ssl); + } + } + + @Override + public final int refCnt() { + return refCnt.refCnt(); + } + + @Override + public final ReferenceCounted retain() { + refCnt.retain(); + return this; + } + + @Override + public final ReferenceCounted retain(int increment) { + refCnt.retain(increment); + return this; + } + + @Override + public final ReferenceCounted touch() { + refCnt.touch(); + return this; + } + + @Override + public final ReferenceCounted touch(Object hint) { + refCnt.touch(hint); + return this; + } + + @Override + public final boolean release() { + return refCnt.release(); + } + + @Override + public final boolean release(int decrement) { + return refCnt.release(decrement); + } + + @Override + public final synchronized SSLSession getHandshakeSession() { + // Javadocs state return value should be: + // null if this instance is not currently handshaking, or if the current handshake has not + // progressed far enough to create a basic SSLSession. Otherwise, this method returns the + // SSLSession currently being negotiated. + switch(handshakeState) { + case NOT_STARTED: + case FINISHED: + return null; + default: + return session; + } + } + + /** + * Returns the pointer to the {@code SSL} object for this {@link ReferenceCountedOpenSslEngine}. + * Be aware that it is freed as soon as the {@link #release()} or {@link #shutdown()} methods are called. + * At this point {@code 0} will be returned. + */ + public final synchronized long sslPointer() { + return ssl; + } + + /** + * Destroys this engine. + */ + public final synchronized void shutdown() { + if (!destroyed) { + destroyed = true; + engineMap.remove(ssl); + SSL.freeSSL(ssl); + ssl = networkBIO = 0; + + isInboundDone = outboundClosed = true; + } + + // On shutdown clear all errors + SSL.clearError(); + } + + /** + * Write plaintext data to the OpenSSL internal BIO + * + * Calling this function with src.remaining == 0 is undefined. + */ + private int writePlaintextData(final ByteBuffer src, int len) { + final int pos = src.position(); + final int limit = src.limit(); + final int sslWrote; + + if (src.isDirect()) { + sslWrote = SSL.writeToSSL(ssl, bufferAddress(src) + pos, len); + if (sslWrote > 0) { + src.position(pos + sslWrote); + } + } else { + ByteBuf buf = alloc.directBuffer(len); + try { + src.limit(pos + len); + + buf.setBytes(0, src); + src.limit(limit); + + sslWrote = SSL.writeToSSL(ssl, memoryAddress(buf), len); + if (sslWrote > 0) { + src.position(pos + sslWrote); + } else { + src.position(pos); + } + } finally { + buf.release(); + } + } + return sslWrote; + } + + /** + * Write encrypted data to the OpenSSL network BIO. + */ + private ByteBuf writeEncryptedData(final ByteBuffer src, int len) { + final int pos = src.position(); + if (src.isDirect()) { + SSL.bioSetByteBuffer(networkBIO, bufferAddress(src) + pos, len, false); + } else { + final ByteBuf buf = alloc.directBuffer(len); + try { + final int limit = src.limit(); + src.limit(pos + len); + buf.writeBytes(src); + // Restore the original position and limit because we don't want to consume from `src`. + src.position(pos); + src.limit(limit); + + SSL.bioSetByteBuffer(networkBIO, memoryAddress(buf), len, false); + return buf; + } catch (Throwable cause) { + buf.release(); + PlatformDependent.throwException(cause); + } + } + return null; + } + + /** + * Read plaintext data from the OpenSSL internal BIO + */ + private int readPlaintextData(final ByteBuffer dst) { + final int sslRead; + final int pos = dst.position(); + if (dst.isDirect()) { + sslRead = SSL.readFromSSL(ssl, bufferAddress(dst) + pos, dst.limit() - pos); + if (sslRead > 0) { + dst.position(pos + sslRead); + } + } else { + final int limit = dst.limit(); + final int len = min(maxEncryptedPacketLength0(), limit - pos); + final ByteBuf buf = alloc.directBuffer(len); + try { + sslRead = SSL.readFromSSL(ssl, memoryAddress(buf), len); + if (sslRead > 0) { + dst.limit(pos + sslRead); + buf.getBytes(buf.readerIndex(), dst); + dst.limit(limit); + } + } finally { + buf.release(); + } + } + + return sslRead; + } + + /** + * Visible only for testing! + */ + final synchronized int maxWrapOverhead() { + return maxWrapOverhead; + } + + /** + * Visible only for testing! + */ + final synchronized int maxEncryptedPacketLength() { + return maxEncryptedPacketLength0(); + } + + /** + * This method is intentionally not synchronized, only use if you know you are in the EventLoop + * thread and visibility on {@link #maxWrapOverhead} is achieved via other synchronized blocks. + */ + final int maxEncryptedPacketLength0() { + return maxWrapOverhead + MAX_PLAINTEXT_LENGTH; + } + + /** + * This method is intentionally not synchronized, only use if you know you are in the EventLoop + * thread and visibility on {@link #maxWrapBufferSize} and {@link #maxWrapOverhead} is achieved + * via other synchronized blocks. + */ + final int calculateMaxLengthForWrap(int plaintextLength, int numComponents) { + return (int) min(maxWrapBufferSize, plaintextLength + (long) maxWrapOverhead * numComponents); + } + + final synchronized int sslPending() { + return sslPending0(); + } + + /** + * It is assumed this method is called in a synchronized block (or the constructor)! + */ + private void calculateMaxWrapOverhead() { + maxWrapOverhead = SSL.getMaxWrapOverhead(ssl); + + // maxWrapBufferSize must be set after maxWrapOverhead because there is a dependency on this value. + // If jdkCompatibility mode is off we allow enough space to encrypt 16 buffers at a time. This could be + // configurable in the future if necessary. + maxWrapBufferSize = jdkCompatibilityMode ? maxEncryptedPacketLength0() : maxEncryptedPacketLength0() << 4; + } + + private int sslPending0() { + // OpenSSL has a limitation where if you call SSL_pending before the handshake is complete OpenSSL will throw a + // "called a function you should not call" error. Using the TLS_method instead of SSLv23_method may solve this + // issue but this API is only available in 1.1.0+ [1]. + // [1] https://www.openssl.org/docs/man1.1.0/ssl/SSL_CTX_new.html + return handshakeState != HandshakeState.FINISHED ? 0 : SSL.sslPending(ssl); + } + + private boolean isBytesAvailableEnoughForWrap(int bytesAvailable, int plaintextLength, int numComponents) { + return bytesAvailable - (long) maxWrapOverhead * numComponents >= plaintextLength; + } + + @Override + public final SSLEngineResult wrap( + final ByteBuffer[] srcs, int offset, final int length, final ByteBuffer dst) throws SSLException { + // Throw required runtime exceptions + if (srcs == null) { + throw new IllegalArgumentException("srcs is null"); + } + if (dst == null) { + throw new IllegalArgumentException("dst is null"); + } + + if (offset >= srcs.length || offset + length > srcs.length) { + throw new IndexOutOfBoundsException( + "offset: " + offset + ", length: " + length + + " (expected: offset <= offset + length <= srcs.length (" + srcs.length + "))"); + } + + if (dst.isReadOnly()) { + throw new ReadOnlyBufferException(); + } + + synchronized (this) { + if (isOutboundDone()) { + // All drained in the outbound buffer + return isInboundDone() || isDestroyed() ? CLOSED_NOT_HANDSHAKING : NEED_UNWRAP_CLOSED; + } + + int bytesProduced = 0; + ByteBuf bioReadCopyBuf = null; + try { + // Setup the BIO buffer so that we directly write the encryption results into dst. + if (dst.isDirect()) { + SSL.bioSetByteBuffer(networkBIO, bufferAddress(dst) + dst.position(), dst.remaining(), + true); + } else { + bioReadCopyBuf = alloc.directBuffer(dst.remaining()); + SSL.bioSetByteBuffer(networkBIO, memoryAddress(bioReadCopyBuf), bioReadCopyBuf.writableBytes(), + true); + } + + int bioLengthBefore = SSL.bioLengthByteBuffer(networkBIO); + + // Explicitly use outboundClosed as we want to drain any bytes that are still present. + if (outboundClosed) { + // If the outbound was closed we want to ensure we can produce the alert to the destination buffer. + // This is true even if we not using jdkCompatibilityMode. + // + // We use a plaintextLength of 2 as we at least want to have an alert fit into it. + // https://tools.ietf.org/html/rfc5246#section-7.2 + if (!isBytesAvailableEnoughForWrap(dst.remaining(), 2, 1)) { + return new SSLEngineResult(BUFFER_OVERFLOW, getHandshakeStatus(), 0, 0); + } + + // There is something left to drain. + // See https://github.com/netty/netty/issues/6260 + bytesProduced = SSL.bioFlushByteBuffer(networkBIO); + if (bytesProduced <= 0) { + return newResultMayFinishHandshake(NOT_HANDSHAKING, 0, 0); + } + // It is possible when the outbound was closed there was not enough room in the non-application + // buffers to hold the close_notify. We should keep trying to close until we consume all the data + // OpenSSL can give us. + if (!doSSLShutdown()) { + return newResultMayFinishHandshake(NOT_HANDSHAKING, 0, bytesProduced); + } + bytesProduced = bioLengthBefore - SSL.bioLengthByteBuffer(networkBIO); + return newResultMayFinishHandshake(NEED_WRAP, 0, bytesProduced); + } + + // Flush any data that may be implicitly generated by OpenSSL (handshake, close, etc..). + SSLEngineResult.HandshakeStatus status = NOT_HANDSHAKING; + // Prepare OpenSSL to work in server mode and receive handshake + if (handshakeState != HandshakeState.FINISHED) { + if (handshakeState != HandshakeState.STARTED_EXPLICITLY) { + // Update accepted so we know we triggered the handshake via wrap + handshakeState = HandshakeState.STARTED_IMPLICITLY; + } + + // Flush any data that may have been written implicitly during the handshake by OpenSSL. + bytesProduced = SSL.bioFlushByteBuffer(networkBIO); + + if (handshakeException != null) { + // TODO(scott): It is possible that when the handshake failed there was not enough room in the + // non-application buffers to hold the alert. We should get all the data before progressing on. + // However I'm not aware of a way to do this with the OpenSSL APIs. + // See https://github.com/netty/netty/issues/6385. + + // We produced / consumed some data during the handshake, signal back to the caller. + // If there is a handshake exception and we have produced data, we should send the data before + // we allow handshake() to throw the handshake exception. + // + // When the user calls wrap() again we will propagate the handshake error back to the user as + // soon as there is no more data to was produced (as part of an alert etc). + if (bytesProduced > 0) { + return newResult(NEED_WRAP, 0, bytesProduced); + } + // Nothing was produced see if there is a handshakeException that needs to be propagated + // to the caller by calling handshakeException() which will return the right HandshakeStatus + // if it can "recover" from the exception for now. + return newResult(handshakeException(), 0, 0); + } + + status = handshake(); + + // Handshake may have generated more data, for example if the internal SSL buffer is small + // we may have freed up space by flushing above. + bytesProduced = bioLengthBefore - SSL.bioLengthByteBuffer(networkBIO); + + if (status == NEED_TASK) { + return newResult(status, 0, bytesProduced); + } + + if (bytesProduced > 0) { + // If we have filled up the dst buffer and we have not finished the handshake we should try to + // wrap again. Otherwise we should only try to wrap again if there is still data pending in + // SSL buffers. + return newResult(mayFinishHandshake(status != FINISHED ? + bytesProduced == bioLengthBefore ? NEED_WRAP : + getHandshakeStatus(SSL.bioLengthNonApplication(networkBIO)) : FINISHED), + 0, bytesProduced); + } + + if (status == NEED_UNWRAP) { + // Signal if the outbound is done or not. + return isOutboundDone() ? NEED_UNWRAP_CLOSED : NEED_UNWRAP_OK; + } + + // Explicit use outboundClosed and not outboundClosed() as we want to drain any bytes that are + // still present. + if (outboundClosed) { + bytesProduced = SSL.bioFlushByteBuffer(networkBIO); + return newResultMayFinishHandshake(status, 0, bytesProduced); + } + } + + final int endOffset = offset + length; + if (jdkCompatibilityMode) { + int srcsLen = 0; + for (int i = offset; i < endOffset; ++i) { + final ByteBuffer src = srcs[i]; + if (src == null) { + throw new IllegalArgumentException("srcs[" + i + "] is null"); + } + if (srcsLen == MAX_PLAINTEXT_LENGTH) { + continue; + } + + srcsLen += src.remaining(); + if (srcsLen > MAX_PLAINTEXT_LENGTH || srcsLen < 0) { + // If srcLen > MAX_PLAINTEXT_LENGTH or secLen < 0 just set it to MAX_PLAINTEXT_LENGTH. + // This also help us to guard against overflow. + // We not break out here as we still need to check for null entries in srcs[]. + srcsLen = MAX_PLAINTEXT_LENGTH; + } + } + + // jdkCompatibilityMode will only produce a single TLS packet, and we don't aggregate src buffers, + // so we always fix the number of buffers to 1 when checking if the dst buffer is large enough. + if (!isBytesAvailableEnoughForWrap(dst.remaining(), srcsLen, 1)) { + return new SSLEngineResult(BUFFER_OVERFLOW, getHandshakeStatus(), 0, 0); + } + } + + // There was no pending data in the network BIO -- encrypt any application data + int bytesConsumed = 0; + // Flush any data that may have been written implicitly by OpenSSL in case a shutdown/alert occurs. + bytesProduced = SSL.bioFlushByteBuffer(networkBIO); + for (; offset < endOffset; ++offset) { + final ByteBuffer src = srcs[offset]; + final int remaining = src.remaining(); + if (remaining == 0) { + continue; + } + + final int bytesWritten; + if (jdkCompatibilityMode) { + // Write plaintext application data to the SSL engine. We don't have to worry about checking + // if there is enough space if jdkCompatibilityMode because we only wrap at most + // MAX_PLAINTEXT_LENGTH and we loop over the input before hand and check if there is space. + bytesWritten = writePlaintextData(src, min(remaining, MAX_PLAINTEXT_LENGTH - bytesConsumed)); + } else { + // OpenSSL's SSL_write keeps state between calls. We should make sure the amount we attempt to + // write is guaranteed to succeed so we don't have to worry about keeping state consistent + // between calls. + final int availableCapacityForWrap = dst.remaining() - bytesProduced - maxWrapOverhead; + if (availableCapacityForWrap <= 0) { + return new SSLEngineResult(BUFFER_OVERFLOW, getHandshakeStatus(), bytesConsumed, + bytesProduced); + } + bytesWritten = writePlaintextData(src, min(remaining, availableCapacityForWrap)); + } + + // Determine how much encrypted data was generated. + // + // Even if SSL_write doesn't consume any application data it is possible that OpenSSL will + // produce non-application data into the BIO. For example session tickets.... + // See https://github.com/netty/netty/issues/10041 + final int pendingNow = SSL.bioLengthByteBuffer(networkBIO); + bytesProduced += bioLengthBefore - pendingNow; + bioLengthBefore = pendingNow; + + if (bytesWritten > 0) { + bytesConsumed += bytesWritten; + + if (jdkCompatibilityMode || bytesProduced == dst.remaining()) { + return newResultMayFinishHandshake(status, bytesConsumed, bytesProduced); + } + } else { + int sslError = SSL.getError(ssl, bytesWritten); + if (sslError == SSL.SSL_ERROR_ZERO_RETURN) { + // This means the connection was shutdown correctly, close inbound and outbound + if (!receivedShutdown) { + closeAll(); + + bytesProduced += bioLengthBefore - SSL.bioLengthByteBuffer(networkBIO); + + // If we have filled up the dst buffer and we have not finished the handshake we should + // try to wrap again. Otherwise we should only try to wrap again if there is still data + // pending in SSL buffers. + SSLEngineResult.HandshakeStatus hs = mayFinishHandshake( + status != FINISHED ? bytesProduced == dst.remaining() ? NEED_WRAP + : getHandshakeStatus(SSL.bioLengthNonApplication(networkBIO)) + : FINISHED); + return newResult(hs, bytesConsumed, bytesProduced); + } + + return newResult(NOT_HANDSHAKING, bytesConsumed, bytesProduced); + } else if (sslError == SSL.SSL_ERROR_WANT_READ) { + // If there is no pending data to read from BIO we should go back to event loop and try + // to read more data [1]. It is also possible that event loop will detect the socket has + // been closed. [1] https://www.openssl.org/docs/manmaster/ssl/SSL_write.html + return newResult(NEED_UNWRAP, bytesConsumed, bytesProduced); + } else if (sslError == SSL.SSL_ERROR_WANT_WRITE) { + // SSL_ERROR_WANT_WRITE typically means that the underlying transport is not writable + // and we should set the "want write" flag on the selector and try again when the + // underlying transport is writable [1]. However we are not directly writing to the + // underlying transport and instead writing to a BIO buffer. The OpenSsl documentation + // says we should do the following [1]: + // + // "When using a buffering BIO, like a BIO pair, data must be written into or retrieved + // out of the BIO before being able to continue." + // + // In practice this means the destination buffer doesn't have enough space for OpenSSL + // to write encrypted data to. This is an OVERFLOW condition. + // [1] https://www.openssl.org/docs/manmaster/ssl/SSL_write.html + return newResult(BUFFER_OVERFLOW, status, bytesConsumed, bytesProduced); + } else if (sslError == SSL.SSL_ERROR_WANT_X509_LOOKUP || + sslError == SSL.SSL_ERROR_WANT_CERTIFICATE_VERIFY || + sslError == SSL.SSL_ERROR_WANT_PRIVATE_KEY_OPERATION) { + + return newResult(NEED_TASK, bytesConsumed, bytesProduced); + } else { + // Everything else is considered as error + throw shutdownWithError("SSL_write", sslError); + } + } + } + return newResultMayFinishHandshake(status, bytesConsumed, bytesProduced); + } finally { + SSL.bioClearByteBuffer(networkBIO); + if (bioReadCopyBuf == null) { + dst.position(dst.position() + bytesProduced); + } else { + assert bioReadCopyBuf.readableBytes() <= dst.remaining() : "The destination buffer " + dst + + " didn't have enough remaining space to hold the encrypted content in " + bioReadCopyBuf; + dst.put(bioReadCopyBuf.internalNioBuffer(bioReadCopyBuf.readerIndex(), bytesProduced)); + bioReadCopyBuf.release(); + } + } + } + } + + private SSLEngineResult newResult(SSLEngineResult.HandshakeStatus hs, int bytesConsumed, int bytesProduced) { + return newResult(OK, hs, bytesConsumed, bytesProduced); + } + + private SSLEngineResult newResult(SSLEngineResult.Status status, SSLEngineResult.HandshakeStatus hs, + int bytesConsumed, int bytesProduced) { + // If isOutboundDone, then the data from the network BIO + // was the close_notify message and all was consumed we are not required to wait + // for the receipt the peer's close_notify message -- shutdown. + if (isOutboundDone()) { + if (isInboundDone()) { + // If the inbound was done as well, we need to ensure we return NOT_HANDSHAKING to signal we are done. + hs = NOT_HANDSHAKING; + + // As the inbound and the outbound is done we can shutdown the engine now. + shutdown(); + } + return new SSLEngineResult(CLOSED, hs, bytesConsumed, bytesProduced); + } + if (hs == NEED_TASK) { + // Set needTask to true so getHandshakeStatus() will return the correct value. + needTask = true; + } + return new SSLEngineResult(status, hs, bytesConsumed, bytesProduced); + } + + private SSLEngineResult newResultMayFinishHandshake(SSLEngineResult.HandshakeStatus hs, + int bytesConsumed, int bytesProduced) throws SSLException { + return newResult(mayFinishHandshake(hs != FINISHED ? getHandshakeStatus() : FINISHED), + bytesConsumed, bytesProduced); + } + + private SSLEngineResult newResultMayFinishHandshake(SSLEngineResult.Status status, + SSLEngineResult.HandshakeStatus hs, + int bytesConsumed, int bytesProduced) throws SSLException { + return newResult(status, mayFinishHandshake(hs != FINISHED ? getHandshakeStatus() : FINISHED), + bytesConsumed, bytesProduced); + } + + /** + * Log the error, shutdown the engine and throw an exception. + */ + private SSLException shutdownWithError(String operations, int sslError) { + return shutdownWithError(operations, sslError, SSL.getLastErrorNumber()); + } + + private SSLException shutdownWithError(String operation, int sslError, int error) { + String errorString = SSL.getErrorString(error); + if (logger.isDebugEnabled()) { + logger.debug("{} failed with {}: OpenSSL error: {} {}", + operation, sslError, error, errorString); + } + + // There was an internal error -- shutdown + shutdown(); + if (handshakeState == HandshakeState.FINISHED) { + return new SSLException(errorString); + } + + SSLHandshakeException exception = new SSLHandshakeException(errorString); + // If we have a handshakeException stored already we should include it as well to help the user debug things. + if (handshakeException != null) { + exception.initCause(handshakeException); + handshakeException = null; + } + return exception; + } + + public final SSLEngineResult unwrap( + final ByteBuffer[] srcs, int srcsOffset, final int srcsLength, + final ByteBuffer[] dsts, int dstsOffset, final int dstsLength) throws SSLException { + + // Throw required runtime exceptions + ObjectUtil.checkNotNull(srcs, "srcs"); + if (srcsOffset >= srcs.length + || srcsOffset + srcsLength > srcs.length) { + throw new IndexOutOfBoundsException( + "offset: " + srcsOffset + ", length: " + srcsLength + + " (expected: offset <= offset + length <= srcs.length (" + srcs.length + "))"); + } + if (dsts == null) { + throw new IllegalArgumentException("dsts is null"); + } + if (dstsOffset >= dsts.length || dstsOffset + dstsLength > dsts.length) { + throw new IndexOutOfBoundsException( + "offset: " + dstsOffset + ", length: " + dstsLength + + " (expected: offset <= offset + length <= dsts.length (" + dsts.length + "))"); + } + long capacity = 0; + final int dstsEndOffset = dstsOffset + dstsLength; + for (int i = dstsOffset; i < dstsEndOffset; i ++) { + ByteBuffer dst = dsts[i]; + if (dst == null) { + throw new IllegalArgumentException("dsts[" + i + "] is null"); + } + if (dst.isReadOnly()) { + throw new ReadOnlyBufferException(); + } + capacity += dst.remaining(); + } + + final int srcsEndOffset = srcsOffset + srcsLength; + long len = 0; + for (int i = srcsOffset; i < srcsEndOffset; i++) { + ByteBuffer src = srcs[i]; + if (src == null) { + throw new IllegalArgumentException("srcs[" + i + "] is null"); + } + len += src.remaining(); + } + + synchronized (this) { + if (isInboundDone()) { + return isOutboundDone() || isDestroyed() ? CLOSED_NOT_HANDSHAKING : NEED_WRAP_CLOSED; + } + + SSLEngineResult.HandshakeStatus status = NOT_HANDSHAKING; + // Prepare OpenSSL to work in server mode and receive handshake + if (handshakeState != HandshakeState.FINISHED) { + if (handshakeState != HandshakeState.STARTED_EXPLICITLY) { + // Update accepted so we know we triggered the handshake via wrap + handshakeState = HandshakeState.STARTED_IMPLICITLY; + } + + status = handshake(); + + if (status == NEED_TASK) { + return newResult(status, 0, 0); + } + + if (status == NEED_WRAP) { + return NEED_WRAP_OK; + } + // Check if the inbound is considered to be closed if so let us try to wrap again. + if (isInboundDone) { + return NEED_WRAP_CLOSED; + } + } + + int sslPending = sslPending0(); + int packetLength; + // The JDK implies that only a single SSL packet should be processed per unwrap call [1]. If we are in + // JDK compatibility mode then we should honor this, but if not we just wrap as much as possible. If there + // are multiple records or partial records this may reduce thrashing events through the pipeline. + // [1] https://docs.oracle.com/javase/7/docs/api/javax/net/ssl/SSLEngine.html + if (jdkCompatibilityMode) { + if (len < SSL_RECORD_HEADER_LENGTH) { + return newResultMayFinishHandshake(BUFFER_UNDERFLOW, status, 0, 0); + } + + packetLength = SslUtils.getEncryptedPacketLength(srcs, srcsOffset); + if (packetLength == SslUtils.NOT_ENCRYPTED) { + throw new NotSslRecordException("not an SSL/TLS record"); + } + + final int packetLengthDataOnly = packetLength - SSL_RECORD_HEADER_LENGTH; + if (packetLengthDataOnly > capacity) { + // Not enough space in the destination buffer so signal the caller that the buffer needs to be + // increased. + if (packetLengthDataOnly > MAX_RECORD_SIZE) { + // The packet length MUST NOT exceed 2^14 [1]. However we do accommodate more data to support + // legacy use cases which may violate this condition (e.g. OpenJDK's SslEngineImpl). If the max + // length is exceeded we fail fast here to avoid an infinite loop due to the fact that we + // won't allocate a buffer large enough. + // [1] https://tools.ietf.org/html/rfc5246#section-6.2.1 + throw new SSLException("Illegal packet length: " + packetLengthDataOnly + " > " + + session.getApplicationBufferSize()); + } else { + session.tryExpandApplicationBufferSize(packetLengthDataOnly); + } + return newResultMayFinishHandshake(BUFFER_OVERFLOW, status, 0, 0); + } + + if (len < packetLength) { + // We either don't have enough data to read the packet length or not enough for reading the whole + // packet. + return newResultMayFinishHandshake(BUFFER_UNDERFLOW, status, 0, 0); + } + } else if (len == 0 && sslPending <= 0) { + return newResultMayFinishHandshake(BUFFER_UNDERFLOW, status, 0, 0); + } else if (capacity == 0) { + return newResultMayFinishHandshake(BUFFER_OVERFLOW, status, 0, 0); + } else { + packetLength = (int) min(MAX_VALUE, len); + } + + // This must always be the case when we reached here as if not we returned BUFFER_UNDERFLOW. + assert srcsOffset < srcsEndOffset; + + // This must always be the case if we reached here. + assert capacity > 0; + + // Number of produced bytes + int bytesProduced = 0; + int bytesConsumed = 0; + try { + srcLoop: + for (;;) { + ByteBuffer src = srcs[srcsOffset]; + int remaining = src.remaining(); + final ByteBuf bioWriteCopyBuf; + int pendingEncryptedBytes; + if (remaining == 0) { + if (sslPending <= 0) { + // We must skip empty buffers as BIO_write will return 0 if asked to write something + // with length 0. + if (++srcsOffset >= srcsEndOffset) { + break; + } + continue; + } else { + bioWriteCopyBuf = null; + pendingEncryptedBytes = SSL.bioLengthByteBuffer(networkBIO); + } + } else { + // Write more encrypted data into the BIO. Ensure we only read one packet at a time as + // stated in the SSLEngine javadocs. + pendingEncryptedBytes = min(packetLength, remaining); + bioWriteCopyBuf = writeEncryptedData(src, pendingEncryptedBytes); + } + try { + for (;;) { + ByteBuffer dst = dsts[dstsOffset]; + if (!dst.hasRemaining()) { + // No space left in the destination buffer, skip it. + if (++dstsOffset >= dstsEndOffset) { + break srcLoop; + } + continue; + } + + int bytesRead = readPlaintextData(dst); + // We are directly using the ByteBuffer memory for the write, and so we only know what has + // been consumed after we let SSL decrypt the data. At this point we should update the + // number of bytes consumed, update the ByteBuffer position, and release temp ByteBuf. + int localBytesConsumed = pendingEncryptedBytes - SSL.bioLengthByteBuffer(networkBIO); + bytesConsumed += localBytesConsumed; + packetLength -= localBytesConsumed; + pendingEncryptedBytes -= localBytesConsumed; + src.position(src.position() + localBytesConsumed); + + if (bytesRead > 0) { + bytesProduced += bytesRead; + + if (!dst.hasRemaining()) { + sslPending = sslPending0(); + // Move to the next dst buffer as this one is full. + if (++dstsOffset >= dstsEndOffset) { + return sslPending > 0 ? + newResult(BUFFER_OVERFLOW, status, bytesConsumed, bytesProduced) : + newResultMayFinishHandshake(isInboundDone() ? CLOSED : OK, status, + bytesConsumed, bytesProduced); + } + } else if (packetLength == 0 || jdkCompatibilityMode) { + // We either consumed all data or we are in jdkCompatibilityMode and have consumed + // a single TLS packet and should stop consuming until this method is called again. + break srcLoop; + } + } else { + int sslError = SSL.getError(ssl, bytesRead); + if (sslError == SSL.SSL_ERROR_WANT_READ || sslError == SSL.SSL_ERROR_WANT_WRITE) { + // break to the outer loop as we want to read more data which means we need to + // write more to the BIO. + break; + } else if (sslError == SSL.SSL_ERROR_ZERO_RETURN) { + // This means the connection was shutdown correctly, close inbound and outbound + if (!receivedShutdown) { + closeAll(); + } + return newResultMayFinishHandshake(isInboundDone() ? CLOSED : OK, status, + bytesConsumed, bytesProduced); + } else if (sslError == SSL.SSL_ERROR_WANT_X509_LOOKUP || + sslError == SSL.SSL_ERROR_WANT_CERTIFICATE_VERIFY || + sslError == SSL.SSL_ERROR_WANT_PRIVATE_KEY_OPERATION) { + return newResult(isInboundDone() ? CLOSED : OK, + NEED_TASK, bytesConsumed, bytesProduced); + } else { + return sslReadErrorResult(sslError, SSL.getLastErrorNumber(), bytesConsumed, + bytesProduced); + } + } + } + + if (++srcsOffset >= srcsEndOffset) { + break; + } + } finally { + if (bioWriteCopyBuf != null) { + bioWriteCopyBuf.release(); + } + } + } + } finally { + SSL.bioClearByteBuffer(networkBIO); + rejectRemoteInitiatedRenegotiation(); + } + + // Check to see if we received a close_notify message from the peer. + if (!receivedShutdown && (SSL.getShutdown(ssl) & SSL.SSL_RECEIVED_SHUTDOWN) == SSL.SSL_RECEIVED_SHUTDOWN) { + closeAll(); + } + + return newResultMayFinishHandshake(isInboundDone() ? CLOSED : OK, status, bytesConsumed, bytesProduced); + } + } + + private SSLEngineResult sslReadErrorResult(int error, int stackError, int bytesConsumed, int bytesProduced) + throws SSLException { + // Check if we have a pending handshakeException and if so see if we need to consume all pending data from the + // BIO first or can just shutdown and throw it now. + // This is needed so we ensure close_notify etc is correctly send to the remote peer. + // See https://github.com/netty/netty/issues/3900 + if (SSL.bioLengthNonApplication(networkBIO) > 0) { + if (handshakeException == null && handshakeState != HandshakeState.FINISHED) { + // we seems to have data left that needs to be transferred and so the user needs + // call wrap(...). Store the error so we can pick it up later. + handshakeException = new SSLHandshakeException(SSL.getErrorString(stackError)); + } + // We need to clear all errors so we not pick up anything that was left on the stack on the next + // operation. Note that shutdownWithError(...) will cleanup the stack as well so its only needed here. + SSL.clearError(); + return new SSLEngineResult(OK, NEED_WRAP, bytesConsumed, bytesProduced); + } + throw shutdownWithError("SSL_read", error, stackError); + } + + private void closeAll() throws SSLException { + receivedShutdown = true; + closeOutbound(); + closeInbound(); + } + + private void rejectRemoteInitiatedRenegotiation() throws SSLHandshakeException { + // As rejectRemoteInitiatedRenegotiation() is called in a finally block we also need to check if we shutdown + // the engine before as otherwise SSL.getHandshakeCount(ssl) will throw an NPE if the passed in ssl is 0. + // See https://github.com/netty/netty/issues/7353 + if (!isDestroyed() && SSL.getHandshakeCount(ssl) > 1 && + // As we may count multiple handshakes when TLSv1.3 is used we should just ignore this here as + // renegotiation is not supported in TLSv1.3 as per spec. + !SslUtils.PROTOCOL_TLS_V1_3.equals(session.getProtocol()) && handshakeState == HandshakeState.FINISHED) { + // TODO: In future versions me may also want to send a fatal_alert to the client and so notify it + // that the renegotiation failed. + shutdown(); + throw new SSLHandshakeException("remote-initiated renegotiation not allowed"); + } + } + + public final SSLEngineResult unwrap(final ByteBuffer[] srcs, final ByteBuffer[] dsts) throws SSLException { + return unwrap(srcs, 0, srcs.length, dsts, 0, dsts.length); + } + + private ByteBuffer[] singleSrcBuffer(ByteBuffer src) { + singleSrcBuffer[0] = src; + return singleSrcBuffer; + } + + private void resetSingleSrcBuffer() { + singleSrcBuffer[0] = null; + } + + private ByteBuffer[] singleDstBuffer(ByteBuffer src) { + singleDstBuffer[0] = src; + return singleDstBuffer; + } + + private void resetSingleDstBuffer() { + singleDstBuffer[0] = null; + } + + @Override + public final synchronized SSLEngineResult unwrap( + final ByteBuffer src, final ByteBuffer[] dsts, final int offset, final int length) throws SSLException { + try { + return unwrap(singleSrcBuffer(src), 0, 1, dsts, offset, length); + } finally { + resetSingleSrcBuffer(); + } + } + + @Override + public final synchronized SSLEngineResult wrap(ByteBuffer src, ByteBuffer dst) throws SSLException { + try { + return wrap(singleSrcBuffer(src), dst); + } finally { + resetSingleSrcBuffer(); + } + } + + @Override + public final synchronized SSLEngineResult unwrap(ByteBuffer src, ByteBuffer dst) throws SSLException { + try { + return unwrap(singleSrcBuffer(src), singleDstBuffer(dst)); + } finally { + resetSingleSrcBuffer(); + resetSingleDstBuffer(); + } + } + + @Override + public final synchronized SSLEngineResult unwrap(ByteBuffer src, ByteBuffer[] dsts) throws SSLException { + try { + return unwrap(singleSrcBuffer(src), dsts); + } finally { + resetSingleSrcBuffer(); + } + } + + @Override + public final synchronized Runnable getDelegatedTask() { + if (isDestroyed()) { + return null; + } + final Runnable task = SSL.getTask(ssl); + if (task == null) { + return null; + } + return new Runnable() { + @Override + public void run() { + if (isDestroyed()) { + // The engine was destroyed in the meantime, just return. + return; + } + try { + task.run(); + } finally { + // The task was run, reset needTask to false so getHandshakeStatus() returns the correct value. + needTask = false; + } + } + }; + } + + @Override + public final synchronized void closeInbound() throws SSLException { + if (isInboundDone) { + return; + } + + isInboundDone = true; + + if (isOutboundDone()) { + // Only call shutdown if there is no outbound data pending. + // See https://github.com/netty/netty/issues/6167 + shutdown(); + } + + if (handshakeState != HandshakeState.NOT_STARTED && !receivedShutdown) { + throw new SSLException( + "Inbound closed before receiving peer's close_notify: possible truncation attack?"); + } + } + + @Override + public final synchronized boolean isInboundDone() { + return isInboundDone; + } + + @Override + public final synchronized void closeOutbound() { + if (outboundClosed) { + return; + } + + outboundClosed = true; + + if (handshakeState != HandshakeState.NOT_STARTED && !isDestroyed()) { + int mode = SSL.getShutdown(ssl); + if ((mode & SSL.SSL_SENT_SHUTDOWN) != SSL.SSL_SENT_SHUTDOWN) { + doSSLShutdown(); + } + } else { + // engine closing before initial handshake + shutdown(); + } + } + + /** + * Attempt to call {@link SSL#shutdownSSL(long)}. + * @return {@code false} if the call to {@link SSL#shutdownSSL(long)} was not attempted or returned an error. + */ + private boolean doSSLShutdown() { + if (SSL.isInInit(ssl) != 0) { + // Only try to call SSL_shutdown if we are not in the init state anymore. + // Otherwise we will see 'error:140E0197:SSL routines:SSL_shutdown:shutdown while in init' in our logs. + // + // See also http://hg.nginx.org/nginx/rev/062c189fee20 + return false; + } + int err = SSL.shutdownSSL(ssl); + if (err < 0) { + int sslErr = SSL.getError(ssl, err); + if (sslErr == SSL.SSL_ERROR_SYSCALL || sslErr == SSL.SSL_ERROR_SSL) { + if (logger.isDebugEnabled()) { + int error = SSL.getLastErrorNumber(); + logger.debug("SSL_shutdown failed: OpenSSL error: {} {}", error, SSL.getErrorString(error)); + } + // There was an internal error -- shutdown + shutdown(); + return false; + } + SSL.clearError(); + } + return true; + } + + @Override + public final synchronized boolean isOutboundDone() { + // Check if there is anything left in the outbound buffer. + // We need to ensure we only call SSL.pendingWrittenBytesInBIO(...) if the engine was not destroyed yet. + return outboundClosed && (networkBIO == 0 || SSL.bioLengthNonApplication(networkBIO) == 0); + } + + @Override + public final String[] getSupportedCipherSuites() { + return OpenSsl.AVAILABLE_CIPHER_SUITES.toArray(new String[0]); + } + + @Override + public final String[] getEnabledCipherSuites() { + final String[] enabled; + synchronized (this) { + if (!isDestroyed()) { + enabled = SSL.getCiphers(ssl); + } else { + return EmptyArrays.EMPTY_STRINGS; + } + } + if (enabled == null) { + return EmptyArrays.EMPTY_STRINGS; + } else { + List enabledList = new ArrayList(); + synchronized (this) { + for (int i = 0; i < enabled.length; i++) { + String mapped = toJavaCipherSuite(enabled[i]); + final String cipher = mapped == null ? enabled[i] : mapped; + if (!OpenSsl.isTlsv13Supported() && SslUtils.isTLSv13Cipher(cipher)) { + continue; + } + enabledList.add(cipher); + } + } + return enabledList.toArray(new String[0]); + } + } + + @Override + public final void setEnabledCipherSuites(String[] cipherSuites) { + checkNotNull(cipherSuites, "cipherSuites"); + + final StringBuilder buf = new StringBuilder(); + final StringBuilder bufTLSv13 = new StringBuilder(); + + CipherSuiteConverter.convertToCipherStrings(Arrays.asList(cipherSuites), buf, bufTLSv13, OpenSsl.isBoringSSL()); + final String cipherSuiteSpec = buf.toString(); + final String cipherSuiteSpecTLSv13 = bufTLSv13.toString(); + + if (!OpenSsl.isTlsv13Supported() && !cipherSuiteSpecTLSv13.isEmpty()) { + throw new IllegalArgumentException("TLSv1.3 is not supported by this java version."); + } + synchronized (this) { + if (!isDestroyed()) { + // TODO: Should we also adjust the protocols based on if there are any ciphers left that can be used + // for TLSv1.3 or for previor SSL/TLS versions ? + try { + // Set non TLSv1.3 ciphers. + SSL.setCipherSuites(ssl, cipherSuiteSpec, false); + + if (OpenSsl.isTlsv13Supported()) { + // Set TLSv1.3 ciphers. + SSL.setCipherSuites(ssl, cipherSuiteSpecTLSv13, true); + } + + } catch (Exception e) { + throw new IllegalStateException("failed to enable cipher suites: " + cipherSuiteSpec, e); + } + } else { + throw new IllegalStateException("failed to enable cipher suites: " + cipherSuiteSpec); + } + } + } + + @Override + public final String[] getSupportedProtocols() { + return OpenSsl.SUPPORTED_PROTOCOLS_SET.toArray(new String[0]); + } + + @Override + public final String[] getEnabledProtocols() { + List enabled = new ArrayList(6); + // Seems like there is no way to explicit disable SSLv2Hello in openssl so it is always enabled + enabled.add(PROTOCOL_SSL_V2_HELLO); + + int opts; + synchronized (this) { + if (!isDestroyed()) { + opts = SSL.getOptions(ssl); + } else { + return enabled.toArray(new String[0]); + } + } + if (isProtocolEnabled(opts, SSL.SSL_OP_NO_TLSv1, PROTOCOL_TLS_V1)) { + enabled.add(PROTOCOL_TLS_V1); + } + if (isProtocolEnabled(opts, SSL.SSL_OP_NO_TLSv1_1, PROTOCOL_TLS_V1_1)) { + enabled.add(PROTOCOL_TLS_V1_1); + } + if (isProtocolEnabled(opts, SSL.SSL_OP_NO_TLSv1_2, PROTOCOL_TLS_V1_2)) { + enabled.add(PROTOCOL_TLS_V1_2); + } + if (isProtocolEnabled(opts, SSL.SSL_OP_NO_TLSv1_3, PROTOCOL_TLS_V1_3)) { + enabled.add(PROTOCOL_TLS_V1_3); + } + if (isProtocolEnabled(opts, SSL.SSL_OP_NO_SSLv2, PROTOCOL_SSL_V2)) { + enabled.add(PROTOCOL_SSL_V2); + } + if (isProtocolEnabled(opts, SSL.SSL_OP_NO_SSLv3, PROTOCOL_SSL_V3)) { + enabled.add(PROTOCOL_SSL_V3); + } + return enabled.toArray(new String[0]); + } + + private static boolean isProtocolEnabled(int opts, int disableMask, String protocolString) { + // We also need to check if the actual protocolString is supported as depending on the openssl API + // implementations it may use a disableMask of 0 (BoringSSL is doing this for example). + return (opts & disableMask) == 0 && OpenSsl.SUPPORTED_PROTOCOLS_SET.contains(protocolString); + } + + /** + * {@inheritDoc} + * TLS doesn't support a way to advertise non-contiguous versions from the client's perspective, and the client + * just advertises the max supported version. The TLS protocol also doesn't support all different combinations of + * discrete protocols, and instead assumes contiguous ranges. OpenSSL has some unexpected behavior + * (e.g. handshake failures) if non-contiguous protocols are used even where there is a compatible set of protocols + * and ciphers. For these reasons this method will determine the minimum protocol and the maximum protocol and + * enabled a contiguous range from [min protocol, max protocol] in OpenSSL. + */ + @Override + public final void setEnabledProtocols(String[] protocols) { + if (protocols == null) { + // This is correct from the API docs + throw new IllegalArgumentException(); + } + int minProtocolIndex = OPENSSL_OP_NO_PROTOCOLS.length; + int maxProtocolIndex = 0; + for (String p: protocols) { + if (!OpenSsl.SUPPORTED_PROTOCOLS_SET.contains(p)) { + throw new IllegalArgumentException("Protocol " + p + " is not supported."); + } + if (p.equals(PROTOCOL_SSL_V2)) { + if (minProtocolIndex > OPENSSL_OP_NO_PROTOCOL_INDEX_SSLV2) { + minProtocolIndex = OPENSSL_OP_NO_PROTOCOL_INDEX_SSLV2; + } + if (maxProtocolIndex < OPENSSL_OP_NO_PROTOCOL_INDEX_SSLV2) { + maxProtocolIndex = OPENSSL_OP_NO_PROTOCOL_INDEX_SSLV2; + } + } else if (p.equals(PROTOCOL_SSL_V3)) { + if (minProtocolIndex > OPENSSL_OP_NO_PROTOCOL_INDEX_SSLV3) { + minProtocolIndex = OPENSSL_OP_NO_PROTOCOL_INDEX_SSLV3; + } + if (maxProtocolIndex < OPENSSL_OP_NO_PROTOCOL_INDEX_SSLV3) { + maxProtocolIndex = OPENSSL_OP_NO_PROTOCOL_INDEX_SSLV3; + } + } else if (p.equals(PROTOCOL_TLS_V1)) { + if (minProtocolIndex > OPENSSL_OP_NO_PROTOCOL_INDEX_TLSv1) { + minProtocolIndex = OPENSSL_OP_NO_PROTOCOL_INDEX_TLSv1; + } + if (maxProtocolIndex < OPENSSL_OP_NO_PROTOCOL_INDEX_TLSv1) { + maxProtocolIndex = OPENSSL_OP_NO_PROTOCOL_INDEX_TLSv1; + } + } else if (p.equals(PROTOCOL_TLS_V1_1)) { + if (minProtocolIndex > OPENSSL_OP_NO_PROTOCOL_INDEX_TLSv1_1) { + minProtocolIndex = OPENSSL_OP_NO_PROTOCOL_INDEX_TLSv1_1; + } + if (maxProtocolIndex < OPENSSL_OP_NO_PROTOCOL_INDEX_TLSv1_1) { + maxProtocolIndex = OPENSSL_OP_NO_PROTOCOL_INDEX_TLSv1_1; + } + } else if (p.equals(PROTOCOL_TLS_V1_2)) { + if (minProtocolIndex > OPENSSL_OP_NO_PROTOCOL_INDEX_TLSv1_2) { + minProtocolIndex = OPENSSL_OP_NO_PROTOCOL_INDEX_TLSv1_2; + } + if (maxProtocolIndex < OPENSSL_OP_NO_PROTOCOL_INDEX_TLSv1_2) { + maxProtocolIndex = OPENSSL_OP_NO_PROTOCOL_INDEX_TLSv1_2; + } + } else if (p.equals(PROTOCOL_TLS_V1_3)) { + if (minProtocolIndex > OPENSSL_OP_NO_PROTOCOL_INDEX_TLSv1_3) { + minProtocolIndex = OPENSSL_OP_NO_PROTOCOL_INDEX_TLSv1_3; + } + if (maxProtocolIndex < OPENSSL_OP_NO_PROTOCOL_INDEX_TLSv1_3) { + maxProtocolIndex = OPENSSL_OP_NO_PROTOCOL_INDEX_TLSv1_3; + } + } + } + synchronized (this) { + if (!isDestroyed()) { + // Clear out options which disable protocols + SSL.clearOptions(ssl, SSL.SSL_OP_NO_SSLv2 | SSL.SSL_OP_NO_SSLv3 | SSL.SSL_OP_NO_TLSv1 | + SSL.SSL_OP_NO_TLSv1_1 | SSL.SSL_OP_NO_TLSv1_2 | SSL.SSL_OP_NO_TLSv1_3); + + int opts = 0; + for (int i = 0; i < minProtocolIndex; ++i) { + opts |= OPENSSL_OP_NO_PROTOCOLS[i]; + } + assert maxProtocolIndex != MAX_VALUE; + for (int i = maxProtocolIndex + 1; i < OPENSSL_OP_NO_PROTOCOLS.length; ++i) { + opts |= OPENSSL_OP_NO_PROTOCOLS[i]; + } + + // Disable protocols we do not want + SSL.setOptions(ssl, opts); + } else { + throw new IllegalStateException("failed to enable protocols: " + Arrays.asList(protocols)); + } + } + } + + @Override + public final SSLSession getSession() { + return session; + } + + @Override + public final synchronized void beginHandshake() throws SSLException { + switch (handshakeState) { + case STARTED_IMPLICITLY: + checkEngineClosed(); + + // A user did not start handshake by calling this method by him/herself, + // but handshake has been started already by wrap() or unwrap() implicitly. + // Because it's the user's first time to call this method, it is unfair to + // raise an exception. From the user's standpoint, he or she never asked + // for renegotiation. + + handshakeState = HandshakeState.STARTED_EXPLICITLY; // Next time this method is invoked by the user, + calculateMaxWrapOverhead(); + // we should raise an exception. + break; + case STARTED_EXPLICITLY: + // Nothing to do as the handshake is not done yet. + break; + case FINISHED: + throw new SSLException("renegotiation unsupported"); + case NOT_STARTED: + handshakeState = HandshakeState.STARTED_EXPLICITLY; + if (handshake() == NEED_TASK) { + // Set needTask to true so getHandshakeStatus() will return the correct value. + needTask = true; + } + calculateMaxWrapOverhead(); + break; + default: + throw new Error(); + } + } + + private void checkEngineClosed() throws SSLException { + if (isDestroyed()) { + throw new SSLException("engine closed"); + } + } + + private static SSLEngineResult.HandshakeStatus pendingStatus(int pendingStatus) { + // Depending on if there is something left in the BIO we need to WRAP or UNWRAP + return pendingStatus > 0 ? NEED_WRAP : NEED_UNWRAP; + } + + private static boolean isEmpty(Object[] arr) { + return arr == null || arr.length == 0; + } + + private static boolean isEmpty(byte[] cert) { + return cert == null || cert.length == 0; + } + + private SSLEngineResult.HandshakeStatus handshakeException() throws SSLException { + if (SSL.bioLengthNonApplication(networkBIO) > 0) { + // There is something pending, we need to consume it first via a WRAP so we don't loose anything. + return NEED_WRAP; + } + + Throwable exception = handshakeException; + assert exception != null; + handshakeException = null; + shutdown(); + if (exception instanceof SSLHandshakeException) { + throw (SSLHandshakeException) exception; + } + SSLHandshakeException e = new SSLHandshakeException("General OpenSslEngine problem"); + e.initCause(exception); + throw e; + } + + /** + * Should be called if the handshake will be failed due a callback that throws an exception. + * This cause will then be used to give more details as part of the {@link SSLHandshakeException}. + */ + final void initHandshakeException(Throwable cause) { + assert handshakeException == null; + handshakeException = cause; + } + + private SSLEngineResult.HandshakeStatus handshake() throws SSLException { + if (needTask) { + return NEED_TASK; + } + if (handshakeState == HandshakeState.FINISHED) { + return FINISHED; + } + + checkEngineClosed(); + + if (handshakeException != null) { + // Let's call SSL.doHandshake(...) again in case there is some async operation pending that would fill the + // outbound buffer. + if (SSL.doHandshake(ssl) <= 0) { + // Clear any error that was put on the stack by the handshake + SSL.clearError(); + } + return handshakeException(); + } + + // Adding the OpenSslEngine to the OpenSslEngineMap so it can be used in the AbstractCertificateVerifier. + engineMap.add(this); + if (lastAccessed == -1) { + lastAccessed = System.currentTimeMillis(); + } + + int code = SSL.doHandshake(ssl); + if (code <= 0) { + int sslError = SSL.getError(ssl, code); + if (sslError == SSL.SSL_ERROR_WANT_READ || sslError == SSL.SSL_ERROR_WANT_WRITE) { + return pendingStatus(SSL.bioLengthNonApplication(networkBIO)); + } + + if (sslError == SSL.SSL_ERROR_WANT_X509_LOOKUP || + sslError == SSL.SSL_ERROR_WANT_CERTIFICATE_VERIFY || + sslError == SSL.SSL_ERROR_WANT_PRIVATE_KEY_OPERATION) { + return NEED_TASK; + } + + // Check if we have a pending exception that was created during the handshake and if so throw it after + // shutdown the connection. + if (handshakeException != null) { + return handshakeException(); + } + + // Everything else is considered as error + throw shutdownWithError("SSL_do_handshake", sslError); + } + // We have produced more data as part of the handshake if this is the case the user should call wrap(...) + if (SSL.bioLengthNonApplication(networkBIO) > 0) { + return NEED_WRAP; + } + // if SSL_do_handshake returns > 0 or sslError == SSL.SSL_ERROR_NAME it means the handshake was finished. + session.handshakeFinished(); + return FINISHED; + } + + private SSLEngineResult.HandshakeStatus mayFinishHandshake(SSLEngineResult.HandshakeStatus status) + throws SSLException { + if (status == NOT_HANDSHAKING && handshakeState != HandshakeState.FINISHED) { + // If the status was NOT_HANDSHAKING and we not finished the handshake we need to call + // SSL_do_handshake() again + return handshake(); + } + return status; + } + + @Override + public final synchronized SSLEngineResult.HandshakeStatus getHandshakeStatus() { + // Check if we are in the initial handshake phase or shutdown phase + if (needPendingStatus()) { + if (needTask) { + // There is a task outstanding + return NEED_TASK; + } + return pendingStatus(SSL.bioLengthNonApplication(networkBIO)); + } + return NOT_HANDSHAKING; + } + + private SSLEngineResult.HandshakeStatus getHandshakeStatus(int pending) { + // Check if we are in the initial handshake phase or shutdown phase + if (needPendingStatus()) { + if (needTask) { + // There is a task outstanding + return NEED_TASK; + } + return pendingStatus(pending); + } + return NOT_HANDSHAKING; + } + + private boolean needPendingStatus() { + return handshakeState != HandshakeState.NOT_STARTED && !isDestroyed() + && (handshakeState != HandshakeState.FINISHED || isInboundDone() || isOutboundDone()); + } + + /** + * Converts the specified OpenSSL cipher suite to the Java cipher suite. + */ + private String toJavaCipherSuite(String openSslCipherSuite) { + if (openSslCipherSuite == null) { + return null; + } + + String version = SSL.getVersion(ssl); + String prefix = toJavaCipherSuitePrefix(version); + return CipherSuiteConverter.toJava(openSslCipherSuite, prefix); + } + + /** + * Converts the protocol version string returned by {@link SSL#getVersion(long)} to protocol family string. + */ + private static String toJavaCipherSuitePrefix(String protocolVersion) { + final char c; + if (protocolVersion == null || protocolVersion.isEmpty()) { + c = 0; + } else { + c = protocolVersion.charAt(0); + } + + switch (c) { + case 'T': + return "TLS"; + case 'S': + return "SSL"; + default: + return "UNKNOWN"; + } + } + + @Override + public final void setUseClientMode(boolean clientMode) { + if (clientMode != this.clientMode) { + throw new UnsupportedOperationException(); + } + } + + @Override + public final boolean getUseClientMode() { + return clientMode; + } + + @Override + public final void setNeedClientAuth(boolean b) { + setClientAuth(b ? ClientAuth.REQUIRE : ClientAuth.NONE); + } + + @Override + public final boolean getNeedClientAuth() { + return clientAuth == ClientAuth.REQUIRE; + } + + @Override + public final void setWantClientAuth(boolean b) { + setClientAuth(b ? ClientAuth.OPTIONAL : ClientAuth.NONE); + } + + @Override + public final boolean getWantClientAuth() { + return clientAuth == ClientAuth.OPTIONAL; + } + + /** + * See SSL_set_verify and + * {@link SSL#setVerify(long, int, int)}. + */ + @UnstableApi + public final synchronized void setVerify(int verifyMode, int depth) { + SSL.setVerify(ssl, verifyMode, depth); + } + + private void setClientAuth(ClientAuth mode) { + if (clientMode) { + return; + } + synchronized (this) { + if (clientAuth == mode) { + // No need to issue any JNI calls if the mode is the same + return; + } + switch (mode) { + case NONE: + SSL.setVerify(ssl, SSL.SSL_CVERIFY_NONE, ReferenceCountedOpenSslContext.VERIFY_DEPTH); + break; + case REQUIRE: + SSL.setVerify(ssl, SSL.SSL_CVERIFY_REQUIRED, ReferenceCountedOpenSslContext.VERIFY_DEPTH); + break; + case OPTIONAL: + SSL.setVerify(ssl, SSL.SSL_CVERIFY_OPTIONAL, ReferenceCountedOpenSslContext.VERIFY_DEPTH); + break; + default: + throw new Error(mode.toString()); + } + clientAuth = mode; + } + } + + @Override + public final void setEnableSessionCreation(boolean b) { + if (b) { + throw new UnsupportedOperationException(); + } + } + + @Override + public final boolean getEnableSessionCreation() { + return false; + } + + @SuppressJava6Requirement(reason = "Usage guarded by java version check") + @Override + public final synchronized SSLParameters getSSLParameters() { + SSLParameters sslParameters = super.getSSLParameters(); + + int version = PlatformDependent.javaVersion(); + if (version >= 7) { + sslParameters.setEndpointIdentificationAlgorithm(endPointIdentificationAlgorithm); + Java7SslParametersUtils.setAlgorithmConstraints(sslParameters, algorithmConstraints); + if (version >= 8) { + if (sniHostNames != null) { + Java8SslUtils.setSniHostNames(sslParameters, sniHostNames); + } + if (!isDestroyed()) { + Java8SslUtils.setUseCipherSuitesOrder( + sslParameters, (SSL.getOptions(ssl) & SSL.SSL_OP_CIPHER_SERVER_PREFERENCE) != 0); + } + + Java8SslUtils.setSNIMatchers(sslParameters, matchers); + } + } + return sslParameters; + } + + @SuppressJava6Requirement(reason = "Usage guarded by java version check") + @Override + public final synchronized void setSSLParameters(SSLParameters sslParameters) { + int version = PlatformDependent.javaVersion(); + if (version >= 7) { + if (sslParameters.getAlgorithmConstraints() != null) { + throw new IllegalArgumentException("AlgorithmConstraints are not supported."); + } + + if (version >= 8) { + if (!isDestroyed()) { + if (clientMode) { + final List sniHostNames = Java8SslUtils.getSniHostNames(sslParameters); + for (String name: sniHostNames) { + SSL.setTlsExtHostName(ssl, name); + } + this.sniHostNames = sniHostNames; + } + if (Java8SslUtils.getUseCipherSuitesOrder(sslParameters)) { + SSL.setOptions(ssl, SSL.SSL_OP_CIPHER_SERVER_PREFERENCE); + } else { + SSL.clearOptions(ssl, SSL.SSL_OP_CIPHER_SERVER_PREFERENCE); + } + } + matchers = sslParameters.getSNIMatchers(); + } + + final String endPointIdentificationAlgorithm = sslParameters.getEndpointIdentificationAlgorithm(); + final boolean endPointVerificationEnabled = isEndPointVerificationEnabled(endPointIdentificationAlgorithm); + + // If the user asks for hostname verification we must ensure we verify the peer. + // If the user disables hostname verification we leave it up to the user to change the mode manually. + if (clientMode && endPointVerificationEnabled) { + SSL.setVerify(ssl, SSL.SSL_CVERIFY_REQUIRED, -1); + } + + this.endPointIdentificationAlgorithm = endPointIdentificationAlgorithm; + algorithmConstraints = sslParameters.getAlgorithmConstraints(); + } + super.setSSLParameters(sslParameters); + } + + private static boolean isEndPointVerificationEnabled(String endPointIdentificationAlgorithm) { + return endPointIdentificationAlgorithm != null && !endPointIdentificationAlgorithm.isEmpty(); + } + + private boolean isDestroyed() { + return destroyed; + } + + final boolean checkSniHostnameMatch(byte[] hostname) { + return Java8SslUtils.checkSniHostnameMatch(matchers, hostname); + } + + @Override + public String getNegotiatedApplicationProtocol() { + return applicationProtocol; + } + + private static long bufferAddress(ByteBuffer b) { + assert b.isDirect(); + if (PlatformDependent.hasUnsafe()) { + return PlatformDependent.directBufferAddress(b); + } + return Buffer.address(b); + } + + private final class DefaultOpenSslSession implements OpenSslSession { + private final OpenSslSessionContext sessionContext; + + // These are guarded by synchronized(OpenSslEngine.this) as handshakeFinished() may be triggered by any + // thread. + private X509Certificate[] x509PeerCerts; + private Certificate[] peerCerts; + + private String protocol; + private String cipher; + private byte[] id; + private long creationTime; + private volatile int applicationBufferSize = MAX_PLAINTEXT_LENGTH; + + // lazy init for memory reasons + private Map values; + + DefaultOpenSslSession(OpenSslSessionContext sessionContext) { + this.sessionContext = sessionContext; + } + + private SSLSessionBindingEvent newSSLSessionBindingEvent(String name) { + return new SSLSessionBindingEvent(session, name); + } + + @Override + public byte[] getId() { + synchronized (ReferenceCountedOpenSslEngine.this) { + if (id == null) { + return EmptyArrays.EMPTY_BYTES; + } + return id.clone(); + } + } + + @Override + public SSLSessionContext getSessionContext() { + return sessionContext; + } + + @Override + public long getCreationTime() { + synchronized (ReferenceCountedOpenSslEngine.this) { + if (creationTime == 0 && !isDestroyed()) { + creationTime = SSL.getTime(ssl) * 1000L; + } + } + return creationTime; + } + + @Override + public long getLastAccessedTime() { + long lastAccessed = ReferenceCountedOpenSslEngine.this.lastAccessed; + // if lastAccessed is -1 we will just return the creation time as the handshake was not started yet. + return lastAccessed == -1 ? getCreationTime() : lastAccessed; + } + + @Override + public void invalidate() { + synchronized (ReferenceCountedOpenSslEngine.this) { + if (!isDestroyed()) { + SSL.setTimeout(ssl, 0); + } + } + } + + @Override + public boolean isValid() { + synchronized (ReferenceCountedOpenSslEngine.this) { + if (!isDestroyed()) { + return System.currentTimeMillis() - (SSL.getTimeout(ssl) * 1000L) < (SSL.getTime(ssl) * 1000L); + } + } + return false; + } + + @Override + public void putValue(String name, Object value) { + ObjectUtil.checkNotNull(name, "name"); + ObjectUtil.checkNotNull(value, "value"); + + final Object old; + synchronized (this) { + Map values = this.values; + if (values == null) { + // Use size of 2 to keep the memory overhead small + values = this.values = new HashMap(2); + } + old = values.put(name, value); + } + + if (value instanceof SSLSessionBindingListener) { + // Use newSSLSessionBindingEvent so we alway use the wrapper if needed. + ((SSLSessionBindingListener) value).valueBound(newSSLSessionBindingEvent(name)); + } + notifyUnbound(old, name); + } + + @Override + public Object getValue(String name) { + ObjectUtil.checkNotNull(name, "name"); + synchronized (this) { + if (values == null) { + return null; + } + return values.get(name); + } + } + + @Override + public void removeValue(String name) { + ObjectUtil.checkNotNull(name, "name"); + + final Object old; + synchronized (this) { + Map values = this.values; + if (values == null) { + return; + } + old = values.remove(name); + } + + notifyUnbound(old, name); + } + + @Override + public String[] getValueNames() { + synchronized (this) { + Map values = this.values; + if (values == null || values.isEmpty()) { + return EmptyArrays.EMPTY_STRINGS; + } + return values.keySet().toArray(new String[0]); + } + } + + private void notifyUnbound(Object value, String name) { + if (value instanceof SSLSessionBindingListener) { + // Use newSSLSessionBindingEvent so we alway use the wrapper if needed. + ((SSLSessionBindingListener) value).valueUnbound(newSSLSessionBindingEvent(name)); + } + } + + /** + * Finish the handshake and so init everything in the {@link OpenSslSession} that should be accessible by + * the user. + */ + @Override + public void handshakeFinished() throws SSLException { + synchronized (ReferenceCountedOpenSslEngine.this) { + if (!isDestroyed()) { + id = SSL.getSessionId(ssl); + + // We need this additional if check because tlsv1.3 isnt always setting the id 7/23/20 + if (id == null || id.length == 0) { + id = Longs.toByteArray(ssl); + } + + cipher = toJavaCipherSuite(SSL.getCipherForSSL(ssl)); + protocol = SSL.getVersion(ssl); + + initPeerCerts(); + selectApplicationProtocol(); + calculateMaxWrapOverhead(); + + handshakeState = HandshakeState.FINISHED; + } else { + throw new SSLException("Already closed"); + } + } + } + + /** + * Init peer certificates that can be obtained via {@link #getPeerCertificateChain()} + * and {@link #getPeerCertificates()}. + */ + private void initPeerCerts() { + // Return the full chain from the JNI layer. + byte[][] chain = SSL.getPeerCertChain(ssl); + if (clientMode) { + if (isEmpty(chain)) { + peerCerts = EMPTY_CERTIFICATES; + x509PeerCerts = EMPTY_JAVAX_X509_CERTIFICATES; + } else { + peerCerts = new Certificate[chain.length]; + x509PeerCerts = new X509Certificate[chain.length]; + initCerts(chain, 0); + } + } else { + // if used on the server side SSL_get_peer_cert_chain(...) will not include the remote peer + // certificate. We use SSL_get_peer_certificate to get it in this case and add it to our + // array later. + // + // See https://www.openssl.org/docs/ssl/SSL_get_peer_cert_chain.html + byte[] clientCert = SSL.getPeerCertificate(ssl); + if (isEmpty(clientCert)) { + peerCerts = EMPTY_CERTIFICATES; + x509PeerCerts = EMPTY_JAVAX_X509_CERTIFICATES; + } else { + if (isEmpty(chain)) { + peerCerts = new Certificate[] {new OpenSslX509Certificate(clientCert)}; + x509PeerCerts = new X509Certificate[] {new OpenSslJavaxX509Certificate(clientCert)}; + } else { + peerCerts = new Certificate[chain.length + 1]; + x509PeerCerts = new X509Certificate[chain.length + 1]; + peerCerts[0] = new OpenSslX509Certificate(clientCert); + x509PeerCerts[0] = new OpenSslJavaxX509Certificate(clientCert); + initCerts(chain, 1); + } + } + } + } + + private void initCerts(byte[][] chain, int startPos) { + for (int i = 0; i < chain.length; i++) { + int certPos = startPos + i; + peerCerts[certPos] = new OpenSslX509Certificate(chain[i]); + x509PeerCerts[certPos] = new OpenSslJavaxX509Certificate(chain[i]); + } + } + + /** + * Select the application protocol used. + */ + private void selectApplicationProtocol() throws SSLException { + ApplicationProtocolConfig.SelectedListenerFailureBehavior behavior = apn.selectedListenerFailureBehavior(); + List protocols = apn.protocols(); + String applicationProtocol; + switch (apn.protocol()) { + case NONE: + break; + // We always need to check for applicationProtocol == null as the remote peer may not support + // the TLS extension or may have returned an empty selection. + case ALPN: + applicationProtocol = SSL.getAlpnSelected(ssl); + if (applicationProtocol != null) { + ReferenceCountedOpenSslEngine.this.applicationProtocol = selectApplicationProtocol( + protocols, behavior, applicationProtocol); + } + break; + case NPN: + applicationProtocol = SSL.getNextProtoNegotiated(ssl); + if (applicationProtocol != null) { + ReferenceCountedOpenSslEngine.this.applicationProtocol = selectApplicationProtocol( + protocols, behavior, applicationProtocol); + } + break; + case NPN_AND_ALPN: + applicationProtocol = SSL.getAlpnSelected(ssl); + if (applicationProtocol == null) { + applicationProtocol = SSL.getNextProtoNegotiated(ssl); + } + if (applicationProtocol != null) { + ReferenceCountedOpenSslEngine.this.applicationProtocol = selectApplicationProtocol( + protocols, behavior, applicationProtocol); + } + break; + default: + throw new Error(); + } + } + + private String selectApplicationProtocol(List protocols, + ApplicationProtocolConfig.SelectedListenerFailureBehavior behavior, + String applicationProtocol) throws SSLException { + if (behavior == ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT) { + return applicationProtocol; + } else { + int size = protocols.size(); + assert size > 0; + if (protocols.contains(applicationProtocol)) { + return applicationProtocol; + } else { + if (behavior == ApplicationProtocolConfig.SelectedListenerFailureBehavior.CHOOSE_MY_LAST_PROTOCOL) { + return protocols.get(size - 1); + } else { + throw new SSLException("unknown protocol " + applicationProtocol); + } + } + } + } + + @Override + public Certificate[] getPeerCertificates() throws SSLPeerUnverifiedException { + synchronized (ReferenceCountedOpenSslEngine.this) { + if (isEmpty(peerCerts)) { + throw new SSLPeerUnverifiedException("peer not verified"); + } + return peerCerts.clone(); + } + } + + @Override + public Certificate[] getLocalCertificates() { + Certificate[] localCerts = ReferenceCountedOpenSslEngine.this.localCertificateChain; + if (localCerts == null) { + return null; + } + return localCerts.clone(); + } + + @Override + public X509Certificate[] getPeerCertificateChain() throws SSLPeerUnverifiedException { + synchronized (ReferenceCountedOpenSslEngine.this) { + if (isEmpty(x509PeerCerts)) { + throw new SSLPeerUnverifiedException("peer not verified"); + } + return x509PeerCerts.clone(); + } + } + + @Override + public Principal getPeerPrincipal() throws SSLPeerUnverifiedException { + Certificate[] peer = getPeerCertificates(); + // No need for null or length > 0 is needed as this is done in getPeerCertificates() + // already. + return ((java.security.cert.X509Certificate) peer[0]).getSubjectX500Principal(); + } + + @Override + public Principal getLocalPrincipal() { + Certificate[] local = ReferenceCountedOpenSslEngine.this.localCertificateChain; + if (local == null || local.length == 0) { + return null; + } + return ((java.security.cert.X509Certificate) local[0]).getIssuerX500Principal(); + } + + @Override + public String getCipherSuite() { + synchronized (ReferenceCountedOpenSslEngine.this) { + if (cipher == null) { + return SslUtils.INVALID_CIPHER; + } + return cipher; + } + } + + @Override + public String getProtocol() { + String protocol = this.protocol; + if (protocol == null) { + synchronized (ReferenceCountedOpenSslEngine.this) { + if (!isDestroyed()) { + protocol = SSL.getVersion(ssl); + } else { + protocol = StringUtil.EMPTY_STRING; + } + } + } + return protocol; + } + + @Override + public String getPeerHost() { + return ReferenceCountedOpenSslEngine.this.getPeerHost(); + } + + @Override + public int getPeerPort() { + return ReferenceCountedOpenSslEngine.this.getPeerPort(); + } + + @Override + public int getPacketBufferSize() { + return maxEncryptedPacketLength(); + } + + @Override + public int getApplicationBufferSize() { + return applicationBufferSize; + } + + @Override + public void tryExpandApplicationBufferSize(int packetLengthDataOnly) { + if (packetLengthDataOnly > MAX_PLAINTEXT_LENGTH && applicationBufferSize != MAX_RECORD_SIZE) { + applicationBufferSize = MAX_RECORD_SIZE; + } + } + } +} diff --git a/src/testing/netty-streaming-cot-client/src/main/java/netty/NettyClientInputTest.java b/src/testing/netty-streaming-cot-client/src/main/java/netty/NettyClientInputTest.java new file mode 100644 index 00000000..cd72cb92 --- /dev/null +++ b/src/testing/netty-streaming-cot-client/src/main/java/netty/NettyClientInputTest.java @@ -0,0 +1,114 @@ +package netty; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.security.KeyStore; + +import javax.net.ssl.KeyManagerFactory; + +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.WriteBufferWaterMark; +import io.netty.channel.epoll.Epoll; +import io.netty.channel.epoll.EpollEventLoopGroup; +import io.netty.channel.epoll.EpollSocketChannel; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.nio.NioSocketChannel; + +/* + * + * This is an example class of how to interface with a gRPC CoT streaming input on TAK Server. + * The client in this example acts as an echo client. It will send back any messages it receives. + * Any clients on TAK Server in the same groups as this client should received the echoed message. + * + * Before Running, make sure to edit @port, @host, @keystoreFile, @keystorePassword, @truststoreFile, @truststorePassword + * The keystore should contain the X509 client cert you want to used for authentication and group assignment + * + */ +public class NettyClientInputTest { + private static final int NUM_AVAIL_CORES = Runtime.getRuntime().availableProcessors(); + public static int highMark; + public static int lowMark; + public static int flushThreshold; + public static int maxOptimalMessagesPerMinute; + private final boolean isEpoll; + private EventLoopGroup workerGroup; + private EventLoopGroup bossGroup; + + + public static class ConnectionMeta { + public int port = 8089; + public String address = "localhost"; + + String keyManager = "SunX509"; + String keystoreType = "JKS"; + // takserver.jks + String keystoreFile = ""; + String keystorePassword = "atakatak"; + + String truststoreType = "JKS"; + // truststore-root.jks + String truststoreFile = ""; + String truststorePassword = "atakatak"; + } + + public static void main(String[] args) { + + try { + ConnectionMeta cm = new ConnectionMeta(); + + NettyClientInputTest client = new NettyClientInputTest(); + client.startClient(cm); + } catch(Exception e) { + System.out.println(e); + } + + + while (true); + } + + private NettyClientInputTest() { + isEpoll = Epoll.isAvailable(); + + int baselineHighMark = 4096; + int baselineMaxOptimalMessagesPerMinute = 250; + + highMark = baselineHighMark * NUM_AVAIL_CORES; + lowMark = highMark / 2; + flushThreshold = lowMark; + maxOptimalMessagesPerMinute = baselineMaxOptimalMessagesPerMinute * NUM_AVAIL_CORES; + } + + private void startClient(ConnectionMeta cm) throws UnknownHostException { + checkAndCreateEventLoopGroups(); + + InetSocketAddress address = new InetSocketAddress(InetAddress.getByName(cm.address), cm.port); + + new Thread(() -> { + try { + Bootstrap clientBootstrap = new Bootstrap(); + clientBootstrap.group(bossGroup) + .channel(isEpoll ? EpollSocketChannel.class : NioSocketChannel.class) + .remoteAddress(address) + .option(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark(lowMark, highMark)) + .handler(new NettyInitializer(cm)); + ChannelFuture channelFuture = clientBootstrap.connect().sync(); + channelFuture.channel().closeFuture().sync(); + } catch (Exception e) { + System.out.println("error initializing netty client " + e); + } + }).start(); + } + + private void checkAndCreateEventLoopGroups() { + if (bossGroup == null) { + bossGroup = isEpoll ? new EpollEventLoopGroup(1) : new NioEventLoopGroup(1); + } + if (workerGroup == null) { + workerGroup = isEpoll ? new EpollEventLoopGroup() : new NioEventLoopGroup(); + } + } +} diff --git a/src/testing/netty-streaming-cot-client/src/main/java/netty/NettyInitializer.java b/src/testing/netty-streaming-cot-client/src/main/java/netty/NettyInitializer.java new file mode 100644 index 00000000..b4269bed --- /dev/null +++ b/src/testing/netty-streaming-cot-client/src/main/java/netty/NettyInitializer.java @@ -0,0 +1,149 @@ +package netty; + +import java.io.FileInputStream; +import java.security.KeyStore; + +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLException; +import javax.net.ssl.TrustManagerFactory; + +import io.netty.channel.ChannelInitializer; +import io.netty.channel.socket.SocketChannel; +import io.netty.handler.codec.bytes.ByteArrayDecoder; +import io.netty.handler.codec.bytes.ByteArrayEncoder; +import io.netty.handler.ssl.ClientAuth; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.SslHandler; +import io.netty.handler.ssl.SslProvider; + +public class NettyInitializer extends ChannelInitializer { + + private SslContext sslContext; + private TrustManagerFactory trustMgrFactory; + private KeyManagerFactory keyMgrFactory; + + private NettyClientInputTest.ConnectionMeta cm; + NettyInitializer(NettyClientInputTest.ConnectionMeta cm) { + this.cm = cm; + } + + @Override + protected void initChannel(SocketChannel channel) throws Exception { + buildClientSslContext(); + + SslHandler sslHandler = sslContext.newHandler(channel.alloc()); + sslHandler.engine().setEnabledProtocols(new String[] {"TLSv1.2", "TLSv1.3"}); + + channel.pipeline() + .addLast("ssl", sslHandler) + .addLast(new ByteArrayDecoder()) + .addLast(new ByteArrayEncoder()) + .addLast(new NioNettyHandler()); + } + + protected SslContext buildClientSslContext() { + try { + initTrust(); + } catch (Exception e) { + System.out.println("Could not init trust " + e); + } + + try { + sslContext = SslContextBuilder.forClient() + .keyManager(keyMgrFactory) + .trustManager(trustMgrFactory) + .clientAuth(ClientAuth.REQUIRE) + .sslProvider(SslProvider.OPENSSL) + .build(); + } catch (SSLException e) { + System.out.println("Could not build client ssl context " + e); + } + + return sslContext; + } + + private void initTrust() throws Exception { + String keyManager = cm.keyManager; + keyMgrFactory = KeyManagerFactory.getInstance(keyManager); + + String keystoreType = cm.keystoreType; + + if (keystoreType == null || keystoreType == "") { + throw new IllegalArgumentException("empty keystore type"); + } + + KeyStore self = KeyStore.getInstance(keystoreType); + + String keystoreFile = cm.keystoreFile; + + if (keystoreFile == null || keystoreFile == "") { + throw new IllegalArgumentException("keystore file name empty"); + } + + String keystorePassword = cm.keystorePassword; + + if (keystorePassword == null || keystorePassword == "") { + throw new IllegalArgumentException("empty keystore password"); + } + + try(FileInputStream fis = new FileInputStream(keystoreFile)) { + // Filename of the keystore file + self.load(fis, keystorePassword.toCharArray()); + } + + // Password of the keystore file + keyMgrFactory.init(self, keystorePassword.toCharArray()); + + // Trust Manager Factory type (e.g., ??) + trustMgrFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + + // initialize trust store + initTrust(trustMgrFactory); + } + + public KeyStore initTrust(TrustManagerFactory trustMgrFactory) { + + KeyStore trust = null; + + try { + String truststoreType = cm.truststoreType; + + if (truststoreType == null || truststoreType == "") { + throw new IllegalArgumentException("empty truststore type"); + } + + // Truststore type (same as keystore types - e.g., "JKS") + trust = KeyStore.getInstance(truststoreType); + + + String truststoreFile = cm.truststoreFile; + + if (truststoreFile == null || truststoreFile == "") { + throw new IllegalArgumentException("empty truststore file name"); + } + + String truststorePassword = cm.truststorePassword; + + if (truststorePassword == null || truststorePassword == "") { + throw new IllegalArgumentException("empty truststore password"); + } + + try(FileInputStream fis = new FileInputStream(truststoreFile)) { + // Filename of the truststore file + trust.load(fis, truststorePassword.toCharArray()); + } + + // NOTE we are not adding any cert revocations like TAK Server does + trustMgrFactory.init(trust); + + + } catch (Exception e) { + System.out.println("exception initializing trust store " + e); + } + + + return trust; + + } +} diff --git a/src/testing/netty-streaming-cot-client/src/main/java/netty/NioNettyHandler.java b/src/testing/netty-streaming-cot-client/src/main/java/netty/NioNettyHandler.java new file mode 100644 index 00000000..06f12dcc --- /dev/null +++ b/src/testing/netty-streaming-cot-client/src/main/java/netty/NioNettyHandler.java @@ -0,0 +1,44 @@ +package netty; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.handler.ssl.SslHandler; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.GenericFutureListener; + +public class NioNettyHandler extends SimpleChannelInboundHandler { + public final static String SA = "<__group name=\"Magenta\" role=\"Team Member\"/>"; + + @Override + public void channelActive(ChannelHandlerContext ctx) { + ctx.pipeline() + .get(SslHandler.class) + .handshakeFuture() + .addListener(new GenericFutureListener>() { + @Override + public void operationComplete(Future future) throws Exception { + if(future.isSuccess()) { + ctx.writeAndFlush(SA.getBytes()); + }else { + ctx.close(); + } + } + }); + + + ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); + executorService.scheduleWithFixedDelay(() -> { + ctx.writeAndFlush(SA.getBytes()); + }, 30, 30, TimeUnit.SECONDS); + } + + @Override + protected void channelRead0(ChannelHandlerContext ctx, byte[] msg) throws Exception { + System.out.print("read " + new String(msg)); + } +} diff --git a/src/testing/takserver-datafeed-load-test/build.gradle b/src/testing/takserver-datafeed-load-test/build.gradle new file mode 100644 index 00000000..deade0cc --- /dev/null +++ b/src/testing/takserver-datafeed-load-test/build.gradle @@ -0,0 +1,24 @@ + +repositories { + maven { + url = 'https://artifacts.tak.gov/artifactory/maven' + credentials { + username = "$takGovUser" + password = "$takGovPassword" + } + } +} + +apply plugin: 'com.github.johnrengelman.shadow' + +// support building a single jar that contains plugin code and also dependencies +shadowJar { } + +dependencies { + + compile group: 'gov.tak', name: 'takserver-plugins', version: takserver_plugins_version, classifier: 'all' + + // add additional depenencies as required for your TAK Server plugin +} + + diff --git a/src/testing/takserver-datafeed-load-test/src/main/java/tak/server/plugins/DataFeedMessageSenderPluginLoadTest.java b/src/testing/takserver-datafeed-load-test/src/main/java/tak/server/plugins/DataFeedMessageSenderPluginLoadTest.java new file mode 100644 index 00000000..8045b18b --- /dev/null +++ b/src/testing/takserver-datafeed-load-test/src/main/java/tak/server/plugins/DataFeedMessageSenderPluginLoadTest.java @@ -0,0 +1,182 @@ +package tak.server.plugins; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import atakmap.commoncommo.protobuf.v1.MessageOuterClass.Message; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@TakServerPlugin( + name = "Data Feed Plugin Load Test", + description = "Load Test Data Feed Plugin") +public class DataFeedMessageSenderPluginLoadTest extends MessageSenderBase { + + private static final Logger logger = LoggerFactory.getLogger(DataFeedMessageSenderPluginLoadTest.class); + + private int numberOfThreads = 1; // EXECUTOR_POOL_SIZE + + Set groups; + private List feedUuids; + + private ScheduledFuture future; + private int numberOfFeeds = 2; // default value + private int numberOfTracksPerFeed = 3; // default value + +// private double messageRatePerTrackInSecond = 0.5; // number of messages per track in second +// private long period; // calculated value based on messageRatePerTrackInSecond, in milliseconds; + + private int delay = 10; // in milliseconds + + private ScheduledExecutorService worker; + + private AtomicInteger count; + private long startTime; + + @SuppressWarnings("unchecked") + public DataFeedMessageSenderPluginLoadTest() { + + if (config.containsProperty("groups")) { + groups = new HashSet((List) config.getProperty("groups")); + } + + if (config.containsProperty("numberOfFeeds")) { + numberOfFeeds = (int)config.getProperty("numberOfFeeds"); + } + + if (config.containsProperty("numberOfTracksPerFeed")) { + numberOfTracksPerFeed = (int)config.getProperty("numberOfTracksPerFeed"); + } + +// if (config.containsProperty("messageRatePerTrackInSecond")) { +// messageRatePerTrackInSecond = (double)config.getProperty("messageRatePerTrackInSecond"); +// } +// period = (long)(1000L/messageRatePerTrackInSecond); // period is in MILLISECONDS +// logger.info("Period in ms: {}", period); + + if (config.containsProperty("delay")) { + delay = (int)config.getProperty("delay"); + } + + if (config.containsProperty("numberOfThreads")) { + numberOfThreads = (int)config.getProperty("numberOfThreads"); + } + + worker = Executors.newScheduledThreadPool(numberOfThreads); + + } + + @Override + public void start() { + + logger.info("Starting plugin " + getClass().getName()); +// logger.info("### Load test params: numberOfFeeds: {}, numberOfTracksPerFeed: {}, messageRatePerTrackInSecond: {}, numberOfThreads: {}", numberOfFeeds, numberOfTracksPerFeed, messageRatePerTrackInSecond, numberOfThreads); + logger.info("### Load test params: numberOfFeeds: {}, numberOfTracksPerFeed: {}, delay: {}, numberOfThreads: {}", numberOfFeeds, numberOfTracksPerFeed, delay, numberOfThreads); + + feedUuids = new ArrayList(); + count = new AtomicInteger(0); + + PluginDataFeedApi pluginDataFeedApi = getPluginDataFeedApi(); + + // create new datafeeds + for (int i = 0 ; i < numberOfFeeds; i++) { +// String feedUuid = UUID.randomUUID().toString(); + String feedUuid = "feedUUID_loadtest_" + (i+1); // DataFeed UUIDs need to be known so that they can be added to missions in the client side + String datafeedName = "loadTestPluginDataFeed_" + (i+1); + List tags = new ArrayList<>(); + tags.add("loadTest"); + logger.info("Creating new datafeed with uuid: {}", feedUuid); + PluginDataFeed myPluginDataFeed = pluginDataFeedApi.create(feedUuid, datafeedName, tags); + logger.info("Successfully created datafeed: {}", myPluginDataFeed.toString()); + + feedUuids.add(feedUuid); + } + + startTime = System.currentTimeMillis(); + +// future = worker.scheduleAtFixedRate (new DataFeedMessageSender(), 3000, period, TimeUnit.MILLISECONDS); + future = worker.scheduleWithFixedDelay(new DataFeedMessageSender(), 3000, delay, TimeUnit.MILLISECONDS); + } + + private class DataFeedMessageSender implements Runnable { + + @Override + public void run() { + + String trackUid = "dummyTrackUid"; // this value will be replaced + String SA = "<__group name=\"Dark Blue\" role=\"Team Member\"/>"; + + int currentCount = count.incrementAndGet(); + + try { + // convert the CoT string into a Message + Message message = getConverter().cotStringToDataMessage(SA, groups, Integer.toString(System.identityHashCode(this))); + + for (int feedIndex = 0; feedIndex < numberOfFeeds; feedIndex++) { + + for (int trackIndex = 0; trackIndex < numberOfTracksPerFeed; trackIndex++) { + + Message.Builder mb = message.toBuilder(); + + mb.getPayloadBuilder().getCotEventBuilder().getDetailBuilder().getStatusBuilder().setBattery(currentCount); + trackUid = "loadTest_" + "feed_" + feedIndex + "_" + "track_" + trackIndex; + mb.getPayloadBuilder().getCotEventBuilder().setUid(trackUid); + + Message m = mb.build(); + + // Send messages to datafeed feedUuid + String feedUuid = feedUuids.get(feedIndex); + send(m, feedUuid); + + logger.debug("Sent message to datafeed UUID: {}, trackUid: {}, count: {}", feedUuid, trackUid, currentCount); + + } + } + + long numberOfMessagesSent = (long)numberOfFeeds * numberOfTracksPerFeed * currentCount; + long timePassedInSecond = (System.currentTimeMillis() - startTime)/1000; + logger.info("### Sent a total of {} messages in {} seconds, rate: {} messages/second", numberOfMessagesSent, timePassedInSecond, numberOfMessagesSent/timePassedInSecond); + + } catch (Exception e) { + logger.error("Error in DataFeedMessageSender",e); + throw new RuntimeException(e); + } + + } + + } + + private void deleteDataFeeds() { + try { + logger.info("Deleting datafeeds used in load testing"); + PluginDataFeedApi pluginDataFeedApi = getPluginDataFeedApi(); + for (String feedUuid : feedUuids) { + pluginDataFeedApi.delete(feedUuid); + } + logger.info("Deleted all datafeeds used in load testing"); + + } catch (Exception e) { + logger.error("Error when deleting datafeeds", e); + } + } + + @Override + public void stop() { + + if (future != null) { + future.cancel(true); + } + + deleteDataFeeds(); + } +} diff --git a/src/testing/takserver-datafeed-load-test/tak.server.plugins.DataFeedMessageSenderPluginLoadTest.yaml b/src/testing/takserver-datafeed-load-test/tak.server.plugins.DataFeedMessageSenderPluginLoadTest.yaml new file mode 100644 index 00000000..39294510 --- /dev/null +++ b/src/testing/takserver-datafeed-load-test/tak.server.plugins.DataFeedMessageSenderPluginLoadTest.yaml @@ -0,0 +1,5 @@ +numberOfFeeds: 50 +numberOfTracksPerFeed: 800 +delay: 1 +numberOfThreads: 3 +system: {archive: true} \ No newline at end of file