diff --git a/.ci/aptPackagesToInstall.txt b/.ci/aptPackagesToInstall.txt new file mode 100644 index 0000000..e69de29 diff --git a/.ci/pythonPackagesToInstallFromGit.txt b/.ci/pythonPackagesToInstallFromGit.txt new file mode 100644 index 0000000..e69de29 diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..c9162b9 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +root = true + +[*] +charset = utf-8 +indent_style = tab +indent_size = 4 +insert_final_newline = true +end_of_line = lf + +[*.{yml,yaml}] +indent_style = space +indent_size = 2 diff --git a/.github/.templateMarker b/.github/.templateMarker new file mode 100644 index 0000000..5e3a3e0 --- /dev/null +++ b/.github/.templateMarker @@ -0,0 +1 @@ +KOLANICH/python_project_boilerplate.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..89ff339 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" + allow: + - dependency-type: "all" diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 0000000..7fe33b3 --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,15 @@ +name: CI +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + build: + runs-on: ubuntu-22.04 + steps: + - name: typical python workflow + uses: KOLANICH-GHActions/typical-python-workflow@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..71fb1b6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +__pycache__ +*.py[co] +/*.egg-info +*.srctrlbm +*.srctrldb +build +dist +.eggs +monkeytype.sqlite3 +/.ipynb_checkpoints diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..5f90db5 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,51 @@ +image: registry.gitlab.com/kolanich-subgroups/docker-images/fixed_python:latest + +variables: + DOCKER_DRIVER: overlay2 + SAST_ANALYZER_IMAGE_TAG: latest + SAST_DISABLE_DIND: "true" + SAST_CONFIDENCE_LEVEL: 5 + CODECLIMATE_VERSION: latest + +include: + - template: SAST.gitlab-ci.yml + - template: Code-Quality.gitlab-ci.yml + - template: License-Management.gitlab-ci.yml + +build: + tags: + - shared + - linux + stage: build + variables: + GIT_DEPTH: "1" + PYTHONUSERBASE: ${CI_PROJECT_DIR}/python_user_packages + + before_script: + - export PATH="$PATH:$PYTHONUSERBASE/bin" # don't move into `variables` + - apt-get update + # todo: + #- apt-get -y install + #- pip3 install --upgrade + #- python3 ./fix_python_modules_paths.py + + script: + - python3 -m build -nw bdist_wheel + - mv ./dist/*.whl ./dist/SaneIO-0.CI-py3-none-any.whl + - pip3 install --upgrade ./dist/*.whl + - coverage run --source=SaneIO -m --branch pytest --junitxml=./rspec.xml ./tests/test.py + - coverage report -m + - coverage xml + + coverage: "/^TOTAL(?:\\s+\\d+){4}\\s+(\\d+%).+/" + + cache: + paths: + - $PYTHONUSERBASE + + artifacts: + paths: + - dist + reports: + junit: ./rspec.xml + cobertura: ./coverage.xml diff --git a/Code_Of_Conduct.md b/Code_Of_Conduct.md new file mode 100644 index 0000000..bcaa2bf --- /dev/null +++ b/Code_Of_Conduct.md @@ -0,0 +1 @@ +No codes of conduct! \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..20f0fa8 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +include UNLICENSE +include *.md +include tests +include .editorconfig diff --git a/ReadMe.md b/ReadMe.md new file mode 100644 index 0000000..68a06ff --- /dev/null +++ b/ReadMe.md @@ -0,0 +1,24 @@ +SaneIO.py [![Unlicensed work](https://raw.githubusercontent.com/unlicense/unlicense.org/master/static/favicon.png)](https://unlicense.org/) +========= +~~[wheel (GHA via `nightly.link`)](https://nightly.link/KOLANICH-libs/SaneIO.py/workflows/CI/master/SaneIO-0.CI-py3-none-any.whl)~~ +~~[![GitHub Actions](https://github.com/KOLANICH-libs/SaneIO.py/workflows/CI/badge.svg)](https://github.com/KOLANICH-libs/SaneIO.py/actions/)~~ +![N∅ hard dependencies](https://shields.io/badge/-N∅_Ъ_deps!-0F0) +[![Libraries.io Status](https://img.shields.io/librariesio/github/KOLANICH-libs/SaneIO.py.svg)](https://libraries.io/github/KOLANICH-libs/SaneIO.py) +[![Code style: antiflash](https://img.shields.io/badge/code%20style-antiflash-FFF.svg)](https://codeberg.org/KOLANICH-tools/antiflash.py) + +A very simplified and limited IO framework. + +1. It is simple to use. It comes at cost. + 1. Count of callback funtions is minimized. + 2. Every object MUST inherit certain classes. + 3. Batteries included. + +2. It is "portable" in the sense the ports of this framework to other languages and platforms should keep the same structure, allowing the programs using it be ported more easily. + 1. The core is decoupled from concrete implementations. + 2. Upper level protocols are "Sans-IO". They don't depend on concrete IO implementations. Instead they depend on the interfaces provided by the framework. + 3. Composition over inheritance -> can be ported C and Rust support + +3. It is composable. The structure is a stack of objects. + * Wanna change the protocol in the stack? Just replace the layer object! + * Wanna access the same server using both TCP and UART? Just add a mux! + * Wanna access multiple upper layer protocols over the same low-level protocol (i.e. TCP)? Again, just add a mux! diff --git a/SaneIO/__init__.py b/SaneIO/__init__.py new file mode 100644 index 0000000..8d1c8b6 --- /dev/null +++ b/SaneIO/__init__.py @@ -0,0 +1 @@ + diff --git a/SaneIO/core/__init__.py b/SaneIO/core/__init__.py new file mode 100644 index 0000000..ec289f5 --- /dev/null +++ b/SaneIO/core/__init__.py @@ -0,0 +1,261 @@ +import typing +from collections.abc import MutableMapping, MutableSequence +from queue import Queue +from warnings import warn + +"""Python AsyncIO is complex shit. Let's wrap it a bit into a simple impl, detached from AsyncIO and IO in general, compatible to SansI/O fashion. +Design goals: be mimimalistic, simple and portable to different languages and architectures. +So composition over inheritance style is preferred.. +""" + + +class SIOLayer: + __slots__ = ("higher", "lower") + + def __init__(self): + self.higher = None # type: typing.Union["SIOResponder", "SIOSender"] + self.lower = None # type: "SIOTransport" + + def bindHigher(self, higher): + self.higher = higher + higher._bindLower(self) + + def unbindHigher(self, higher): + self.higher._unbindLower(self) + self.higher = None + + def _bindLower(self, lower: "SIOTransport"): + self.lower = lower + + def _unbindLower(self, lower: "SIOTransport"): + self.lower = None + + @property + def lowestLevelResourceID(self) -> typing.Optional[typing.Any]: + """The unique lowest level stuff that the layer uniquiely takes. It can be a TCP or UDP port, or file descriptor. It is intended to be used as a key for muxers. + `None` means the layer cannot be multiplexed. Usually it is because I have not invented a good way how to multiplex the lower level resources for now.""" + return self.lower.lowestLevelResourceID + + +class SIOResponder(SIOLayer): + """The entity that can receive data and return responses to them. Previous level is ALWAYS provided as a pointer. It cannot initiate connections itself.""" + + __slots__ = () + + def onReceive(self, lowerLevelTransport: "SIOTransport", data: bytes): + raise NotImplementedError(self.__class__, "onReceive") + + +class SIOSender(SIOLayer): + __slots__ = () + + def sendBytes(self, data: bytes): + raise NotImplementedError(self.__class__, "sendBytes") + + +class SIOTransport(SIOResponder, SIOSender): + """The entity that can unwrap the lower level of protocol into the higher level. + Important: Servers also belong here. + By default it just passes through to the lower and upper layers""" + + __slots__ = () + + def onReceive(self, proto, commandRaw: bytes): + return self.higher.onReceive(proto, commandRaw) + + def sendBytes(self, data: bytes): + return self.lower.sendBytes(data) + + +class SIO1HiNLoMux(SIOTransport, MutableMapping): + """A multiplexor from higher level to lower ones. + One higher level corresponds to multiple lower levels. All the messages sent from higher levels are copied to all the lower levels. Any message from any of lower levels is passed indiscriminately to higher levels.""" + + __slots__ = () + + def __init__(self): + super().__init__() + self.lower = {} + + def __getitem__(self, k): + return self.lower[k] + + def __setitem__(self, k, v): + self.lower[k] = v + + def __delitem__(self, k): + del self.lower[k] + + def __iter__(self): + return self.lower.keys() + + def __len__(self): + return len(self.lower) + + def _bindLower(self, lower: "SIOTransport"): + self[lower.lowestLevelResourceID] = lower + + def _unbindLower(self, lower: "SIOTransport"): + del self[lower.lowestLevelResourceID] + + def lowestLevelResourceID(self): + return None + + def onReceive(self, lowerLevelTransport: "SIOTransport", data: bytes): + return self.higher.onReceive(self, data) + + def sendBytes(self, data: bytes): + for el in self.lower.values(): + el.sendBytes(data) + + +class HigherMultiplexableSIOResponder(SIOResponder): + """A mixin class that defines the facilities needed to map raw bytes to upper levels""" + + __slots__ = () + + def ifThisResponder(self, data: bytes): + raise NotImplementedError + + +class SIONHi1LoMux(SIOTransport, MutableMapping): + """A multiplexor from lower level to higher ones. + Multiple higher levels correspond to single lower level. All the messages received get through the upper level target identification. Then when the level is identified, the message is sent to it.""" + + __slots__ = () + + def __init__(self): + super().__init__() + self.higher = [] + + def __getitem__(self, k): + return self.higher[k] + + def __setitem__(self, k, v): + self.higher[k] = v + + def __delitem__(self, k): + del self.higher[k] + + def __iter__(self): + return self.higher + + def __len__(self): + return len(self.higher) + + def bindHigher(self, higher: HigherMultiplexableSIOResponder): + self.higher.append(higher) + + def unbindHigher(self, higher: HigherMultiplexableSIOResponder): + del self[a.index(higher)] + + def identifyHigherLevelTarget(self, data: bytes) -> HigherMultiplexableSIOResponder: + for el in self: + if el.ifThisResponder(data): + return el + + def onReceive(self, lowerLevelTransport: SIOTransport, data: bytes): + higherLevelTransport = self.identifyHigherLevelTarget(data) + return higherLevelTransport.onReceive(self, data) + + def sendBytes(self, data: bytes): + self.lower.sendBytes(data) + + +class HigherMultiplexableSIOResponder(SIOResponder): + """A mixin class that defines the facilities needed to map raw bytes to upper levels""" + + __slots__ = () + + NHI_1LO_MUX_MARKER_BYTES = None # type: bytes + NHI_1LO_MUX_MARKER_POSITION = None # type: slice + + @classmethod + def getMarkerSlice(cls) -> slice: + return slice(cls.NHI_1LO_MUX_MARKER_POSITION, cls.NHI_1LO_MUX_MARKER_POSITION + len(cls.NHI_1LO_MUX_MARKER_BYTES)) + + def ifThisResponder(self, data: bytes): + slc = self.__class__.getMarkerSlice() + marker = data[slc] + return marker == self.__class__.NHI_1LO_MUX_MARKER_BYTES + + +class SIOIdentificationBytesNHi1LoMux(SIONHi1LoMux): + __slots__ = ("markBytesSlice",) + + def __init__(self, markBytesSlice: slice): + self.markBytesSlice = markBytesSlice + self.higher = {} + + def __iter__(self): + return self.higher.values() + + def identifyHigherLevelTarget(self, data: bytes): + marker = data[self.markBytesSlice] + return self.higher[marker] + + def bindHigher(self, higher: HigherMultiplexableSIOResponder): + assert higher.getMarkerSlice() != self.markBytesSlice, "This " + repr(higher) + " has incompatible location of marker slice: higher.getMarkerSlice() (" + higher.getMarkerSlice() + ") != self.markBytesSlice (" + self.markBytesSlice + ")" + self[higher.NHI_1LO_MUX_MARKER_BYTES] = higher + + def unbindHigher(self, higher: HigherMultiplexableSIOResponder): + del self[higher.NHI_1LO_MUX_MARKER_BYTES] + + +class UnsafelyMuxableSIO1HiNLoMux(SIO1HiNLoMux): + """A mux that can be muxed, but unsafely""" + + def lowestLevelResourceID(self): + return id(self) + + +class HalfDuplex_SIOTransport(SIOTransport): + """The SIOTransport that works on byte-by-byte. These protocols usually use in-band signalling, so we have to handle them. So we process the input byte-by-byte.""" + + __slots__ = ("tx",) + + immediateSend = True + if not immediateSend: + warn("Some software requires the responses to be sent within a strict time windows (50 ms by default, but no more than 999). If we delay sending using the queue (immediateSend = False), we don't fit this time window.") + + def __init__(self): + super().__init__() + self.tx = Queue() + + def canSend(self) -> bool: + """Determines if the remote party has finished its communication, so we can send own one""" + raise NotImplementedError + + def sendCommandsInCertainStates(self): + # print("self.inCommand", self.inCommand) + if self.canSend(): + while not self.tx.empty(): + commandB = self.tx.get() + # print("commandB", commandB) + self.lower.sendBytes(commandB) + self.tx.task_done() + # print("written") + + def sendBytes(self, data: bytes): + if self.__class__.immediateSend: + return self.lower.sendBytes(data) + else: + self.tx.put(data) + self.sendCommandsInCertainStates() + + +class StreamingHalfDuplexInBandSignalling_SIOTransport(HalfDuplex_SIOTransport): + __slots__ = () + + def receiveByte(self, b: int): + raise NotImplementedError + + def filterSentBytes(self, b: bytes) -> bytes: + raise NotImplementedError + + def onReceive(self, lowerLevelTransport: "SIOTransport", data: bytes): + for b in data: + self.receiveByte(b) + + def sendBytes(self, data: bytes): + return super().sendBytes(self.filterSentBytes(data)) diff --git a/SaneIO/protocols/serial.py b/SaneIO/protocols/serial.py new file mode 100644 index 0000000..2831b74 --- /dev/null +++ b/SaneIO/protocols/serial.py @@ -0,0 +1,43 @@ +import asyncio +import typing +from pathlib import Path + +import serial +import serial_asyncio + +from ..core import SIOTransport +from ..utils.asyncIO import AsyncIOSIOTransportServerConnectionAdapter + + +class AsyncSIO_UARTServer(AsyncIOSIOTransportServerConnectionAdapter): + __slots__ = ("readyFut",) + + def __init__(self): + super().__init__() + self.readyFut = asyncio.Future() + + def prepare(self): + self.transport.serial.read() + self.readyFut.set_result(True) + + @property + def lowestLevelResourceID(self) -> str: + """The unique lowest level stuff that the layer uniquiely takes. It can be a TCP or UDP port, or file descriptor. It is intended to be used as a key for muxers""" + return self.transport.serial.port + + @classmethod + # Async function because there are no async CTORs in python! + async def create(cls, port: typing.Union[Path, str]): + sioTransport = cls() + coro = serial_asyncio.create_serial_connection( + asyncio.get_event_loop(), + lambda: sioTransport, + port, + # baudrate=9600, + # bytesize=8, # CS8 + # parity=serial.PARITY_EVEN, # PARENB + # stopbits=1, + ) + transport, protocol = await coro + await sioTransport.readyFut + return sioTransport diff --git a/SaneIO/protocols/tcp.py b/SaneIO/protocols/tcp.py new file mode 100644 index 0000000..7e627f2 --- /dev/null +++ b/SaneIO/protocols/tcp.py @@ -0,0 +1,29 @@ +import asyncio +import typing +from pathlib import Path +from socket import AddressFamily + +from ..core import SIOTransport +from ..utils.asyncIO import AsyncIOSIOMuxedTransportServerConnectionAdapter, AsyncIOSIOTransportServerAdapter + + +class AsyncSIO_TCPServerConnection(AsyncIOSIOMuxedTransportServerConnectionAdapter): + __slots__ = () + + @property + def lowestLevelResourceID(self) -> str: + """The unique lowest level stuff that the layer uniquiely takes. It can be a TCP or UDP port, or file descriptor. It is intended to be used as a key for muxers""" + t = self.transport + return t.get_extra_info("sockname") + t.get_extra_info("peername") + + +class AsyncSIO_TCPServer(AsyncIOSIOTransportServerAdapter): + __slots__ = ("mux", "server") + + CONNECTION_CLASS = AsyncSIO_TCPServerConnection + + @classmethod + def SERVER_FACTORY(cls, protocolFactory, *args, **kwargs): + l = asyncio.get_event_loop() + s = l.create_server(protocolFactory, *args, **kwargs) + return s diff --git a/SaneIO/utils/asyncIO.py b/SaneIO/utils/asyncIO.py new file mode 100644 index 0000000..1261d0e --- /dev/null +++ b/SaneIO/utils/asyncIO.py @@ -0,0 +1,89 @@ +import asyncio + +from ..core import SIO1HiNLoMux, SIOSender, SIOTransport + + +class AsyncIOSIOTransportConnectionAdapter(asyncio.Protocol, SIOSender): + """A class to wrap `asyncio` transports and protocols. + You must implement something: + 1. calling `self.higher.onReceive` + 2. calling `self.transport.write` + 3. property returning `lowestLevelResourceID` from the underlying resource""" + + __slots__ = ("transport",) + + def __init__(self, transport: asyncio.Transport = None): + self.transport = transport + + ###### Sans I/O callbacks###### + + def sendBytes(self, data: bytes): + self.transport.write(data) + + +class AsyncIOSIOTransportServerConnectionAdapter(AsyncIOSIOTransportConnectionAdapter): + __slots__ = () + + ###### Asyncio I/O callbacks######## + + def data_received(self, data: bytes): + self.higher.onReceive(self, data) + + def connection_made(self, transport: asyncio.Transport): + self.transport = transport + self.prepare() + + def connection_lost(self, exc): + self.transport.loop.stop() + + def eof_received(self): + pass + + def pause_writing(self): + pass + + def resume_writing(self): + pass + + ############################### + + def prepare(self): + """Sets the lower asyncio transport into a sane clean state""" + pass + + +class AsyncIOSIOMuxedTransportServerConnectionAdapter(AsyncIOSIOTransportServerConnectionAdapter): + __slots__ = ("mux",) + + def __init__(self, mux): + super().__init__() + self.mux = mux + + ###### Asyncio I/O callbacks######## + + def connection_made(self, transport: asyncio.Transport): + self.transport = transport + self.prepare() + self.bindHigher(self.mux) + + def connection_lost(self, exc): + self.unbindHigher(self.mux) + + +class AsyncIOSIOTransportServerAdapter(asyncio.Protocol): + __slots__ = ("mux", "server") + + CONNECTION_CLASS = AsyncIOSIOMuxedTransportServerConnectionAdapter + SERVER_FACTORY = None + + def __init__(self, mux: SIO1HiNLoMux): + self.mux = mux + self.server = None # populated in factory + + @classmethod + # Async function because there are no async CTORs in python! + async def create(cls, mux: SIO1HiNLoMux, *args, **kwargs): + sioServer = cls(mux) + sioServer.server = await cls.SERVER_FACTORY(lambda: cls.CONNECTION_CLASS(mux), *args, **kwargs) + await sioServer.server.start_serving() + return sioServer diff --git a/UNLICENSE b/UNLICENSE new file mode 100644 index 0000000..efb9808 --- /dev/null +++ b/UNLICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6ff624a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,35 @@ +[build-system] +requires = ["setuptools>=61.2.0", "wheel", "setuptools_scm[toml]>=3.4.3"] +build-backend = "setuptools.build_meta" + +[project] +name = "SaneIO" +readme = "ReadMe.md" +description = "A simple and limited IO framework \"for humans\", because raw `asyncio` is not enough simple to use." +authors = [{name = "KOLANICH"}] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Other Environment", + "Intended Audience :: Developers", + "License :: Public Domain", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Software Development :: Libraries :: Python Modules", +] +keywords = ["SaneIO"] +license = {text = "Unlicense"} +requires-python = ">=3.4" +dynamic = ["version"] + +[project.urls] +Homepage = "https://codeberg.org/KOLANICH-libs/SaneIO.py" + +[tool.setuptools] +zip-safe = true + +[tool.setuptools.packages.find] +include = ["SaneIO", "SaneIO.*"] + +[tool.setuptools_scm] diff --git a/tests/tests.py b/tests/tests.py new file mode 100755 index 0000000..805a3ef --- /dev/null +++ b/tests/tests.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path +import unittest +import itertools, re +import colorama + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from collections import OrderedDict + +dict = OrderedDict + +import SaneIO +from SaneIO import * + + +class Tests(unittest.TestCase): + + def testSimple(self): + raise NotImplementedError + + +if __name__ == "__main__": + unittest.main()