Python, CLI example

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "egui_code_editor"
 authors = ["Roman Chumak <p4ymak@yandex.ru>"]
-version = "0.2.2"
+version = "0.2.3"
 edition = "2021"
 license = "MIT"
 repository = "https://github.com/p4ymak/egui_code_editor"

README.md

@@ -2,7 +2,7 @@
 
 Text Editor Widget for [egui](https://github.com/emilk/egui) with numbered lines and simple syntax highlighting based on keywords sets.
 
-## Usage
+## Usage with egui
 
 ```rust
 use egui_code_editor::{CodeEditor, ColorTheme, Syntax};
@@ -17,6 +17,56 @@ CodeEditor::default()
   .show(ui, &mut self.code);
 ```
 
+## Usage as lexer
+
+*Cargo.toml*
+
+```toml
+[dependencies]
+egui_code_editor = { version = "0.2" , default-features = false }
+colorful = "0.2.2"
+```
+
+```rust
+use colorful::{Color, Colorful};
+use egui_code_editor::{Syntax, Token, TokenType};
+
+fn color(token: TokenType) -> Color {
+    match token {
+        TokenType::Comment(_) => Color::Grey37,
+        TokenType::Function => Color::Yellow3b,
+        TokenType::Keyword => Color::IndianRed1c,
+        TokenType::Literal => Color::NavajoWhite1,
+        TokenType::Numeric(_) => Color::MediumPurple,
+        TokenType::Punctuation(_) => Color::Orange3,
+        TokenType::Special => Color::Cyan,
+        TokenType::Str(_) => Color::Green,
+        TokenType::Type => Color::GreenYellow,
+        TokenType::Whitespace(_) => Color::White,
+        TokenType::Unknown => Color::Pink1,
+    }
+}
+
+fn main() {
+    let text = r#"// Code Editor
+CodeEditor::default()
+    .id_source("code editor")
+    .with_rows(12)
+    .with_fontsize(14.0)
+    .with_theme(self.theme)
+    .with_syntax(self.syntax.to_owned())
+    .with_numlines(true)
+    .vscroll(true)
+    .show(ui, &mut self.code);
+    "#;
+
+    let syntax = Syntax::rust();
+    for token in Token::default().tokens(&syntax, text) {
+        print!("{}", token.buffer().color(color(token.ty())));
+    }
+}
+```
+
 
 ## Themes
 

examples/demo.rs

@@ -14,7 +14,7 @@ const THEMES: [ColorTheme; 8] = [
     ColorTheme::SONOKAI,
 ];
 
-const SYNTAXES: [SyntaxDemo; 4] = [
+const SYNTAXES: [SyntaxDemo; 5] = [
     SyntaxDemo::new(
         "Lua",
         r#"-- Binary Search
@@ -29,6 +29,28 @@ function binarySearch(list, value)
     return search(1,#list)
 end"#,
     ),
+    SyntaxDemo::new(
+        "Python",
+        r#"from collections.abc import Iterable
+from typing import Protocol
+
+class Combiner(Protocol):
+    def __call__(self, *vals: bytes, maxlen: int | None = None) -> list[bytes]: ...
+
+def batch_proc(data: Iterable[bytes], cb_results: Combiner) -> bytes:
+    for item in data:
+        ...
+
+def good_cb(*vals: bytes, maxlen: int | None = None) -> list[bytes]:
+    ...
+"""
+def bad_cb(*vals: bytes, maxitems: int | None) -> list[bytes]:
+    ...
+"""
+batch_proc([], good_cb)  # OK
+batch_proc([], bad_cb)   # Error! Argument 2 has incompatible type because of
+                         # different name and kind in the callback"#,
+    ),
     SyntaxDemo::new(
         "Rust",
         r#"// Code Editor
@@ -81,6 +103,7 @@ impl SyntaxDemo {
         match self.name {
             "Assembly" => Syntax::asm(),
             "Lua" => Syntax::lua(),
+            "Python" => Syntax::python(),
             "Rust" => Syntax::rust(),
             "Shell" => Syntax::shell(),
             "SQL" => Syntax::sql(),
@@ -119,7 +142,7 @@ struct CodeEditorDemo {
 }
 impl CodeEditorDemo {
     fn new(_cc: &CreationContext) -> Self {
-        let rust = SYNTAXES[1];
+        let rust = SYNTAXES[2];
         CodeEditorDemo {
             code: rust.example.to_string(),
             theme: ColorTheme::GRUVBOX,

src/highlighting.rs

@@ -1,7 +1,8 @@
 use super::syntax::{Syntax, TokenType, QUOTES, SEPARATORS};
 use std::mem;
 
-#[derive(Default, Debug, PartialEq, PartialOrd)]
+#[derive(Default, Debug, PartialEq, PartialOrd, Eq, Ord)]
+/// Lexer and Token
 pub struct Token {
     ty: TokenType,
     buffer: String,
@@ -65,6 +66,7 @@ impl Token {
     }
 
     #[cfg(feature = "egui")]
+    /// Syntax highlighting
     pub fn highlight(&mut self, editor: &CodeEditor, text: &str) -> LayoutJob {
         *self = Token::default();
         let mut job = LayoutJob::default();
@@ -77,6 +79,7 @@ impl Token {
         job
     }
 
+    /// Lexer
     pub fn tokens(&mut self, syntax: &Syntax, text: &str) -> Vec<Self> {
         let mut tokens: Vec<Self> = text
             .chars()

src/syntax/mod.rs

@@ -1,6 +1,7 @@
 #![allow(dead_code)]
 pub mod asm;
 pub mod lua;
+pub mod python;
 pub mod rust;
 pub mod shell;
 pub mod sql;
@@ -14,7 +15,7 @@ pub const QUOTES: [char; 3] = ['\'', '"', '`'];
 type MultiLine = bool;
 type Float = bool;
 
-#[derive(Default, Clone, Copy, PartialEq, PartialOrd)]
+#[derive(Default, Clone, Copy, PartialEq, PartialOrd, Eq, Ord)]
 #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
 pub enum TokenType {
     Comment(MultiLine),

src/syntax/python.rs

@@ -0,0 +1,36 @@
+use super::Syntax;
+use std::collections::BTreeSet;
+
+impl Syntax {
+    pub fn python() -> Syntax {
+        Syntax {
+            language: "Python",
+            case_sensitive: true,
+            comment: "#",
+            comment_multiline: [r#"'''"#, r#"'''"#],
+            keywords: BTreeSet::from([
+                "and", "as", "assert", "break", "class", "continue", "def", "del", "elif", "else",
+                "except", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda",
+                "nonlocal", "not", "or", "pass", "raise", "return", "try", "while", "with",
+                "yield",
+            ]),
+            types: BTreeSet::from([
+                "bool",
+                "int",
+                "float",
+                "complex",
+                "str",
+                "list",
+                "tuple",
+                "range",
+                "bytes",
+                "bytearray",
+                "memoryview",
+                "dict",
+                "set",
+                "frozenset",
+            ]),
+            special: BTreeSet::from(["False", "None", "True"]),
+        }
+    }
+}