diff --git a/Cargo.lock b/Cargo.lock index b9c690a..843e09f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -114,7 +114,7 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "genin" -version = "0.4.13" +version = "0.4.14" dependencies = [ "clap", "env_logger", diff --git a/Cargo.toml b/Cargo.toml index 0848687..fb2fae0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "genin" -version = "0.4.13" +version = "0.4.14" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/README.md b/README.md index cd48093..1121305 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,8 @@ Download and unzip the archive for the desired architecture. Universal executable: ```shell -curl -sLO https://binary.picodata.io/repository/raw/genin/bin/genin-0.4.13-x86_64-musl.tar.gz -tar -xvf genin-0.4.13-x86_64-musl.tar.gz ; sudo install genin /usr/local/bin/ +curl -sLO https://binary.picodata.io/repository/raw/genin/bin/genin-0.4.14-x86_64-musl.tar.gz +tar -xvf genin-0.4.14-x86_64-musl.tar.gz ; sudo install genin /usr/local/bin/ ``` --- @@ -84,11 +84,11 @@ sudo yum install -y genin 2. If you want to install `rpm` packages directly without adding our repository. ```shell -sudo rpm -i https://binary.picodata.io/repository/yum/el/8/x86_64/os/genin-0.4.13-1.el8.x86_64.rpm +sudo rpm -i https://binary.picodata.io/repository/yum/el/8/x86_64/os/genin-0.4.14-1.el8.x86_64.rpm ``` RHEL 7.x, CentOS 7.x ```shell -sudo rpm -i https://binary.picodata.io/repository/yum/el/7/x86_64/os/genin-0.4.13-1.el7.x86_64.rpm +sudo rpm -i https://binary.picodata.io/repository/yum/el/7/x86_64/os/genin-0.4.14-1.el7.x86_64.rpm ``` --- @@ -116,7 +116,7 @@ sudo apt install -y genin 2. Downloading and installing the package directly: ```shell -curl -sLO https://binary.picodata.io/repository/raw/genin/deb/genin-0.4.13.amd64.deb && sudo dpkg -i genin-0.4.13.amd64.deb +curl -sLO https://binary.picodata.io/repository/raw/genin/deb/genin-0.4.14.amd64.deb && sudo dpkg -i genin-0.4.14.amd64.deb ``` --- @@ -142,7 +142,7 @@ sudo apt install -y genin 2. Downloading and installing the package directly: ```shell -curl -sLO https://binary.picodata.io/repository/raw/genin/deb/genin-0.4.13.amd64.deb && sudo dpkg -i genin-0.4.13.amd64.deb +curl -sLO https://binary.picodata.io/repository/raw/genin/deb/genin-0.4.14.amd64.deb && sudo dpkg -i genin-0.4.14.amd64.deb ``` --- @@ -162,8 +162,8 @@ brew install genin Use the following command to grab and install Genin in macOS (10.10+) wihtout homebrew: ```shell -curl -L https://binary.picodata.io/repository/raw/genin/apple/genin-0.4.13-darwin-amd64.zip -o genin-0.4.13-darwin-amd64.zip -unzip genin-0.4.13-darwin-amd64.zip -d ~/bin/ +curl -L https://binary.picodata.io/repository/raw/genin/apple/genin-0.4.14-darwin-amd64.zip -o genin-0.4.14-darwin-amd64.zip +unzip genin-0.4.14-darwin-amd64.zip -d ~/bin/ ``` > **Note:** The application can then be found under the `~/bin` directory. > Make sure the directory is in your `$PATH`. @@ -181,8 +181,8 @@ brew install genin@0.3.8 #### Windows Use the following command to grab and install Genin in Windows 7 64 bit or newer: ```shell -curl.exe -L https://binary.picodata.io/repository/raw/genin/windows/genin-0.4.13-darwin-amd64.zip -o genin-0.4.13-windows-amd64.zip -unzip.exe genin-0.4.13-windows-amd64.zip -d %HOME%/.cargo/bin/ +curl.exe -L https://binary.picodata.io/repository/raw/genin/windows/genin-0.4.14-darwin-amd64.zip -o genin-0.4.14-windows-amd64.zip +unzip.exe genin-0.4.14-windows-amd64.zip -d %HOME%/.cargo/bin/ ``` > **Note:** The application can then be found under the `.cargo/bin` folder inside > your user profile folder. Make sure it is in your `%PATH%`. @@ -261,8 +261,8 @@ hosts: # in this example, both hosts are in the same cloud data center config: # (optional) begin binary and http port, by default 8081, 3031 # ports can be defined on all levels - http: 8081 # (optional) http port to start counting from - binary: 3031 # (optional) binary port to start counting from + http_port: 8081 # (optional) http port to start counting from + binary_port: 3031 # (optional) binary port to start counting from hosts: - name: host-1 # (mandatory) hostname or domain name config: diff --git a/README.ru.md b/README.ru.md index 7fe64ac..20dfa70 100644 --- a/README.ru.md +++ b/README.ru.md @@ -49,8 +49,8 @@ Genin уже заранее скомпилирован под разные ар Универсальный исполняемый файл: ```shell -curl -sLO https://binary.picodata.io/repository/raw/genin/bin/genin-0.4.13-x86_64-musl.tar.gz -tar -xvf genin-0.4.13-x86_64-musl.tar.gz ; sudo install genin /usr/local/bin/ +curl -sLO https://binary.picodata.io/repository/raw/genin/bin/genin-0.4.14-x86_64-musl.tar.gz +tar -xvf genin-0.4.14-x86_64-musl.tar.gz ; sudo install genin /usr/local/bin/ ``` --- @@ -87,11 +87,11 @@ sudo yum install -y genin 2. Так же вы можете установить пакет `rpm` напрямую без добавления нашего репозитория. RHEL 8.x, CentOS 8.x, Rockylinux 8.x, recent Fedora version ```shell -sudo rpm -i https://binary.picodata.io/repository/yum/el/8/x86_64/os/genin-0.4.13-1.el8.x86_64.rpm +sudo rpm -i https://binary.picodata.io/repository/yum/el/8/x86_64/os/genin-0.4.14-1.el8.x86_64.rpm ``` RHEL 7.x, CentOS 7.x ```shell -sudo rpm -i https://binary.picodata.io/repository/yum/el/7/x86_64/os/genin-0.4.13-1.el7.x86_64.rpm +sudo rpm -i https://binary.picodata.io/repository/yum/el/7/x86_64/os/genin-0.4.14-1.el7.x86_64.rpm ``` > **Note:** будьте внимательны, так как при выборе не правильной версии ос могут быть ошибки > при установке `rpm` @@ -118,7 +118,7 @@ sudo apt install -y genin 2. Загрузкой и установкой пакета напрямую: ```shell -curl -sLO https://binary.picodata.io/repository/raw/genin/deb/genin-0.4.13.amd64.deb && sudo dpkg -i genin-0.4.13.amd64.deb +curl -sLO https://binary.picodata.io/repository/raw/genin/deb/genin-0.4.14.amd64.deb && sudo dpkg -i genin-0.4.14.amd64.deb ``` --- @@ -143,7 +143,7 @@ sudo apt install -y genin 2. Загрузкой и установкой пакета напрямую: ```shell -curl -sLO https://binary.picodata.io/repository/raw/genin/deb/genin-0.4.13.amd64.deb && sudo dpkg -i genin-0.4.13.amd64.deb +curl -sLO https://binary.picodata.io/repository/raw/genin/deb/genin-0.4.14.amd64.deb && sudo dpkg -i genin-0.4.14.amd64.deb ``` --- @@ -163,8 +163,8 @@ brew install genin Для установки без помощи homebrew используйте следующие команды для загрузки и установки Genin на macOS (10.10+): ```shell -curl -sLO https://binary.picodata.io/repository/raw/genin/osx/genin-0.4.13-x86_64-macosx.tar.gz -unzip genin-0.4.13-darwin-amd64.zip -d ~/bin/ +curl -sLO https://binary.picodata.io/repository/raw/genin/osx/genin-0.4.14-x86_64-macosx.tar.gz +unzip genin-0.4.14-darwin-amd64.zip -d ~/bin/ ``` --- @@ -188,8 +188,8 @@ brew install genin@0.3.8 Используйте следующие команды для скачивания и установки Genin на операционных системах Windows 7 64 и новее. ```shell -curl.exe -sLO https://binary.picodata.io/repository/raw/genin/win/genin-0.4.13-win64.zip -unzip.exe genin-0.4.13-win64.zip -d %HOME%/.cargo/bin/ +curl.exe -sLO https://binary.picodata.io/repository/raw/genin/win/genin-0.4.14-win64.zip +unzip.exe genin-0.4.14-win64.zip -d %HOME%/.cargo/bin/ ``` > **Note:** Genin будет распакован в директорию `.cargo/bin` которая находится в домашнем > каталоге важего пользователя. Перед использованием приложения пожалуйста удостоверьтесь @@ -199,7 +199,7 @@ unzip.exe genin-0.4.13-win64.zip -d %HOME%/.cargo/bin/ ``` genin --version ``` -Если вы видите сообщение `genin 0.4.13` значит установка прошла успешно. +Если вы видите сообщение `genin 0.4.14` значит установка прошла успешно. --- ## Руководство по использованию diff --git a/src/task.rs b/src/task.rs index 472ecf6..9a0d7da 100644 --- a/src/task.rs +++ b/src/task.rs @@ -6,12 +6,14 @@ pub mod serde_genin; pub mod vars; use log::info; +use regex::RegexBuilder; use serde_yaml::{Mapping, Value}; +use std::collections::HashMap; use std::convert::TryFrom; use std::error::Error; use std::fmt; use std::fs::File; -use std::io::Read; +use std::io::{Read, Write}; use crate::error::{GeninError, GeninErrorKind}; use crate::task::cluster::fs::{TryMap, IO, UPGRADE_YAML}; @@ -50,6 +52,74 @@ pub fn run_v2() -> Result<(), Box> { std::env::var("RUST_LOG").unwrap_or_else(|_| "warn".into()) ); + // Cluster init comments + + let comments = [ + ( + "replicasets_count".to_string(), + "# How many masters we want, by default equal 1".to_string(), + ), + ( + "replication_factor".to_string(), + "# Number of replicas in replicaset, default for router 0".to_string(), + ), + ( + "address".to_string(), + "# Host or instance address (may be IP or URI)".to_string(), + ), + ( + "http_port".to_string(), + "# Specify http port to start counting from".to_string(), + ), + ( + "binary_port".to_string(), + "# Specify binary port to start counting from".to_string(), + ), + ( + "weight".to_string(), + "# Vshard replicaset weight (matters only if `vshard-storage` role is enabled)" + .to_string(), + ), + ( + "zone".to_string(), + "# Zone parameter for ansible cartridge playbook".to_string(), + ), + ( + "config".to_string(), + "# Config with arbitrary key-values pairs".to_string(), + ), + ( + "mode".to_string(), + "# Failover mode (stateful, eventual, disabled)".to_string(), + ), + ( + "state_provider".to_string(), + "# What is serve failover (stateboard, stateful)".to_string(), + ), + ( + "stateboard_params".to_string(), + "# Params for chosen in state_provider failover type".to_string(), + ), + ( + "ansible_user".to_string(), + "# Username under which the ansible will connect to the servers".to_string(), + ), + ( + "ansible_password".to_string(), + "# Ansible user password".to_string(), + ), + ( + "cartridge_cluster_cookie".to_string(), + "# Cookie for connecting to the administrative console of the instances".to_string(), + ), + ( + "cartridge_package_path".to_string(), + "# Path to the application package".to_string(), + ), + ] + .into_iter() + .collect::>(); + // The idea of the first step of creating a task: // - create FsInteration // - map FsInteration as: @@ -71,7 +141,37 @@ pub fn run_v2() -> Result<(), Box> { }) })? .print_input() - .serialize_input()?; + .try_map(|io| { + if let IO { + input: Some(cluster), + output: Some(mut file), + } = io + { + let mut text = serde_yaml::to_string(&cluster) + .map_err(|err| GeninError::new(GeninErrorKind::Deserialization, err))?; + + for (k, v) in comments { + let comment = + RegexBuilder::new(&format!(r"(?P^.+{}:[ ]*[^#<> ]+)$", k)) + .multi_line(true) + .build() + .unwrap(); + text = comment + .replace_all(&text, &format!("$key {}", v)) + .to_string(); + + file.write(text.as_bytes()).map_err(|err| { + GeninError::new(GeninErrorKind::Deserialization, err) + })?; + } + + return Ok(IO { + input: Some(()), + output: Some(()), + }); + } + Err(GeninError::new(GeninErrorKind::EmptyField, "TODO")) + })?; } Some(("build", args)) => { IO::from(args) @@ -81,7 +181,6 @@ pub fn run_v2() -> Result<(), Box> { args.get_flag("force"), )? .deserialize_input::()? - //.map_err(|err|ClusterError::from(err))? .print_input() .try_map(|IO { input, output }| { Inventory::try_from(&input).map(|inventory| IO { @@ -168,7 +267,7 @@ pub fn run_v2() -> Result<(), Box> { pub trait Validate { type Type: fmt::Debug + Default + 'static; - type Error: fmt::Debug; + type Error: fmt::Debug + ToString; fn validate(bytes: &[u8]) -> Result; diff --git a/src/task/cluster.rs b/src/task/cluster.rs index 43cca18..de52322 100644 --- a/src/task/cluster.rs +++ b/src/task/cluster.rs @@ -7,6 +7,7 @@ pub mod topology; use clap::ArgMatches; use indexmap::IndexMap; use log::{debug, trace}; +use regex::{Captures, RegexBuilder}; use serde::ser::SerializeStruct; use serde::{Deserialize, Serialize}; use serde_yaml::Value; @@ -411,7 +412,7 @@ impl Validate for Cluster { type Error = serde_yaml::Error; fn validate(bytes: &[u8]) -> Result { - serde_yaml::from_slice(bytes) + serde_yaml::from_str(&check_placeholders(bytes)?) } fn whole_block(bytes: &[u8]) -> String { @@ -419,6 +420,45 @@ impl Validate for Cluster { } } +pub fn check_placeholders(slice: &[u8]) -> Result { + let text = String::from_utf8_lossy(slice).to_string(); + let reg = RegexBuilder::new(r"(?P^.+:) +(<<.*>>) *(?:# *([^#:]+)$)*") + .multi_line(true) + .build() + .map_err(|err| serde::de::Error::custom(err))?; + let captures = reg.captures_iter(&text).collect::>(); + + if captures.is_empty() { + return Ok(text); + } + + let mut result = format!("\n{}", text); + + for c in captures { + let placeholder = c.get(2).map(|r| r.as_str().to_string()).unwrap_or_default(); + let comment = c + .get(3) + .map(|r| r.as_str().to_string()) + .unwrap_or("Please replace or remove!".to_string()); + + result = reg + .replace( + &result, + &format!( + "$key Err({})", + format!( + "The placeholder {} was not replaced! {}", + placeholder, comment + ) + .as_error() + ), + ) + .to_string(); + } + + Err(serde::de::Error::custom(&result)) +} + impl Cluster { pub fn spread(self) -> Self { Self { @@ -715,7 +755,7 @@ impl std::fmt::Debug for InvalidCluster { Value::Null => {} Value::Mapping(_) => {} _ => { - formatter.write_str("\nvars:: ")?; + formatter.write_str("\nvars: ")?; formatter.write_str("Vars must be a mapping".as_error().as_str())?; formatter.write_str("\n")?; } diff --git a/src/task/cluster/fs.rs b/src/task/cluster/fs.rs index 2df45c1..fee0c88 100644 --- a/src/task/cluster/fs.rs +++ b/src/task/cluster/fs.rs @@ -318,7 +318,7 @@ impl IO { } impl IO { - pub fn serialize_input(self) -> Result, Box> { + pub fn serialize_input(self) -> Result, Box> { if let IO { input, output: Some(mut writer), @@ -327,7 +327,7 @@ impl IO { serde_yaml::to_writer(&mut writer, &input)?; Ok(IO { input, - output: Some(()), + output: Some(writer), }) } else { Err(GeninError::new( diff --git a/src/task/cluster/test.rs b/src/task/cluster/test.rs index 8b36734..8484825 100644 --- a/src/task/cluster/test.rs +++ b/src/task/cluster/test.rs @@ -808,8 +808,9 @@ topology: let result = format!("{:?}", serde_genin::from_slice::(bytes)); - let error_str = "Err(while parsing a block mapping, did not find expected key \ - at line 6 column 5(Err(\u{1b}[93m\u{1b}4\"Mallformed content\"\u{1b}[0m)))"; + let error_str = "Err(while parsing a block mapping, did not find expected key at \ + line 6 column 5(Err(while parsing a block mapping, did not find expected key at \ + line 6 column 5)))"; assert_eq!(result, error_str) } @@ -891,3 +892,36 @@ hosts: println!("{:?}", result); } + +#[test] +fn placeholders_in_config() { + let bytes = r" +--- +topology: + - name: router + replicasets_count: <> # указываем здесь количество репликасетов хранилищ из сайзинга + roles: + - router + - failover-coordinator + - name: storage + replicasets_count: <> + replication_factor: <> #для инсталляции в одном ЦОД это число должно быть 2, для инсталяции в двух ЦОД - 4 + roles: + - storage +".as_bytes(); + + let result = format!("{:?}", serde_genin::from_slice::(bytes)); + + let error_str = "Err(Data did not match any variant of cluster configuration\ + (Err(\n\n---\ntopology:\n - name: router\n replicasets_count: \ + Err(\u{1b}[93m\u{1b}4\"The placeholder <> was not replaced! \ + указываем здесь количество репликасетов хранилищ из сайзинга\"\u{1b}[0m)\n \ + roles:\n - router\n - failover-coordinator\n - name: storage\n \ + replicasets_count: Err(\u{1b}[93m\u{1b}4\"The placeholder <> \ + was not replaced! Please replace or remove!\"\u{1b}[0m)\n replication_factor: \ + Err(\u{1b}[93m\u{1b}4\"The placeholder <> was not replaced! \ + для инсталляции в одном ЦОД это число должно быть 2, для инсталяции в двух \ + ЦОД - 4\"\u{1b}[0m)\n roles:\n - storage\n)))"; + + assert_eq!(result, error_str) +} diff --git a/src/task/cluster/topology/test.rs b/src/task/cluster/topology/test.rs index 11031b2..d36e035 100644 --- a/src/task/cluster/topology/test.rs +++ b/src/task/cluster/topology/test.rs @@ -7,7 +7,7 @@ use crate::task::cluster::{ Role, }, name::Name, - topology::{Topology, TopologySet}, Cluster, + topology::{Topology, TopologySet}, }; #[test] @@ -99,4 +99,3 @@ roles: assert_eq!(topology_member_str, topology_member_model_str); } - diff --git a/src/task/serde_genin.rs b/src/task/serde_genin.rs index 04322af..59f3a82 100644 --- a/src/task/serde_genin.rs +++ b/src/task/serde_genin.rs @@ -5,8 +5,6 @@ use std::fmt::Display; use crate::task; -use super::AsError; - pub struct Error { message: String, inner: T, @@ -49,6 +47,8 @@ where { serde_yaml::from_slice::(slice).map_err(|err| Error { message: err.to_string(), - inner: T::validate(slice).map_err(|_| MallformedContent("Mallformed content".as_error())), + inner: T::validate(slice).map_err(|err| { + MallformedContent(err.to_string()) + }), }) }