diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..1ec47bc --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +/usebruno +/venv +/uploads +/exports +/.vscode +/.local +/.github +/.config +/.cache \ No newline at end of file diff --git a/.env b/.env index 6ae1618..ee41672 100644 --- a/.env +++ b/.env @@ -1 +1,3 @@ -SECRET_KEY=DEFAULT_KEY \ No newline at end of file +NB_WORKERS=6 +MAXTIME=120 +SECRET_KEY=DEFAULT_KEY diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml new file mode 100644 index 0000000..4adeffd --- /dev/null +++ b/.github/workflows/ruff.yml @@ -0,0 +1,8 @@ +name: Ruff +on: [ push, pull_request ] +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/ruff-action@v3 diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 2755c4e..a9f6db4 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.gitignore b/.gitignore index 348bdd0..294849a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ __pycache__ .bash_history +*.swp +.viminfo /.cache/ /.config/ .local @@ -10,16 +12,21 @@ uploads/ .fleet/ .vscode/ .~lock.*# +.python_history exports/ /config* /lotemplate/unittest/files/content/*.unittest.txt /lotemplate/unittest/files/content/*.unittest.odt +/lotemplate/unittest/files/content/*.unittest.html /lotemplate/unittest/files/content/debug.docx /lotemplate/unittest/files/content/debug.odt /lotemplate/unittest/files/content/debug.json /lotemplate/unittest/files/content/*.unittest.odt /lotemplate/unittest/files/content/*.unittest.pdf /venv +/lotemplate/unittest/files/content/e89fbedb61af3994184da3e5340bd9e9-calc_variables.ods.json /output*.* .fontconfig/ docker-compose.override.yml +/tmpfile/** +!/tmpfile/.keep diff --git a/API/utils.py b/API/utils.py index 739d5c4..85a410b 100644 --- a/API/utils.py +++ b/API/utils.py @@ -2,51 +2,40 @@ Copyright (C) 2023 Probesys """ -from flask import * +from flask import Response, send_file import lotemplate as ot -import configargparse as cparse import glob import os import sys -import subprocess -from time import sleep from typing import Union -from zipfile import ZipFile - -p = cparse.ArgumentParser(default_config_files=['config.yml', 'config.ini', 'config']) -p.add_argument('--config', '-c', is_config_file=True, help='Configuration file path') -p.add_argument('--host', default="localhost", help='Host address to use for the libreoffice connection') -p.add_argument('--port', default="2002", help='Port to use for the libreoffice connexion') -args = p.parse_known_args()[0] - -os.makedirs("uploads", exist_ok=True) -os.makedirs("exports", exist_ok=True) -subprocess.call( - f'soffice "--accept=socket,host={args.host},port={args.port};urp;StarOffice.ServiceManager" &', shell=True) -sleep(3) -cnx = ot.Connexion(args.host, args.port) +host='localhost' +port='200' +gworkers=0 +scannedjson='' +maxtime=60 +def start_soffice(workers,jsondir,maxt=60): + global gworkers + global my_lo + global scannedjson + global maxtime + maxtime=maxt + scannedjson=jsondir + gworkers=workers + os.makedirs("uploads", exist_ok=True) + os.makedirs("exports", exist_ok=True) + os.makedirs(scannedjson, exist_ok=True) + clean_temp_files() + my_lo=ot.start_multi_office(nb_env=workers) -def restart_soffice() -> None: - """ - simply restart the soffice process - :return: None - """ - - clean_temp_files() - subprocess.call( - f'soffice "--accept=socket,host={cnx.host},port={cnx.port};urp;StarOffice.ServiceManager" &', - shell=True - ) - sleep(2) - try: - cnx.restart() - except: - pass +def connexion(): + global my_lo + cnx= ot.randomConnexion(my_lo) + return cnx def clean_temp_files(): """ @@ -103,7 +92,6 @@ def error_format(exception: Exception, message: str = None) -> dict: { 'error': type(exception).__name__, 'code': exception.code if isinstance(exception, ot.errors.LotemplateError) else type(exception).__name__, - # 'message': message or str(exception), 'message': message or exception_message, 'variables': exception.infos if isinstance(exception, ot.errors.LotemplateError) else {} } @@ -148,23 +136,16 @@ def save_file(directory: str, f, name: str, error_caught=False) -> Union[tuple[d i += 1 f.stream.seek(0) f.save(f"uploads/{directory}/{name}") + + + cnx = connexion() + global scannedjson try: - with ot.Template(f"uploads/{directory}/{name}", cnx, True) as temp: + with ot.TemplateFromExt(f"uploads/{directory}/{name}", cnx, True,scannedjson) as temp: values = temp.variables except ot.errors.TemplateError as e: delete_file(directory, name) return error_format(e), 415 - except ot.errors.UnoException as e: - delete_file(directory, name) - restart_soffice() - if error_caught: - return ( - error_format(e, "Internal server error on file opening. Please checks the README file, section " - "'Unsolvable problems' for more informations."), - 500 - ) - else: - return save_file(directory, f, name, True) except Exception as e: delete_file(directory, name) return error_format(e), 500 @@ -180,24 +161,14 @@ def scan_file(directory: str, file: str, error_caught=False) -> Union[tuple[dict :param error_caught: specify if an error was already caught :return: a json and optionally an int which represent the status code to return """ - - try: - with ot.Template(f"uploads/{directory}/{file}", cnx, True) as temp: + cnx = connexion() + global scannedjson + with ot.TemplateFromExt(f"uploads/{directory}/{file}", cnx, True,scannedjson) as temp: variables = temp.variables - except ot.errors.UnoException as e: - restart_soffice() - if error_caught: - return ( - error_format(e, "Internal server error on file opening. Please checks the README file, section " - "'Unsolvable problems' for more informations."), - 500 - ) - else: - return scan_file(directory, file, True) return {'file': file, 'message': "Successfully scanned", 'variables': variables} -def fill_file(directory: str, file: str, json, error_caught=False) -> Union[tuple[dict, int], dict, Response]: +def fill_file(directory: str, file: str, json, error_caught=False) -> Union[tuple[dict, int], dict, tuple[str,Response]]: """ fill the specified file @@ -208,61 +179,48 @@ def fill_file(directory: str, file: str, json, error_caught=False) -> Union[tupl :return: a json and optionally an int which represent the status code to return """ - if type(json) != list or not json: - return error_sim("JsonSyntaxError", 'api_invalid_base_value_type', "The json should be a non-empty array"), 415 - + if isinstance(json, list): + json=json[0] + print("####\nUsing a list of dict is DEPRECATED, you must directly send the dict.") + print("See documentation.\n#######") + cnx = connexion() + global scannedjson try: - with ot.Template(f"uploads/{directory}/{file}", cnx, True) as temp: - - exports = [] - - for elem in json: - - length = len(elem) - is_name_present = type(elem.get("name")) is str - is_variables_present = type(elem.get("variables")) is dict - is_page_break_present = type(elem.get("page_break")) is bool - - if ( - not is_name_present - or not is_variables_present - or ((length > 2 and not is_page_break_present) or (length > 3 and is_page_break_present)) - ): - return error_sim( - "JsonSyntaxError", - 'api_invalid_instance_syntax', - "Each instance of the array in the json should be an object containing only 'name' - " - "a non-empty string, 'variables' - a non-empty object, and, optionally, 'page_break' - " - "a boolean."), 415 - - try: - json_variables = ot.convert_to_datas_template(elem["variables"]) - temp.search_error(json_variables) - temp.fill(elem["variables"]) - if elem.get('page_break', False): - temp.page_break() - exports.append(temp.export("exports/" + elem["name"], should_replace=( - True if len(json) == 1 else False))) - except Exception as e: - return error_format(e), 415 - - if len(exports) == 1: - return send_file(exports[0], download_name=exports[0].split("/")[-1]) - else: - with ZipFile('exports/export.zip', 'w') as zipped: - for elem2 in exports: - zipped.write(elem2, elem2.split("/")[-1]) - return send_file('exports/export.zip', 'export.zip') - except ot.errors.UnoException as e: - restart_soffice() - if error_caught: - return ( - error_format(e, "Internal server error on file opening. Please checks the README file, section " - "'Unsolvable problems' for more informations."), - 500 - ) - else: - return fill_file(directory, file, json, True) - - -clean_temp_files() + with ot.TemplateFromExt(f"uploads/{directory}/{file}", cnx, True,scannedjson) as temp: + + length = len(json) + is_name_present = type(json.get("name")) is str + is_variables_present = type(json.get("variables")) is dict + is_page_break_present = type(json.get("page_break")) is bool + + if ( + not is_name_present + or not is_variables_present + or ((length > 2 and not is_page_break_present) or (length > 3 and is_page_break_present)) + ): + return 415, error_sim( + "JsonSyntaxError", + 'api_invalid_instance_syntax', + "Each instance of the array in the json should be an object containing only 'name' - " + "a non-empty string, 'variables' - a non-empty object, and, optionally, 'page_break' - " + "a boolean.") + + try: + json_variables = ot.convert_to_datas_template(json["variables"]) + temp.search_error(json_variables) + temp.fill(json["variables"]) + if json.get('page_break', False): + temp.page_break() + export_file=temp.export(json["name"],"exports") + export_name=json["name"] + except Exception as e: + if 'export_name' in locals(): + return ( export_file,error_format(e)) + else: + return ( "nofile",error_format(e)) + + return (export_file,send_file(export_file, export_name)) + + except Exception as e: + return error_format(e), 500 + diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e573898 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,75 @@ +Versions +======== + +Note : the upgrade from version 1.x to 2.x is easy. There is no reason to stay to version 1.x. + +The upgrade documentation is in the file [UPGRADE.md](UPGRADE.md). + +Versions 2.x +------------ + +- v2.0.0 : 01/01/2025 + - BC Break (easy to fix) : see [UPGRADE.md](UPGRADE.md) + - We can now generate Calc / Excel files (from Calc templates) + - Is multiThreaded : we can generate several files at the same time + - Performances improvements + - No BC Breaks for the templates + - upgrade debian, LibreOffice, Python libs versions + - for devs : added "use bruno" requests inside the repository + +Versions 1.x +------------ + +- v1.6.1 : 2024-04-12 : bugfix + - fix the issue https://github.com/Probesys/lotemplate/issues/34 : too many endif bugg +- v1.6.0 : 2024-04-11 + - allow put variables inside headers and footers + - fix a bug when a variable is both inside the text content and inside a table (it should not arrive, but it is fixed) + - a new unit test system based on PDF converted to text in order to test contents that are not converted to text with a simple saveAs +- v1.5.2 : 2024-02-24 : Better README + - Rewrite for a betterdocker DockerFile without bug +- v1.5.1 : 2024-02-16 : Better README + - Rewriting of the README file +- v1.5.0 : 2024-02-12 : syntax error detection + - add syntax error detection in if statements + - add syntax error detection in for statements + - come back to default libreoffice of Debian Bookworm (removed backports, incompatibility) +- v1.4.1 : 2023-11-20 : micro-feature for counter and fix possible bug + - use counters for counting elements of a list + - fix possible bug with reset and last. +- v1.4.0, 2023-11-17 : counters + - add a counter system inside templates + - add better scan for if statement. Raises an error if there is too many endif in the template. + - speedup html statement replacement and scanning + - speedup for statement replacement and scanning + - tests of for scanning + - internal : add scan testing inside content unit tests +- v1.3.0, 2023-11-16 : + - major refactoring. No evolution for the user. + - new unit tests on tables and images + - no BC Break (theoretically) +- v1.2.8, 2023-09-01 : + - fix bug in TextShape var replacement +- v1.2.7, 2023-08-30 : + - Upgrade to debian bookworm slim +- v1.2.6, 2023-08-30 : + - new comparators for if statements : ===, !==, CONTAINS, NOT_CONTAINS + - variables of type "html" are now supported and copied as HTML +- v1.2.5, 2023-07-17 : temporary fix for detecting endhtml and endfor +- v1.2.4, 2023-07-09 : fix major bug in if statement scanning +- v1.2.3, 2023-07-07 : no endif detection, performance improvement in if statement +- v1.2.2, 2023-06-09 : bugfix html statement scan missing +- v1.2.1, 2023-06-05 : little fix for CI +- v1.2.0, 2023-06-04 : if statements inside for +- v1.1.0, 2023-05-23 : recursive if statement +- v1.0.1, 2023-05-05 : workaround, fix in html formatting +- v1.0.0, 2023-05-03 : if statement, for statement, html statement +- not numbered : about may 2022 : first version + +### Possible futur evolutions + +- Possibly to add dynamic images in tables +- another way to make image variables that would be compatible with Microsoft Word and maybe other formats (example : set the variable name in the 'alternative text' field) +- key system for each institution for security +- handle bulleted lists using table like variables +- use variable formatting instead of the one of the character before diff --git a/Dockerfile b/Dockerfile index 043115b..885c041 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,7 @@ -FROM debian:bookworm-slim as prod +FROM debian:trixie-slim as prod + +RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache + RUN --mount=type=cache,id=apt-cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,id=apt-lib,target=/var/lib/apt,sharing=locked \ --mount=type=cache,id=debconf,target=/var/cache/debconf,sharing=locked \ diff --git a/README.md b/README.md index 85da15d..508c350 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,23 @@ LOTemplate (for Libre Office Template) ====================================== +**Warning** : This readme is for the version 2.x of LoTemplate. There are breaking changes between versions 1.x and 2.x. See [UPGRADE.md](UPGRADE.md) documentation. You can also see the [CHANGELOG.md](CHANGELOG.md) for the versions. + + [![Unittest](https://github.com/Probesys/lotemplate/actions/workflows/unittest.yml/badge.svg)](https://github.com/Probesys/lotemplate/actions/workflows/unittest.yml) -LOTemplate is document generator used to create documents programatically (ODT, DOCX, PDF) from a template and a json file. +Principles +---------- + +LOTemplate is document generator used to create documents programatically (ODT, DOCX,ODS, XLSX, PDF) from an office template and a json file. ```mermaid +--- +title: Word / Writer document +--- + flowchart LR - template["Template
(DOCX or ODT)"] + template["Word Template
(DOCX or ODT)"] json["Data
(JSON)"] lotemplate["LO Template
(accessible by API or CLI)"] generatedFile["Generated File
(PDF, DOCX, ODT, RTF,...)"] @@ -17,33 +27,53 @@ flowchart LR lotemplate --> generatedFile ``` +```mermaid +--- +title: Excel / Calc document +--- + + flowchart LR + calc_template["Excel Template
(ODS or XLSX)"] + calc_json["Data
(JSON)"] + calc_lotemplate["LO Template
(accessible by API or CLI)"] + calc_generatedFile["Generated File
(ODS, XLSX, PDF, csv,...)"] + + calc_template --> calc_lotemplate + calc_json --> calc_lotemplate + calc_lotemplate --> calc_generatedFile +``` + What makes this tool different from others are the following features : -* The templates are in DOCX or ODT (Word or Libre Office) format -* Template can have complex structures (variables, loop, conditions, counters, html,...) +* The templates are in office format (ods,odt, docx, xlsx, ... ) format +* Word Template can have complex structures (variables, loop, conditions, counters, html,...) * The tool can scan the template to extract the variables sheet * The tool can be called by an API, a CLI or a python module. +* The tool uses a real LibreOffice headless to fill the templates. Then the output formats are all the LibreOffice supported formats (docx, xlsx, pdf, odt, ods, text, rtf, html, ...) The tool is written in Python and use a real LibreOffice headless to fill the templates. -Quick start +Table of content +---------------- + +* [Principles](#installation) +* [Quick Start](#quick_start) +* [API and CLI Usage](#api-and-cli-usage) +* [DOCX and ODT Template syntax and examples](#docx-and-odt-template-syntax) +* [XLSX and ODS Template syntax and examples](#xlsx-and-ods-template-syntax) +* [Supported formats](#supported-formats) +* [Doc for developpers of lotemplate](#doc-for-devs) +* [Unsolvable problems](#unsolvable-problems) +* [Installation without Docker](#installation_without_docker) +* [External documentations](#external-documentations) + + +Quick start ----------- ### Run the project with docker compose -Create a docker-compose.yml - -```yaml -version: '3' -services: - lotemplate: - image: probesys38/lotemplate:v1.5.0 - volumes: - - lotemplate-uploads:/app/uploads - environment: - - SECRET_KEY=lopassword - command: "gunicorn -w 4 -b 0.0.0.0:8000 app:app" -``` +Use the docker-compose.yml at the root of the project. Configure the .env file run the service @@ -55,7 +85,7 @@ docker-compose up -d ```bash # creation of a directory -curl -X PUT -H 'secret_key: lopassword' -H 'directory: test_dir1' http://localhost:8000/ +curl -X PUT -H 'secretkey: lopassword' -H 'directory: test_dir1' http://localhost:8000/ # {"directory":"test_dir1","message":"Successfully created"} ``` @@ -73,14 +103,14 @@ Upload this file to lotemplate ```bash # upload a template -curl -X PUT -H 'secret_key: lopassword' -F file=@/tmp/basic_test.odt http://localhost:8000/test_dir1 +curl -X PUT -H 'secretkey: lopassword' -F file=@/tmp/basic_test.odt http://localhost:8000/test_dir1 # {"file":"basic_test.odt","message":"Successfully uploaded","variables":{"my_tag":{"type":"text","value":""},"other_tag":{"type":"text","value":""}}} # generate a file titi.odt from a template and a json content curl -X POST \ - -H 'secret_key: lopassword' \ + -H 'secretkey: lopassword' \ -H 'Content-Type: application/json' \ - -d '[{"name":"my_file.odt","variables":{"my_tag":{"type":"text","value":"foo"},"other_tag":{"type":"text","value":"bar"}}}]' \ + -d '{"name":"my_file.odt","variables":{"my_tag":{"type":"text","value":"foo"},"other_tag":{"type":"text","value":"bar"}}}' \ --output titi.odt http://localhost:8000/test_dir1/basic_test.odt ``` @@ -95,59 +125,9 @@ My tag is foo ``` -Table of content ----------------- - -* [Installation](#installation) -* [Basic Usage](#basic-usage) -* [Template syntax and examples](#template-syntax) -* [Supported formats](#supported-formats) -* [Doc for developpers of lotemplate](#doc-for-devs) -* [Unsolvable problems](#unsolvable-problems) -* [External documentations](#external-documentations) -* [Versions](#versions) - -Installation ---------------------------------------- - -### Requirements - -For Docker use of the API, you can skip this step. - -- LibreOffice (the console-line version will be enough) -- python3.8 or higher -- python3-uno -- some python packages specified in [requirement.txt](requirements.txt) that you can install with - `pip install -r requirements.txt`. `Flask` and `Werkzeug` are optional, as they are used only for the API. - -```bash -# on debian bookworm, you can use these commands -apt update -apt -y -t install bash python3 python3-uno python3-pip libreoffice-nogui -pip install -r requirements.txt -``` - -### Run the API -Run the following command on your server : -```shell -python3 -m flask run -``` - -or simply - -```shell -flask run -``` - -or, for Docker deployment: - -```shell -docker-compose up -``` - -Basic Usage +API and CLI Usage ------------------------------------- ### With the API @@ -156,21 +136,21 @@ docker-compose up ```bash # creation of a directory -curl -X PUT -H 'secret_key: my_secret_key' -H 'directory: test_dir1' http://lotemplate:8000/ +curl -X PUT -H 'secretkey: my_secret_key' -H 'directory: test_dir1' http://lotemplate:8000/ # {"directory":"test_dir1","message":"Successfully created"} -curl -X PUT -H 'secret_key: my_secret_key' -H 'directory: test_dir2' http://lotemplate:8000/ +curl -X PUT -H 'secretkey: my_secret_key' -H 'directory: test_dir2' http://lotemplate:8000/ # {"directory":"test_dir2","message":"Successfully created"} # look at the created directories -curl -X GET -H 'secret_key: my_secret_key' http://lotemplate:8000/ +curl -X GET -H 'secretkey: my_secret_key' http://lotemplate:8000/ # ["test_dir2","test_dir1"] # delete a directory (and it's content -curl -X DELETE -H 'secret_key: my_secret_key' http://lotemplate:8000/test_dir2 +curl -X DELETE -H 'secretkey: my_secret_key' http://lotemplate:8000/test_dir2 # {"directory":"test_dir2","message":"The directory and all his content has been deleted"} # look at the directories -curl -X GET -H 'secret_key: my_secret_key' http://lotemplate:8000/ +curl -X GET -H 'secretkey: my_secret_key' http://lotemplate:8000/ # ["test_dir1"] ``` @@ -186,15 +166,15 @@ Upload this file to lotemplate ```bash # upload a template -curl -X PUT -H 'secret_key: my_secret_key' -F file=@/tmp/basic_test.odt http://lotemplate:8000/test_dir1 +curl -X PUT -H 'secretkey: my_secret_key' -F file=@/tmp/basic_test.odt http://lotemplate:8000/test_dir1 {"file":"basic_test.odt","message":"Successfully uploaded","variables":{"my_tag":{"type":"text","value":""},"other_tag":{"type":"text","value":""}}} # analyse an existing file and get variables -curl -X GET -H 'secret_key: my_secret_key' http://lotemplate:8000/test_dir1/basic_test.odt +curl -X GET -H 'secretkey: my_secret_key' http://lotemplate:8000/test_dir1/basic_test.odt # {"file":"basic_test.odt","message":"Successfully scanned","variables":{"my_tag":{"type":"text","value":""},"other_tag":{"type":"text","value":""}}} # generate a file titi.odt from a template and a json content - curl -X POST -H 'secret_key: my_secret_key' -H 'Content-Type: application/json' -d '[{"name":"my_file.odt","variables":{"my_tag":{"type":"text","value":"foo"},"other_tag":{"type":"text","value":"bar"}}}]' --output titi.odt http://lotemplate:8000/test_dir1/basic_test.odt + curl -X POST -H 'secretkey: my_secret_key' -H 'Content-Type: application/json' -d '{"name":"my_file.odt","variables":{"my_tag":{"type":"text","value":"foo"},"other_tag":{"type":"text","value":"bar"}}}' --output titi.odt http://lotemplate:8000/test_dir1/basic_test.odt ``` After the operation, you get the file titi.odt with this content : @@ -209,7 +189,7 @@ let’s see if the tag foo is replaced and this bar is detected. Then use the following routes : -*all routes take a secret key in the header, key `secret_key`, that correspond to the secret key configured in the +*all routes take a secret key in the header, key `secretkey`, that correspond to the secret key configured in the [.env](.env) file. If no secret key is configured, the secret key isn't required at request.* - `/` @@ -260,9 +240,9 @@ positional arguments: optional arguments: -h, --help show this help message and exit - --json_file JSON_FILE [JSON_FILE ...], -jf JSON_FILE [JSON_FILE ...] + --json_file JSON_FILE , -jf JSON_FILE Json files that must fill the template, if any - --json JSON [JSON ...], -j JSON [JSON ...] + --json JSON , -j JSON Json strings that must fill the template, if any --output OUTPUT, -o OUTPUT Names of the filled files, if the template should @@ -272,7 +252,8 @@ optional arguments: --host HOST Host address to use for the libreoffice connection --port PORT Port to use for the libreoffice connexion --scan, -s Specify if the program should just scan the template - and return the information, or fill it. + --cpu Specify the number of libreoffice to start, default 0 is + the number of CPU and return the information, or fill it. --force_replacement, -f Specify if the program should ignore the scan's result ``` @@ -290,7 +271,7 @@ of an array to dynamically add rows. Then pass the file, and the completed json to fill it. -Template syntax and examples +DOCX and ODT Template syntax and examples ---------------------------------------------------------- ### text variables @@ -756,22 +737,225 @@ we displayed [counter.last iterator] solutions ``` +XLSX and ODS Template syntax and examples +---------------------------------------------------------- + +The idea is to generate a real CALC / Excel file from a template, with potentially several sheets, variables to replace, dynamic tables, operations, etc. + +The replacements are done in all the sheets of the document. + +### Simple variables replacements + +If you have a excel template like this : + + + + + + + + + + + + + + +
Name$myname
Hours$myhours
days=B2/7 (displays "#VALUE!")
+ + +And a json file like this : + +```json +{ + "name": "simple_vars_result.xlsx", + "variables": { + "myhours": { + "type": "text", + "value": "12" + }, + "myname": { + "type": "text", + "value": "Gérard" + } + } +} +``` + +You obtain a excel like this + + + + + + + + + + + + + + +
NameGérard
Hours12
days=B2/7 (displays "1.71428571428571")
+ +You can format the cells, add formulas, etc. Everything is kept in the final document. + +### Dynamic tables + +#### Example with a document LibreOffice Calc as a template + +Dynamic tables are a bit more tricky. We are using "Named ranges". + +Lets say you have a calc (.ods) document like this : + + + + + + + + + + + + + + + + + + + + +
ArticleUnit priceQuantityTotal
&name&unitPrice&quantity=B2*C2
Total Price=SUM(INDEX(loop_down_article, ,4))
+ +Note 1 : the `&` is used to indicate that the cell content is a array. +Note 2 : the SUM is strange : it is related to a "name range" + +In order to explain to lotemplate that you want to duplicate the line 2 to the bottom, you have to create a named range that is named "loop_down_article". For that : +- select the cells that you want to be duplicated (A2 to D2 in this example) +- go to Sheet -> Named ranges and expressions -> Define +- give a name to the range (loop_down_article in this example) + +![named_ranges_screenshot_1](doc/assets/named_ranges_1.png) + +![named_ranges_screenshot_2](doc/assets/named_ranges_2.png) + +Note : in the name of the range, "loop_down_" says that we want to insert the lines to the bottom. There is only two possibilities : "loop_down_" and "loop_right_". Le last part (article in the example) is only here to give a unique name to the range. + +Now that we have a named range, we can use it to calculate the total price of the lines. The formula `=SUM(INDEX(loop_down_article, ,4))` will sum all the values of the 4th column of the named range "loop_down_article". + +Then you can use the following json file : + +```json +{ + "name": "calc_table_formula.html", + "variables": { + "loop_down_article": { + "type": "object", + "value": { + "name": { + "type": "table", + "value": [ + "appel", + "banana", + "melon", + "lemon" + ] + }, + "unitPrice": { + "type": "table", + "value": [ + "1", + "1.5", + "3.2", + "0.8" + ] + }, + "quantity": { + "type": "table", + "value": [ + "4", + "6", + "2", + "1" + ] + } + } + } + } +} +``` + +The result of the generation will be : + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ArticleUnit priceQuantityTotal
apple14=B2*C2 (display 4)
banana1.56=B2*C2 (display 9)
melon3.22=B2*C2 (display 6.4)
lemon0.81=B2*C2 (display 0.8)
Total Price=SUM(INDEX(loop_down_article, ,4)) (display 20.2)
+ +#### Example with a document Excel as a template + +The principle is exactly the same as for LibreOffice Calc exept for the creation of the named range. + +![excel_named_range screenshot 1](doc/assets/excel_named_range_1.png) + +![excel_named_range screenshot 2](doc/assets/excel_named_range_2.png) + +(sorry, my excel is in french...) + Supported formats ------------------------------------------------- ### Import -| Format | ODT, OTT | HTML | DOC, DOCX | RTF | TXT | OTHER | -|-------------------------|----------|------|-----------|-----|-----|-------| -| text variables support | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | -| dynamic tables support | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | -| image variables support | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | +| Format | ODT, OTT |ODS, ODST |XLSX, XLS | HTML | DOC, DOCX | RTF | TXT | OTHER | +|-------------------------|----------|----------|----------|------|-----------|-----|-----|-------| +| text variables support | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | +| dynamic tables support | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | +| image variables support | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ### Export +For Writer odt, pdf, html, docx. +For Calc +ods, xls, xlsx, html, csv -Other formats can be easily added by adding the format information in the dictionary `formats` in -[lotemplate/classes.py](lotemplate/classes.py) > Template > export(). - +Other formats can be easily added by adding the format information in the dictionary `formats` of the respective classes Format information can be found on the [unoconv repo](https://github.com/unoconv/unoconv/blob/94161ec11ef583418a829fca188c3a878567ed84/unoconv#L391). @@ -800,96 +984,60 @@ cp docker-compose.override.yml.example docker-compose.override.yml docker-compose up ``` -Unsolvable problems ------------------------------------------------------ -The error `UnoException` happens frequently and -unpredictably, and this error stops the soffice processus -(please note that the API try to re-launch the process by itself). This error, particularly annoying, is unfortunately -impossible to fix, since it can be caused by multiples soffice (LibreOffice) bugs. -Here is a non-exhaustive list of cases that ***can*** cause this bug : -- The soffice process was simply closed after the connection is established. -- The `.~lock.[FILENAME].odt#` file is present in the folder where the document is open. This file is created when the - file is currently edited via libreoffice, and deleted when the programs in which it is edited are - closed. The program try to avoid this error by deleting this file at document opening. -- The first line of the document is occupied by a table or another dynamic element - (just jump a line, it will solve the problem) -- The background of document is an image, and is overlaid by many text fields -- The document is an invalid file (e.g: the file is an image), and the bridge crashes instead of return the proper - error. +Installation without Docker +--------------------------------------- + +### Requirements + +For Docker use of the API, you can skip this step. + +- LibreOffice (the console-line version will be enough) +- python3.8 or higher +- python3-uno +- some python packages specified in [requirement.txt](requirements.txt) that you can install with + `pip install -r requirements.txt`. `Flask` and `Werkzeug` are optional, as they are used only for the API. -The amount of memory used by soffice can increase with its use, even when open files are properly closed (which is the -case). Again, this is a bug in LibreOffice/soffice that has existed for years. +```bash +# on debian bookworm, you can use these commands +apt update +apt -y -t install bash python3 python3-uno python3-pip libreoffice-nogui +pip install -r requirements.txt +``` + +### Run the API + +Run the following command on your server : + +```shell +python3 -m flask run +``` + +or simply + +```shell +flask run +``` + +or, for Docker deployment: + +```shell +docker-compose up +``` -For trying to fix these problems, you can try: -- Use the most recent stable release of LibreOffice (less memory, more stable, fewer crashes) External documentations --------------------------------------------------------- + +For Pyuno + +- [LibreOffice SDK API Reference](https://api.libreoffice.org/docs/idl/ref/index.html) +- [LibreOffice 24.2 API Documentation](https://api.libreoffice.org/) + +- [Libreoffice Development Wiki](https://wiki.documentfoundation.org/Development) - [JODConverter wiki for list formats compatibles with LibreOffice](https://github.com/sbraconnier/jodconverter/wiki/Getting-Started) - [The unoconv source code, written in python with PyUNO](https://github.com/unoconv/unoconv/blob/master/unoconv) - [Unoconv source code for list formats - and properties - compatible with LibreOffice for export](https://github.com/unoconv/unoconv/blob/94161ec11ef583418a829fca188c3a878567ed84/unoconv#L391) -- [OpenOffice Python Bridge information and code exemples](http://www.openoffice.org/udk/python/python-bridge.html) -- [com.sun.star Java API docs (On which pyuno is based - but is not identical)](https://www.openoffice.org/api/docs/common/ref/com/sun/star/module-ix.html) -- [Java LibreOffice Programming Book](http://fivedots.coe.psu.ac.th/~ad/jlop) -- [Deploying Flask](https://flask.palletsprojects.com/en/2.0.x/deploying/) -- [Flask documentation - quickstart](https://flask.palletsprojects.com/en/2.0.x/quickstart/) -- [Flask documentation - upload](https://flask.palletsprojects.com/en/2.0.x/patterns/fileuploads/) - -Versions -------------------------------- - -- v1.6.1 : 2024-04-12 : bugfix - - fix the issue https://github.com/Probesys/lotemplate/issues/34 : too many endif buggy -- v1.6.0 : 2024-04-11 - - allow put variables inside headers and footers - - fix a bug when a variable is both inside the text content and inside a table (it should not arrive, but it is fixed) - - a new unit test system based on PDF converted to text in order to test contents that are not converted to text with a simple saveAs -- v1.5.2 : 2024-02-24 : Better README - - Rewrite for a betterdocker DockerFile without bug -- v1.5.1 : 2024-02-16 : Better README - - Rewriting of the README file -- v1.5.0 : 2024-02-12 : syntax error detection - - add syntax error detection in if statements - - add syntax error detection in for statements - - come back to default libreoffice of Debian Bookworm (removed backports, incompatibility) -- v1.4.1 : 2023-11-20 : micro-feature for counter and fix possible bug - - use counters for counting elements of a list - - fix possible bug with reset and last. -- v1.4.0, 2023-11-17 : counters - - add a counter system inside templates - - add better scan for if statement. Raises an error if there is too many endif in the template. - - speedup html statement replacement and scanning - - speedup for statement replacement and scanning - - tests of for scanning - - internal : add scan testing inside content unit tests -- v1.3.0, 2023-11-16 : - - major refactoring. No evolution for the user. - - new unit tests on tables and images - - no BC Break (theoretically) -- v1.2.8, 2023-09-01 : - - fix bug in TextShape var replacement -- v1.2.7, 2023-08-30 : - - Upgrade to debian bookworm slim -- v1.2.6, 2023-08-30 : - - new comparators for if statements : ===, !==, CONTAINS, NOT_CONTAINS - - variables of type "html" are now supported and copied as HTML -- v1.2.5, 2023-07-17 : temporary fix for detecting endhtml and endfor -- v1.2.4, 2023-07-09 : fix major bug in if statement scanning -- v1.2.3, 2023-07-07 : no endif detection, performance improvement in if statement -- v1.2.2, 2023-06-09 : bugfix html statement scan missing -- v1.2.1, 2023-06-05 : little fix for CI -- v1.2.0, 2023-06-04 : if statements inside for -- v1.1.0, 2023-05-23 : recursive if statement -- v1.0.1, 2023-05-05 : workaround, fix in html formatting -- v1.0.0, 2023-05-03 : if statement, for statement, html statement -- not numbered : about may 2022 : first version - -### Possible futur evolutions - -- Possibly to add dynamic images in tables -- another way to make image variables that would be compatible with Microsoft Word and maybe other formats (example : set the variable name in the 'alternative text' field) -- key system for each institution for security -- handle bulleted lists using table like variables -- use variable formatting instead of the one of the character before + + diff --git a/UPGRADE.md b/UPGRADE.md new file mode 100644 index 0000000..d133526 --- /dev/null +++ b/UPGRADE.md @@ -0,0 +1,37 @@ +Upgrades +======== + +From 1.x to 2.x +--------------- + +### API : secretkey field + +In all the API requests, the field secret_key is now named secretkey (due to upgrade of Flask). + +before : + +```bash +curl -X GET -H 'secret_key: my_secret_key' http://lotemplate:8000/ +``` + +after : + +```bash +curl -X GET -H 'secretkey: my_secret_key' http://lotemplate:8000/ +``` + +### API : JSON to send in order to generate file is not an array anymore + +Look at the following example : The JSON sent was an array in the previous version. Now it is directly the dict that was inside the array. + +before : + +```bash + curl -X POST -H 'secret_key: my_secret_key' -H 'Content-Type: application/json' -d '[{"name":"my_file.odt","variables":{"my_tag":{"type":"text","value":"foo"},"other_tag":{"type":"text","value":"bar"}}}]' --output titi.odt http://lotemplate:8000/test_dir1/basic_test.odt +``` + +after : + +```bash + curl -X POST -H 'secretkey: my_secret_key' -H 'Content-Type: application/json' -d '{"name":"my_file.odt","variables":{"my_tag":{"type":"text","value":"foo"},"other_tag":{"type":"text","value":"bar"}}}' --output titi.odt http://lotemplate:8000/test_dir1/basic_test.odt +``` \ No newline at end of file diff --git a/app.py b/app.py index 8bcba8b..5db6669 100644 --- a/app.py +++ b/app.py @@ -2,22 +2,25 @@ Copyright (C) 2023 Probesys """ -from flask import * +from flask import Flask,request, jsonify,after_this_request,send_file from werkzeug.utils import secure_filename import os -from shutil import copyfile, rmtree +from shutil import copyfile, rmtree +from os.path import isfile, join +from os import listdir from API import utils +from lotemplate.utils import get_cached_json +from lotemplate import statistic_open_document,clean_old_open_document app = Flask(__name__) - @app.route("/", methods=['PUT', 'GET']) def main_route(): - if request.headers.get('secret_key', '') != os.environ.get('SECRET_KEY', ''): + if request.headers.get('secretkey', '') != os.environ.get('SECRET_KEY', ''): return utils.error_sim( - 'ApiError', 'invalid_secret_key', f"The secret key is invalid or not given", {'key': 'secret_key'}), 401 + 'ApiError', 'invalid_secretkey', "The secret key is invalid or not given", {'key': 'secret_key'}), 401 if request.method == 'PUT': if 'directory' not in request.headers: return utils.error_sim( @@ -34,11 +37,27 @@ def main_route(): return jsonify(os.listdir("uploads")) +@app.route("/stats") +def stats_route(): + if request.headers.get('secretkey', '') != os.environ.get('SECRET_KEY', ''): + return utils.error_sim( + 'ApiError', 'invalid_secretkey', "The secret key is invalid or not given", {'key': 'secret_key'}), 401 + else: + return statistic_open_document(utils.my_lo,utils.maxtime) + +@app.route("/clean_lo") +def clean_route(): + if request.headers.get('secretkey', '') != os.environ.get('SECRET_KEY', ''): + return utils.error_sim( + 'ApiError', 'invalid_secretkey', "The secret key is invalid or not given", {'key': 'secret_key'}), 401 + else: + return clean_old_open_document(utils.my_lo,utils.maxtime) + @app.route("/", methods=['PUT', 'DELETE', 'PATCH', 'GET']) def directory_route(directory): - if request.headers.get('secret_key', '') != os.environ.get('SECRET_KEY', ''): + if request.headers.get('secretkey', '') != os.environ.get('SECRET_KEY', ''): return utils.error_sim( - 'ApiError', 'invalid_secret_key', f"The secret key is invalid or not given", {'key': 'secret_key'}), 401 + 'ApiError', 'invalid_secretkey', "The secret key is invalid or not given", {'key': 'secret_key'}), 401 if not os.path.isdir(f"uploads/{directory}") and request.method != 'PUT': return utils.error_sim( 'ApiError', 'dir_not_found', f"the specified directory {repr(directory)} doesn't exist", @@ -60,6 +79,12 @@ def directory_route(directory): {'key': 'file'}), 400 return utils.save_file(directory, f, secure_filename(f.filename)) elif request.method == 'DELETE': + onlyfiles = [f for f in listdir("uploads/"+directory) if isfile(join("uploads/"+directory, f))] + json_cache_dir=utils.scannedjson + for file in onlyfiles: + cachedjson=get_cached_json(json_cache_dir,"uploads/"+directory+"/"+file) + if os.path.exists(cachedjson): + os.remove(cachedjson) rmtree(f"uploads/{directory}") return {'directory': directory, 'message': 'The directory and all his content has been deleted'} elif request.method == 'PATCH': @@ -80,9 +105,17 @@ def directory_route(directory): @app.route("//", methods=['GET', 'PATCH', 'DELETE', 'POST']) def file_route(directory, file): - if request.headers.get('secret_key', '') != os.environ.get('SECRET_KEY', ''): + @after_this_request + def delete_tmp_file(response): + if request.method == 'POST': + try: + os.remove(file) + except Exception: + print("Error delete file " + str(file)) + return response + if request.headers.get('secretkey', '') != os.environ.get('SECRET_KEY', ''): return utils.error_sim( - 'ApiError', 'invalid_secret_key', f"The secret key is invalid or not given", {'key': 'secret_key'}), 401 + 'ApiError', 'invalid_secretkey', "The secret key is invalid or not given", {'key': 'secret_key'}), 401 if not os.path.isdir(f"uploads/{directory}"): return utils.error_sim( 'ApiError', 'dir_not_found', f"the specified directory {repr(directory)} doesn't exist", @@ -109,17 +142,24 @@ def file_route(directory, file): elif request.method == 'POST': if not request.json: return utils.error_sim('ApiError', 'missing_json', "You must provide a json in the body"), 400 - return utils.fill_file(directory, file, request.json) + + file ,response = utils.fill_file(directory, file, request.json) + return response elif request.method == 'DELETE': - os.remove(f"uploads/{directory}/{file}") + json_cache_dir=utils.scannedjson + cachedjson=get_cached_json(json_cache_dir,"uploads/"+directory+"/"+file) + if os.path.exists(cachedjson): + os.remove(cachedjson) + if os.path.exists(f"uploads/{directory}/{file}"): + os.remove(f"uploads/{directory}/{file}") return {'directory': directory, 'file': file, 'message': "File successfully deleted"} @app.route("///download") def download_route(directory, file): - if request.headers.get('secret_key', '') != os.environ.get('SECRET_KEY', ''): + if request.headers.get('secretkey', '') != os.environ.get('SECRET_KEY', ''): return utils.error_sim( - 'ApiError', 'invalid_secret_key', f"The secret key is invalid or not given", {'key': 'secret_key'}), 401 + 'ApiError', 'invalid_secretkey', "The secret key is invalid or not given", {'key': 'secret_key'}), 401 if not os.path.isdir(f"uploads/{directory}"): return utils.error_sim( 'ApiError', 'dir_not_found', f"the specified directory {repr(directory)} doesn't exist", diff --git a/doc/assets/excel_named_range_1.png b/doc/assets/excel_named_range_1.png new file mode 100644 index 0000000..326b671 Binary files /dev/null and b/doc/assets/excel_named_range_1.png differ diff --git a/doc/assets/excel_named_range_2.png b/doc/assets/excel_named_range_2.png new file mode 100644 index 0000000..91dd966 Binary files /dev/null and b/doc/assets/excel_named_range_2.png differ diff --git a/doc/assets/named_ranges_1.png b/doc/assets/named_ranges_1.png new file mode 100644 index 0000000..21dbdbb Binary files /dev/null and b/doc/assets/named_ranges_1.png differ diff --git a/doc/assets/named_ranges_2.png b/doc/assets/named_ranges_2.png new file mode 100644 index 0000000..b54e8a8 Binary files /dev/null and b/doc/assets/named_ranges_2.png differ diff --git a/docker-compose.yml b/docker-compose.yml index cc955a5..add6d1f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3' services: lo_api: build: @@ -11,4 +10,6 @@ services: - "8000:8000" environment: - SECRET_KEY=$SECRET_KEY - command: "gunicorn -w 4 -b 0.0.0.0:8000 app:app" + - NB_WORKERS=$NB_WORKERS + - MAXTIME=$MAXTIME + command: "gunicorn -b 0.0.0.0:8000 app:app" diff --git a/gunicorn.conf.py b/gunicorn.conf.py new file mode 100644 index 0000000..5fa87f0 --- /dev/null +++ b/gunicorn.conf.py @@ -0,0 +1,10 @@ +from API import utils +import os + +workers=int(os.environ.get('NB_WORKERS', 4)) +maxtime=int(os.environ.get('MAXTIME', 60)) +my_lo=[] +scannedjson='uploads/scannnedjson' +def on_starting(server): + + utils.start_soffice(workers,scannedjson,maxtime) diff --git a/lotemplate/CalcTemplate.py b/lotemplate/CalcTemplate.py new file mode 100644 index 0000000..ba6a7e6 --- /dev/null +++ b/lotemplate/CalcTemplate.py @@ -0,0 +1,139 @@ +""" +Copyright (C) 2023 Probesys + + +The classes used for document connexion and manipulation +""" + +__all__ = ( + 'CalcTemplate', +) +from typing import Union + + + +from . import errors + +from lotemplate.Statement.CalcTableStatement import CalcTableStatement +from .Template import Template +from lotemplate.Statement.CalcSearchStatement import CalcTextStatement +from jsondiff import diff + +class CalcTemplate(Template): + formats = { + "ods": "calc8", + "pdf": "calc_pdf_Export", + "html": "HTML (StarCalc)", + "csv": "Text - txt - csv (StarCalc)", + 'xls': 'MS Excel 2003 XML', + 'xlsx': 'Calc MS Excel 2007 XML' + } + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def __str__(self): + return str(self.file_name) + + def __repr__(self): + return repr(self.file_url) + + def __getitem__(self, item): + return self.variables[item] if self.variables else None + + + def validDocType(self,doc): + + if not doc or not doc.supportsService('com.sun.star.sheet.SpreadsheetDocument'): + self.close() + raise errors.TemplateError( + 'invalid_format', + f"The given format ({self.file_name.split('.')[-1]!r}) is invalid, or the file is already open by " + f"an other process (accepted formats: ODS, OTS, XLS, XLSX, CSV)", + dict(format=self.file_name.split('.')[-1]) + ) + return doc + + + + + def scan(self, **kwargs) -> dict[str: dict[str, Union[str, list[str]]]]: + """ + scans the variables contained in the template. Supports text, tables and images + + :return: list containing all the variables founded in the template + """ + + #should_close = kwargs.get("should_close", False) + texts = {} + #(Pdb) self.doc.getSheets().getElementNames() + for sheet in self.doc.getSheets(): + texts = texts | CalcTextStatement.scan(sheet) + tables=CalcTableStatement.scan(self.doc) + #texts = CalcTextStatement.scan_Document_text(self.doc) + #pdb.set_trace() + return texts | tables + + + def search_error(self, json_vars: dict[str, dict[str, Union[str, list[str]]]]) -> None: + """ + find out which variable is a problem, and raise the required error + + :param json_vars: the given json variables + :return: None + """ + notdiff=diff(json_vars, self.variables) + if not notdiff: + return + else: + self.close() + raise errors.JsonComparaisonError( + 'missing_required_variable', + f"There is one or more missing variables in the json {repr(notdiff)}", + {"error" : repr(notdiff)} + ) + + # when parsing the template, we assume that all vars are of type text. But it can also be of type html. + # So we check if types are equals or if type in json is "html" while type in template is "text" + json_incorrect = [key for key in self.variables if (json_vars[key]['type'] != self.variables[key]['type']) and (json_vars[key]['type'] != "html" or self.variables[key]['type']!="text")] + if json_incorrect: + self.close() + raise errors.JsonComparaisonError( + 'incorrect_value_type', + f"The variable {json_incorrect[0]!r} should be of type " + f"{self.variables[json_incorrect[0]]['type']!r}, like in the template, but is of type " + f"{json_vars[json_incorrect[0]]['type']!r}", + dict(variable=json_incorrect[0], actual_variable_type=json_vars[json_incorrect[0]]['type'], + expected_variable_type=self.variables[json_incorrect[0]]['type']) + ) + + template_missing = [key for key in set(json_vars) - set(self.variables)] + json_vars_without_template_missing = {key: json_vars[key] for key in json_vars if key not in template_missing} + if json_vars_without_template_missing == self.variables: + return + + + def fill(self, variables: dict[str, dict[str, Union[str, list[str]]]]) -> None: + """ + Fills a template copy with the given values + + :param variables: the values to fill in the template + :return: None + """ + + ### + ### main calls + ### + objects={} + for var, details in sorted(variables.items(), key=lambda s: -len(s[0])): + if details['type'] == 'text': + for sheet in self.doc.getSheets(): + CalcTextStatement.fill(sheet, "$" + var, details['value']) + elif details['type'] == "object" and CalcTableStatement.isTableVar(var) : + objects[var]=details + for var, details in objects.items(): + CalcTableStatement.fill(self.doc, var, details['value']) + diff --git a/lotemplate/Statement/CalcSearchStatement.py b/lotemplate/Statement/CalcSearchStatement.py new file mode 100644 index 0000000..b726047 --- /dev/null +++ b/lotemplate/Statement/CalcSearchStatement.py @@ -0,0 +1,69 @@ +import re +from com.sun.star.lang import XComponent +import regex + +class CalcTextStatement: + text_regex_str = r'\$(\w+(\(((?:\\.|.)*?)\))?)' + text_regex = regex.compile(text_regex_str) + table_regex_str = r'\&(\w+(\(((?:\\.|.)*?)\))?)' + table_regex = regex.compile(table_regex_str) + + def __init__(self, text_string): + self.text_string = text_string + + + + def scan(component: XComponent, get_table=False) -> dict[str, dict[str, str]]: + """ + scan for text in the given doc + + :param doc: the document to scan + :return: the scanned variables + """ + if component.getImplementationName()=="ScNamedRangeObj": + #doc= component.getReferredCells().getSpreadsheet() + doc=component.getReferredCells() + regex_to_use=CalcTextStatement.table_regex_str + else: + doc=component + regex_to_use=CalcTextStatement.text_regex_str + + plain_vars = {} + search = doc.createReplaceDescriptor() + search.SearchString = regex_to_use + search.SearchRegularExpression = True + search.SearchCaseSensitive = False + founded = doc.findAll(search) + var_table = {} + if founded: + for x_found in founded: + Arraytext = x_found.getDataArray() + + #cursor = text.createTextCursorByRange(x_found) + for Array in Arraytext: + for text in Array: + for result in re.findall(search.SearchString,text): + plain_vars[result[0]] = {'type': 'text', 'value': ''} + var_table[result[0]] = {'type': 'table', 'value':[]} + + + return var_table if get_table else plain_vars + + def fill(doc: XComponent, variable: str, value: str) -> None: + """ + Fills all the text-related content + + :param doc: the document to fill + :param variable: the variable to search + :param value: the value to replace with + :return: None + """ + #pdb.set_trace() + #print("var="+variable+" value="+value+" "+str(doc) ) + search = doc.createReplaceDescriptor() + search.SearchString = variable + search.ReplaceString = value + doc.replaceAll(search) + + + diff --git a/lotemplate/Statement/CalcTableStatement.py b/lotemplate/Statement/CalcTableStatement.py new file mode 100644 index 0000000..cfe8492 --- /dev/null +++ b/lotemplate/Statement/CalcTableStatement.py @@ -0,0 +1,115 @@ +from com.sun.star.lang import XComponent +from typing import Union +from lotemplate.Statement.CalcSearchStatement import CalcTextStatement +from com.sun.star.sheet.CellInsertMode import DOWN, RIGHT +from com.sun.star.sheet.CellDeleteMode import UP, LEFT +import re + + +def incr_chr(c): + return chr(ord(c) + 1) if c != 'Z' else 'A' + +def incr_str(s,numb): + for i in range(numb): + lpart = s.rstrip('Z') + num_replacements = len(s) - len(lpart) + new_s = lpart[:-1] + incr_chr(lpart[-1]) if lpart else 'A' + new_s += 'A' * num_replacements + s=new_s + return new_s + + +class CalcTableStatement: + table_pattern = re.compile("^loop_(down|right)_(.+)",re.IGNORECASE) + + def isTableVar(var): + if re.match(CalcTableStatement.table_pattern,var): + return True + else: + return False + + def scan(doc: XComponent, get_list=False) -> Union[dict, list]: + """ + scan for tables in the given doc + + :param get_list: indicates if the function should return a list + of variables or the formatted dictionary of variables + :param doc: the document to scan + :return: the scanned variables + """ + + def scan_range(myrange) -> None: + """ + scan for variables in the given cell + + :param cell: the cell to scan + :return: None + """ + nonlocal doc + return CalcTextStatement.scan(myrange,True) + tab_vars = {} + for NamedRange in doc.NamedRanges: + if re.match(CalcTableStatement.table_pattern,NamedRange.getName()): + named = scan_range(NamedRange) + tab_vars[NamedRange.getName()] = {'type': 'object', 'value': named} + + return tab_vars + + def fill(doc: XComponent, variable: str, value) -> None: + """ + Fills all the table-related content + + :param doc: the document to fill + :param variable: the variable to search + :param value: the value to replace with + :return: None + """ + myrange=doc.NamedRanges.getByName(variable) + #mycellrange=myrange.getReferredCells() + mycellrangeaddr=myrange.getReferredCells().getRangeAddress() + + + mycontent,finalcol,finalrow =myrange.getContent().rsplit('$', 2) + + StartColumn=myrange.getReferredCells().getRangeAddress().StartColumn + StartRow=myrange.getReferredCells().getRangeAddress().StartRow + EndColumn=myrange.getReferredCells().getRangeAddress().EndColumn + EndRow=myrange.getReferredCells().getRangeAddress().EndRow + maxlen=max([len(value[x]['value']) for x in value]) + sheet=doc.getSheets()[mycellrangeaddr.Sheet] + + match = re.match(CalcTableStatement.table_pattern, variable) + direction = match.group(1) + if direction=="right": + size=1+EndColumn-StartColumn + left=True + decale=RIGHT + delete=LEFT + elif direction=="down": + size=1+EndRow-StartRow + left=False + decale=DOWN + delete=UP + + for i in reversed(range(maxlen)): + sheet.insertCells(mycellrangeaddr,decale) + copycell=sheet.getCellByPosition(StartColumn,StartRow) + rangetocopy=sheet.getCellRangeByPosition(StartColumn,StartRow,EndColumn,EndRow ) + sheet.copyRange(copycell.CellAddress,doc.NamedRanges.getByName(variable).getReferredCells().getRangeAddress()) + for key, mylist in value.items(): + try: + CalcTextStatement.fill(rangetocopy,'&'+key,mylist['value'][i]) + except IndexError: + CalcTextStatement.fill(rangetocopy,'&'+key,"") + + if left: + myrange.setContent(mycontent+'$'+incr_str(finalcol,maxlen*size)+'$'+finalrow) + mycellrangeaddr.EndColumn=mycellrangeaddr.EndColumn+maxlen*size + mycellrangeaddr.StartColumn=mycellrangeaddr.StartColumn+maxlen*size + + else: + myrange.setContent(mycontent+'$'+finalcol+'$'+str(int(finalrow)+(maxlen-1)*size)) + mycellrangeaddr.EndRow=mycellrangeaddr.EndRow+maxlen*size + mycellrangeaddr.StartRow=mycellrangeaddr.StartRow+maxlen*size + + sheet.removeRange(mycellrangeaddr,delete) diff --git a/lotemplate/Statement/CounterStatement.py b/lotemplate/Statement/CounterStatement.py index e920be2..d310995 100644 --- a/lotemplate/Statement/CounterStatement.py +++ b/lotemplate/Statement/CounterStatement.py @@ -55,13 +55,13 @@ def scan_counter(doc: XComponent) -> None: scan for counter statement. No return. We just verify that there is and endif for each if statement """ - def compute_counter(x_found): - """ - Compute the counter statement. - """ - counter_text = x_found.getText() - counter_cursor = counter_text.createTextCursorByRange(x_found) - cursor_statement = CounterStatement(counter_cursor.String) + #def compute_counter(x_found): + # """ + # Compute the counter statement. + # """ + # counter_text = x_found.getText() + #counter_cursor = counter_text.createTextCursorByRange(x_found) + #cursor_statement = CounterStatement(counter_cursor.String) def find_counter_to_compute(doc, search, x_found): """ @@ -70,7 +70,7 @@ def find_counter_to_compute(doc, search, x_found): if x_found is None: return None - compute_counter(x_found) + # compute_counter(x_found) # searching for the next counter statement. x_found_after = doc.findNext(x_found.End, search) @@ -139,9 +139,9 @@ def find_counter_to_compute(doc, search, x_found): if x_found is None: return None - text = x_found.getText() - cursor = text.createTextCursorByRange(x_found) - str = cursor.String + #text = x_found.getText() + #cursor = text.createTextCursorByRange(x_found) + #str = cursor.String compute_counter(x_found) diff --git a/lotemplate/Statement/ForStatement.py b/lotemplate/Statement/ForStatement.py index af22047..951aecc 100644 --- a/lotemplate/Statement/ForStatement.py +++ b/lotemplate/Statement/ForStatement.py @@ -77,7 +77,7 @@ def scan_single_for(doc: XComponent, local_x_found) -> str: scan for a single for statement """ for_statement = ForStatement(local_x_found.getString()) - position_in_text = len(for_statement.for_string) + #position_in_text = len(for_statement.for_string) endfor_search = doc.createSearchDescriptor() endfor_search.SearchString = ForStatement.end_regex diff --git a/lotemplate/Statement/HtmlStatement.py b/lotemplate/Statement/HtmlStatement.py index f278226..73312b2 100644 --- a/lotemplate/Statement/HtmlStatement.py +++ b/lotemplate/Statement/HtmlStatement.py @@ -1,4 +1,3 @@ -import re from sorcery import dict_of import lotemplate.errors as errors from com.sun.star.lang import XComponent @@ -24,7 +23,7 @@ def scan_single_html(local_x_found) -> None: """ scan for a single for statement """ - html_statement = HtmlStatement(local_x_found.getString()) + #html_statement = HtmlStatement(local_x_found.getString()) endhtml_search = doc.createSearchDescriptor() endhtml_search.SearchString = HtmlStatement.end_regex endhtml_search.SearchRegularExpression = True @@ -34,7 +33,7 @@ def scan_single_html(local_x_found) -> None: cursor = local_x_found.getText().createTextCursorByRange(local_x_found) raise errors.TemplateError( 'no_endhtml_found', - f"The statement [html] has no endhtml", + "The statement [html] has no endhtml", dict_of(cursor.String) ) @@ -55,7 +54,7 @@ def html_replace(template, doc: XComponent) -> None: """ def compute_html(doc, local_x_found): - html_statement = HtmlStatement(local_x_found.getString()) + #html_statement = HtmlStatement(local_x_found.getString()) endhtml_search = doc.createSearchDescriptor() endhtml_search.SearchString = HtmlStatement.end_regex endhtml_search.SearchRegularExpression = True @@ -65,7 +64,7 @@ def compute_html(doc, local_x_found): cursor = local_x_found.getText().createTextCursorByRange(local_x_found) raise errors.TemplateError( 'no_endhtml_found', - f"The statement [html] has no endhtml", + "The statement [html] has no endhtml", dict_of(cursor.String) ) diff --git a/lotemplate/Statement/IfStatement.py b/lotemplate/Statement/IfStatement.py index d02b846..32b10ff 100644 --- a/lotemplate/Statement/IfStatement.py +++ b/lotemplate/Statement/IfStatement.py @@ -132,10 +132,12 @@ def compute_if(x_found, x_found_endif): match = re.search(IfStatement.start_regex, if_cursor.String, re.IGNORECASE) if match is None: + c_string=dict_of(if_cursor.String) + doc.close(True) raise errors.TemplateError( 'syntax_error_in_if_statement', - f"The statement {if_cursor.String} has a Syntax Error", - dict_of(if_cursor.String) + f"The statement {c_string} has a Syntax Error", + dict_of(c_string) ) if_cursor.String = '' @@ -163,10 +165,12 @@ def find_if_to_compute(doc, search, x_found): x_found_endif = doc.findNext(x_found.End, endif_search) if x_found_endif is None: cursor = x_found.getText().createTextCursorByRange(x_found) + c_string=dict_of(cursor.String) + doc.close(True) raise errors.TemplateError( 'no_endif_found', - f"The statement {cursor.String} has no endif", - dict_of(cursor.String) + f"The statement {c_string} has no endif", + c_string ) compute_if(x_found, x_found_endif) @@ -186,9 +190,10 @@ def find_if_to_compute(doc, search, x_found): str = globalCursor.String match = re.search(IfStatement.end_regex, str, re.IGNORECASE) if match is not None: + doc.close(True) raise errors.TemplateError( 'too_many_endif_found', - f"The document has too many endif", + "The document has too many endif", {} ) @@ -251,10 +256,12 @@ def find_if_to_compute(doc, search, x_found): x_found_endif = doc.findNext(x_found.End, endif_search) if x_found_endif is None: cursor = x_found.getText().createTextCursorByRange(x_found) + c_string=dict_of(cursor.String) + doc.close(True) raise errors.TemplateError( 'no_endif_found', - f"The statement {cursor.String} has no endif", - dict_of(cursor.String) + f"The statement {c_string} has no endif", + dict_of(c_string) ) compute_if(x_found, x_found_endif) diff --git a/lotemplate/Statement/ImageStatement.py b/lotemplate/Statement/ImageStatement.py index 398de43..0502c6d 100644 --- a/lotemplate/Statement/ImageStatement.py +++ b/lotemplate/Statement/ImageStatement.py @@ -1,4 +1,4 @@ -from com.sun.star.beans import PropertyValue, UnknownPropertyException +from com.sun.star.beans import PropertyValue from com.sun.star.lang import XComponent import regex from urllib import request diff --git a/lotemplate/Statement/TableStatement.py b/lotemplate/Statement/TableStatement.py index c4ddc6d..1da8ca3 100644 --- a/lotemplate/Statement/TableStatement.py +++ b/lotemplate/Statement/TableStatement.py @@ -57,7 +57,7 @@ def scan_cell(cell) -> None: except errors.TemplateError as e: raise e continue - except RuntimeException as e: + except RuntimeException: continue return list_tab_vars if get_list else tab_vars diff --git a/lotemplate/Statement/TextStatement.py b/lotemplate/Statement/TextStatement.py index 1dfd9f0..b72ba99 100644 --- a/lotemplate/Statement/TextStatement.py +++ b/lotemplate/Statement/TextStatement.py @@ -1,6 +1,6 @@ import re from com.sun.star.lang import XComponent -from com.sun.star.beans import PropertyValue, UnknownPropertyException +from com.sun.star.beans import UnknownPropertyException import regex from lotemplate.Statement.ForStatement import ForStatement from lotemplate.Statement.TableStatement import TableStatement diff --git a/lotemplate/Template.py b/lotemplate/Template.py new file mode 100644 index 0000000..9f26c2b --- /dev/null +++ b/lotemplate/Template.py @@ -0,0 +1,242 @@ +""" +Copyright (C) 2023 Probesys + + +The classes used for document connexion and manipulation +""" + +__all__ = ( + 'Template', +) + +import os +from typing import Union +from sorcery import dict_of +import unohelper +from com.sun.star.beans import PropertyValue +from com.sun.star.io import IOException +from com.sun.star.lang import IllegalArgumentException, DisposedException +from com.sun.star.uno import RuntimeException + +from . import errors +#from . import Connexion +from .utils import get_file_url,get_cached_json + + +import uuid +import shutil +import json + +class Template: + + TMPDIR='/tmp' + + formats = {} + tmp_file= '' + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def __del__(self): + #print('destroy#########################################################') + self.close() + if os.path.exists(self.tmp_file): + os.remove(self.tmp_file) + + + def __str__(self): + return str(self.file_name) + + def __repr__(self): + return repr(self.file_url) + + def __getitem__(self, item): + return self.variables[item] if self.variables else None + + def open_doc_from_url(self): + try: + doc = self.cnx.desktop.loadComponentFromURL(self.file_url, "_blank", 0, ()) + except DisposedException as e: + self.close() + raise errors.UnoException( + 'bridge_exception', + f"The connection bridge on '{self.cnx.host}:{self.cnx.port}' crashed on file opening." + f"Please restart the soffice process. For more informations on what caused this bug and how to avoid " + f"it, please read the README file, section 'Unsolvable Problems'.", + dict_of(self.cnx.host, self.cnx.port) + ) from e + except IllegalArgumentException: + self.close() + raise errors.FileNotFoundError( + 'file_not_found', + f"the given file does not exist or has not been found (file {self.file_path!r})", + dict_of(self.file_path) + ) from None + except RuntimeException as e: + self.close() + raise errors.UnoException( + 'connection_closed', + f"The previously established connection with the soffice process on '{self.cnx.host}:{self.cnx.port}' " + f"has been closed, or ran into an unknown error. Please restart the soffice process, and retry.", + dict_of(self.cnx.host, self.cnx.port) + ) from e + self.validDocType(doc) + return doc + + def validDocType(self,doc): + pass + + def __init__(self, file_path: str, cnx, should_scan: bool, + json_cache_dir=None, author=''): + """ + An object representing a LibreOffice/OpenOffice template that you can fill, scan, export and more + + :param file_path: the path of the document + :param cnx: the connection object to the bridge + :param should_scan: indicates if the document should be scanned at initialisation + """ + if os.path.exists(file_path): + self.cnx = cnx + self.file_name = file_path.split("/")[-1] + self.file_dir = "/".join(file_path.split("/")[:-1]) + self.file_path = file_path + self.file_tmp_name=str(uuid.uuid4())+'_'+self.file_name + self.tmp_file=Template.TMPDIR+"/"+self.file_tmp_name + + shutil.copy(file_path, self.tmp_file) + + self.file_url = get_file_url(self.tmp_file) + self.variables = None + self.doc = None + self.doc = self.open_doc_from_url() + self.doc.getDocumentProperties().resetUserData(author) + #print("number of opendocument"+str(len(list(self.cnx.desktop.getComponents())))) + #print([print(a.getURL()) for a in list(self.cnx.desktop.getComponents())]) + if json_cache_dir: + cachedjson=get_cached_json(json_cache_dir,file_path) + if os.path.exists(cachedjson) and should_scan : + try: + with open(cachedjson) as f: + self.variables = json.load(f) + return + except Exception: + pass + self.variables = self.scan(should_close=True) + if json_cache_dir: + with open(cachedjson, 'w') as f: + json.dump(self.variables, f, ensure_ascii=False) + else: + self.close() + raise errors.FileNotFoundError( + 'file_not_found', + f"the given file does not exist or has not been found (file {file_path!r})", + dict_of(file_path)) + + def scan(self, **kwargs) -> dict[str: dict[str, Union[str, list[str]]]]: + """ + scans the variables contained in the template. Supports text, tables and images + + :return: list containing all the variables founded in the template + """ + + pass + + def search_error(self, json_vars: dict[str, dict[str, Union[str, list[str]]]]) -> None: + """ + find out which variable is a problem, and raise the required error + + :param json_vars: the given json variables + :return: None + """ + if json_vars == self.variables: + return + + json_missing = [key for key in set(self.variables) - set(json_vars)] + if json_missing: + self.close() + raise errors.JsonComparaisonError( + 'missing_required_variable', + f"The variable {json_missing[0]!r}, present in the template, " + f"isn't present in the json.", + dict(variable=json_missing[0]) + ) + + # when parsing the template, we assume that all vars are of type text. But it can also be of type html. + # So we check if types are equals or if type in json is "html" while type in template is "text" + json_incorrect = [key for key in self.variables if (json_vars[key]['type'] != self.variables[key]['type']) and (json_vars[key]['type'] != "html" or self.variables[key]['type']!="text")] + if json_incorrect: + self.close() + raise errors.JsonComparaisonError( + 'incorrect_value_type', + f"The variable {json_incorrect[0]!r} should be of type " + f"{self.variables[json_incorrect[0]]['type']!r}, like in the template, but is of type " + f"{json_vars[json_incorrect[0]]['type']!r}", + dict(variable=json_incorrect[0], actual_variable_type=json_vars[json_incorrect[0]]['type'], + expected_variable_type=self.variables[json_incorrect[0]]['type']) + ) + + template_missing = [key for key in set(json_vars) - set(self.variables)] + json_vars_without_template_missing = {key: json_vars[key] for key in json_vars if key not in template_missing} + if json_vars_without_template_missing == self.variables: + return + + def fill(self, variables: dict[str, dict[str, Union[str, list[str]]]]) -> None: + pass + + def close(self) -> None: + """ + close the template + + :return: None + """ + if not self: + return + try: + if self.doc: + self.doc.close(True) + except Exception: + pass + try: + os.remove(self.tmp_file) + os.remove(Template.TMPDIR+ "/.~lock." + self.file_tmp_name + "#") + except FileNotFoundError: + pass + + def export(self, filename: str, dirname=None, no_uid=None ) -> Union[str, None]: + """ + Exports the newly generated document, if any. + + :param name: the path/name with file extension of the file to export. + file type is automatically deducted from it. + :return: the full path of the exported document, or None if there is no document to export + """ + file_type = filename.split(".")[-1] + if no_uid: + path = os.getcwd() + "/" + dirname + '/' + filename + else: + path = os.getcwd() + "/" + dirname + '/' + str(uuid.uuid4()) +filename + url = unohelper.systemPathToFileUrl(path) + + # list of available convert filters + # cf https://help.libreoffice.org/latest/he/text/shared/guide/convertfilters.html + try: + self.doc.storeToURL(url, (PropertyValue("FilterName", 0, self.formats[file_type], 0),)) + except KeyError: + self.close() + raise errors.ExportError('invalid_format', + f"Invalid export format {file_type!r}.", dict_of(file_type)) from None + except IOException as error: + self.close() + + raise errors.ExportError( + 'unknown_error', + f"Unable to save document to {path!r} : error {error.value!r}", + dict_of(path, error) + ) from error + + return path + + diff --git a/lotemplate/WriterTemplate.py b/lotemplate/WriterTemplate.py new file mode 100644 index 0000000..290a2ff --- /dev/null +++ b/lotemplate/WriterTemplate.py @@ -0,0 +1,184 @@ +""" +Copyright (C) 2023 Probesys + + +The classes used for document connexion and manipulation +""" + +__all__ = ( + 'Template', +) + +from typing import Union +from sorcery import dict_of + +import uno + +from com.sun.star.beans import PropertyValue +from com.sun.star.text.ControlCharacter import PARAGRAPH_BREAK +from com.sun.star.style.BreakType import PAGE_AFTER + +from . import errors + + +from .Template import Template + +from lotemplate.Statement.ForStatement import ForStatement +from lotemplate.Statement.HtmlStatement import HtmlStatement +from lotemplate.Statement.IfStatement import IfStatement +from lotemplate.Statement.TextStatement import TextStatement +from lotemplate.Statement.TableStatement import TableStatement +from lotemplate.Statement.ImageStatement import ImageStatement +from lotemplate.Statement.CounterStatement import CounterManager + +__all__ = ( + 'WriterTemplate', +) + + +class WriterTemplate(Template): + + formats = { + "odt": "writer8", + "pdf": "writer_pdf_Export", + "html": "HTML (StarWriter)", + "docx": "Office Open XML Text", + "txt": "Text (encoded)", + 'rtf': 'Rich Text Format' + } + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + + def __str__(self): + return str(self.file_name) + + def __repr__(self): + return repr(self.file_url) + + def __getitem__(self, item): + return self.variables[item] if self.variables else None + + def validDocType(self,doc): + + + if not doc or not doc.supportsService('com.sun.star.text.GenericTextDocument'): + if doc: + doc.close(True) + raise errors.TemplateError( + 'invalid_format', + f"The given format ({self.file_name.split('.')[-1]!r}) is invalid, or the file is already open by " + f"an other process (accepted formats: ODT, OTT, DOC, DOCX, HTML, RTF or TXT)", + dict(format=self.file_name.split('.')[-1]) + ) + return doc + + + def pasteHtml(self, html_string, cursor): + """ + copy the html string as html at the location of the cursor + :param html_string: + :param cursor: + :return: + """ + # horrible hack : there is a bug with the "paste HTML" function of libreoffice, so we have to add + # a   at the beginning of the string to make it work. Without that, the first element of a list + #
  • ...
is displayed without the bullet point. This is the less visible workaround I found. + html_string = ' ' + html_string + input_stream = self.cnx.ctx.ServiceManager.createInstanceWithContext("com.sun.star.io.SequenceInputStream", + self.cnx.ctx) + input_stream.initialize((uno.ByteSequence(html_string.encode()),)) + prop1 = PropertyValue() + prop1.Name = "FilterName" + prop1.Value = "HTML (StarWriter)" + prop2 = PropertyValue() + prop2.Name = "InputStream" + prop2.Value = input_stream + cursor.insertDocumentFromURL("private:stream", (prop1, prop2)) + + def scan(self, **kwargs) -> dict[str: dict[str, Union[str, list[str]]]]: + """ + scans the variables contained in the template. Supports text, tables and images + + :return: list containing all the variables founded in the template + """ + + #should_close = kwargs.get("should_close", False) + + texts = TextStatement.scan_text(self.doc) + # we use another document for if statement scanning because it modifies the file + IfStatement.scan_if(template = self) + tables = TableStatement.scan_table(self.doc) + images = ImageStatement.scan_image(self.doc) + fors = ForStatement.scan_for(self.doc) + HtmlStatement.scan_html(self.doc) + CounterManager.scan_counter(self.doc) + + variables_list = list(texts.keys()) + list(tables.keys()) + list(images.keys()) + list(fors.keys()) + duplicates = [variable for variable in variables_list if variables_list.count(variable) > 1] + + if duplicates: + first_type = "text" if duplicates[0] in texts.keys() else "image" + second_type = "table" if duplicates[0] in tables.keys() else "image" + self.close() + raise errors.TemplateError( + 'duplicated_variable', + f"The variable {duplicates[0]!r} is mentioned two times, but " + f"for two different types: {first_type!r}, and {second_type!r}", + dict_of(first_type, second_type, variable=duplicates[0]) + ) + + return texts | tables | images | fors + + + def fill(self, variables: dict[str, dict[str, Union[str, list[str]]]]) -> None: + """ + Fills a template copy with the given values + + :param variables: the values to fill in the template + :return: None + """ + + + ### + ### main calls + ### + ForStatement.for_replace(self.doc, variables) + + IfStatement.if_replace(self.doc, variables) + + for var, details in sorted(variables.items(), key=lambda s: -len(s[0])): + if details['type'] == 'text': + TextStatement.text_fill(self.doc, "$" + var, details['value']) + elif details['type'] == 'image': + ImageStatement.image_fill(self.doc, self.cnx.graphic_provider, "$" + var, details['value']) + elif details['type'] == 'html': + HtmlStatement.html_fill(template=self, doc=self.doc, variable="$" + var, value=details['value']) + + HtmlStatement.html_replace(template=self, doc=self.doc) + + TableStatement.tables_fill(self.doc, variables, '$', '&') + + CounterManager.counter_replace(self.doc) + + + def page_break(self) -> None: + """ + Add a page break to the document + + :return: None + """ + + if not self.doc: + return + + cursor = self.doc.Text.createTextCursor() + cursor.gotoEnd(False) + cursor.collapseToEnd() + cursor.BreakType = PAGE_AFTER + self.doc.Text.insertControlCharacter(cursor, PARAGRAPH_BREAK, False) + diff --git a/lotemplate/__init__.py b/lotemplate/__init__.py index dfcfdc7..de4a292 100644 --- a/lotemplate/__init__.py +++ b/lotemplate/__init__.py @@ -15,13 +15,21 @@ __all__ = ( 'Connexion', 'Template', - 'errors', + 'CalcTemplate', + 'WriterTemplate', 'convert_to_datas_template', 'is_network_based', 'get_file_url', + 'TemplateFromExt', + 'start_multi_office', + 'randomConnexion', + 'clean_old_open_document', + 'statistic_open_document', ) -from .errors import * -from .utils import * -from .classes import * - +from .connexion import Connexion +from .utils import convert_to_datas_template,is_network_based,get_file_url +from .Template import Template +from .WriterTemplate import WriterTemplate +from .CalcTemplate import CalcTemplate +from .lofunction import TemplateFromExt,start_multi_office,randomConnexion,clean_old_open_document,statistic_open_document diff --git a/lotemplate/classes.py b/lotemplate/classes.py deleted file mode 100644 index 040ecf9..0000000 --- a/lotemplate/classes.py +++ /dev/null @@ -1,397 +0,0 @@ -""" -Copyright (C) 2023 Probesys - - -The classes used for document connexion and manipulation -""" - -__all__ = ( - 'Connexion', - 'Template', -) - -import os -from typing import Union -from sorcery import dict_of - -import uno -import unohelper -from com.sun.star.beans import PropertyValue -from com.sun.star.io import IOException -from com.sun.star.lang import IllegalArgumentException, DisposedException -from com.sun.star.connection import NoConnectException -from com.sun.star.uno import RuntimeException -from com.sun.star.text.ControlCharacter import PARAGRAPH_BREAK -from com.sun.star.style.BreakType import PAGE_AFTER - -from . import errors -from .utils import * - -from lotemplate.Statement.ForStatement import ForStatement -from lotemplate.Statement.HtmlStatement import HtmlStatement -from lotemplate.Statement.IfStatement import IfStatement -from lotemplate.Statement.TextStatement import TextStatement -from lotemplate.Statement.TableStatement import TableStatement -from lotemplate.Statement.ImageStatement import ImageStatement -from lotemplate.Statement.CounterStatement import CounterManager - -class Connexion: - - def __repr__(self): - return ( - f"" - ) - - def __str__(self): - return f"Connexion host {self.host}, port {self.port}" - - def __init__(self, host: str, port: str): - """ - An object representing the connexion between the script and the LibreOffice/OpenOffice processus - - :param host: the address of the host to connect to - :param port: the host port to connect to - """ - - self.host = host - self.port = port - self.local_ctx = uno.getComponentContext() - try: - self.ctx = self.local_ctx.ServiceManager.createInstanceWithContext( - "com.sun.star.bridge.UnoUrlResolver", self.local_ctx - ).resolve(f"uno:socket,host={host},port={port};urp;StarOffice.ComponentContext") - except (NoConnectException, RuntimeException) as e: - raise errors.UnoException( - 'connection_error', - f"Couldn't find/connect to the soffice process on \'{host}:{port}\'. " - f"Make sure the soffice process is correctly running with correct host and port informations. " - f"Read the README file, section 'Executing the script' for more informations about how to " - f"run the script.", dict_of(host, port) - ) from e - self.desktop = self.ctx.ServiceManager.createInstanceWithContext("com.sun.star.frame.Desktop", self.ctx) - self.graphic_provider = self.ctx.ServiceManager.createInstance('com.sun.star.graphic.GraphicProvider') - - def restart(self) -> None: - """ - Restart the connexion - - :return: None - """ - - self.__init__(self.host, self.port) - -class Template: - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - - def __str__(self): - return str(self.file_name) - - def __repr__(self): - return repr(self.file_url) - - def __getitem__(self, item): - return self.variables[item] if self.variables else None - - def open_doc_from_url(self): - try: - doc = self.cnx.desktop.loadComponentFromURL(self.file_url, "_blank", 0, ()) - except DisposedException as e: - self.close() - raise errors.UnoException( - 'bridge_exception', - f"The connection bridge on '{self.cnx.host}:{self.cnx.port}' crashed on file opening." - f"Please restart the soffice process. For more informations on what caused this bug and how to avoid " - f"it, please read the README file, section 'Unsolvable Problems'.", - dict_of(self.cnx.host, self.cnx.port) - ) from e - except IllegalArgumentException: - self.close() - raise errors.FileNotFoundError( - 'file_not_found', - f"the given file does not exist or has not been found (file {self.file_path!r})", - dict_of(self.file_path) - ) from None - except RuntimeException as e: - self.close() - raise errors.UnoException( - 'connection_closed', - f"The previously established connection with the soffice process on '{self.cnx.host}:{self.cnx.port}' " - f"has been closed, or ran into an unknown error. Please restart the soffice process, and retry.", - dict_of(self.cnx.host, self.cnx.port) - ) from e - - if not doc or not doc.supportsService('com.sun.star.text.GenericTextDocument'): - self.close() - raise errors.TemplateError( - 'invalid_format', - f"The given format ({self.file_name.split('.')[-1]!r}) is invalid, or the file is already open by " - f"an other process (accepted formats: ODT, OTT, DOC, DOCX, HTML, RTF or TXT)", - dict(format=self.file_name.split('.')[-1]) - ) - return doc - - - def __init__(self, file_path: str, cnx: Connexion, should_scan: bool): - """ - An object representing a LibreOffice/OpenOffice template that you can fill, scan, export and more - - :param file_path: the path of the document - :param cnx: the connection object to the bridge - :param should_scan: indicates if the document should be scanned at initialisation - """ - - self.cnx = cnx - self.file_name = file_path.split("/")[-1] - self.file_dir = "/".join(file_path.split("/")[:-1]) - self.file_path = file_path - self.file_url = get_file_url(file_path) - self.new = None - self.variables = None - self.doc = None - try: - os.remove(self.file_dir + "/.~lock." + self.file_name + "#") - except FileNotFoundError: - pass - self.doc = self.open_doc_from_url() - self.variables = self.scan(should_close=True) if should_scan else None - - def pasteHtml(self, html_string, cursor): - """ - copy the html string as html at the location of the cursor - :param html_string: - :param cursor: - :return: - """ - # horrible hack : there is a bug with the "paste HTML" function of libreoffice, so we have to add - # a   at the beginning of the string to make it work. Without that, the first element of a list - #
  • ...
is displayed without the bullet point. This is the less visible workaround I found. - html_string = ' ' + html_string - input_stream = self.cnx.ctx.ServiceManager.createInstanceWithContext("com.sun.star.io.SequenceInputStream", - self.cnx.ctx) - input_stream.initialize((uno.ByteSequence(html_string.encode()),)) - prop1 = PropertyValue() - prop1.Name = "FilterName" - prop1.Value = "HTML (StarWriter)" - prop2 = PropertyValue() - prop2.Name = "InputStream" - prop2.Value = input_stream - cursor.insertDocumentFromURL("private:stream", (prop1, prop2)) - - def scan(self, **kwargs) -> dict[str: dict[str, Union[str, list[str]]]]: - """ - scans the variables contained in the template. Supports text, tables and images - - :return: list containing all the variables founded in the template - """ - - should_close = kwargs.get("should_close", False) - - texts = TextStatement.scan_text(self.doc) - # we use another document for if statement scanning because it modifies the file - IfStatement.scan_if(template = self) - tables = TableStatement.scan_table(self.doc) - images = ImageStatement.scan_image(self.doc) - fors = ForStatement.scan_for(self.doc) - HtmlStatement.scan_html(self.doc) - CounterManager.scan_counter(self.doc) - - variables_list = list(texts.keys()) + list(tables.keys()) + list(images.keys()) + list(fors.keys()) - duplicates = [variable for variable in variables_list if variables_list.count(variable) > 1] - - if duplicates: - first_type = "text" if duplicates[0] in texts.keys() else "image" - second_type = "table" if duplicates[0] in tables.keys() else "image" - if should_close: - self.close() - raise errors.TemplateError( - 'duplicated_variable', - f"The variable {duplicates[0]!r} is mentioned two times, but " - f"for two different types: {first_type!r}, and {second_type!r}", - dict_of(first_type, second_type, variable=duplicates[0]) - ) - - return texts | tables | images | fors - - def search_error(self, json_vars: dict[str, dict[str, Union[str, list[str]]]]) -> None: - """ - find out which variable is a problem, and raise the required error - - :param json_vars: the given json variables - :return: None - """ - - if json_vars == self.variables: - return - - json_missing = [key for key in set(self.variables) - set(json_vars)] - if json_missing: - raise errors.JsonComparaisonError( - 'missing_required_variable', - f"The variable {json_missing[0]!r}, present in the template, " - f"isn't present in the json.", - dict(variable=json_missing[0]) - ) - - # when parsing the template, we assume that all vars are of type text. But it can also be of type html. - # So we check if types are equals or if type in json is "html" while type in template is "text" - json_incorrect = [key for key in self.variables if (json_vars[key]['type'] != self.variables[key]['type']) and (json_vars[key]['type'] != "html" or self.variables[key]['type']!="text")] - if json_incorrect: - raise errors.JsonComparaisonError( - 'incorrect_value_type', - f"The variable {json_incorrect[0]!r} should be of type " - f"{self.variables[json_incorrect[0]]['type']!r}, like in the template, but is of type " - f"{json_vars[json_incorrect[0]]['type']!r}", - dict(variable=json_incorrect[0], actual_variable_type=json_vars[json_incorrect[0]]['type'], - expected_variable_type=self.variables[json_incorrect[0]]['type']) - ) - - template_missing = [key for key in set(json_vars) - set(self.variables)] - json_vars_without_template_missing = {key: json_vars[key] for key in json_vars if key not in template_missing} - if json_vars_without_template_missing == self.variables: - return - - def fill(self, variables: dict[str, dict[str, Union[str, list[str]]]]) -> None: - """ - Fills a template copy with the given values - - :param variables: the values to fill in the template - :return: None - """ - - if self.new: - self.new.dispose() - self.new.close(True) - - try: - self.new = (self.cnx.desktop.loadComponentFromURL(self.file_url, "_blank", 0, ())) - except DisposedException as e: - raise errors.UnoException( - 'bridge_exception', - f"The connection bridge on '{self.cnx.host}:{self.cnx.port}' crashed on file opening." - f"Please restart the soffice process. For more informations on what caused this bug and how to " - f"avoid it, please read the README file, section 'Unsolvable Problems'.", - dict_of(self.cnx.host, self.cnx.port) - ) from e - except RuntimeException as e: - raise errors.UnoException( - 'connection_closed', - f"The previously established connection with the soffice process on " - f"'{self.cnx.host}:{self.cnx.port}' has been closed, or ran into an unknown error. " - f"Please restart the soffice process, and retry.", - dict_of(self.cnx.host, self.cnx.port) - ) from e - - ### - ### main calls - ### - ForStatement.for_replace(self.new, variables) - - IfStatement.if_replace(self.new, variables) - - for var, details in sorted(variables.items(), key=lambda s: -len(s[0])): - if details['type'] == 'text': - TextStatement.text_fill(self.new, "$" + var, details['value']) - elif details['type'] == 'image': - ImageStatement.image_fill(self.new, self.cnx.graphic_provider, "$" + var, details['value']) - elif details['type'] == 'html': - HtmlStatement.html_fill(template=self, doc=self.new, variable="$" + var, value=details['value']) - - HtmlStatement.html_replace(template=self, doc=self.new) - - TableStatement.tables_fill(self.new, variables, '$', '&') - - CounterManager.counter_replace(self.new) - - def export(self, name: str, should_replace=False) -> Union[str, None]: - """ - Exports the newly generated document, if any. - - :param should_replace: precise if the exported file should replace the fils with the same name - :param name: the path/name with file extension of the file to export. - file type is automatically deducted from it. - :return: the full path of the exported document, or None if there is no document to export - """ - - if not self.new: - return - - file_type = name.split(".")[-1] - path = os.getcwd() + "/" + name if name != '/' else name - path_without_num = path - if not should_replace: - i = 1 - while os.path.isfile(path): - path = path_without_num[:-(len(file_type) + 1)] + f"_{i}." + file_type - i += 1 - - url = unohelper.systemPathToFileUrl(path) - - # list of available convert filters - # cf https://help.libreoffice.org/latest/he/text/shared/guide/convertfilters.html - formats = { - "odt": "writer8", - "pdf": "writer_pdf_Export", - "html": "HTML (StarWriter)", - "docx": "Office Open XML Text", - "txt": "Text (encoded)", - 'rtf': 'Rich Text Format' - } - - try: - self.new.storeToURL(url, (PropertyValue("FilterName", 0, formats[file_type], 0),)) - - except KeyError: - raise errors.ExportError('invalid_format', - f"Invalid export format {file_type!r}.", dict_of(file_type)) from None - except IOException as error: - raise errors.ExportError( - 'unknown_error', - f"Unable to save document to {path!r} : error {error.value!r}", - dict_of(path, error) - ) from error - - return path - - def close(self) -> None: - """ - close the template - - :return: None - """ - - if not self: - return - if self.new: - self.new.dispose() - self.new.close(True) - self.new = None - if self.doc: - self.doc.dispose() - self.doc.close(True) - try: - os.remove(self.file_dir + "/.~lock." + self.file_name + "#") - except FileNotFoundError: - pass - - def page_break(self) -> None: - """ - Add a page break to the document - - :return: None - """ - - if not self.new: - return - - cursor = self.new.Text.createTextCursor() - cursor.gotoEnd(False) - cursor.collapseToEnd() - cursor.BreakType = PAGE_AFTER - self.new.Text.insertControlCharacter(cursor, PARAGRAPH_BREAK, False) diff --git a/lotemplate/connexion.py b/lotemplate/connexion.py new file mode 100644 index 0000000..c62f0fe --- /dev/null +++ b/lotemplate/connexion.py @@ -0,0 +1,110 @@ +""" +Copyright (C) 2023 Probesys + + +The classes used for document connexion and manipulation +""" + +__all__ = ( + 'Connexion', + 'start_office', +) + +import os +from sorcery import dict_of +import shlex +import subprocess +import uno +from com.sun.star.connection import NoConnectException +from com.sun.star.uno import RuntimeException +from time import sleep +from . import errors +#from .utils import * +from .WriterTemplate import WriterTemplate +from .CalcTemplate import CalcTemplate + +def start_office(host:str="localhost",port:str="2000"): + """ + start one process LibreOffice + + :param host: define host in the UNO connect-string --accept + :param port: define port in the UNO connect-string --accept + environnement had to be different for each environnement + """ + subprocess.Popen( + shlex.split('soffice \ + -env:UserInstallation="file:///tmp/LibO_Process'+port+'" \ + -env:UserInstallation="file:///tmp/LibO_Process'+port+'" \ + "--accept=socket,host="'+host+',port='+port+';urp;" \ + --headless --nologo --terminate_after_init \ + --norestore " '), shell=False, stdin = subprocess.PIPE, + stdout = subprocess.PIPE,) + return host, port,'file:///tmp/LibO_Process'+str(port) + + +class Connexion: + + def __repr__(self): + return ( + f"" + ) + + def __str__(self): + return f"Connexion host {self.host}, port {self.port}" + + def __init__(self, host: str, port: str): + """ + An object representing the connexion between the script and the LibreOffice/OpenOffice processus + + :param host: the address of the host to connect to + :param port: the host port to connect to + """ + + self.host = host + self.port = port + self.local_ctx = uno.getComponentContext() + for attempt in range(3): + try: + self.ctx = self.local_ctx.ServiceManager.createInstanceWithContext( + "com.sun.star.bridge.UnoUrlResolver", self.local_ctx + ).resolve(f"uno:socket,host={host},port={port};urp;StarOffice.ComponentContext") + except (NoConnectException, RuntimeException) as e: + if attempt==0: + sleep(4) + elif attempt<2: + start_office(host,port) + sleep(5) + else: + raise errors.UnoException( + 'connection_error', + f"Couldn't find/connect to the soffice process on \'{host}:{port}\'. " + f"Make sure the soffice process is correctly running with correct host and port informations. " + f"Read the README file, section 'Executing the script' for more informations about how to " + f"run the script.", dict_of(host, port) + ) from e + else: + break + self.desktop = self.ctx.ServiceManager.createInstanceWithContext("com.sun.star.frame.Desktop", self.ctx) + self.graphic_provider = self.ctx.ServiceManager.createInstance('com.sun.star.graphic.GraphicProvider') + + def restart(self) -> None: + """ + Restart the connexion + + :return: None + """ + print( "##### RESTART One Office#####") + self.__init__(self.host, self.port) + +def TemplateFromExt(file_path: str, cnx: Connexion, should_scan: bool): + + filename, file_extension = os.path.splitext(file_path) + ods_ext=('.xls','.xlsx','.ods') + if file_extension in ods_ext: + document = CalcTemplate(file_path, cnx , should_scan) + else: + document = WriterTemplate(file_path, cnx , should_scan) + return document + diff --git a/lotemplate/lofunction.py b/lotemplate/lofunction.py new file mode 100644 index 0000000..d358d8b --- /dev/null +++ b/lotemplate/lofunction.py @@ -0,0 +1,103 @@ +""" +Copyright (C) 2023 Probesys + + +The classes used for document connexion and manipulation +""" + +__all__ = ( + 'TemplateFromExt', + 'start_multi_office', + 'randomConnexion', +) + +import os +from .WriterTemplate import WriterTemplate +from .CalcTemplate import CalcTemplate +from .connexion import Connexion,start_office +import random +from datetime import datetime + +def TemplateFromExt(file_path: str, cnx, should_scan: bool,json_cache_dir=None): + + filename, file_extension = os.path.splitext(file_path) + ods_ext=('.xls','.xlsx','.ods') + if file_extension in ods_ext: + document = CalcTemplate(file_path, cnx , should_scan,json_cache_dir) + else: + document = WriterTemplate(file_path, cnx , should_scan,json_cache_dir) + return document + + +def randomConnexion(lstOffice): + host,port,lodir = random.choice(lstOffice) + return Connexion(host,port) + +def start_multi_office(host:str="localhost",start_port:int=2000,nb_env:int=1): + """ + start a nb_env of process LibreOffice + + :param host: define host in the UNO connect-string --accept + :param port: define port in the UNO connect-string --accept + :param nb_env: number of process to launch + :return: list of (host,port,lo dir) + """ + if nb_env <= 0: + raise TypeError("%s is an invalid positive int value" % nb_env) + soffices=[] + port=start_port + for i in range(nb_env): + soffices.append(start_office(host,str(port))) + port=port+1 + return soffices + +def clean_old_open_document(lstOffice, max_time): + """ + clean open document open for too long and where a sure are not use anymore + :lstOffice list of host,port + :max_time: Maximum time in second afeter wich were consider the document + not use anymore + :checkfile: add a check to see if the file still exist if not just close + the doc. + """ + counter=0 + for host,port,lodir in lstOffice: + cnx=Connexion(host,port) + for doc in list(cnx.desktop.getComponents()): + #print("number of opendocument"+str(len(list(cnx.desktop.getComponents())))) + #print(doc) + url=doc.getURL() + file=url[7:] + try: + delta=(datetime.now() - + datetime.fromtimestamp(os.path.getmtime(file))).seconds + if delta > int(max_time): + counter += 1 + doc.close(True) + os.remove(file) + except FileNotFoundError: + counter += 1 + doc.close(True) + return({"nb_clean":counter}) + +def statistic_open_document(lstOffice, max_time,): + mylist=[] + for host,port,lodir in lstOffice: + cnx=Connexion(host,port) + baddoc={"tooold":[],"missing":[]} + cnxdict={"maxtime":max_time,"hosts":host,"port":port,} + for doc in list(cnx.desktop.getComponents()): + #print(doc) + url=doc.getURL() + file=url[7:] + try: + delta=(datetime.now() - + datetime.fromtimestamp(os.path.getmtime(file))).seconds + if delta > int(max_time): + baddoc["tooold"].append(file) + except FileNotFoundError: + baddoc["missing"].append(file) + cnxdict["baddoc"]=baddoc + mylist.append(cnxdict) + return mylist + diff --git a/lotemplate/unittest/files/content/calc_table.expected.html b/lotemplate/unittest/files/content/calc_table.expected.html new file mode 100644 index 0000000..c539a65 --- /dev/null +++ b/lotemplate/unittest/files/content/calc_table.expected.html @@ -0,0 +1,491 @@ + + + + + + + + + + + + + +
+

+

Overview

+ maF1
+ Feuille2
+ Feuille3
+ matab
+ large_row
+ vtab
+ vtab_large
+ +

+
+

Sheet 1: maF1

 + + + + + + + + + + + + + + + + + +
test
RORO aa recond
RORO aa recond
teezt
riri ccc
+ +
+

Sheet 2: Feuille2

 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
RORO AA


riri bbroobar

ryval
















































recond
























+ +
+

Sheet 3: Feuille3

 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

punbhtttc
perle1101012
chat2.541012
chien1022024




48

























mon appart



+ +
+

Sheet 4: matab

 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

punbhtttc
velo291821.6
pied2.541012
skate4289.6
woiture

00




43.2
+ +
+

Sheet 5: large_row

 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

punbhtttc
velo2
1821.6


9

pied2.5
1012


4

skate4
89.6


2





43.2
+ +
+

Sheet 6: vtab

 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
nomleonardoRaffaelloDonatelloMichelangelo
prénomde Vinci Sanziodi Niccolò di Betto Bardidi Lodovico Buonarroti Simoni
age 12131414
poid100110111112433
posteninja1ninja2ninja3ninja4
+ +
+

Sheet 7: vtab_large

 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
nomleonardo
Raffaello
Donatello
Michelangelo

prénom
de Vinci
Sanzio
di Niccolò di Betto Bardi
di Lodovico Buonarroti Simoni
age 12
13
14
14

poid100100110110111111112112866
posteninja1
ninja2
ninja3
ninja4

+ + + + diff --git a/lotemplate/unittest/files/content/calc_table.json b/lotemplate/unittest/files/content/calc_table.json new file mode 100644 index 0000000..7a5dfec --- /dev/null +++ b/lotemplate/unittest/files/content/calc_table.json @@ -0,0 +1,131 @@ +{ + "TOTO": { + "type": "text", + "value": "RORO" + }, + "second": { + "type": "text", + "value": "recond" + }, + "titi": { + "type": "text", + "value": "riri" + }, + "toto": { + "type": "text", + "value": "roro" + }, + "myvar": { + "type": "text", + "value": "ryval" + }, + "foobar": { + "type": "text", + "value": "roobar" + }, + "f3": { + "type": "text", + "value": "mon appart" + }, + "loop_down_tab1": { + "type": "object", + "value": { + "nom": { + "type": "table", + "value": ["perle","chat","chien"] + }, + "prix": { + "type": "table", + "value": ["1","2.5","10"] + }, + "nb": { + "type": "table", + "value": ["10","4","2"] + } + } + }, + "loop_down_tab2": { + "type": "object", + "value": { + "produit": { + "type": "table", + "value": ["velo","pied","skate","woiture"] + }, + "prix": { + "type": "table", + "value": ["2","2.5","4"] + }, + "nb": { + "type": "table", + "value": ["9","4","2"] + } + } + }, + "loop_right_tab3": { + "type": "object", + "value": { + "nom": { + "type": "table", + "value": ["leonardo","Raffaello","Donatello","Michelangelo"] + }, + "prenom": { + "type": "table", + "value": ["de Vinci ","Sanzio","di Niccolò di Betto Bardi","di Lodovico Buonarroti Simoni"] + }, + "Age": { + "type": "table", + "value": ["12","13","14","14"] + }, + "poid": { + "type": "table", + "value": ["100","110","111","112"] + }, + "poste": { + "type": "table", + "value": ["ninja1","ninja2","ninja3","ninja4"] + } + } + }, + "loop_down_large1": { + "type": "object", + "value": { + "produit": { + "type": "table", + "value": ["velo","pied","skate"] + }, + "prix": { + "type": "table", + "value": ["2","2.5","4"] + }, + "nb": { + "type": "table", + "value": ["9","4","2"] + } + } + }, + "loop_right_large2": { + "type": "object", + "value": { + "nom": { + "type": "table", + "value": ["leonardo","Raffaello","Donatello","Michelangelo"] + }, + "prenom": { + "type": "table", + "value": ["de Vinci ","Sanzio","di Niccolò di Betto Bardi","di Lodovico Buonarroti Simoni"] + }, + "Age": { + "type": "table", + "value": ["12","13","14","14"] + }, + "poid": { + "type": "table", + "value": ["100","110","111","112"] + }, + "poste": { + "type": "table", + "value": ["ninja1","ninja2","ninja3","ninja4"] + } + } + } +} diff --git a/lotemplate/unittest/files/content/calc_table.ods b/lotemplate/unittest/files/content/calc_table.ods new file mode 100644 index 0000000..6bd0d45 Binary files /dev/null and b/lotemplate/unittest/files/content/calc_table.ods differ diff --git a/lotemplate/unittest/files/content/calc_variables.expected.html b/lotemplate/unittest/files/content/calc_variables.expected.html new file mode 100644 index 0000000..caf7730 --- /dev/null +++ b/lotemplate/unittest/files/content/calc_variables.expected.html @@ -0,0 +1,181 @@ + + + + + + + + + + + + + +
+

+

Overview

+ maF1
+ Feuille2
+ +

+
+

Sheet 1: maF1

 + + + + + + + + + + + + + + + + + +
test
RORO aa recond
RORO aa recond
teezt
riri ccc
+ +
+

Sheet 2: Feuille2

 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
RORO AA


riri bbroobar

ryval
















































recond
























+ + + + diff --git a/lotemplate/unittest/files/content/calc_variables.json b/lotemplate/unittest/files/content/calc_variables.json new file mode 100644 index 0000000..800f071 --- /dev/null +++ b/lotemplate/unittest/files/content/calc_variables.json @@ -0,0 +1 @@ +{"TOTO": {"type": "text", "value": "RORO"}, "second": {"type": "text", "value": "recond"}, "titi": {"type": "text", "value": "riri"}, "toto": {"type": "text", "value": "roro"}, "myvar": {"type": "text", "value": "ryval"}, "foobar": {"type": "text", "value": "roobar"}} diff --git a/lotemplate/unittest/files/content/calc_variables.ods b/lotemplate/unittest/files/content/calc_variables.ods new file mode 100644 index 0000000..e1aef40 Binary files /dev/null and b/lotemplate/unittest/files/content/calc_variables.ods differ diff --git a/lotemplate/unittest/files/content/e89fbedb61af3994184da3e5340bd9e9-calc_variables.ods.expected.json b/lotemplate/unittest/files/content/e89fbedb61af3994184da3e5340bd9e9-calc_variables.ods.expected.json new file mode 100644 index 0000000..3488f0d --- /dev/null +++ b/lotemplate/unittest/files/content/e89fbedb61af3994184da3e5340bd9e9-calc_variables.ods.expected.json @@ -0,0 +1 @@ +{"TOTO": {"type": "text", "value": ""}, "second": {"type": "text", "value": ""}, "titi": {"type": "text", "value": ""}, "toto": {"type": "text", "value": ""}, "myvar": {"type": "text", "value": ""}, "foobar": {"type": "text", "value": ""}} \ No newline at end of file diff --git a/lotemplate/unittest/files/content/html.expected.txt b/lotemplate/unittest/files/content/html.expected.txt index 4435938..b2aace3 100644 --- a/lotemplate/unittest/files/content/html.expected.txt +++ b/lotemplate/unittest/files/content/html.expected.txt @@ -29,8 +29,6 @@ Address :  123   -perso 1 -string 1 -perso 2 -lastname with < and > +perso 1 string 1 +perso 2 lastname with < and > diff --git a/lotemplate/unittest/files/content/table.expected.txt b/lotemplate/unittest/files/content/table.expected.txt index 0e67f24..55250ab 100644 --- a/lotemplate/unittest/files/content/table.expected.txt +++ b/lotemplate/unittest/files/content/table.expected.txt @@ -1,26 +1,10 @@  -Column1 -Column2 -Column3 -a -static -A -b -static -B -c -static -C -d -static -D -e -static -E -f -static -F -g -static -G +Column1 Column2 Column3 +a static A +b static B +c static C +d static D +e static E +f static F +g static G diff --git a/lotemplate/unittest/files/templates/calc_variables.ods b/lotemplate/unittest/files/templates/calc_variables.ods new file mode 100644 index 0000000..e1aef40 Binary files /dev/null and b/lotemplate/unittest/files/templates/calc_variables.ods differ diff --git a/lotemplate/unittest/test_cachedjson.py b/lotemplate/unittest/test_cachedjson.py new file mode 100644 index 0000000..385435e --- /dev/null +++ b/lotemplate/unittest/test_cachedjson.py @@ -0,0 +1,22 @@ +""" +Copyright (C) 2023 Probesys +""" +import unittest + +import lotemplate as ot +import filecmp +import os + +cnx = ot.start_multi_office() + + +class Test_calc(unittest.TestCase): + + def test_cachedjson(self): + cachejson="lotemplate/unittest/files/content/e89fbedb61af3994184da3e5340bd9e9-calc_variables.ods.json" + if os.path.isfile(cachejson): + os.remove(cachejson) + ot.TemplateFromExt("lotemplate/unittest/files/templates/calc_variables.ods",ot.randomConnexion(cnx),True,json_cache_dir='lotemplate/unittest/files/content/') + self.assertTrue(filecmp.cmp(cachejson,"lotemplate/unittest/files/content/e89fbedb61af3994184da3e5340bd9e9-calc_variables.ods.expected.json", shallow=False)) + + diff --git a/lotemplate/unittest/test_calc.py b/lotemplate/unittest/test_calc.py new file mode 100644 index 0000000..6240f5a --- /dev/null +++ b/lotemplate/unittest/test_calc.py @@ -0,0 +1,31 @@ +""" +Copyright (C) 2023 Probesys +""" +import unittest + +import lotemplate as ot +from lotemplate.unittest.test_function import compare_files_html + +cnx = ot.start_multi_office() + + +class Test_calc(unittest.TestCase): + + def test_scan(self): + self.assertEqual( + {"TOTO": {"type": "text", "value": ""}, "second": {"type": + "text", "value": ""}, "titi": {"type": "text", "value": + ""}, "toto": {"type": "text", "value": ""}, "myvar": + {"type": "text", "value": ""}, "foobar": {"type": "text", + "value": ""}}, + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/calc_variables.ods", ot.randomConnexion(cnx), False)).scan()) + doc.close() + + def test_var(self): + self.assertTrue(compare_files_html('calc_variables',cnx)) + + def test_table(self): + self.assertTrue(compare_files_html('calc_table',cnx)) + + + diff --git a/lotemplate/unittest/test_comparaison.py b/lotemplate/unittest/test_comparaison.py index 2be9972..d256cb1 100644 --- a/lotemplate/unittest/test_comparaison.py +++ b/lotemplate/unittest/test_comparaison.py @@ -4,22 +4,14 @@ import unittest import lotemplate as ot -import test_json_convertion -from time import sleep -import subprocess +from lotemplate.unittest.test_function import to_data -subprocess.call(f'soffice "--accept=socket,host=localhost,port=2002;urp;StarOffice.ServiceManager" &', shell=True) -sleep(2) -cnx = ot.Connexion("localhost", "2002") - - -def to_data(file: str): - return ot.convert_to_datas_template(test_json_convertion.file_to_dict(file)) +cnx=ot.start_multi_office() class Text(unittest.TestCase): - temp = ot.Template("lotemplate/unittest/files/comparaison/text_vars.odt", cnx, True) + temp = ot.TemplateFromExt("lotemplate/unittest/files/comparaison/text_vars.odt", ot.randomConnexion(cnx), True) def test_valid(self): self.temp.search_error(to_data("lotemplate/unittest/files/comparaison/text_vars_valid.json")) @@ -41,7 +33,7 @@ def test_invalid_incorrect_value(self): temp.close() - temp_tab = ot.Template("lotemplate/unittest/files/comparaison/static_tab.odt", cnx, True) + temp_tab = ot.TemplateFromExt("lotemplate/unittest/files/comparaison/static_tab.odt", ot.randomConnexion(cnx), True) def test_tab_valid(self): self.temp_tab.search_error(to_data("lotemplate/unittest/files/comparaison/static_tab_valid.json")) @@ -51,7 +43,7 @@ def test_tab_valid(self): class Tables(unittest.TestCase): - temp = ot.Template("lotemplate/unittest/files/comparaison/two_row_tab_varied.odt", cnx, True) + temp = ot.TemplateFromExt("lotemplate/unittest/files/comparaison/two_row_tab_varied.odt", ot.randomConnexion(cnx), True) def test_valid(self): self.temp.search_error(to_data("lotemplate/unittest/files/comparaison/two_row_tab_varied_valid.json")) @@ -73,7 +65,7 @@ def test_invalid_unknown_variable(self): class Images(unittest.TestCase): - temp = ot.Template("lotemplate/unittest/files/comparaison/img_vars.odt", cnx, True) + temp = ot.TemplateFromExt("lotemplate/unittest/files/comparaison/img_vars.odt", ot.randomConnexion(cnx), True) def test_valid(self): self.temp.search_error(to_data("lotemplate/unittest/files/comparaison/img_vars_valid.json")) diff --git a/lotemplate/unittest/test_content.py b/lotemplate/unittest/test_content.py index ecfd560..6104eaa 100644 --- a/lotemplate/unittest/test_content.py +++ b/lotemplate/unittest/test_content.py @@ -3,136 +3,66 @@ """ import unittest -import json -import filecmp -import os -import urllib.request import lotemplate as ot -from time import sleep -import subprocess -from pypdf import PdfReader - -subprocess.call(f'soffice "--accept=socket,host=localhost,port=2002;urp;StarOffice.ServiceManager" &', shell=True) -sleep(2) -cnx = ot.Connexion("localhost", "2002") - - -def file_to_dict(file_path: str) -> dict: - if ot.is_network_based(file_path): - return json.loads(urllib.request.urlopen(file_path).read()) - else: - with open(file_path) as f: - return json.loads(f.read()) - - -def to_data(file: str): - return ot.convert_to_datas_template(file_to_dict(file)) - - -def compare_files(name: str, format: str = 'txt'): - if format not in ['txt', 'pdf']: - return False - - base_path = 'lotemplate/unittest/files/content' - - def get_filename(ext: str): - return base_path + '/' + name + '.' + ext - - temp = None - if os.path.isfile(get_filename('odt')): - temp = ot.Template(get_filename('odt'), cnx, True) - if os.path.isfile(get_filename('docx')): - temp = ot.Template(get_filename('docx'), cnx, True) - - if temp is None: - if name == 'debug': - return True - else: - raise FileNotFoundError('No file found for ' + name) - - temp.scan() - temp.search_error(to_data(get_filename('json'))) - temp.fill(file_to_dict(get_filename('json'))) - - if os.path.isfile(get_filename('unittest.'+format)): - os.remove(get_filename('unittest.'+format)) - temp.export(get_filename('unittest.'+format), True) - # temp.close() - if os.path.isfile(get_filename('unittest.odt')): - os.remove(get_filename('unittest.odt')) - temp.export(get_filename('unittest.odt'), True) - temp.close() - - # The PDF format is used to test some documents with headers or footers that are not supported by the text saveAs from - # LibreOffice. The PDF is then converted to text to compare with the expected text. - if format == 'pdf': - # convert to text - reader = PdfReader(get_filename('unittest.pdf')) - text = "" - for page in reader.pages: - text += page.extract_text() + "\n" - if os.path.isfile(get_filename('unittest.txt')): - os.remove(get_filename('unittest.txt')) - with open(get_filename('unittest.txt'), 'w') as f: - f.write(text) - - response = filecmp.cmp(get_filename('unittest.txt'), get_filename('expected.txt')) - return response +from test_function import compare_files + + +cnx=ot.start_multi_office() class Text(unittest.TestCase): def test_html(self): - self.assertTrue(compare_files('html')) + self.assertTrue(compare_files('html',cnx=cnx)) def test_html_missing_endhtml(self): with self.assertRaises(ot.errors.TemplateError): - self.assertTrue(compare_files('html_missing_endhtml')) + self.assertTrue(compare_files('html_missing_endhtml',cnx=cnx)) def test_for(self): - self.assertTrue(compare_files('for')) + self.assertTrue(compare_files('for',cnx=cnx)) def test_for_inside_if(self): - self.assertTrue(compare_files('for_inside_if')) + self.assertTrue(compare_files('for_inside_if',cnx=cnx)) def test_vars(self): - self.assertTrue(compare_files('text_vars')) + self.assertTrue(compare_files('text_vars',cnx=cnx)) def test_if(self): - self.assertTrue(compare_files('if')) + self.assertTrue(compare_files('if',cnx=cnx)) def test_if_empty(self): - self.assertTrue(compare_files('if_empty')) + self.assertTrue(compare_files('if_empty',cnx=cnx)) def test_if_contains(self): - self.assertTrue(compare_files('if_contains')) + self.assertTrue(compare_files('if_contains',cnx=cnx)) def test_function_variable(self): - self.assertTrue(compare_files('function_variable')) + self.assertTrue(compare_files('function_variable',cnx=cnx)) def test_if_recursive(self): - self.assertTrue(compare_files('if_recursive')) + self.assertTrue(compare_files('if_recursive',cnx=cnx)) def test_if_inside_for(self): - self.assertTrue(compare_files('if_inside_for')) + self.assertTrue(compare_files('if_inside_for',cnx=cnx)) def test_html_vars(self): - self.assertTrue(compare_files('html_vars')) + self.assertTrue(compare_files('html_vars',cnx=cnx)) def test_table(self): - self.assertTrue(compare_files('table')) + self.assertTrue(compare_files('table',cnx=cnx)) def test_image(self): - self.assertTrue(compare_files('image')) + self.assertTrue(compare_files('image',cnx=cnx)) def test_counter(self): - self.assertTrue(compare_files('counter')) + self.assertTrue(compare_files('counter',cnx=cnx)) def test_text_var_in_header(self): - self.assertTrue(compare_files('text_var_in_header', 'pdf')) + self.assertTrue(compare_files('text_var_in_header', 'pdf',cnx=cnx)) def test_too_many_endif_strange(self): - self.assertTrue(compare_files('too_many_endif_strange')) + self.assertTrue(compare_files('too_many_endif_strange',cnx=cnx)) def test_debug(self): - self.assertTrue(compare_files('debug')) + self.assertTrue(compare_files('debug',cnx=cnx)) diff --git a/lotemplate/unittest/test_function.py b/lotemplate/unittest/test_function.py new file mode 100644 index 0000000..4d4516c --- /dev/null +++ b/lotemplate/unittest/test_function.py @@ -0,0 +1,122 @@ +""" +Copyright (C) 2023 Probesys +""" + + +import lotemplate as ot +import filecmp +import os +import json +from pypdf import PdfReader + + + +def file_to_dict(file_path: str) -> dict: + with open(file_path) as f: + return json.loads(f.read()) + + + +def to_data(file: str): + return ot.convert_to_datas_template(file_to_dict(file)) + + +def compare_files_html(name: str, cnx ): + + base_path = 'lotemplate/unittest/files/content' + + def get_filename(ext: str): + return base_path + '/' + name + '.' + ext + + temp = None + if os.path.isfile(get_filename('ods')): + temp = ot.TemplateFromExt(get_filename('ods'), ot.randomConnexion(cnx), True) + + if temp is None: + if name == 'debug': + return True + else: + raise FileNotFoundError('No file found for ' + name) + + temp.scan() + temp.search_error(to_data(get_filename('json'))) + temp.fill(file_to_dict(get_filename('json'))) + + if os.path.isfile(get_filename('unittest.html')): + os.remove(get_filename('unittest.html')) + temp.export(name+'.unittest.html',base_path, True) + temp.close() + with open(get_filename('unittest.html'), 'r+') as fp: + # read an store all lines into list + lines = fp.readlines() + # move file pointer to the beginning of a file + fp.seek(0) + # truncate the file + fp.truncate() + + # start writing lines + # iterate line and line number + for number, line in enumerate(lines): + # delete line number 8,9,10 + # note: list index start from 0 + if number not in [ 7,8,9]: + fp.write(line) + + + response = filecmp.cmp(get_filename('unittest.html'), + get_filename('expected.html')) + return response + + + +def compare_files(name: str, format: str = 'txt',cnx = None): + if format not in ['txt', 'pdf']: + return False + + base_path = 'lotemplate/unittest/files/content' + + def get_filename(ext: str): + return base_path + '/' + name + '.' + ext + + temp = None + if os.path.isfile(get_filename('odt')): + temp = ot.TemplateFromExt(get_filename('odt'), ot.randomConnexion(cnx), True) + if os.path.isfile(get_filename('docx')): + temp = ot.TemplateFromExt(get_filename('docx'), ot.randomConnexion(cnx), True) + + if temp is None: + if name == 'debug': + return True + else: + raise FileNotFoundError('No file found for ' + name) + + temp.scan() + temp.search_error(to_data(get_filename('json'))) + temp.fill(file_to_dict(get_filename('json'))) + + if os.path.isfile(get_filename('unittest.'+format)): + os.remove(get_filename('unittest.'+format)) + temp.export(name+'.unittest.'+format,base_path, True) + # temp.close() + if os.path.isfile(get_filename('unittest.odt')): + os.remove(get_filename('unittest.odt')) + temp.export(name+'.unittest.odt',base_path, True) + temp.close() + + # The PDF format is used to test some documents with headers or footers that are not supported by the text saveAs from + # LibreOffice. The PDF is then converted to text to compare with the expected text. + if format == 'pdf': + # convert to text + reader = PdfReader(get_filename('unittest.pdf')) + text = "" + for page in reader.pages: + text += page.extract_text() + "\n" + if os.path.isfile(get_filename('unittest.txt')): + os.remove(get_filename('unittest.txt')) + with open(get_filename('unittest.txt'), 'w') as f: + f.write(text) + + response = filecmp.cmp(get_filename('unittest.txt'), get_filename('expected.txt')) + return response + + diff --git a/lotemplate/unittest/test_json_convertion.py b/lotemplate/unittest/test_json_convertion.py index 33088aa..31378b2 100644 --- a/lotemplate/unittest/test_json_convertion.py +++ b/lotemplate/unittest/test_json_convertion.py @@ -2,19 +2,13 @@ Copyright (C) 2023 Probesys """ -import json import unittest -import urllib.request import lotemplate as ot +import json +from .test_function import file_to_dict -def file_to_dict(file_path: str) -> dict: - if ot.is_network_based(file_path): - return json.loads(urllib.request.urlopen(file_path).read()) - else: - with open(file_path) as f: - return json.loads(f.read()) - +cnx = ot.start_multi_office() class Generic(unittest.TestCase): @@ -128,9 +122,9 @@ class Tables(unittest.TestCase): def test_valid(self): self.assertEqual( { - "var": {"type": "table", "value": [""]}, - "var1": {"type": "table", "value": [""]}, - "var2": {"type": "table", "value": [""]} + "var": {"type": "table", "value": []}, + "var1": {"type": "table", "value": []}, + "var2": {"type": "table", "value": []} }, ot.convert_to_datas_template(file_to_dict("lotemplate/unittest/files/jsons/tab_valid.json")) ) diff --git a/lotemplate/unittest/test_template_scan.py b/lotemplate/unittest/test_template_scan.py index f8e2172..924a57a 100644 --- a/lotemplate/unittest/test_template_scan.py +++ b/lotemplate/unittest/test_template_scan.py @@ -4,14 +4,11 @@ import unittest import lotemplate as ot -from time import sleep -import subprocess -subprocess.call(f'soffice "--accept=socket,host=localhost,port=2002;urp;StarOffice.ServiceManager" &', shell=True) -sleep(2) -cnx = ot.Connexion("localhost", "2002") +cnx=ot.start_multi_office() + class Text(unittest.TestCase): def test_noformat(self): @@ -24,7 +21,7 @@ def test_noformat(self): "aet": {"type": "text", "value": ""}, "h": {"type": "text", "value": ""} }, - (doc := ot.Template("lotemplate/unittest/files/templates/text_vars_noformat.odt", cnx, False)).scan()) + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/text_vars_noformat.odt", ot.randomConnexion(cnx), False)).scan()) doc.close() def test_format(self): @@ -37,34 +34,34 @@ def test_format(self): "aet": {"type": "text", "value": ""}, "h": {"type": "text", "value": ""} }, - (doc := ot.Template("lotemplate/unittest/files/templates/text_vars.odt", cnx, False)).scan()) + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/text_vars.odt", ot.randomConnexion(cnx), False)).scan()) doc.close() def test_text_var_in_header(self): self.assertEqual( {"my_var": {"type": "text", "value": ""}}, - (doc := ot.Template("lotemplate/unittest/files/templates/text_var_in_header.odt", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/text_var_in_header.odt", ot.randomConnexion(cnx), False)).scan() ) doc.close() def test_static_table(self): self.assertEqual( {"var1": {"type": "text", "value": ""}, "var2": {"type": "text", "value": ""}}, - (doc := ot.Template("lotemplate/unittest/files/templates/static_tab.odt", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/static_tab.odt", ot.randomConnexion(cnx), False)).scan() ) doc.close() def test_function_variable(self): self.assertEqual( {"test(\"jean\")": {"type": "text", "value": ""}}, - (doc := ot.Template("lotemplate/unittest/files/templates/function_variable.odt", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/function_variable.odt", ot.randomConnexion(cnx), False)).scan() ) doc.close() def test_for_variable(self): self.assertEqual( {"tutu": {"type": "array", "value": []}}, - (doc := ot.Template("lotemplate/unittest/files/templates/for.odt", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/for.odt", ot.randomConnexion(cnx), False)).scan() ) doc.close() @@ -73,7 +70,7 @@ class Images(unittest.TestCase): def test_one_image(self): self.assertEqual( {"image": {"type": "image", "value": ""}}, - (doc := ot.Template("lotemplate/unittest/files/templates/img_vars.odt", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/img_vars.odt", ot.randomConnexion(cnx), False)).scan() ) doc.close() @@ -84,15 +81,14 @@ def test_multiple_images(self): "image2": {"type": "image", "value": ""}, "image3": {"type": "image", "value": ""} }, - (doc := ot.Template("lotemplate/unittest/files/templates/multiple_img_vars.odt", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/multiple_img_vars.odt", ot.randomConnexion(cnx), False)).scan() ) doc.close() class Ifs(unittest.TestCase): def test_no_endif(self): with self.assertRaises(ot.errors.TemplateError): - (doc := ot.Template("lotemplate/unittest/files/templates/invalid_if_statement.odt", cnx, False)).scan() - doc.close() + (ot.TemplateFromExt("lotemplate/unittest/files/templates/invalid_if_statement.odt", ot.randomConnexion(cnx), False)).scan() class Tables(unittest.TestCase): @@ -100,45 +96,42 @@ class Tables(unittest.TestCase): def test_multiple_row(self): self.assertEqual( {"var1": {"type": "table", "value": [""]}, "var2": {"type": "table", "value": [""]}}, - (doc := ot.Template("lotemplate/unittest/files/templates/multiple_row_tab.odt", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/multiple_row_tab.odt", ot.randomConnexion(cnx), False)).scan() ) doc.close() def test_one_row_varied(self): self.assertEqual( {"var": {"type": "table", "value": [""]}}, - (doc := ot.Template("lotemplate/unittest/files/templates/one_row_tab_varied.odt", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/one_row_tab_varied.odt", ot.randomConnexion(cnx), False)).scan() ) doc.close() def test_two_row_varied(self): self.assertEqual( {"var1": {"type": "table", "value": [""]}}, - (doc := ot.Template("lotemplate/unittest/files/templates/two_row_tab_varied.odt", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/two_row_tab_varied.odt", ot.randomConnexion(cnx), False)).scan() ) doc.close() def test_invalid_var(self): with self.assertRaises(ot.errors.TemplateError): - (doc := ot.Template("lotemplate/unittest/files/templates/invalid_var_tab.odt", cnx, False)).scan() - doc.close() + (ot.TemplateFromExt("lotemplate/unittest/files/templates/invalid_var_tab.odt", ot.randomConnexion(cnx), False)).scan() def test_invalid_vars(self): with self.assertRaises(ot.errors.TemplateError): - (doc := ot.Template("lotemplate/unittest/files/templates/invalid_vars_tab.odt", cnx, False)).scan() - doc.close() + (ot.TemplateFromExt("lotemplate/unittest/files/templates/invalid_vars_tab.odt", ot.randomConnexion(cnx), False)).scan() def test_for(self): self.assertEqual( {'tutu': {'type': 'array', 'value': []}}, - (doc := ot.Template("lotemplate/unittest/files/templates/for.odt", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/for.odt", ot.randomConnexion(cnx), False)).scan() ) doc.close() def test_for_missing_endfor(self): with self.assertRaises(ot.errors.TemplateError): - (doc := ot.Template("lotemplate/unittest/files/templates/for_missing_endfor.odt", cnx, False)).scan() - doc.close() + (ot.TemplateFromExt("lotemplate/unittest/files/templates/for_missing_endfor.odt", ot.randomConnexion(cnx), False)).scan() def test_two_tabs_varied(self): self.assertEqual( @@ -150,7 +143,7 @@ def test_two_tabs_varied(self): "4": {"type": "table", "value": [""]}, "5": {"type": "table", "value": [""]} }, - (doc := ot.Template("lotemplate/unittest/files/templates/two_tabs_varied.odt", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/two_tabs_varied.odt", ot.randomConnexion(cnx), False)).scan() ) doc.close() @@ -161,7 +154,7 @@ def test_function_variable(self): "test": {"type": "table", "value": [""]}, "test2": {"type": "text", "value": ""}, }, - (doc := ot.Template("lotemplate/unittest/files/templates/function_variable_tab.odt", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/function_variable_tab.odt", ot.randomConnexion(cnx), False)).scan() ) doc.close() @@ -183,7 +176,7 @@ def test_multiple_variables(self): "static2": {"type": "text", "value": ""}, "static3": {"type": "text", "value": ""} }, - (doc := ot.Template("lotemplate/unittest/files/templates/multiple_variables.odt", cnx, False)).scan()) + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/multiple_variables.odt", ot.randomConnexion(cnx), False)).scan()) doc.close() def test_multiple_pages(self): @@ -203,26 +196,22 @@ def test_multiple_pages(self): "date": {"type": "text", "value": ""}, "lieu": {"type": "text", "value": ""} }, - (doc := ot.Template("lotemplate/unittest/files/templates/multiple_pages.odt", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/multiple_pages.odt", ot.randomConnexion(cnx), False)).scan() ) doc.close() def test_online_empty_doc(self): - self.assertEqual( - {}, - (doc := ot.Template( - "https://www.mtsac.edu/webdesign/accessible-docs/word/example03.docx", cnx, False)).scan() - ) - doc.close() + with self.assertRaises(ot.errors.FileNotFoundError): + ( ot.TemplateFromExt( + "https://www.mtsac.edu/webdesign/accessible-docs/word/example03.docx", ot.randomConnexion(cnx), False)) def test_invalid_path(self): with self.assertRaises(ot.errors.FileNotFoundError): - ot.Template("bfevg", cnx, True) + ot.TemplateFromExt("bfevg", ot.randomConnexion(cnx), True) def test_duplicated_variable(self): with self.assertRaises(ot.errors.TemplateError): - (doc := ot.Template("lotemplate/unittest/files/templates/duplicated_variables.odt", cnx, False)).scan() - doc.close() + (ot.TemplateFromExt("lotemplate/unittest/files/templates/duplicated_variables.odt", ot.randomConnexion(cnx), False)).scan() class OtherFormats(unittest.TestCase): @@ -237,7 +226,7 @@ def test_ott(self): "signature": {"type": "text", "value": ""}, "photo": {"type": "image", "value": ""} }, - (doc := ot.Template("lotemplate/unittest/files/templates/format.ott", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/format.ott", ot.randomConnexion(cnx), False)).scan() ) doc.close() @@ -251,14 +240,14 @@ def test_docx(self): "signature": {"type": "text", "value": ""}, "photo": {"type": "image", "value": ""} }, - (doc := ot.Template("lotemplate/unittest/files/templates/format.docx", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/format.docx", ot.randomConnexion(cnx), False)).scan() ) doc.close() def test_text(self): self.assertEqual( {"signature": {"type": "text", "value": ""}}, - (doc := ot.Template("lotemplate/unittest/files/templates/format.txt", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/format.txt", ot.randomConnexion(cnx), False)).scan() ) doc.close() @@ -272,13 +261,13 @@ def test_html(self): "signature": {"type": "text", "value": ""}, "photo": {"type": "image", "value": ""} }, - (doc := ot.Template("lotemplate/unittest/files/templates/format.html", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/format.html", ot.randomConnexion(cnx), False)).scan() ) doc.close() def test_html_without_endhtml(self): with self.assertRaises(ot.errors.TemplateError): - (doc := ot.Template("lotemplate/unittest/files/templates/html_without_endhtml.odt", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/html_without_endhtml.odt", ot.randomConnexion(cnx), False)).scan() doc.close() @@ -291,12 +280,12 @@ def test_rtf(self): "prenon": {"type": "text", "value": ""}, "signature": {"type": "text", "value": ""} }, - (doc := ot.Template("lotemplate/unittest/files/templates/format.rtf", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/format.rtf", ot.randomConnexion(cnx), False)).scan() ) doc.close() def test_for_inside_if(self): - doc = ot.Template("lotemplate/unittest/files/content/for_inside_if.odt", cnx, False) + doc = ot.TemplateFromExt("lotemplate/unittest/files/content/for_inside_if.odt", ot.randomConnexion(cnx), False) self.assertEqual( { 'tata': {'type': 'text', 'value': ''}, @@ -316,31 +305,30 @@ def test_for_inside_if(self): def test_if_too_many_endif(self): with self.assertRaises(ot.errors.TemplateError) as cm: - (doc := ot.Template("lotemplate/unittest/files/templates/if_too_many_endif.odt", cnx, False)).scan() - doc.close() + (ot.TemplateFromExt("lotemplate/unittest/files/templates/if_too_many_endif.odt", ot.randomConnexion(cnx), False)).scan() self.assertEqual(cm.exception.code, "too_many_endif_found") def test_if_syntax_error(self): with self.assertRaises(ot.errors.TemplateError) as cm: - (doc := ot.Template("lotemplate/unittest/files/templates/if_syntax_error.odt", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/if_syntax_error.odt", ot.randomConnexion(cnx), False)).scan() doc.close() self.assertEqual(cm.exception.code, "syntax_error_in_if_statement") def test_for_syntax_error(self): with self.assertRaises(ot.errors.TemplateError) as cm: - (doc := ot.Template("lotemplate/unittest/files/templates/for_syntax_error.odt", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/for_syntax_error.odt", ot.randomConnexion(cnx), False)).scan() doc.close() self.assertEqual(cm.exception.code, "syntax_error_in_for_statement") def test_if_syntax_error_no_endif(self): with self.assertRaises(ot.errors.TemplateError) as cm: - (doc := ot.Template("lotemplate/unittest/files/templates/if_syntax_error_no_endif.odt", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/if_syntax_error_no_endif.odt", ot.randomConnexion(cnx), False)).scan() doc.close() self.assertEqual(cm.exception.code, "no_endif_found") def test_invalid(self): with self.assertRaises(ot.errors.TemplateError): - (doc := ot.Template("lotemplate/unittest/files/templates/invalid_format.jpg", cnx, False)).scan() + (doc := ot.TemplateFromExt("lotemplate/unittest/files/templates/invalid_format.jpg", ot.randomConnexion(cnx), False)).scan() doc.close() diff --git a/lotemplate/utils.py b/lotemplate/utils.py index 7af1eea..757e3bd 100644 --- a/lotemplate/utils.py +++ b/lotemplate/utils.py @@ -8,7 +8,8 @@ __all__ = ( 'convert_to_datas_template', 'is_network_based', - 'get_file_url' + 'get_file_url', + 'get_cached_json' ) import functools @@ -20,9 +21,16 @@ from sorcery import dict_of from copy import deepcopy +import hashlib from . import errors + +def get_cached_json(json_cache_dir:str, filepath:str): + filename = filepath.split("/")[-1] + with open(filepath,'rb') as office: + return json_cache_dir+"/"+(hashlib.md5(office.read()).hexdigest())+'-'+filename+".json" + def convert_to_datas_template(json) -> dict[dict[str: Union[str, list]]]: """ converts a dictionary of variables for filling a template to a dictionary of variables types, @@ -121,10 +129,10 @@ def recursive_check_type(rec_type, rec_value): f"{repr(get_type(rec_type, is_type=True))}", dict(varible=var_name, expected_variable_value_type=get_type(rec_type, is_type=True)) ) - if rec_type.__origin__ == list: + if rec_type.__origin__ is list: for element in rec_value: recursive_check_type(rec_type.__args__[0], element) - if rec_type.__origin__ == dict: + if rec_type.__origin__ is dict: for key, value in rec_value: recursive_check_type(rec_type.__args__[0], key) recursive_check_type(rec_type.__args__[1], value) @@ -142,6 +150,17 @@ def recursive_check_type(rec_type, rec_value): return f(var_name, var_value) return wrapper_check_type + @check_type + def get_cleaned_object(var_name: str, var_value: dict ) -> dict: + """ + clean a table variable + :param var_name: the variable name + :param var_value: the table + :return: the cleaned table + """ + return convert_to_datas_template(var_value) + + @check_type def get_cleaned_table(var_name: str, var_value: list[str]) -> list[str]: """ @@ -150,7 +169,7 @@ def get_cleaned_table(var_name: str, var_value: list[str]) -> list[str]: :param var_value: the table :return: the cleaned table """ - return [""] + return [] @check_type def get_cleaned_image(var_name: str, var_value: str) -> str: @@ -217,7 +236,7 @@ def get_cleaned_array(var_name: str, var_value: list) -> list: template = {} for variable_name, variable_infos in json.items(): - if type(variable_infos) != dict: + if type(variable_infos) is not dict: raise errors.JsonSyntaxError( 'invalid_variable_base_value_type', f"The value type {repr(get_type(variable_infos))} isn't accepted in variable, only objects " @@ -241,7 +260,7 @@ def get_cleaned_array(var_name: str, var_value: list) -> list: dict_of(variable_name, information=invalid_infos[0]) ) - if type(variable_infos['type']) != str: + if type(variable_infos['type']) is not str: raise errors.JsonSyntaxError( 'invalid_variable_type_value_type', f"The 'type' information is supposed to be string, not a {repr(get_type(variable_infos['type']))} " diff --git a/lotemplate_cli.py b/lotemplate_cli.py index b45b64c..f5beab7 100755 --- a/lotemplate_cli.py +++ b/lotemplate_cli.py @@ -12,9 +12,7 @@ import urllib.error import sys import traceback -import subprocess -from time import sleep - +import os def set_arguments() -> cparse.Namespace: """ @@ -24,11 +22,11 @@ def set_arguments() -> cparse.Namespace: """ p = cparse.ArgumentParser(default_config_files=['config.yml', 'config.ini', 'config']) - p.add_argument('template_file', + p.add_argument('--template_file', '-t', help="Template file to scan or fill") - p.add_argument('--json_file', '-jf', nargs='+', default=[], + p.add_argument('--json_file', '-jf', help="Json files that must fill the template, if any") - p.add_argument('--json', '-j', nargs='+', default=[], + p.add_argument('--json', '-j', help="Json strings that must fill the template, if any") p.add_argument('--output', '-o', default="output.pdf", help="Names of the filled files, if the template should be filled. supported formats: " @@ -36,28 +34,48 @@ def set_arguments() -> cparse.Namespace: p.add_argument('--config', '-c', is_config_file=True, help='Configuration file path') p.add_argument('--host', default="localhost", help='Host address to use for the libreoffice connection') p.add_argument('--port', default="2002", help='Port to use for the libreoffice connexion') + p.add_argument('--cpu', default="0", help='number of libreoffice start, default 0 is the number of CPU') + p.add_argument('--clean', action='store_true', + help="Specify if the program should all to old open connection") + p.add_argument('--maxtime', default="60" , help='number of second before considering a document open for too long') + p.add_argument('--stats', action='store_true', + help="return statistic about open files that should not be") p.add_argument('--scan', '-s', action='store_true', help="Specify if the program should just scan the template and return the information, or fill it.") p.add_argument('--force_replacement', '-f', action='store_true', help="Specify if the program should ignore the scan's result") - return p.parse_args() - + p.add_argument('--json_cache_dir',nargs='?', help="Specify a cache for the scanned json") + args=p.parse_args() + if not args.scan and not args.clean and not args.template_file and not args.stats: + p.error(" #######You need at minimun --scan or --clean or --template_file") + elif (args.scan and not args.template_file): + p.error(" ####### with --scan you need --template_file") + return args if __name__ == '__main__': - # get the necessaries arguments args = set_arguments() - # run soffice - subprocess.call( - f'soffice "--accept=socket,host={args.host},port={args.port};urp;StarOffice.ServiceManager" &', shell=True) - sleep(2) + if args.cpu == "0": + nb_process=len(os.sched_getaffinity(0)) + else: + nb_process=int(args.cpu) - # establish the connection to the server - connexion = ot.Connexion(args.host, args.port) + my_lo=ot.start_multi_office(nb_env=nb_process) + if args.clean: + print(json.dumps(ot.clean_old_open_document(my_lo, args.maxtime))) + exit() + + if args.stats: + print(json.dumps(ot.statistic_open_document(my_lo, args.maxtime))) + + exit() + + # establish the connection to the server + connexion = ot.randomConnexion(my_lo) # generate the document to operate and its parameters - document = ot.Template(args.template_file, connexion, not args.force_replacement) + document = ot.TemplateFromExt(args.template_file, connexion, not args.force_replacement,args.json_cache_dir) # prints scan result in json format if it should if args.scan: @@ -68,35 +86,31 @@ def set_arguments() -> cparse.Namespace: # get the specified jsons json_dict = {} - for elem in args.json_file: - if ot.is_network_based(elem): - json_dict[elem] = json.loads(urllib.request.urlopen(elem).read()) + if args.json_file: + if ot.is_network_based(args.json_file): + json_variables = json.loads(urllib.request.urlopen(args.json_file).read()) else: - with open(elem) as f: - json_dict[elem] = json.loads(f.read()) - for index, elem in enumerate(args.json): - json_dict[f"json_{index}"] = json.loads(elem) - - for json_name, json_variables in json_dict.items(): - - try: - # scan for errors - document.search_error(ot.convert_to_datas_template(json_variables)) - - # fill and export the document - document.fill(json_variables) - print( - f"File {repr(json_name)}: Document saved as " + - repr(document.export( - args.output if len(json_dict) == 1 else - ".".join(args.output.split(".")[:-1]) + '_' + ( - json_name.split("/")[-1][:-5] if json_name.split("/")[-1][-5:] == ".json" - else json_name.split("/")[-1] - ) + "." + args.output.split(".")[-1] - )) - ) - except Exception as exception: - print(f'Ignoring exception on json {repr(json_name)}:', file=sys.stderr) - traceback.print_exception(type(exception), exception, exception.__traceback__, file=sys.stderr) - continue + with open(args.json_file) as f: + json_variables = json.loads(f.read()) + if args.json: + json_variables = json.loads(args.json) + + try: + # scan for errors + document.search_error(ot.convert_to_datas_template(json_variables)) + #pdb.set_trace() + # fill and export the document + document.fill(json_variables) + filename=os.path.basename(args.output) + path=os.path.dirname(args.output) + if not path: + path='exports' + + print( + "Document saved as " + + repr(document.export( filename, path, True)) + ) + except Exception as exception: + print('Ignoring exception on json :', file=sys.stderr) + traceback.print_exception(type(exception), exception, exception.__traceback__, file=sys.stderr) document.close() diff --git a/requirements.txt b/requirements.txt index d38e4bc..adff2c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,10 @@ -configargparse~=1.5.1 -Flask~=2.2.2 -python-dotenv~=0.21.0 -Pillow~=9.4.0 -gunicorn~=20.1.0 -Werkzeug~=2.2.2 -sorcery~=0.2.1 -regex~=2022.10.31 -pypdf~=4.2.0 \ No newline at end of file +configargparse~=1.7 +Flask~=3.1 +python-dotenv~=1.0.1 +Pillow~=11.0.0 +gunicorn~=23.0.0 +Werkzeug~=3.1.2 +sorcery~=0.2.2 +regex~=2024.11.6 +pypdf~=5.1.0 +jsondiff~=2.2 diff --git a/usebruno/README.md b/usebruno/README.md new file mode 100644 index 0000000..1539731 --- /dev/null +++ b/usebruno/README.md @@ -0,0 +1,6 @@ +Use Bruno for testing your API +============================== + +We are using the tool "Use Bruno" [https://www.usebruno.com/](https://www.usebruno.com/) to test our API. + +This directory contains a list of example requests we are using to test the API in developpement. \ No newline at end of file diff --git a/usebruno/bruno.json b/usebruno/bruno.json new file mode 100644 index 0000000..d587668 --- /dev/null +++ b/usebruno/bruno.json @@ -0,0 +1,9 @@ +{ + "version": "1", + "name": "lotemplate", + "type": "collection", + "ignore": [ + "node_modules", + ".git" + ] +} \ No newline at end of file diff --git a/usebruno/create_dir.bru b/usebruno/create_dir.bru new file mode 100644 index 0000000..7171dd7 --- /dev/null +++ b/usebruno/create_dir.bru @@ -0,0 +1,16 @@ +meta { + name: create_dir + type: http + seq: 2 +} + +put { + url: {{baseUrl}}/ + body: none + auth: none +} + +headers { + secretkey: DEFAULT_KEY + directory: test_dir1 +} diff --git a/usebruno/del_dir.bru b/usebruno/del_dir.bru new file mode 100644 index 0000000..204a4b4 --- /dev/null +++ b/usebruno/del_dir.bru @@ -0,0 +1,15 @@ +meta { + name: del_dir + type: http + seq: 1 +} + +delete { + url: {{baseUrl}}/test_dir1 + body: none + auth: none +} + +headers { + secretkey: DEFAULT_KEY +} diff --git a/usebruno/environments/dev.bru b/usebruno/environments/dev.bru new file mode 100644 index 0000000..2de9da6 --- /dev/null +++ b/usebruno/environments/dev.bru @@ -0,0 +1,3 @@ +vars { + baseUrl: http://localhost:8000 +} diff --git a/usebruno/generate_loop_formula_calc_result.bru b/usebruno/generate_loop_formula_calc_result.bru new file mode 100644 index 0000000..7ab3b45 --- /dev/null +++ b/usebruno/generate_loop_formula_calc_result.bru @@ -0,0 +1,55 @@ +meta { + name: generate_loop_formula_calc_result + type: http + seq: 12 +} + +post { + url: {{baseUrl}}/test_dir1/calc_table_formula.ods + body: json + auth: none +} + +headers { + secretkey: DEFAULT_KEY +} + +body:json { + { + "name": "calc_table_formula.html", + "variables": { + "loop_down_article": { + "type": "object", + "value": { + "name": { + "type": "table", + "value": [ + "appel", + "banana", + "melon", + "lemon" + ] + }, + "unitPrice": { + "type": "table", + "value": [ + "1", + "1.5", + "3.2", + "0.8" + ] + }, + "quantity": { + "type": "table", + "value": [ + "4", + "6", + "2", + "1" + ] + } + } + } + } + } +} diff --git a/usebruno/generate_loop_formula_excel_result.bru b/usebruno/generate_loop_formula_excel_result.bru new file mode 100644 index 0000000..f5af562 --- /dev/null +++ b/usebruno/generate_loop_formula_excel_result.bru @@ -0,0 +1,55 @@ +meta { + name: generate_loop_formula_excel_result + type: http + seq: 14 +} + +post { + url: {{baseUrl}}/test_dir1/calc_table_formula.xlsx + body: json + auth: none +} + +headers { + secretkey: DEFAULT_KEY +} + +body:json { + { + "name": "calc_table_formula.html", + "variables": { + "loop_down_article": { + "type": "object", + "value": { + "name": { + "type": "table", + "value": [ + "appel", + "banana", + "melon", + "lemon" + ] + }, + "unitPrice": { + "type": "table", + "value": [ + "1", + "1.5", + "3.2", + "0.8" + ] + }, + "quantity": { + "type": "table", + "value": [ + "4", + "6", + "2", + "1" + ] + } + } + } + } + } +} diff --git a/usebruno/generate_simple_vars_xlsx_result.bru b/usebruno/generate_simple_vars_xlsx_result.bru new file mode 100644 index 0000000..9c29f16 --- /dev/null +++ b/usebruno/generate_simple_vars_xlsx_result.bru @@ -0,0 +1,35 @@ +meta { + name: generate_simple_vars_xlsx_result + type: http + seq: 10 +} + +post { + url: {{baseUrl}}/test_dir1/simple_vars.xlsx + body: json + auth: none +} + +headers { + secretkey: DEFAULT_KEY +} + +body:json { + { + "name": "simple_vars_result.html", + "variables": { + "myhours": { + "type": "text", + "value": "12" + }, + "myname": { + "type": "text", + "value": "Gérard" + }, + "otherName": { + "type": "text", + "value": "Martin" + } + } + } +} diff --git a/usebruno/generate_writer_result.bru b/usebruno/generate_writer_result.bru new file mode 100644 index 0000000..398a484 --- /dev/null +++ b/usebruno/generate_writer_result.bru @@ -0,0 +1,79 @@ +meta { + name: generate_writer_result + type: http + seq: 7 +} + +post { + url: {{baseUrl}}/test_dir1/if_inside_for.odt + body: json + auth: none +} + +headers { + secretkey: DEFAULT_KEY +} + +body:json { + { + "name": "my_file.txt", + "variables": { + "tutu": { + "type": "array", + "value": [ + { + "type": "person", + "firstName": "perso 1", + "lastName": "string 1", + "address": { + "street1": "8 rue de la paix", + "street2": "", + "zip": "75008", + "city": "Paris", + "state": "Ile de France" + } + }, + { + "type": "person", + "firstName": "perso 2", + "lastName": "lastname with < and >", + "address": { + "street1": "12 avenue Jean Jaurès", + "street2": "", + "zip": "38000", + "city": "Grenoble", + "state": "Isère" + } + }, + { + "type": "company", + "name": "my_company", + "address": { + "street1": "12 avenue Gambetta", + "street2": "", + "zip": "38000", + "city": "Grenoble", + "state": "Isère" + } + } + ] + }, + "outsidefor": { + "type": "text", + "value": "foo" + }, + "odoo(tutu.0.city)": { + "type": "text", + "value": "Paris" + }, + "odoo(tutu.1.city)": { + "type": "text", + "value": "Grenoble" + }, + "odoo(tutu.2.city)": { + "type": "text", + "value": "Grenoble" + } + } + } +} diff --git a/usebruno/get_dirs.bru b/usebruno/get_dirs.bru new file mode 100644 index 0000000..a0b1817 --- /dev/null +++ b/usebruno/get_dirs.bru @@ -0,0 +1,15 @@ +meta { + name: get_dirs + type: http + seq: 3 +} + +get { + url: {{baseUrl}}/ + body: none + auth: none +} + +headers { + secretkey: DEFAULT_KEY +} diff --git a/usebruno/scan_simple_vars_xlsx_template.bru b/usebruno/scan_simple_vars_xlsx_template.bru new file mode 100644 index 0000000..6a8db32 --- /dev/null +++ b/usebruno/scan_simple_vars_xlsx_template.bru @@ -0,0 +1,15 @@ +meta { + name: scan_simple_vars_xlsx_template + type: http + seq: 9 +} + +get { + url: {{baseUrl}}/test_dir1/simple_vars.xlsx + body: none + auth: none +} + +headers { + secretkey: DEFAULT_KEY +} diff --git a/usebruno/scan_writer_template.bru b/usebruno/scan_writer_template.bru new file mode 100644 index 0000000..4cf7bb6 --- /dev/null +++ b/usebruno/scan_writer_template.bru @@ -0,0 +1,15 @@ +meta { + name: scan_writer_template + type: http + seq: 5 +} + +get { + url: {{baseUrl}}/test_dir1/if_inside_for.odt + body: none + auth: none +} + +headers { + secretkey: DEFAULT_KEY +} diff --git a/usebruno/template_files/calc_table_formula.ods b/usebruno/template_files/calc_table_formula.ods new file mode 100644 index 0000000..fc942c6 Binary files /dev/null and b/usebruno/template_files/calc_table_formula.ods differ diff --git a/usebruno/template_files/calc_table_formula.xlsx b/usebruno/template_files/calc_table_formula.xlsx new file mode 100644 index 0000000..7453e6f Binary files /dev/null and b/usebruno/template_files/calc_table_formula.xlsx differ diff --git a/usebruno/template_files/if_inside_for.odt b/usebruno/template_files/if_inside_for.odt new file mode 100644 index 0000000..8d91c10 Binary files /dev/null and b/usebruno/template_files/if_inside_for.odt differ diff --git a/usebruno/template_files/simple_vars.xlsx b/usebruno/template_files/simple_vars.xlsx new file mode 100644 index 0000000..c356e39 Binary files /dev/null and b/usebruno/template_files/simple_vars.xlsx differ diff --git a/usebruno/upload_simple_vars_xlsx_template.bru b/usebruno/upload_simple_vars_xlsx_template.bru new file mode 100644 index 0000000..41be04f --- /dev/null +++ b/usebruno/upload_simple_vars_xlsx_template.bru @@ -0,0 +1,19 @@ +meta { + name: upload_simple_vars_xlsx_template + type: http + seq: 8 +} + +put { + url: {{baseUrl}}/test_dir1 + body: multipartForm + auth: none +} + +headers { + secretkey: DEFAULT_KEY +} + +body:multipart-form { + file: @file(template_files/simple_vars.xlsx) +} diff --git a/usebruno/upload_table_formula_calc_template.bru b/usebruno/upload_table_formula_calc_template.bru new file mode 100644 index 0000000..f2ab314 --- /dev/null +++ b/usebruno/upload_table_formula_calc_template.bru @@ -0,0 +1,19 @@ +meta { + name: upload_table_formula_calc_template + type: http + seq: 11 +} + +put { + url: {{baseUrl}}/test_dir1 + body: multipartForm + auth: none +} + +headers { + secretkey: DEFAULT_KEY +} + +body:multipart-form { + file: @file(template_files/calc_table_formula.ods) +} diff --git a/usebruno/upload_table_formula_excel_template.bru b/usebruno/upload_table_formula_excel_template.bru new file mode 100644 index 0000000..c04297f --- /dev/null +++ b/usebruno/upload_table_formula_excel_template.bru @@ -0,0 +1,19 @@ +meta { + name: upload_table_formula_excel_template + type: http + seq: 13 +} + +put { + url: {{baseUrl}}/test_dir1 + body: multipartForm + auth: none +} + +headers { + secretkey: DEFAULT_KEY +} + +body:multipart-form { + file: @file(template_files/calc_table_formula.xlsx) +} diff --git a/usebruno/upload_writer_template.bru b/usebruno/upload_writer_template.bru new file mode 100644 index 0000000..2dcbd4d --- /dev/null +++ b/usebruno/upload_writer_template.bru @@ -0,0 +1,19 @@ +meta { + name: upload_writer_template + type: http + seq: 4 +} + +put { + url: {{baseUrl}}/test_dir1 + body: multipartForm + auth: none +} + +headers { + secretkey: DEFAULT_KEY +} + +body:multipart-form { + file: @file(template_files/if_inside_for.odt) +}