diff --git a/README.md b/README.md index cf6d6ac..19c636a 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,11 @@ # texd - [![Go Reference](https://pkg.go.dev/badge/github.com/digineo/texd.svg)](https://pkg.go.dev/github.com/digineo/texd) [![Test, Lint, Release](https://github.com/digineo/texd/actions/workflows/test.yml/badge.svg)](https://github.com/digineo/texd/actions/workflows/test.yml) [![Coverage](https://codecov.io/gh/digineo/texd/branch/master/graph/badge.svg)](https://codecov.io/gh/digineo/texd) [![Go Report Card](https://goreportcard.com/badge/github.com/digineo/texd)](https://goreportcard.com/report/github.com/digineo/texd) [![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/digineo/texd/master/LICENSE) - +[![Docker Pulls](https://img.shields.io/docker/pulls/digineode/texd)](https://hub.docker.com/r/digineode/texd) texd is a TeXaS (TeX as (a) service) solution, designed for your internal document generation, i.e. on your own servers. @@ -28,582 +27,12 @@ Several technologies make scaling in any dimension relatively easy: distributions simultaneously) - using HTTP enables redundancy and/or load balancing without much effort -## Operation Modes - -texd is designed to be run/deployed in 2½ different ways: - -### Local Mode - -This is primarily for (local) testing and development. You download and start texd locally, provide -a TeX distribution, and texd will compile documents on your host machine. - -To start texd in this mode, execute: - -```console -$ texd -``` - -### Ephemeral Containers - -Here, you still download and run texd locally, but document rendering will happen in an short-lived -Docker container, using a specific Docker image (`texlive/texlive:latest` will do just fine, but you -could easily build a smaller one using e.g. a Debian base image). - -To run in container mode, run: - -```console -$ texd texlive/texlive:latest -``` - -This will pull the specified image, if it doesn't exist yet. Note that you need to give texd -access to `/var/run/docker.sock`, in order to allow it to pull the image and create containers. - -You may provide multiple image names and switch on a per-request basis (see HTTP API below). In -this case, the first image is used as default image: - -```console -$ texd \ - texlive/texlive:latest \ - registry.gitlab.com/islandoftex/images/texlive:TL2014-historic \ - ghcr.io/yourcompany/texlive-prod -``` - -### CI Service - -This runs texd within a Docker container, and is primarily targeted for CI pipelines, but can be a -viable alternative to the local mode. In fact, this mode is functionally equivalent to the -*local mode*, with the one exception (texd being packaged and started in a container). - -To run texd as Docker service, use this command: - -```console -$ docker run --rm -t -p localhost:2201:2201 digineode/texd:latest -``` - -When using Gitlab CI, you can add this line to your `.gitlab-ci.yml`: - -```yml -services: - - name: digineode/texd:latest - alias: texd - -variables: - # reconfigure test application to use this endpoint - # (this is specific to your application!) - TEXD_ENDPOINT: http://texd:2201/render -``` - -This image is based on `texlive/texlive:latest`. - -## CLI Options - -Calling texd with options works in any mode; these commands are equivalent: - -```console -$ texd -h -$ texd texlive/texlive:latest -h -$ docker run --rm -t digineode/texd:latest -h -``` - -- `--help`, `-h` - - Prints a short option listing and exits. - -- `--version`, `-v` - - Prints version information and exits. - -- `--listen-address=ADDR`, `-b ADDR` (Default: `:2201`) - - Specifies host address (optional) and port number for the HTTP API to bind to. Valid values are, - among others: - - - `:2201` (bind to all addresses on port 2201) - - `localhost:2201` (bind only to localhost on port 2201) - - `[fe80::dead:c0ff:fe42:beef%eth0]:2201` (bind to a link-local IPv6 address on a specific - interface) - -- `--tex-engine=ENGINE`, `-X ENGINE` (Default: `xelatex`) - - TeX engine used to compile documents. Can be overridden on a per-request basis (see HTTP API - below). Supported engines are `xelatex`, `lualatex`, and `pdflatex`. - -- `--compile-timeout=DURATION`, `-t DURATION` (Default: `1m`) - - Maximum duration for a document rendering process before it is killed by texd. The value must be - acceptable by Go's `ParseDuruation` function. - -- `--parallel-jobs=NUM`, `-P NUM` (Default: number of cores) - - Concurrency level. PDF rendering is inherently single threaded, so limiting the document - processing to the number of cores is a good start. - -- `--queue-wait=DURATION`, `-w DURATION` (Default: `10s`) - - Time to wait in queue before aborting. When <= 0, clients will immediately receive a "full queue" - response. - -- `--job-directory=PATH`, `-D PATH` (Default: OS temp directory) - - Place to put job sub directories in. The path must exist and it must be writable. - -- `--pull` (Default: omitted) - - Always pulls Docker images. By default, images are only pulled when they don't exist locally. - - This has no effect when no image tags are given to the command line. - -> Note: This option listing might be outdated. Run `texd --help` to get the up-to-date listing. - -## HTTP API - -### Render a document - -To create a PDF document from an input `.tex` file, send a HTTP POST to the `/render` endpoint. -You may encode the payload as `multipart/form-data` or `application/x-www-form-encoded`, however -the latter is not recommended. - -Assuming, you have a `input.tex` in the current directory, you can issue the following command -to send that file to your texd instance, and save the result in a file named `output.pdf`: - -```console -$ curl -X POST \ - -F "input.tex=Guessing the input file (click to show details) - -- only filenames starting with alphanumeric character and ending in `.tex` are considered - (`foo.tex`, `00-intro.tex` will be considered, but not `_appendix.tex`, `figure.png`) -- files in sub directories are ignored (e.g. `chapters/a.tex`) -- if only one file in the root directory remains, it is taken as main input - - otherwise search for a file containing a line starting with: - - either `%!texd` at the beginning of the file - - or `\documentclass` somewhere in the first KiB - - if no match, consider (in order): - - `input.tex` - - `main.tex` - - `document.tex` - - - -If no main input file can be determined, texd will abort with an error. - -#### URL Parameters - -- `input=` - instructs texd to skip guessing main input file and use the specified one. - The filename must be present in the body. - -- `engine=` - specifies which TeX engine to run. Supported engines are: - - - `xelatex` (default) - - `lualatex` - - `pdflatex` - - Note that the default can be changed with a CLI option (e.g. `--tex-engine=lualatex`). - -- `image=` - selects Docker image for document processing. - - This is only available in *ephemeral container* mode. The image name must match the ones listed - in the texd command invocation, i.e. you can't select arbitrary images. - - If you provide an unknown image name, you will receive a 404 Not Found response. In *local* and - *CI service* mode, this parameter only logged, but will otherwise be ignored. - -- `errors=` - tries to retrieve the compilation log, in case of compilation errors. - Acceptable detail levels are: - - - *empty* (or `errors` completely absent), to return a JSON description (default) - - `condensed`, to return only the TeX error message from the log file - - `full`, to return the full log file as `text/plain` response - - The "condensed" form extracts only the lines from the error log which start with a `!`. Due to - the way TeX works, these lines may not paint the full picture, as TeX's log lines generally don't - exceed a certain line length, and wrapped lines won't get another `!` prefix. - - Note that this parameter changes the response content to a plain text file if you select `full` - or `condensed`, and not a JSON response as in all other cases. - -#### Successful response - -If compilation succeeds, you'll receive a status 200 OK, with content type `application/pdf`, and -the PDF file as response body. - -```http -HTTP/1.1 200 OK -Content-Type: application/pdf -Content-Length: 1234 - -%PDF/1.5... -``` - -#### Failure responses - -If the request was accepted, but could not complete due to errors, you will by default receive a 422 -Unprocessable Entity response with content type `application/json`, and an error description in -JSON format: - -```http -HTTP/1.1 422 Unprocessable Entity -Content-Type: application/json -Content-Length: 154 - -{ - "error": "latexmk call failed with status 1", - "category": "compilation", - "output": "[truncated output log]" -} -``` - -The fields `error` and `category` represent a short error description and an error category, -respectively. - -Possible, known error categories are currently: - -- *input* - one or more files are invalid (e.g. file was discarded after path normalization), - or the main input file could not be determined. - -- *compilation* - `latexmk` exited with an error (likely due to invalid or missing input files). - -- *queue* - texd won't accept new render jobs, if its internal queue is at capacity. In this case - wait for a few moments to give texd a chance to catch up and then try again. - -- *reference* - texd could not find the provided reference store entries. The missing references - are listed in the response; you need to repeat the request with those files included. - -Additional fields, like `log` for compilation failures, might be present. - -> Note: The JSON response is pretty-printed only for this README. Expect the actual response to -> be minified. - -If you set `errors=full`, you may receive a plain text file with the compilation log: - -
Show response (click to open) - -```http -HTTP/1.1 422 Unprocessable Entity -Content-Type: text/plain -Content-Length: 3156 - -This is XeTeX, Version 3.141592653-2.6-0.999993 (TeX Live 2021) (preloaded format=xelatex 2022.3.6) 12 MAR 2022 13:57 -entering extended mode - restricted \write18 enabled. - %&-line parsing enabled. -... ommitting some lines ... -! LaTeX Error: File `missing.tex' not found. - -Type X to quit or to proceed, -or enter new name. (Default extension: tex) - -Enter file name: -! Emergency stop. - - -l.3 \input{missing.tex} - ^^M -*** (cannot \read from terminal in nonstop modes) -``` - -
- -For `errors=condensed`, you'll only receive the lines starting with `!` (with this prefix removed): - -
Show response (click to open) - -```http -HTTP/1.1 422 Unprocessable Entity -Content-Type: text/plain -Content-Length: 59 - -LaTeX Error: File `missing.tex' not found. -Emergency stop. -``` - -
- -### Status and Configuration - -texd has a number of runtime configuration knobs and internal state variables, which may or may not -of interest for API consumers. To receive a current snapshot, query `/status`: - -```console -$ curl -i http://localhost:2201/status -HTTP/1.1 200 OK -Content-Type: application/json; charset=utf-8 -Content-Length: 287 - -{ - "version": "0.0.0", - "mode": "container", - "images": ["texlive/texlive:latest"], - "timeout": 60, - "engines": ["xelatex","pdflatex","lualatex"], - "default_engine": "xelatex", - "queue": { - "length": 0, - "capacity": 16 - } -} -``` - -### Metrics - -For monitoring, texd provides a Prometheus endpoint at `/metrics`: - -```console -$ curl -i http://localhost:2201/metrics -Content-Type: text/plain; version=0.0.4; charset=utf-8 - -... -``` - -The metrics include Go runtime information, as well as texd specific metrics: - -| Metric name | Type | Description | -|:------------|:-----|:------------| -| `texd_processed_total{status="success"}` | counter | Number of documents processed. | -| `texd_processed_total{status="failure"}` | counter | Number of rendering errors, including timeouts. | -| `texd_processed_total{status="rejected"}` | counter | Number of rejected requests, due to full job queue. | -| `texd_processed_total{status="aborted"}` | counter | Number of aborted requests, usually due to timeouts. | -| `texd_processing_duration_seconds` | histogram | Overview of processing time per document. | -| `texd_input_file_size_bytes{type=?}` | histogram | Overview of input file sizes. Type is either "tex" (for .tex, .cls, .sty, and similar files), "asset" (for images and fonts), "data" (for CSV files), or "other" (for unknown files) | -| `texd_output_file_size_bytes` | histogram | Overview of output file sizes. | -| `texd_job_queue_length` | gauge | Length of rendering queue, i.e. how many documents are waiting for processing. | -| `texd_job_queue_usage_ratio` | gauge | Queue capacity indicator (0.0 = empty, 1.0 = full). | -| `texd_info{version="0.0.0", mode="local", ...}` | constant | Various version and configuration information. | - - -Metrics related to processing also have an `engine=?` label indicating the TeX engine ("xelatex", -"lualatex", or "pdflatex"), and an `image=?` label indicating the Docker image. - -### Simple Web UI - -You can try compiling TeX documents directly in your browser: Visit http://localhost:2201, and -you'll be greeted with a very basic, but functional UI. - -Please note, that this UI is *not* built to work in every browser. It intentionally does not -use fancy build tools. It's just a simple HTML file, built by hand, using Bootstrap 5 for -aesthetics and Vue 3 for interaction. Both Bootstrap and Vue are bundled with texd, so you won't -need internet access for this to work. - -If your browser does not support modern features like ES2022 proxies, `Object.entries`, `fetch`, -and `` elements, you're out of luck. (Maybe upgrade your browser?) -Anyway, consider the UI only as demonstrator for the API. - -## Reference store - -texd has the ability to re-use previously sent material. This allows you to reduce the amount -of data you need to transmit with each render request. Following a back-of-the-envelope calculation: - -- If you want to generate 1000 documents, each including a font with 400 kB in size, and a logo - file with 100 kB in size, you will need to transmit 500 MB of the same two files in total. -- If you can re-use those two assets, you would only need to transmit them once, and use a reference - hash for each subsequent request. The total then reduces 1×500 kB (complete assets for the first - request) + 999×100 Byte (50 Byte per reference hash for subsequent requests) = 599.9 kB. - -The feature in texd parlance is called "reference store", and you may think of it as a cache. It -saves files server-side (e.g. on disk) and retrieves them on-demand, if you request such a file -reference. - -A reference hash is simply the Base64-encoded SHA256 checksum of the file contents, prefixed with -"sha256:". (Canonically, we use the URL-safe alphabet without padding for the Base64 encoder, but -texd also accepts the standard alphabet, and padding characters are ignored in both cases.) - -To *use* a file reference, you need to set a special content type in the request, and include the -reference hash instead of the file contents. The content type must be `application/x.texd; ref=use`. - -The resulting HTTP request should then look something like this: - -```http -POST /render HTTP/1.1 -Content-Type: multipart/form-data; boundary=boundary - ---boundary -Content-Disposition: form-data; name=input.tex; filename=input.tex -Content-Type: application/octet-stream - -[content of input.tex omitted] ---boundary -Content-Disposition: form-data; name=logo.pdf; filename=logo.pdf -Content-Type: application/x.texd; ref=use - -sha256:p5w-x0VQUh2kXyYbbv1ubkc-oZ0z7aZYNjSKVVzaZuo= ---boundary-- -``` - -For unknown reference hashes, texd will respond with an error, and list all unknown references: - -```http -HTTP/1.1 422 Unprocessable Entity -Content-Type: application/json - -{ - "category": "reference", - "error": "unknown file references", - "reference": [ - "sha256:p5w-x0VQUh2kXyYbbv1ubkc-oZ0z7aZYNjSKVVzaZuo=" - ] -} -``` - -In such a case, you can repeat you HTTP request, and change the `ref=use` to `ref=store` for -matching documents: - -```http -POST /render HTTP/1.1 -Content-Type: multipart/form-data; boundary=boundary - ---boundary -Content-Disposition: form-data; name=input.tex; filename=input.tex -Content-Type: application/octet-stream - -[content of input.tex omitted] ---boundary -Content-Disposition: form-data; name=logo.pdf; filename=logo.pdf -Content-Type: application/x.texd; ref=store - -[content of logo.pdf omitted] ---boundary-- -``` - -### Server configuration - -By default, the reference store is not enabled. You must enable it explicitly, by providing -a command line flag. Assuming you have a local directory `./refs`, you instruct texd to use -this directory for references: - -```console -$ texd --reference-store=dir://./refs -``` - -The actual syntax is `--reference-store=DSN`, where storage adapters are identified through and -configured with a DSN (*data source name*, a URL). Currently there are only handful implementations: - -1. The `dir://` adapter ([docs][docs-dir]), which stores reference files on disk in a specified - directory. Coincidentally, this adapter also provides an in-memory adapter (`memory://`), - courtesy of the [spf13/afero][afero] package. - -2. The `memcached://` adapter ([docs][docs-memcached]), which stores, you may have guessed it, - reference files in a [Memcached][memcached] instance or cluster. - -3. The `nop://` adapter ([docs][docs-nop]), which―for the sake of completeness sake―implements a - no-op store (i.e. attempts to store reference file into is, or load files from it fail silently). - This adapter is used as fallback if you don't configure any other adapter. - -[docs-dir]: https://pkg.go.dev/github.com/digineo/texd/refstore/dir -[afero]: https://github.com/spf13/afero -[docs-memcached]: https://pkg.go.dev/github.com/digineo/texd/refstore/memcached -[memcached]: https://memcached.org/ -[docs-nop]: https://pkg.go.dev/github.com/digineo/texd/refstore/nop - -It is not unfeasible to imagine further adapters being available in the future, such as additional -key/value stores (`redis://`), object storages (`s3://`, `minio://`), or even RDBMS (`postgresql://`, -`mariadb://`). - -### Data retention - -texd supports three different retention policies: - -1. `keep` (or `none`) will keep all file references forever. This is the default setting. -2. `purge-on-start` (or just `purge`) will delete file references once on startup. -3. `access` will keep an access list with LRU semantics, and delete file references, either if - a max. number of items is reached, or if the total size of items exceeds a threshold, or both. - -To select a specific retention policy, use the `--retention-policy` CLI flag: - -```console -$ texd --reference-store=dir://./refs --retention-policy=purge -``` - -To configure the access list (`--retention-policy=access`), you can adopt the quota to your needs: - -``` -$ texd --reference-store=dir://./refs \ - --retention-policy=access \ - --rp-access-items=1000 \ - --rp-access-size=100MB -``` - -Notes: - -- The default quota for the max. number of items (`--rp-access-items`) is 1000. -- The default quota for the max. total file size (`--rp-access-size`) is 100MB. -- Total file size is measured in bytes, common suffixes (100KB, 2MiB, 1.3GB) work as expected. -- To disable either limit, set the value to 0 (e.g. `--rp-access-items=0`). -- It is an error to disable both limits (in this case just use `--retention-policy=keep`). -- Currently, only the `dir://` (and `memory://`) adapter support a retention policy; the - `memcached://` adapter delegates this responsibility to the Memcached server. - -## History - -texd came to life because I've build dozens of Rails applications, which all needed to build PDF -documents in one form or another (from recipies, to invoices, order confirmations, reports and -technical documentation). Each server basically needed a local TeX installation (weighing in at -several 100 MB, up to several GB). Compiling many LaTeX documents also became a bottleneck for -applications running on otherwise modest hardware (or cloud VMs), as this process is also -computationally expensive. - -Over time I've considered using alternatives for PDF generation (Prawn, HexaPDF, gofpdf, SILE, iText -PDF, to name but a few), and found that the quality of the rendered PDF is far inferior to the ones -generated by LaTeX. Other times, the licensing costs are astronomical, or the library doesn't -support some layouting feature, or the library in an early alpha stage or already abandoned... - -I'll admit that writing TeX templates for commercial settings is a special kind of pain-inducing -form of art. But looking back at using LaTeX for now over a decade, I still feel it's worth it. - - -## Future - -One wishlist's item is asynchronous rendering: Consider rendering monthly invoices on the first -of each month; depending on the amount of customers/contracts/invoice positions, this can easily -mean you need to render a few thousand PDF documents. - -Usually, the PDF generation is not time critical, i.e. they should finish in a reasonable amount of -time (say, within the next 6h to ensure timely delivery to the customer via email). For this to -work, the client could provide a callback URL to which texd sends the PDF via HTTP POST when -the rendering is finished. +## Documentation -Of course, this will also increase complexity on both sides: The client must be network-reachable -itself, an keep track of rendering request in order to associate the PDF to the correct invoice; -texd on the other hand would need a priority queue (processing async documents only if no sync -documents are enqueued), and it would need to store the callback URL somewhere. +The [latest documentation](./docs/index.md) can be found on GitHub. +If you have a running texd instance, head to http://localhost:2201/docs to view the documentation +for your instance. ## Related work diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..d2f1376 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,12 @@ +# texd documentation + +1. [Operation Modes](./operation-modes.md) +2. [CLI Options](./cli-options.md) +3. HTTP API + 1. [Render a document](./http-api/render.md) + 2. [Status and Configuration](./http-api/status.md) + 3. [Metrics](./http-api/metrics.md) + 4. [Simple Web UI](./http-api/web-ui.md) +4. [Reference Store](./reference-store.md) +5. [History](./history.md) +6. [Future](./future.md) diff --git a/docs/cli-options.md b/docs/cli-options.md new file mode 100644 index 0000000..f2b83a2 --- /dev/null +++ b/docs/cli-options.md @@ -0,0 +1,60 @@ +# CLI Options + +Calling texd with options works in any [operation mode](./operation-modes.md); +these commands are equivalent: + +```console +$ texd -h +$ texd texlive/texlive:latest -h +$ docker run --rm -t digineode/texd:latest -h +``` + +- `--help`, `-h` + + Prints a short option listing and exits. + +- `--version`, `-v` + + Prints version information and exits. + +- `--listen-address=ADDR`, `-b ADDR` (Default: `:2201`) + + Specifies host address (optional) and port number for the HTTP API to bind to. Valid values are, + among others: + + - `:2201` (bind to all addresses on port 2201) + - `localhost:2201` (bind only to localhost on port 2201) + - `[fe80::dead:c0ff:fe42:beef%eth0]:2201` (bind to a link-local IPv6 address on a specific + interface) + +- `--tex-engine=ENGINE`, `-X ENGINE` (Default: `xelatex`) + + TeX engine used to compile documents. Can be overridden on a per-request basis (see HTTP API + below). Supported engines are `xelatex`, `lualatex`, and `pdflatex`. + +- `--compile-timeout=DURATION`, `-t DURATION` (Default: `1m`) + + Maximum duration for a document rendering process before it is killed by texd. The value must be + acceptable by Go's `ParseDuruation` function. + +- `--parallel-jobs=NUM`, `-P NUM` (Default: number of cores) + + Concurrency level. PDF rendering is inherently single threaded, so limiting the document + processing to the number of cores is a good start. + +- `--queue-wait=DURATION`, `-w DURATION` (Default: `10s`) + + Time to wait in queue before aborting. When <= 0, clients will immediately receive a "full queue" + response. + +- `--job-directory=PATH`, `-D PATH` (Default: OS temp directory) + + Place to put job sub directories in. The path must exist and it must be writable. + +- `--pull` (Default: omitted) + + Always pulls Docker images. By default, images are only pulled when they don't exist locally. + + This has no effect when no image tags are given to the command line. + +> Note: This option listing might be outdated. Run `texd --help` to get the up-to-date listing. diff --git a/docs/docs.go b/docs/docs.go new file mode 100644 index 0000000..73252eb --- /dev/null +++ b/docs/docs.go @@ -0,0 +1,215 @@ +package docs + +import ( + "bytes" + "embed" + "fmt" + "html/template" + "io" + "log" + "net/http" + "strings" + + toc "github.com/abhinav/goldmark-toc" + chromahtml "github.com/alecthomas/chroma/formatters/html" + "github.com/digineo/texd" + "github.com/microcosm-cc/bluemonday" + "github.com/yuin/goldmark" + highlighting "github.com/yuin/goldmark-highlighting" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/extension" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/renderer/html" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" + "gopkg.in/yaml.v3" +) + +//go:embed *.md **/*.md +var sources embed.FS + +//go:embed docs.yml +var config []byte + +//go:embed docs.html +var rawLayout string +var tplLayout = template.Must(template.New("layout").Parse(rawLayout)) + +type page struct { + Title string + Breadcrumbs []string + TOC *toc.TOC + CSS []byte + Body []byte + File string + Route string + Children []*page +} + +type pageRoutes map[string]*page + +func getRoutes(urlPrefix string) (pageRoutes, error) { + var menu page + dec := yaml.NewDecoder(bytes.NewReader(config)) + dec.KnownFields(true) + if err := dec.Decode(&menu); err != nil { + return nil, err + } + + urlPrefix = strings.TrimSuffix(urlPrefix, "/") + return menu.init(urlPrefix, make(pageRoutes)) +} + +func (pg *page) init(urlPrefix string, r pageRoutes, crumbs ...string) (pageRoutes, error) { + if pg.File != "" { + if r := strings.TrimSuffix(pg.File, ".md"); r == "README" { + pg.Route = urlPrefix + } else { + pg.Route = urlPrefix + "/" + r + } + r[pg.Route] = pg + err := pg.parseFile(urlPrefix) + if err != nil { + return nil, err + } + } + if pg.Title != "" { + pg.Breadcrumbs = append([]string{pg.Title}, crumbs...) + } + for _, child := range pg.Children { + _, err := child.init(urlPrefix, r, pg.Breadcrumbs...) + if err != nil { + return nil, err + } + } + return r, nil +} + +type localLinkTransformer struct { + prefix string +} + +var _ parser.ASTTransformer = (*localLinkTransformer)(nil) + +func (link *localLinkTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) { + _ = ast.Walk(node, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + if entering && n.Kind() == ast.KindLink { + if l, ok := n.(*ast.Link); ok { + link.rewrite(l) + } + } + return ast.WalkContinue, nil + }) +} + +const ( + localLinkPrefix = "./" + localLinkSuffix = ".md" +) + +func (link *localLinkTransformer) rewrite(l *ast.Link) { + dst := string(l.Destination) + if strings.HasPrefix(dst, localLinkPrefix) && strings.HasSuffix(dst, localLinkSuffix) { + dst = strings.TrimPrefix(dst, localLinkPrefix) + dst = strings.TrimSuffix(dst, localLinkSuffix) + l.Destination = []byte(link.prefix + "/" + dst) + } +} + +var sanitize = func() func(io.Reader) *bytes.Buffer { + p := bluemonday.UGCPolicy() + p.AllowAttrs("class").Globally() + return p.SanitizeReader +}() + +func (pg *page) parseFile(urlPrefix string) error { + raw, err := sources.ReadFile(pg.File) + if err != nil { + return err + } + + var css, body bytes.Buffer + md := goldmark.New( + goldmark.WithParserOptions( + parser.WithAutoHeadingID(), + parser.WithASTTransformers(util.PrioritizedValue{ + Value: &localLinkTransformer{urlPrefix}, + Priority: 999, + }), + ), + goldmark.WithRendererOptions( + html.WithUnsafe(), + ), + goldmark.WithExtensions( + extension.GFM, + highlighting.NewHighlighting( + highlighting.WithCSSWriter(&css), + highlighting.WithStyle("github"), + highlighting.WithFormatOptions( + chromahtml.WithLineNumbers(true), + chromahtml.WithClasses(true), + ), + ), + ), + ) + + doc := md.Parser().Parse(text.NewReader(raw)) + tree, err := toc.Inspect(doc, raw) + if err != nil { + return err + } + if pg.Title == "" { + if len(tree.Items) > 0 { + pg.Title = string(tree.Items[0].Title) + } + } + if err := md.Renderer().Render(&body, raw, doc); err != nil { + return err + } + pg.TOC = tree + pg.CSS = css.Bytes() + pg.Body = sanitize(&body).Bytes() + return nil +} + +func Handler(prefix string) (http.Handler, error) { + type pageVars struct { + Version string + Title string + CSS template.CSS + Content template.HTML + } + + routes, err := getRoutes(prefix) + if err != nil { + return nil, fmt.Errorf("failed to build docs: %w", err) + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + pg := routes[r.URL.Path] + if pg == nil { + http.NotFound(w, r) + return + } + + var buf bytes.Buffer + err := tplLayout.Execute(&buf, &pageVars{ + Version: texd.Version(), + Title: strings.Join(pg.Breadcrumbs, " · "), + CSS: template.CSS(pg.CSS), + Content: template.HTML(pg.Body), + }) + + if err != nil { + log.Println(err) + code := http.StatusInternalServerError + http.Error(w, http.StatusText(code), code) + return + } + + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(http.StatusOK) + _, _ = buf.WriteTo(w) + }), nil +} diff --git a/docs/docs.html b/docs/docs.html new file mode 100644 index 0000000..d864afc --- /dev/null +++ b/docs/docs.html @@ -0,0 +1,39 @@ + + + + + + + {{ .Title }} + + + + + + +
+ + +
+ {{ .Content }} +
+
+ + diff --git a/docs/docs.yml b/docs/docs.yml new file mode 100644 index 0000000..616fd06 --- /dev/null +++ b/docs/docs.yml @@ -0,0 +1,14 @@ +--- +file: README.md +children: + - file: operation-modes.md + - file: cli-options.md + - title: HTTP API + children: + - file: http-api/render.md + - file: http-api/status.md + - file: http-api/metrics.md + - file: http-api/web-ui.md + - file: reference-store.md + - file: history.md + - file: future.md diff --git a/docs/future.md b/docs/future.md new file mode 100644 index 0000000..8d17d86 --- /dev/null +++ b/docs/future.md @@ -0,0 +1,15 @@ +# Future + +One wishlist's item is asynchronous rendering: Consider rendering monthly invoices on the first +of each month; depending on the amount of customers/contracts/invoice positions, this can easily +mean you need to render a few thousand PDF documents. + +Usually, the PDF generation is not time critical, i.e. they should finish in a reasonable amount of +time (say, within the next 6h to ensure timely delivery to the customer via email). For this to +work, the client could provide a callback URL to which texd sends the PDF via HTTP POST when +the rendering is finished. + +Of course, this will also increase complexity on both sides: The client must be network-reachable +itself, an keep track of rendering request in order to associate the PDF to the correct invoice; +texd on the other hand would need a priority queue (processing async documents only if no sync +documents are enqueued), and it would need to store the callback URL somewhere. diff --git a/docs/history.md b/docs/history.md new file mode 100644 index 0000000..9b463fc --- /dev/null +++ b/docs/history.md @@ -0,0 +1,16 @@ +# History + +texd came to life because I've build dozens of Rails applications, which all needed to build PDF +documents in one form or another (from recipies, to invoices, order confirmations, reports and +technical documentation). Each server basically needed a local TeX installation (weighing in at +several 100 MB, up to several GB). Compiling many LaTeX documents also became a bottleneck for +applications running on otherwise modest hardware (or cloud VMs), as this process is also +computationally expensive. + +Over time I've considered using alternatives for PDF generation (Prawn, HexaPDF, gofpdf, SILE, iText +PDF, to name but a few), and found that the quality of the rendered PDF is far inferior to the ones +generated by LaTeX. Other times, the licensing costs are astronomical, or the library doesn't +support some layouting feature, or the library in an early alpha stage or already abandoned... + +I'll admit that writing TeX templates for commercial settings is a special kind of pain-inducing +form of art. But looking back at using LaTeX for now over a decade, I still feel it's worth it. diff --git a/docs/http-api/metrics.md b/docs/http-api/metrics.md new file mode 100644 index 0000000..36e54eb --- /dev/null +++ b/docs/http-api/metrics.md @@ -0,0 +1,29 @@ +# Metrics + +For monitoring, texd provides a Prometheus endpoint at `/metrics`: + +```console +$ curl -i http://localhost:2201/metrics +Content-Type: text/plain; version=0.0.4; charset=utf-8 + +... +``` + +The metrics include Go runtime information, as well as texd specific metrics: + +| Metric name | Type | Description | +|:------------|:-----|:------------| +| `texd_processed_total{status="success"}` | counter | Number of documents processed. | +| `texd_processed_total{status="failure"}` | counter | Number of rendering errors, including timeouts. | +| `texd_processed_total{status="rejected"}` | counter | Number of rejected requests, due to full job queue. | +| `texd_processed_total{status="aborted"}` | counter | Number of aborted requests, usually due to timeouts. | +| `texd_processing_duration_seconds` | histogram | Overview of processing time per document. | +| `texd_input_file_size_bytes{type=?}` | histogram | Overview of input file sizes. Type is either "tex" (for .tex, .cls, .sty, and similar files), "asset" (for images and fonts), "data" (for CSV files), or "other" (for unknown files) | +| `texd_output_file_size_bytes` | histogram | Overview of output file sizes. | +| `texd_job_queue_length` | gauge | Length of rendering queue, i.e. how many documents are waiting for processing. | +| `texd_job_queue_usage_ratio` | gauge | Queue capacity indicator (0.0 = empty, 1.0 = full). | +| `texd_info{version="0.0.0", mode="local", ...}` | constant | Various version and configuration information. | + + +Metrics related to processing also have an `engine=?` label indicating the TeX engine ("xelatex", +"lualatex", or "pdflatex"), and an `image=?` label indicating the Docker image. diff --git a/docs/http-api/render.md b/docs/http-api/render.md new file mode 100644 index 0000000..7df1f7c --- /dev/null +++ b/docs/http-api/render.md @@ -0,0 +1,199 @@ +# Render a document + +To create a PDF document from an input `.tex` file, send a HTTP POST to the `/render` endpoint. +You may encode the payload as `multipart/form-data` or `application/x-www-form-encoded`, however +the latter is not recommended. + +Assuming, you have a `input.tex` in the current directory, you can issue the following command +to send that file to your texd instance, and save the result in a file named `output.pdf`: + +```console +$ curl -X POST \ + -F "input.tex=Guessing the input file (click to show details) + +- only filenames starting with alphanumeric character and ending in `.tex` are considered + (`foo.tex`, `00-intro.tex` will be considered, but not `_appendix.tex`, `figure.png`) +- files in sub directories are ignored (e.g. `chapters/a.tex`) +- if only one file in the root directory remains, it is taken as main input + - otherwise search for a file containing a line starting with: + - either `%!texd` at the beginning of the file + - or `\documentclass` somewhere in the first KiB + - if no match, consider (in order): + - `input.tex` + - `main.tex` + - `document.tex` + + + +If no main input file can be determined, texd will abort with an error. + +## URL Parameters + +- `input=` - instructs texd to skip guessing main input file and use the specified one. + The filename must be present in the body. + +- `engine=` - specifies which TeX engine to run. Supported engines are: + + - `xelatex` (default) + - `lualatex` + - `pdflatex` + + Note that the default can be changed with a CLI option (e.g. `--tex-engine=lualatex`). + +- `image=` - selects Docker image for document processing. + + This is only available in *ephemeral container* mode. The image name must match the ones listed + in the texd command invocation, i.e. you can't select arbitrary images. + + If you provide an unknown image name, you will receive a 404 Not Found response. In *local* and + *CI service* mode, this parameter only logged, but will otherwise be ignored. + +- `errors=` - tries to retrieve the compilation log, in case of compilation errors. + Acceptable detail levels are: + + - *empty* (or `errors` completely absent), to return a JSON description (default) + - `condensed`, to return only the TeX error message from the log file + - `full`, to return the full log file as `text/plain` response + + The "condensed" form extracts only the lines from the error log which start with a `!`. Due to + the way TeX works, these lines may not paint the full picture, as TeX's log lines generally don't + exceed a certain line length, and wrapped lines won't get another `!` prefix. + + Note that this parameter changes the response content to a plain text file if you select `full` + or `condensed`, and not a JSON response as in all other cases. + +## Successful response + +If compilation succeeds, you'll receive a status 200 OK, with content type `application/pdf`, and +the PDF file as response body. + +```http +HTTP/1.1 200 OK +Content-Type: application/pdf +Content-Length: 1234 + +%PDF/1.5... +``` + +## Failure responses + +If the request was accepted, but could not complete due to errors, you will by default receive a 422 +Unprocessable Entity response with content type `application/json`, and an error description in +JSON format: + +```http +HTTP/1.1 422 Unprocessable Entity +Content-Type: application/json +Content-Length: 154 + +{ + "error": "latexmk call failed with status 1", + "category": "compilation", + "output": "[truncated output log]" +} +``` + +The fields `error` and `category` represent a short error description and an error category, +respectively. + +Possible, known error categories are currently: + +- *input* - one or more files are invalid (e.g. file was discarded after path normalization), + or the main input file could not be determined. + +- *compilation* - `latexmk` exited with an error (likely due to invalid or missing input files). + +- *queue* - texd won't accept new render jobs, if its internal queue is at capacity. In this case + wait for a few moments to give texd a chance to catch up and then try again. + +- *reference* - texd could not find the provided reference store entries. The missing references + are listed in the response; you need to repeat the request with those files included. + +Additional fields, like `log` for compilation failures, might be present. + +> Note: The JSON response is pretty-printed only for this README. Expect the actual response to +> be minified. + +If you set `errors=full`, you may receive a plain text file with the compilation log: + +
Show response (click to open) + +```http +HTTP/1.1 422 Unprocessable Entity +Content-Type: text/plain +Content-Length: 3156 + +This is XeTeX, Version 3.141592653-2.6-0.999993 (TeX Live 2021) (preloaded format=xelatex 2022.3.6) 12 MAR 2022 13:57 +entering extended mode + restricted \write18 enabled. + %&-line parsing enabled. +... ommitting some lines ... +! LaTeX Error: File `missing.tex' not found. + +Type X to quit or to proceed, +or enter new name. (Default extension: tex) + +Enter file name: +! Emergency stop. + + +l.3 \input{missing.tex} + ^^M +*** (cannot \read from terminal in nonstop modes) +``` + +
+ +For `errors=condensed`, you'll only receive the lines starting with `!` (with this prefix removed): + +
Show response (click to open) + +```http +HTTP/1.1 422 Unprocessable Entity +Content-Type: text/plain +Content-Length: 59 + +LaTeX Error: File `missing.tex' not found. +Emergency stop. +``` + +
diff --git a/docs/http-api/status.md b/docs/http-api/status.md new file mode 100644 index 0000000..38503e6 --- /dev/null +++ b/docs/http-api/status.md @@ -0,0 +1,24 @@ +# Status and Configuration + +texd has a number of runtime configuration knobs and internal state variables, which may or may not +of interest for API consumers. To receive a current snapshot, query `/status`: + +```console +$ curl -i http://localhost:2201/status +HTTP/1.1 200 OK +Content-Type: application/json; charset=utf-8 +Content-Length: 287 + +{ + "version": "0.0.0", + "mode": "container", + "images": ["texlive/texlive:latest"], + "timeout": 60, + "engines": ["xelatex","pdflatex","lualatex"], + "default_engine": "xelatex", + "queue": { + "length": 0, + "capacity": 16 + } +} +``` diff --git a/docs/http-api/web-ui.md b/docs/http-api/web-ui.md new file mode 100644 index 0000000..059299a --- /dev/null +++ b/docs/http-api/web-ui.md @@ -0,0 +1,13 @@ +# Simple Web UI + +You can try compiling TeX documents directly in your browser: Visit http://localhost:2201, and +you'll be greeted with a very basic, but functional UI. + +Please note, that this UI is *not* built to work in every browser. It intentionally does not +use fancy build tools. It's just a simple HTML file, built by hand, using Bootstrap 5 for +aesthetics and Vue 3 for interaction. Both Bootstrap and Vue are bundled with texd, so you won't +need internet access for this to work. + +If your browser does not support modern features like ES2022 proxies, `Object.entries`, `fetch`, +and `` elements, you're out of luck. (Maybe upgrade your browser?) +Anyway, consider the UI only as demonstrator for the API. diff --git a/docs/operation-modes.md b/docs/operation-modes.md new file mode 100644 index 0000000..bc36ee4 --- /dev/null +++ b/docs/operation-modes.md @@ -0,0 +1,66 @@ +# Operation Modes + +texd is designed to be run/deployed in 2½ different ways: + +## Local Mode + +This is primarily for (local) testing and development. You download and start texd locally, provide +a TeX distribution, and texd will compile documents on your host machine. + +To start texd in this mode, execute: + +```console +$ texd +``` + +## Ephemeral Containers + +Here, you still download and run texd locally, but document rendering will happen in an short-lived +Docker container, using a specific Docker image (`texlive/texlive:latest` will do just fine, but you +could easily build a smaller one using e.g. a Debian base image). + +To run in container mode, run: + +```console +$ texd texlive/texlive:latest +``` + +This will pull the specified image, if it doesn't exist yet. Note that you need to give texd +access to `/var/run/docker.sock`, in order to allow it to pull the image and create containers. + +You may provide multiple image names and switch on a per-request basis (see HTTP API below). In +this case, the first image is used as default image: + +```console +$ texd \ + texlive/texlive:latest \ + registry.gitlab.com/islandoftex/images/texlive:TL2014-historic \ + ghcr.io/yourcompany/texlive-prod +``` + +## CI Service + +This runs texd within a Docker container, and is primarily targeted for CI pipelines, but can be a +viable alternative to the local mode. In fact, this mode is functionally equivalent to the +*local mode*, with the one exception (texd being packaged and started in a container). + +To run texd as Docker service, use this command: + +```console +$ docker run --rm -t -p localhost:2201:2201 digineode/texd:latest +``` + +When using Gitlab CI, you can add this line to your `.gitlab-ci.yml`: + +```yml +services: + - name: digineode/texd:latest + alias: texd + +variables: + # reconfigure test application to use this endpoint + # (this is specific to your application!) + TEXD_ENDPOINT: http://texd:2201/render +``` + +This image is based on `texlive/texlive:latest`. diff --git a/docs/reference-store.md b/docs/reference-store.md new file mode 100644 index 0000000..faec3e3 --- /dev/null +++ b/docs/reference-store.md @@ -0,0 +1,143 @@ +# Reference store + +texd has the ability to re-use previously sent material. This allows you to reduce the amount +of data you need to transmit with each render request. Following a back-of-the-envelope calculation: + +- If you want to generate 1000 documents, each including a font with 400 kB in size, and a logo + file with 100 kB in size, you will need to transmit 500 MB of the same two files in total. +- If you can re-use those two assets, you would only need to transmit them once, and use a reference + hash for each subsequent request. The total then reduces 1×500 kB (complete assets for the first + request) + 999×100 Byte (50 Byte per reference hash for subsequent requests) = 599.9 kB. + +The feature in texd parlance is called "reference store", and you may think of it as a cache. It +saves files server-side (e.g. on disk) and retrieves them on-demand, if you request such a file +reference. + +A reference hash is simply the Base64-encoded SHA256 checksum of the file contents, prefixed with +"sha256:". (Canonically, we use the URL-safe alphabet without padding for the Base64 encoder, but +texd also accepts the standard alphabet, and padding characters are ignored in both cases.) + +To *use* a file reference, you need to set a special content type in the request, and include the +reference hash instead of the file contents. The content type must be `application/x.texd; ref=use`. + +The resulting HTTP request should then look something like this: + +```http +POST /render HTTP/1.1 +Content-Type: multipart/form-data; boundary=boundary + +--boundary +Content-Disposition: form-data; name=input.tex; filename=input.tex +Content-Type: application/octet-stream + +[content of input.tex omitted] +--boundary +Content-Disposition: form-data; name=logo.pdf; filename=logo.pdf +Content-Type: application/x.texd; ref=use + +sha256:p5w-x0VQUh2kXyYbbv1ubkc-oZ0z7aZYNjSKVVzaZuo= +--boundary-- +``` + +For unknown reference hashes, texd will respond with an error, and list all unknown references: + +```http +HTTP/1.1 422 Unprocessable Entity +Content-Type: application/json + +{ + "category": "reference", + "error": "unknown file references", + "reference": [ + "sha256:p5w-x0VQUh2kXyYbbv1ubkc-oZ0z7aZYNjSKVVzaZuo=" + ] +} +``` + +In such a case, you can repeat you HTTP request, and change the `ref=use` to `ref=store` for +matching documents: + +```http +POST /render HTTP/1.1 +Content-Type: multipart/form-data; boundary=boundary + +--boundary +Content-Disposition: form-data; name=input.tex; filename=input.tex +Content-Type: application/octet-stream + +[content of input.tex omitted] +--boundary +Content-Disposition: form-data; name=logo.pdf; filename=logo.pdf +Content-Type: application/x.texd; ref=store + +[content of logo.pdf omitted] +--boundary-- +``` + +## Server configuration + +By default, the reference store is not enabled. You must enable it explicitly, by providing +a command line flag. Assuming you have a local directory `./refs`, you instruct texd to use +this directory for references: + +```console +$ texd --reference-store=dir://./refs +``` + +The actual syntax is `--reference-store=DSN`, where storage adapters are identified through and +configured with a DSN (*data source name*, a URL). Currently there are only handful implementations: + +1. The `dir://` adapter ([docs][docs-dir]), which stores reference files on disk in a specified + directory. Coincidentally, this adapter also provides an in-memory adapter (`memory://`), + courtesy of the [spf13/afero][afero] package. + +2. The `memcached://` adapter ([docs][docs-memcached]), which stores, you may have guessed it, + reference files in a [Memcached][memcached] instance or cluster. + +3. The `nop://` adapter ([docs][docs-nop]), which―for the sake of completeness sake―implements a + no-op store (i.e. attempts to store reference file into is, or load files from it fail silently). + This adapter is used as fallback if you don't configure any other adapter. + +[docs-dir]: https://pkg.go.dev/github.com/digineo/texd/refstore/dir +[afero]: https://github.com/spf13/afero +[docs-memcached]: https://pkg.go.dev/github.com/digineo/texd/refstore/memcached +[memcached]: https://memcached.org/ +[docs-nop]: https://pkg.go.dev/github.com/digineo/texd/refstore/nop + +It is not unfeasible to imagine further adapters being available in the future, such as additional +key/value stores (`redis://`), object storages (`s3://`, `minio://`), or even RDBMS (`postgresql://`, +`mariadb://`). + +## Data retention + +texd supports three different retention policies: + +1. `keep` (or `none`) will keep all file references forever. This is the default setting. +2. `purge-on-start` (or just `purge`) will delete file references once on startup. +3. `access` will keep an access list with LRU semantics, and delete file references, either if + a max. number of items is reached, or if the total size of items exceeds a threshold, or both. + +To select a specific retention policy, use the `--retention-policy` CLI flag: + +```console +$ texd --reference-store=dir://./refs --retention-policy=purge +``` + +To configure the access list (`--retention-policy=access`), you can adopt the quota to your needs: + +``` +$ texd --reference-store=dir://./refs \ + --retention-policy=access \ + --rp-access-items=1000 \ + --rp-access-size=100MB +``` + +Notes: + +- The default quota for the max. number of items (`--rp-access-items`) is 1000. +- The default quota for the max. total file size (`--rp-access-size`) is 100MB. +- Total file size is measured in bytes, common suffixes (100KB, 2MiB, 1.3GB) work as expected. +- To disable either limit, set the value to 0 (e.g. `--rp-access-items=0`). +- It is an error to disable both limits (in this case just use `--retention-policy=keep`). +- Currently, only the `dir://` (and `memory://`) adapter support a retention policy; the + `memcached://` adapter delegates this responsibility to the Memcached server. diff --git a/go.mod b/go.mod index a52cd21..b665d46 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,15 @@ module github.com/digineo/texd go 1.18 require ( + github.com/abhinav/goldmark-toc v0.2.1 + github.com/alecthomas/chroma v0.10.0 github.com/bahlo/generic-list-go v0.2.0 github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d github.com/docker/docker v20.10.17+incompatible github.com/docker/go-units v0.4.0 github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 + github.com/microcosm-cc/bluemonday v1.0.19 github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 github.com/opencontainers/image-spec v1.0.2 github.com/prometheus/client_golang v1.12.2 @@ -16,21 +19,27 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.0 github.com/thediveo/enumflag v0.10.1 + github.com/yuin/goldmark v1.4.13 + github.com/yuin/goldmark-highlighting v0.0.0-20220208100518-594be1970594 go.uber.org/zap v1.21.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect + github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dlclark/regexp2 v1.4.0 // indirect github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.6 // indirect + github.com/gorilla/css v1.0.0 // indirect github.com/kr/text v0.2.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/morikuni/aec v1.0.0 // indirect @@ -54,7 +63,6 @@ require ( golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.1.0 // indirect ) diff --git a/go.sum b/go.sum index fad6146..0e552d7 100644 --- a/go.sum +++ b/go.sum @@ -43,12 +43,18 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/abhinav/goldmark-toc v0.2.1 h1:QJsKKGbdVeCWYMB11hSkNuZLuIzls7Y4KBZfwTkBB90= +github.com/abhinav/goldmark-toc v0.2.1/go.mod h1:aq1IZ9qN85uFYpowec98iJrFkEHYT4oeFD1SC0qd8d0= +github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= +github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -60,7 +66,6 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d h1:pVrfxiGfwelyab6n21ZBkbkmbevaf+WvMIiR7sr97hw= github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= @@ -89,6 +94,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/digineo/afero v1.8.3-0.20220715171204-b7c01b3e0267 h1:yjyMHADZ7cyDMxzJdF09tVdowOTwfb+cys5WdtU/5k4= github.com/digineo/afero v1.8.3-0.20220715171204-b7c01b3e0267/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= +github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v20.10.17+incompatible h1:JYCuMrWaVNophQTOrMMoSwudOVEfcegoZZrleKc1xwE= @@ -189,6 +196,8 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= @@ -233,6 +242,8 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/microcosm-cc/bluemonday v1.0.19 h1:OI7hoF5FY4pFz2VA//RN8TfM0YJ2dJcl4P4APrCWy6c= +github.com/microcosm-cc/bluemonday v1.0.19/go.mod h1:QNzV2UbLK2/53oIIwTOyLUSABMkjZ4tqiyC1g/DyqxE= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= @@ -345,7 +356,13 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.3/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.5/go.mod h1:rmuwmfZ0+bvzB24eSC//bk1R1Zp3hM0OXYv/G2LIilg= +github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark-highlighting v0.0.0-20220208100518-594be1970594 h1:yHfZyN55+5dp1wG7wDKv8HQ044moxkyGq12KFFMFDxg= +github.com/yuin/goldmark-highlighting v0.0.0-20220208100518-594be1970594/go.mod h1:U9ihbh+1ZN7fR5Se3daSPoz1CGF9IYtSvWwVQtnzGHU= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= diff --git a/service/assets/texd.js b/service/assets/texd.js index e683444..8bad096 100644 --- a/service/assets/texd.js +++ b/service/assets/texd.js @@ -65,7 +65,7 @@ const app = Vue.createApp({ }, beforeMount() { - setInterval(this.fetchStatus, 1000) + setInterval(this.fetchStatus, 5000) this.fetchStatus() }, diff --git a/service/service.go b/service/service.go index 98f4869..7ee8478 100644 --- a/service/service.go +++ b/service/service.go @@ -9,6 +9,7 @@ import ( "net/http" "time" + "github.com/digineo/texd/docs" "github.com/digineo/texd/exec" "github.com/digineo/texd/metrics" "github.com/digineo/texd/refstore" @@ -88,6 +89,12 @@ func (svc *service) routes() http.Handler { r.HandleFunc("/", HandleUI).Methods(http.MethodGet) r.PathPrefix("/assets/").Handler(HandleAssets()).Methods(http.MethodGet) + if h, err := docs.Handler("/docs"); err != nil { + svc.Logger().Warn("documentation unavailable", zap.Error(err)) + } else { + r.PathPrefix("/docs").Handler(h).Methods(http.MethodGet) + } + render := http.Handler(http.HandlerFunc(svc.HandleRender)) if max := svc.maxJobSize; max > 0 { render = http.MaxBytesHandler(render, max) diff --git a/service/ui.go b/service/ui.go index c743828..82b7b08 100644 --- a/service/ui.go +++ b/service/ui.go @@ -3,6 +3,7 @@ package service import ( "bytes" "embed" + "io" "net/http" ) @@ -13,12 +14,11 @@ var uiHTML []byte var assets embed.FS func HandleUI(res http.ResponseWriter, req *http.Request) { - buf := bytes.NewBuffer(uiHTML) - res.Header().Set("Content-Type", mimeTypeHTML) res.Header().Set("X-Content-Type-Options", "nosniff") res.WriteHeader(http.StatusOK) - _, _ = buf.WriteTo(res) + + _, _ = io.Copy(res, bytes.NewReader(uiHTML)) } func HandleAssets() http.Handler { diff --git a/service/ui.html b/service/ui.html index 507a4ee..fa32787 100644 --- a/service/ui.html +++ b/service/ui.html @@ -13,9 +13,18 @@
-