From 09e0a9497d5a388cdfa8bb3a4915a55ba2560392 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sat, 17 Jul 2021 14:44:21 +1200 Subject: [PATCH 1/4] basic api docs --- .envrc | 1 + .readthedocs.yaml | 8 ++ docs/api-v2.md | 234 ++++++++++++++++++++++++++++++++++++++++++ docs/index.md | 3 + docs/requirements.txt | 1 + mkdocs.yml | 4 + shell.nix | 2 +- 7 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 .envrc create mode 100644 .readthedocs.yaml create mode 100644 docs/api-v2.md create mode 100644 docs/index.md create mode 100644 docs/requirements.txt create mode 100644 mkdocs.yml diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..17d6464 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use_nix \ No newline at end of file diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..e651ad5 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,8 @@ +version: 2 +mkdocs: + configuration: mkdocs.yml + +python: + version: 3.7 + install: + - requirements: docs/requirements.txt diff --git a/docs/api-v2.md b/docs/api-v2.md new file mode 100644 index 0000000..111b514 --- /dev/null +++ b/docs/api-v2.md @@ -0,0 +1,234 @@ +# API + +Piston exposes an API for managing packages and executing user-defined code. + +The API is broken in to 2 main sections - packages and jobs. + +The API is exposed from the container, by default on port 2000, at `/api/v2/`. + +All inputs are validated, and if an error occurs, a 4xx or 5xx status code is returned. +In this case, a JSON payload is sent back containing the error message as `message` + +## Runtimes + +### `GET /api/v2/runtimes` + +Returns a list of available languages, including the version, runtime and aliases. + +#### Response + +- `[].language`: Name of the language +- `[].version`: Version of the runtime +- `[].aliases`: List of alternative names that can be used for the language +- `[].runtime` (_optional_): Name of the runtime used to run the langage, only provided if alternative runtimes exist for the language + +#### Example + +``` +GET /api/v2/runtimes +``` + +```json +HTTP/1.1 200 OK +Content-Type: application/json + +[ + { + "language": "bash", + "version": "5.1.0", + "aliases": ["sh"] + }, + { + "language": "javascript", + "version": "15.10.0", + "aliases": ["node-javascript", "node-js", "javascript", "js"], + "runtime": "node" + } +] +``` + +## Execute + +### `POST /api/v2/execute` + +Runs the given code, using the given runtime and arguments, returning the result. + +#### Request + +- `language`: Name or alias of a language listed in [runtimes](#runtimes) +- `version`: SemVer version selector of a language listed in [runtimes](#runtimes) +- `files`: An array of files which should be uploaded into the job context +- `files[].name` (_optional_): Name of file to be written, if none a random name is picked +- `files[].content`: Content of file to be written +- `stdin` (_optional_): Text to pass into stdin of the program. Defaults to blank string. +- `args` (_optional_): Arguments to pass to the program. Defaults to none +- `run_timeout` (_optional_): The maximum allowed time in milliseconds for the compile stage to finish before bailing out. Must be a number, less than or equal to the configured maximum timeout. +- `compile_timeout` (_optional_): The maximum allowed time in milliseconds for the run stage to finish before bailing out. Must be a number, less than or equal to the configured maximum timeout. Defaults to maximum. +- `compile_memory_limit` (_optional_): The maximum amount of memory the compile stage is allowed to use in bytes. Must be a number, less than or equal to the configured maximum. Defaults to maximum, or `-1` (no limit) if none is configured. +- `run_memory_limit` (_optional_): The maximum amount of memory the run stage is allowed to use in bytes. Must be a number, less than or equal to the configured maximum. Defaults to maximum, or `-1` (no limit) if none is configured. + +#### Response + +- `language`: Name (not alias) of the runtime used +- `version`: Version of the used runtime +- `run`: Results from the run stage +- `run.stdout`: stdout from run stage process +- `run.stderr`: stderr from run stage process +- `run.output`: stdout and stderr combined in order of data from run stage process +- `run.code`: Exit code from run process, or null if signal is not null +- `run.signal`: Signal from run process, or null if code is not null +- `compile` (_optional_): Results from the compile stage, only provided if the runtime has a compile stage +- `compile.stdout`: stdout from compile stage process +- `compile.stderr`: stderr from compile stage process +- `compile.output`: stdout and stderr combined in order of data from compile stage process +- `compile.code`: Exit code from compile process, or null if signal is not null +- `compile.signal`: Signal from compile process, or null if code is not null + +#### Example + +```json +POST /api/v2/execute +Content-Type: application/json + +{ + "language": "js", + "version": "15.10.0", + "files": [ + { + "name": "my_cool_code.js", + "content": "console.log(process.argv)" + } + ], + "stdin": "", + "args": ["1", "2", "3"], + "compile_timeout": 10000, + "run_timeout": 3000, + "compile_memory_limit": -1, + "run_memory_limit": -1 +} +``` + +```json +HTTP/1.1 200 OK +Content-Type: application/json + +{ + "run": { + "stdout": "[\n '/piston/packages/node/15.10.0/bin/node',\n '/piston/jobs/e87afa0d-6c2a-40b8-a824-ffb9c5c6cb64/my_cool_code.js',\n '1',\n '2',\n '3'\n]\n", + "stderr": "", + "code": 0, + "signal": null, + "output": "[\n '/piston/packages/node/15.10.0/bin/node',\n '/piston/jobs/e87afa0d-6c2a-40b8-a824-ffb9c5c6cb64/my_cool_code.js',\n '1',\n '2',\n '3'\n]\n" + }, + "language": "javascript", + "version": "15.10.0" +} +``` + +## Packages + +### `GET /api/v2/packages` + +Returns a list of all possible packages, and whether their installation status. + +#### Response + +- `[].language`: Name of the contained runtime +- `[].language_version`: Version of the contained runtime +- `[].installed`: Status on the package being installed + +#### Example + +``` +GET /api/v2/packages +``` + +```json +HTTP/1.1 200 OK +Content-Type: application/json + +[ + { + "language": "node", + "language_version": "15.10.0", + "installed": true + }, + { + "language": "bash", + "language_version": "5.1.0", + "installed": true + } +] +``` + +### `POST /api/v2/packages` + +Install the given package. + +#### Request + +- `language`: Name of package from [package list](#get-apiv2packages) +- `version`: SemVer version selector for package from [package list](#get-apiv2packages) + +#### Response + +- `language`: Name of package installed +- `version`: Version of package installed + +#### Example + +```json +POST /api/v2/packages +Content-Type: application/json + +{ + "language": "bash", + "version": "5.x" +} +``` + +```json +HTTP/1.1 200 OK +Content-Type: application/json + +{ + "language": "bash", + "version": "5.1.0" +} +``` + +### `DELETE /api/v2/packages` + +Uninstall the given package. + +#### Request + +- `language`: Name of package from [package list](#get-apiv2packages) +- `version`: SemVer version selector for package from [package list](#get-apiv2packages) + +#### Response + +- `language`: Name of package uninstalled +- `version`: Version of package uninstalled + +#### Example + +```json +DELETE /api/v2/packages +Content-Type: application/json + +{ + "language": "bash", + "version": "5.x" +} +``` + +```json +HTTP/1.1 200 OK +Content-Type: application/json + +{ + "language": "bash", + "version": "5.1.0" +} +``` diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..a8e5891 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,3 @@ +# Piston + +These docs are a WIP diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..53dbf05 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1 @@ +mkdocs==1.1.2 \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..a9a86fe --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,4 @@ +site_name: Piston Documentation +nav: + - Home: index.md + - API: api-v2.md diff --git a/shell.nix b/shell.nix index 47b2d12..339bead 100644 --- a/shell.nix +++ b/shell.nix @@ -1,5 +1,5 @@ { pkgs ? import {} }: pkgs.mkShell { # nativeBuildInputs is usually what you want -- tools you need to run - nativeBuildInputs = with pkgs; [ nodejs-15_x jq ]; + nativeBuildInputs = with pkgs; [ nodejs-15_x jq mkdocs ]; } From 33ba39cbc7073e39b21fe3ef775f95fa944da002 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sat, 17 Jul 2021 14:46:40 +1200 Subject: [PATCH 2/4] docs: rtd theme --- mkdocs.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index a9a86fe..2030962 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,4 +1,7 @@ -site_name: Piston Documentation +site_name: Piston nav: - Home: index.md - API: api-v2.md + +theme: + name: readthedocs From acb590bf39786b24f90c92dca8959e6e070440bc Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sat, 17 Jul 2021 14:49:02 +1200 Subject: [PATCH 3/4] readme: add docs link --- readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 193d90b..01f3a2a 100644 --- a/readme.md +++ b/readme.md @@ -33,7 +33,8 @@ Supported Languages • Principles • Security • - License + License • + Documentation --- @@ -69,7 +70,6 @@ The following are approved and endorsed extensions/utilities to the core Piston - [Piston4J](https://github.com/the-codeboy/Piston4J), a Java wrapper for accessing the Piston API. - [Pyston](https://github.com/ffaanngg/pyston), a Python wrapper for accessing the Piston API. -
# Public API From 968390d5b69973ebdd56a35bb545fc864c53a586 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Sat, 17 Jul 2021 15:35:43 +1200 Subject: [PATCH 4/4] docs: configuration options --- docs/configuration.md | 147 ++++++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 8 +++ 2 files changed, 155 insertions(+) create mode 100644 docs/configuration.md diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..1388e9d --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,147 @@ +# Configuration + +Piston provides many different configuration options to tweak Piston to meet your needs. + +Configuration is specified through environment variables, prefixed with `PISTON_`. + +## Log Level + +```yaml +key: PISTON_LOG_LEVEL +default: INFO +``` + +Level of log output to provide. + +One of `DEBUG`, `INFO`, `WARN`, `ERROR` or `NONE` + +## Bind Address + +```yaml +key: PISTON_BIND_ADDRESS +default: 0.0.0.0:2000 +``` + +Port and IP address to bind the Piston API to. + + +!!! warning + Changing this value is not recommended. + + This changes the bind address inside the container, and thus serves no purpose when running in a container + +## Data Directory + +```yaml +key: PISTON_DATA_DIRECTORY +default: /piston +``` + +Absolute path to piston related data, including packages and job contexts. + + +!!! warning + Changing this value is not recommended. + + Some packages require absolute paths on disk at build time. + Due to this, some packages may break when changing this parameter. + +## Runner GID/UID range + +```yaml +key: + - PISTON_RUNNER_UID_MIN + - PISTON_RUNNER_UID_MAX + - PISTON_RUNNER_GID_MIN + - PISTON_RUNNER_GID_MAX +default: + - 1001 + - 1500 + - 1001 + - 1500 +``` + +UID and GID ranges to use when executing jobs. + + +!!! warning + Changing this value is not recommended. + + The piston container creates 500 users and groups by default, and reserves user/group 1000 for running the API. + Any processes run by these users will be killed when cleaning up a job. + +## Disable Networking + +```yaml +key: PISTON_DISABLE_NETWORKING +default: true +``` + +Disallows access to `socket` syscalls, effectively disabling networking for jobs run by piston. + +## Max Process Count + +```yaml +key: PISTON_MAX_PROCESS_COUNT +default: 64 +``` + +Maximum number of processess allowed to to have open for a job. + +Resists against exhausting the process table, causing a full system lockup. + +## Output Max Side + +```yaml +key: PISTON_OUTPUT_MAX_SIZE +default: 1024 +``` + +Maximum size of stdio buffers for each job. + +Resist against run-away output which could lead to memory exhaustion. + +## Max Open Files + +```yaml +key: PISTON_MAX_OPEN_FILES +default: 64 +``` + +Maximum number of open files at a given time by a job. + +Resists against writing many smaller files to exhaust inodes. + +## Max File Size + +```yaml +key: PISTON_MAX_FILE_SIZE +default: 10000000 #10MB +``` + +Maximum size for a singular file written to disk. + +Resists against large file writes to exhaust disk space. + +## Compile/Run memory limits + +```yaml +key: + - PISTON_COMPILE_MEMORY_LIMIT + - PISTON_RUN_MEMORY_LIMIT +default: -1 +``` + +Maximum memory allowed by a stage in bytes. +Use -1 for unlimited memory usage. + +Useful for running memory-limited contests. + +## Repository URL + +```yaml +key: PISTON_REPO_URL +default: https://github.com/engineer-man/piston/releases/download/pkgs/index +``` + +URL for repository index, where packages will be downloaded from. diff --git a/mkdocs.yml b/mkdocs.yml index 2030962..148ba91 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,7 +1,15 @@ site_name: Piston nav: - Home: index.md + - Configuration: configuration.md - API: api-v2.md theme: name: readthedocs + highlightjs: true + hljs_languages: + - yaml + - json + +markdown_extensions: + - admonition