diff --git a/packages/brainfuck/2.7.3/build.sh b/packages/brainfuck/2.7.3/build.sh index 50c1868..796b573 100644 --- a/packages/brainfuck/2.7.3/build.sh +++ b/packages/brainfuck/2.7.3/build.sh @@ -6,9 +6,6 @@ PREFIX=$(realpath $(dirname $0)) git clone https://github.com/fabianishere/brainfuck.git cd brainfuck -git checkout 06f84462e0a96487670a4b8024e3ec531e0377ee - - mkdir -p build cd build diff --git a/packages/clojure/1.10.3/build.sh b/packages/clojure/1.10.3/build.sh deleted file mode 100755 index 4633daf..0000000 --- a/packages/clojure/1.10.3/build.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -# Installation location -PREFIX=$(realpath $(dirname $0)) - -# Clojure depends on Java (build and runtime) -mkdir -p java -cd java -curl "https://download.java.net/java/GA/jdk15.0.2/0d1cfde4252546c6931946de8db48ee2/7/GPL/openjdk-15.0.2_linux-x64_bin.tar.gz" -o java.tar.gz -tar xzf java.tar.gz --strip-components=1 -rm java.tar.gz -cd .. - -# Clojure depends on Maven (build) -mkdir -p maven -cd maven -curl "https://apache.claz.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz" -o maven.tar.gz -tar xzf maven.tar.gz --strip-components=1 -rm maven.tar.gz -cd .. - -# Adding java and maven to the path for building -export PATH=$PWD/java/bin:$PWD/maven/bin:$PATH -export JAVA_HOME=$PWD/java - -# Clojure download -mkdir -p build -cd build -git clone -q "https://github.com/clojure/clojure.git" . -git checkout -b clojure-1.10.3 aaf73b12467df80f5db3e086550a33fee0e1b39e # commit for 1.10.3 release - -# Build using maven -mvn -Plocal -Dmaven.test.skip=true package - -# Get ridda that m2 bloat from Maven and remove Maven itself -cd ../ -rm -rf ~/.m2 -rm -rf maven/ - -# Move the jar for easier reference and cleanup -mkdir -p bin -mv build/clojure.jar bin -rm -rf build diff --git a/packages/clojure/1.10.3/environment b/packages/clojure/1.10.3/environment deleted file mode 100644 index e7ef3b1..0000000 --- a/packages/clojure/1.10.3/environment +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -# Clojure requires JAVA_HOME to be set and java binary to be in the path -export JAVA_HOME=$PWD/java -export CLOJURE_PATH=$PWD/bin -export PATH=$PWD/java/bin:$PATH diff --git a/packages/clojure/1.10.3/metadata.json b/packages/clojure/1.10.3/metadata.json deleted file mode 100644 index 64f83fa..0000000 --- a/packages/clojure/1.10.3/metadata.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "language": "clojure", - "version": "1.10.3", - "aliases": ["clojure","clj"], - "author": "Dan Vargas <danvargas46@gmail.com>" -} diff --git a/packages/clojure/1.10.3/run b/packages/clojure/1.10.3/run deleted file mode 100644 index a8901af..0000000 --- a/packages/clojure/1.10.3/run +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -# Run clojure with Java referencing the clojure jar location -java -jar $CLOJURE_PATH/clojure.jar "$@" diff --git a/packages/clojure/1.10.3/test.clj b/packages/clojure/1.10.3/test.clj deleted file mode 100644 index 7f50202..0000000 --- a/packages/clojure/1.10.3/test.clj +++ /dev/null @@ -1,5 +0,0 @@ -(ns clojure.examples.main - (:gen-class)) -(defn main [] - (println "OK")) -(main) \ No newline at end of file diff --git a/packages/dotnet/5.0.201/build.sh b/packages/dotnet/5.0.201/build.sh deleted file mode 100644 index c685668..0000000 --- a/packages/dotnet/5.0.201/build.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -curl "https://download.visualstudio.microsoft.com/download/pr/73a9cb2a-1acd-4d20-b864-d12797ca3d40/075dbe1dc3bba4aa85ca420167b861b6/dotnet-sdk-5.0.201-linux-x64.tar.gz" -Lo dotnet.tar.gz -tar xzf dotnet.tar.gz --strip-components=1 -rm dotnet.tar.gz - -# Cache nuget packages -export DOTNET_CLI_HOME=$PWD -./dotnet new console -o cache_application -# This calls a restore on the global-packages index ($DOTNET_CLI_HOME/.nuget/packages) -# If we want to allow more packages, we could add them to this cache_application - -rm -rf cache_application -# Get rid of it, we don't actually need the application - just the restore \ No newline at end of file diff --git a/packages/dotnet/5.0.201/compile b/packages/dotnet/5.0.201/compile deleted file mode 100644 index 1500acb..0000000 --- a/packages/dotnet/5.0.201/compile +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -export DOTNET_CLI_HOME=$PWD -export HOME=$PWD - -dotnet build --help > /dev/null # Shut the thing up - -dotnet new console -o . --no-restore -rm Program.cs - -dotnet restore --source $DOTNET_ROOT/.nuget/packages -dotnet build --no-restore \ No newline at end of file diff --git a/packages/dotnet/5.0.201/environment b/packages/dotnet/5.0.201/environment deleted file mode 100644 index 596d56e..0000000 --- a/packages/dotnet/5.0.201/environment +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -# Put 'export' statements here for environment variables -export DOTNET_ROOT=$PWD -export PATH=$DOTNET_ROOT:$PATH \ No newline at end of file diff --git a/packages/dotnet/5.0.201/metadata.json b/packages/dotnet/5.0.201/metadata.json deleted file mode 100644 index 72f2e79..0000000 --- a/packages/dotnet/5.0.201/metadata.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "language": "dotnet", - "version": "5.0.201", - "aliases": ["cs", "csharp"], - "author": "Thomas Hobson <git@hexf.me>" -} diff --git a/packages/dotnet/5.0.201/run b/packages/dotnet/5.0.201/run deleted file mode 100644 index 774a08a..0000000 --- a/packages/dotnet/5.0.201/run +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -# Put instructions to run the runtime -export DOTNET_CLI_HOME=$PWD - -shift -dotnet bin/Debug/net5.0/$(basename $(realpath .)).dll "$@" \ No newline at end of file diff --git a/packages/dotnet/5.0.201/test.cs b/packages/dotnet/5.0.201/test.cs deleted file mode 100644 index e91a2a2..0000000 --- a/packages/dotnet/5.0.201/test.cs +++ /dev/null @@ -1,9 +0,0 @@ -using System; - -public class Test -{ - public static void Main(string[] args) - { - Console.WriteLine("OK"); - } -} \ No newline at end of file diff --git a/packages/python/2.7.18/build.sh b/packages/python/2.7.18/build.sh index 131ba96..1cc8e23 100755 --- a/packages/python/2.7.18/build.sh +++ b/packages/python/2.7.18/build.sh @@ -17,7 +17,5 @@ make install -j$(nproc) cd .. rm -rf build -bin/pip2 install -U pip==20.3.* -# Upgrade pip to latest supported version -bin/pip2 install numpy scipy pycrypto whoosh bcrypt passlib + diff --git a/packages/python/3.5.10/build.sh b/packages/python/3.5.10/build.sh index bced203..445f96a 100755 --- a/packages/python/3.5.10/build.sh +++ b/packages/python/3.5.10/build.sh @@ -18,4 +18,4 @@ cd .. rm -rf build -bin/pip3 install numpy scipy pandas pycrypto whoosh bcrypt passlib + diff --git a/packages/python/3.9.1/build.sh b/packages/python/3.9.1/build.sh index fcae7c7..ec2a2d5 100755 --- a/packages/python/3.9.1/build.sh +++ b/packages/python/3.9.1/build.sh @@ -18,4 +18,4 @@ cd .. rm -rf build -bin/pip3 install numpy scipy pandas pycrypto whoosh bcrypt passlib + diff --git a/packages/vlang/0.1.13/build.sh b/packages/vlang/0.1.13/build.sh deleted file mode 100755 index 127bac9..0000000 --- a/packages/vlang/0.1.13/build.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -# Cloning vlang source -git clone https://github.com/vlang/v -cd v - -# Building and installing vlang -make \ No newline at end of file diff --git a/packages/vlang/0.1.13/environment b/packages/vlang/0.1.13/environment deleted file mode 100644 index 638b0c2..0000000 --- a/packages/vlang/0.1.13/environment +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -# Put 'export' statements here for environment variables -export PATH="$PWD/v:$PATH" \ No newline at end of file diff --git a/packages/vlang/0.1.13/metadata.json b/packages/vlang/0.1.13/metadata.json deleted file mode 100644 index 855f7bb..0000000 --- a/packages/vlang/0.1.13/metadata.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "language": "vlang", - "version": "0.1.13", - "author": "Shivansh-007 <shivansh-007@outlook.com>", - "aliases": ["v"] -} diff --git a/packages/vlang/0.1.13/run b/packages/vlang/0.1.13/run deleted file mode 100644 index d8fa7b2..0000000 --- a/packages/vlang/0.1.13/run +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -# Export environment variables -export VMODULES="$PWD" -export TMPDIR="$PWD" - -# Put instructions to run the runtime -v run "$@" \ No newline at end of file diff --git a/packages/vlang/0.1.13/test.v b/packages/vlang/0.1.13/test.v deleted file mode 100644 index fb5de5e..0000000 --- a/packages/vlang/0.1.13/test.v +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - println('OK') -} \ No newline at end of file diff --git a/readme.md b/readme.md index 9d1fb11..e671b93 100644 --- a/readme.md +++ b/readme.md @@ -83,162 +83,111 @@ so we can discuss potentially getting you an unlimited key. # Getting Started -## All In One - ### Host System Package Dependencies - Docker - Docker Compose - Node JS -### After system dependencies are installed, clone this repository: +#### After system dependencies are installed, clone this repository: ```sh # clone and enter repo git clone https://github.com/engineer-man/piston ``` -### Installation +#### Installation -```sh -docker-compose up -d piston_api -# Start the API container - -npm install -g yarn -cd cli && yarn && cd - -# Install all the dependencies for the cli -``` - -## Just Piston (no CLI) - -### Host System Package Dependencies - -- Docker - -### Installation - -```sh -echo "$GITHUB_TOKEN" | docker login https://docker.pkg.github.com -u "$GITHUB_USERNAME" --password-stdin -# Change out the $GITHUB_TOKEN and $GITHUB_USERNAME with appropritate values - -docker run -v $PWD:'/piston' --tmpfs /piston/jobs -dit -p 6969:6969 --privileged --name piston_api docker.pkg.github.com/engineer-man/piston/api:latest -``` +- docker-compose up +#### CLI Usage +- `cli/execute [language] [file path] [args]` <br> # Usage ### CLI -The CLI is the main tool used for installing packages within piston, but also supports running code. - -You can execute the cli with `cli/index.js`. - ```sh -# List all available packages -cli/index.js ppman list - -# Install python 3.9.1 -cli/index.js ppman install python 3.9.1 - -# Run a python script -echo 'print("Hello world!")' > test.py -cli/index.js run python 3.9.1 test.py -``` - -If you are operating on a remote machine, add the `-u` flag like so: - -```sh -cli/index.js -u http://piston.server:6969 ppman list +lxc/execute [language] [file path] [args] ``` ### API +To use the API, it must first be started. Please note that if root is required to access +LXC then the API must also be running as root. To start the API, run the following: -The container exposes an API on port 6969 by default. -This is used by the CLI to carry out running jobs and package managment. +``` +cd api +./start +``` -#### Runtimes Endpoint -`GET /runtimes` -This endpoint will return the supported languages along with the current version, author and aliases. To execute -code for a particular language using the `/jobs` endpoint, either the name or one of the aliases must -be provided, along with the version. -Multiple versions of the same language may be present at the same time, and may be selected when running a job. +For your own local installation, the API is available at: + +``` +http://127.0.0.1:2000 +``` + +#### Versions Endpoint +`GET /versions` +This endpoint will return the supported languages along with the current version and aliases. To execute +code for a particular language using the `/execute` endpoint, either the name or one of the aliases must +be provided. ```json HTTP/1.1 200 OK Content-Type: application/json [ - { - "language": "bash", - "version": "5.1.0", - "author": "Thomas Hobson <git@hexf.me>", - "aliases": [ - "sh" - ] - }, - { - "language": "brainfuck", - "version": "2.7.3", - "author": "Thomas Hobson <git@hexf.me>", - "aliases": [ - "bf" - ] - }, - ... + { + "name": "awk", + "aliases": ["awk"], + "version": "1.3.3" + }, + { + "name": "bash", + "aliases": ["bash"], + "version": "4.4.20" + }, + { + "name": "c", + "aliases": ["c"], + "version": "7.5.0" + } ] ``` #### Execute Endpoint -`POST /jobs` +`POST /execute` This endpoint requests execution of some arbitrary code. -- `language` (**required**) The language to use for execution, must be a string and must be installed. -- `version` (**required**) The version of the language to use for execution, must be a string containing a SemVer selector for the version or the specific version number to use. -- `files` (**required**) An array of files containing code or other data that should be used for execution. -- `files[].name` (**required**) The name of the file to upload, must be a string containing no path. -- `files[].content` (**required**) The content of the files to upload, must be a string containing text to write. -- `main` (**required**) The name of one of the files provided that should be considered the main source file which will be used as the entrypoint, must be a string and be the name of a file in `files`. -- `stdin` (**required**) The text to pass as stdin to the program. Must be a string, can be left blank. -- `args` (**required**) The arguments to pass to the program. Must be an array. -- `compile_timeout` (**required**) The maximum time allowed for the compile stage to finish before bailing out in milliseconds. Must be a number. -- `run_timeout` (**required**) The maximum time allowed for the run stage to finish before bailing out in milliseconds. Must be a number. - +- `language` (**required**) The language to use for execution, must be a string and supported by Piston (see list below). +- `source` (**required**) The source code to execute, must be a string. +- `stdin` (*optional*) The text to pass as stdin to the program. Must be a string or left out of the request. +- `args` (*optional*) The arguments to pass to the program. Must be an array or left out of the request. ```json { "language": "js", - "version": "15.10.0", - "files":[ - { - "name": "my_cool_code.js", - "content": "console.log(process.argv)" - } - ], - "main": "my_cool_code.js", + "source": "console.log(process.argv)", "stdin": "", "args": [ "1", "2", "3" - ], - "compile_timeout": 10000, - "run_timeout": 3000 + ] } ``` -A typical response upon successful execution will contain 1 or 2 keys `run` and `compile`. -`compile` will only be present if the language requested requires a compile stage. - -Each of these keys has an identical structure, containing both a `stdout` and `stderr` key, which is a string containing the text outputted during the stage into each buffer. -It also contains the `code` and `signal` which was returned from each process. +A typical response upon successful execution will contain the `language`, `version`, `output` which +is a combination of both `stdout` and `stderr` but in chronological order according to program output, +as well as separate `stdout` and `stderr`. ```json HTTP/1.1 200 OK Content-Type: application/json { - "run": { - "stdout": "[\n '/piston/packages/node/15.10.0/bin/node',\n '/piston/jobs/9501b09d-0105-496b-b61a-e5148cf66384/my_cool_code.js',\n '1',\n '2',\n '3'\n]\n", - "stderr": "", - "code": 0, - "signal": null - } + "ran": true, + "language": "js", + "version": "12.13.0", + "output": "[ '/usr/bin/node',\n '/tmp/code.code',\n '1',\n '2',\n '3' ]", + "stdout": "[ '/usr/bin/node',\n '/tmp/code.code',\n '1',\n '2',\n '3' ]", + "stderr": "" } ``` If a problem exists with the request, a `400` status code is returned and the reason in the `message` key. @@ -247,35 +196,43 @@ HTTP/1.1 400 Bad Request Content-Type: application/json { - "message": "html-5.0.0 runtime is unknown" + "message": "Supplied language is not supported by Piston" } ``` <br> -# Principle of Operation +# Supported Languages -Piston uses Docker as the primary mechanism for sandboxing. There is an API within the container written in Node -which takes in execution requests and executees them within the container safely. -High level, the API writes any source code to a temporary directory in `/piston/jobs`. +`python`,`php`,`node` + + +<br> +<!-- +# Principle of Operation +Piston utilizes LXC as the primary mechanism for sandboxing. There is a small API written in Node which takes +in execution requests and executes them in the container. High level, the API writes +a temporary source and args file to `/tmp` and that gets mounted read-only along with the execution scripts into the container. The source file is either ran or compiled and ran (in the case of languages like c, c++, c#, go, etc.). <br> - +<!-- # Security -Docker provides a great deal of security out of the box in that it's separate from the system. +LXC provides a great deal of security out of the box in that it's separate from the system. Piston takes additional steps to make it resistant to various privilege escalation, denial-of-service, and resource saturation threats. These steps include: - Disabling outgoing network interaction -- Capping max processes at 256 by default (resists `:(){ :|: &}:;`, `while True: os.fork()`, etc.) +- Capping max processes at 64 (resists `:(){ :|: &}:;`, `while True: os.fork()`, etc.) - Capping max files at 2048 (resists various file based attacks) +- Mounting all resources read-only (resists `sudo rm -rf --no-preserve-root /`) - Cleaning up all temp space after each execution (resists out of drive space attacks) - Running as a variety of unprivileged users - Capping runtime execution at 3 seconds - Capping stdout to 65536 characters (resists yes/no bombs and runaway output) - SIGKILLing misbehaving code - +--> <br> +<!-- Someone please do this --> # License Piston is licensed under the MIT license. diff --git a/repo/Dockerfile b/repo/Dockerfile index a4fcf0a..be968fa 100644 --- a/repo/Dockerfile +++ b/repo/Dockerfile @@ -1,6 +1,6 @@ FROM debian:buster-slim -RUN apt-get update && apt-get install -y unzip autoconf build-essential libssl-dev pkg-config zlib1g-dev libargon2-dev libsodium-dev libcurl4-openssl-dev sqlite3 libsqlite3-dev libonig-dev libxml2 libxml2-dev bc curl git linux-headers-amd64 perl xz-utils python3 python3-pip gnupg jq zlib1g-dev cmake cmake-doc extra-cmake-modules build-essential gcc binutils bash coreutils util-linux pciutils usbutils coreutils binutils findutils grep libncurses5-dev libncursesw5-dev python3-pip libgmp-dev libmpfr-dev python2 && \ +RUN apt-get update && apt-get install -y unzip autoconf build-essential libssl-dev pkg-config zlib1g-dev libargon2-dev libsodium-dev libcurl4-openssl-dev sqlite3 libsqlite3-dev libonig-dev libxml2 libxml2-dev bc curl git linux-headers-amd64 perl xz-utils python3 python3-pip gnupg jq zlib1g-dev cmake cmake-doc extra-cmake-modules build-essential gcc binutils bash coreutils util-linux pciutils usbutils coreutils binutils findutils grep libncurses5-dev libncursesw5-dev python3-pip && \ ln -sf /bin/bash /bin/sh && \ rm -rf /var/lib/apt/lists/*