readme: add cli dependencies

* pkg(vlang-0.1.13): Add vlang 0.1.13

* Export environment variables: VMODULES and TMPDIR environment vars to the current working dir in the run script.

Co-authored-by: Shivansh-007 <Shivansh-007@users.noreply.github.com>

packages/brainfuck/2.7.3/build.sh

@@ -6,6 +6,9 @@ PREFIX=$(realpath $(dirname $0))
 git clone https://github.com/fabianishere/brainfuck.git
 
 cd brainfuck
+git checkout 06f84462e0a96487670a4b8024e3ec531e0377ee
+
+
 mkdir -p build
 
 cd build

packages/clojure/1.10.3/build.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# Installation location
+PREFIX=$(realpath $(dirname $0))
+
+# Clojure depends on Java (build and runtime)
+mkdir -p java
+cd java
+curl "https://download.java.net/java/GA/jdk15.0.2/0d1cfde4252546c6931946de8db48ee2/7/GPL/openjdk-15.0.2_linux-x64_bin.tar.gz" -o java.tar.gz
+tar xzf java.tar.gz --strip-components=1
+rm java.tar.gz
+cd ..
+
+# Clojure depends on Maven (build)
+mkdir -p maven
+cd maven
+curl "https://apache.claz.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz" -o maven.tar.gz
+tar xzf maven.tar.gz --strip-components=1
+rm maven.tar.gz
+cd ..
+
+# Adding java and maven to the path for building
+export PATH=$PWD/java/bin:$PWD/maven/bin:$PATH
+export JAVA_HOME=$PWD/java
+
+# Clojure download
+mkdir -p build
+cd build
+git clone -q "https://github.com/clojure/clojure.git" .
+git checkout -b clojure-1.10.3 aaf73b12467df80f5db3e086550a33fee0e1b39e # commit for 1.10.3 release
+
+# Build using maven
+mvn -Plocal -Dmaven.test.skip=true package
+
+# Get ridda that m2 bloat from Maven and remove Maven itself
+cd ../
+rm -rf ~/.m2
+rm -rf maven/
+
+# Move the jar for easier reference and cleanup
+mkdir -p bin
+mv build/clojure.jar bin
+rm -rf build

packages/clojure/1.10.3/environment

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# Clojure requires JAVA_HOME to be set and java binary to be in the path
+export JAVA_HOME=$PWD/java
+export CLOJURE_PATH=$PWD/bin
+export PATH=$PWD/java/bin:$PATH

packages/clojure/1.10.3/metadata.json

@@ -0,0 +1,6 @@
+{
+  "language": "clojure",
+  "version": "1.10.3",
+  "aliases": ["clojure","clj"],
+  "author": "Dan Vargas <danvargas46@gmail.com>"
+}

packages/clojure/1.10.3/run

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+# Run clojure with Java referencing the clojure jar location
+java -jar $CLOJURE_PATH/clojure.jar "$@"

packages/clojure/1.10.3/test.clj

@@ -0,0 +1,5 @@
+(ns clojure.examples.main
+   (:gen-class))
+(defn main []
+   (println "OK"))
+(main)

packages/dotnet/5.0.201/build.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+curl "https://download.visualstudio.microsoft.com/download/pr/73a9cb2a-1acd-4d20-b864-d12797ca3d40/075dbe1dc3bba4aa85ca420167b861b6/dotnet-sdk-5.0.201-linux-x64.tar.gz" -Lo dotnet.tar.gz
+tar xzf dotnet.tar.gz --strip-components=1
+rm dotnet.tar.gz
+
+# Cache nuget packages
+export DOTNET_CLI_HOME=$PWD
+./dotnet new console -o cache_application
+# This calls a restore on the global-packages index ($DOTNET_CLI_HOME/.nuget/packages)
+# If we want to allow more packages, we could add them to this cache_application
+
+rm -rf cache_application
+# Get rid of it, we don't actually need the application - just the restore

packages/dotnet/5.0.201/compile

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+export DOTNET_CLI_HOME=$PWD
+export HOME=$PWD
+
+dotnet build --help > /dev/null # Shut the thing up
+
+dotnet new console -o . --no-restore
+rm Program.cs
+
+dotnet restore --source $DOTNET_ROOT/.nuget/packages
+dotnet build --no-restore

packages/dotnet/5.0.201/environment

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+# Put 'export' statements here for environment variables
+export DOTNET_ROOT=$PWD
+export PATH=$DOTNET_ROOT:$PATH

packages/dotnet/5.0.201/metadata.json

@@ -0,0 +1,6 @@
+{
+  "language": "dotnet",
+  "version": "5.0.201",
+  "aliases": ["cs", "csharp"],
+  "author": "Thomas Hobson <git@hexf.me>"
+}

packages/dotnet/5.0.201/run

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+# Put instructions to run the runtime
+export DOTNET_CLI_HOME=$PWD
+
+shift
+dotnet bin/Debug/net5.0/$(basename $(realpath .)).dll "$@"

packages/dotnet/5.0.201/test.cs

@@ -0,0 +1,9 @@
+using System;
+
+public class Test
+{
+    public static void Main(string[] args)
+    {
+        Console.WriteLine("OK");
+    }
+}

packages/python/2.7.18/build.sh

@@ -17,5 +17,7 @@ make install -j$(nproc)
 cd ..
 
 rm -rf build 
+bin/pip2 install -U pip==20.3.*
+# Upgrade pip to latest supported version
 
-
+bin/pip2 install numpy scipy pycrypto whoosh bcrypt passlib

packages/python/3.5.10/build.sh

@@ -18,4 +18,4 @@ cd ..
 
 rm -rf build 
 
-
+bin/pip3 install numpy scipy pandas pycrypto whoosh bcrypt passlib

packages/python/3.9.1/build.sh

@@ -18,4 +18,4 @@ cd ..
 
 rm -rf build 
 
-
+bin/pip3 install numpy scipy pandas pycrypto whoosh bcrypt passlib

packages/vlang/0.1.13/build.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+# Cloning vlang source
+git clone https://github.com/vlang/v
+cd v
+
+# Building and installing vlang
+make

packages/vlang/0.1.13/environment

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+# Put 'export' statements here for environment variables
+export PATH="$PWD/v:$PATH"

packages/vlang/0.1.13/metadata.json

@@ -0,0 +1,6 @@
+{
+    "language": "vlang",
+    "version": "0.1.13",
+    "author": "Shivansh-007 <shivansh-007@outlook.com>",
+    "aliases": ["v"]
+}

packages/vlang/0.1.13/run

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+# Export environment variables
+export VMODULES="$PWD"
+export TMPDIR="$PWD"
+
+# Put instructions to run the runtime
+v run "$@"

packages/vlang/0.1.13/test.v

@@ -0,0 +1,3 @@
+fn main() {
+  println('OK')
+}

readme.md

@@ -83,111 +83,162 @@ so we can discuss potentially getting you an unlimited key.
 
 # Getting Started
 
+## All In One
+
 ### Host System Package Dependencies
 
 - Docker
 - Docker Compose
 - Node JS
 
-#### After system dependencies are installed, clone this repository:
+### After system dependencies are installed, clone this repository:
 
 ```sh
 # clone and enter repo
 git clone https://github.com/engineer-man/piston
 ```
 
-#### Installation
+### Installation
 
-- docker-compose up
+```sh
+docker-compose up -d piston_api
+# Start the API container
+
+npm install -g yarn
+cd cli && yarn && cd -
+# Install all the dependencies for the cli
+```
+
+## Just Piston (no CLI)
+
+### Host System Package Dependencies
+
+- Docker
+
+### Installation
+
+```sh
+echo "$GITHUB_TOKEN" | docker login https://docker.pkg.github.com -u "$GITHUB_USERNAME" --password-stdin
+# Change out the $GITHUB_TOKEN and $GITHUB_USERNAME with appropritate values
+
+docker run -v $PWD:'/piston' --tmpfs /piston/jobs -dit -p 6969:6969 --privileged --name piston_api docker.pkg.github.com/engineer-man/piston/api:latest
+```
 
-#### CLI Usage
-- `cli/execute [language] [file path] [args]`
 <br>
 
 # Usage
 
 ### CLI
 
+The CLI is the main tool used for installing packages within piston, but also supports running code.
+
+You can execute the cli with `cli/index.js`.
+
 ```sh
-lxc/execute [language] [file path] [args]
+# List all available packages
+cli/index.js ppman list
+
+# Install python 3.9.1
+cli/index.js ppman install python 3.9.1
+
+# Run a python script
+echo 'print("Hello world!")' > test.py
+cli/index.js run python 3.9.1 test.py
+```
+
+If you are operating on a remote machine, add the `-u` flag like so:
+
+```sh
+cli/index.js -u http://piston.server:6969 ppman list
 ```
 
 ### API
-To use the API, it must first be started. Please note that if root is required to access
-LXC then the API must also be running as root. To start the API, run the following:
 
-```
-cd api
-./start
-```
+The container exposes an API on port 6969 by default.
+This is used by the CLI to carry out running jobs and package managment.
 
-For your own local installation, the API is available at:
-
-```
-http://127.0.0.1:2000
-```
-
-#### Versions Endpoint
-`GET /versions`
-This endpoint will return the supported languages along with the current version and aliases. To execute
-code for a particular language using the `/execute` endpoint, either the name or one of the aliases must
-be provided.
+#### Runtimes Endpoint
+`GET /runtimes`
+This endpoint will return the supported languages along with the current version, author and aliases. To execute
+code for a particular language using the `/jobs` endpoint, either the name or one of the aliases must
+be provided, along with the version.
+Multiple versions of the same language may be present at the same time, and may be selected when running a job.
 ```json
 HTTP/1.1 200 OK
 Content-Type: application/json
 
 [
-    {
-        "name": "awk",
-        "aliases": ["awk"],
-        "version": "1.3.3"
-    },
-    {
-        "name": "bash",
-        "aliases": ["bash"],
-        "version": "4.4.20"
-    },
-    {
-        "name": "c",
-        "aliases": ["c"],
-        "version": "7.5.0"
-    }
+  {
+    "language": "bash",
+    "version": "5.1.0",
+    "author": "Thomas Hobson <git@hexf.me>",
+    "aliases": [
+      "sh"
+    ]
+  },
+  {
+    "language": "brainfuck",
+    "version": "2.7.3",
+    "author": "Thomas Hobson <git@hexf.me>",
+    "aliases": [
+      "bf"
+    ]
+  },
+  ...
 ]
 ```
 
 #### Execute Endpoint
-`POST /execute`
+`POST /jobs`
 This endpoint requests execution of some arbitrary code.
-- `language` (**required**) The language to use for execution, must be a string and supported by Piston (see list below).
-- `source` (**required**) The source code to execute, must be a string.
-- `stdin` (*optional*) The text to pass as stdin to the program. Must be a string or left out of the request.
-- `args` (*optional*) The arguments to pass to the program. Must be an array or left out of the request.
+- `language` (**required**) The language to use for execution, must be a string and must be installed.
+- `version` (**required**) The version of the language to use for execution, must be a string containing a SemVer selector for the version or the specific version number to use.
+- `files` (**required**) An array of files containing code or other data that should be used for execution.
+- `files[].name` (**required**) The name of the file to upload, must be a string containing no path.
+- `files[].content` (**required**) The content of the files to upload, must be a string containing text to write.
+- `main` (**required**) The name of one of the files provided that should be considered the main source file which will be used as the entrypoint, must be a string and be the name of a file in `files`.
+- `stdin` (**required**) The text to pass as stdin to the program. Must be a string, can be left blank.
+- `args` (**required**) The arguments to pass to the program. Must be an array.
+- `compile_timeout` (**required**) The maximum time allowed for the compile stage to finish before bailing out in milliseconds. Must be a number.
+- `run_timeout` (**required**) The maximum time allowed for the run stage to finish before bailing out in milliseconds. Must be a number.
+
 ```json
 {
     "language": "js",
-    "source": "console.log(process.argv)",
+    "version": "15.10.0",
+    "files":[
+        {
+            "name": "my_cool_code.js",
+            "content": "console.log(process.argv)"
+        }
+    ],
+    "main": "my_cool_code.js",
     "stdin": "",
     "args": [
         "1",
         "2",
         "3"
-    ]
+    ],
+    "compile_timeout": 10000,
+    "run_timeout": 3000
 }
 ```
-A typical response upon successful execution will contain the `language`, `version`, `output` which
-is a combination of both `stdout` and `stderr` but in chronological order according to program output,
-as well as separate `stdout` and `stderr`.
+A typical response upon successful execution will contain 1 or 2 keys `run` and `compile`.
+`compile` will only be present if the language requested requires a compile stage.
+
+Each of these keys has an identical structure, containing both a `stdout` and `stderr` key, which is a string containing the text outputted during the stage into each buffer.
+It also contains the `code` and `signal` which was returned from each process.
 ```json
 HTTP/1.1 200 OK
 Content-Type: application/json
 
 {
-    "ran": true,
-    "language": "js",
-    "version": "12.13.0",
-    "output": "[ '/usr/bin/node',\n  '/tmp/code.code',\n  '1',\n  '2',\n  '3' ]",
-    "stdout": "[ '/usr/bin/node',\n  '/tmp/code.code',\n  '1',\n  '2',\n  '3' ]",
-    "stderr": ""
+  "run": {
+    "stdout": "[\n  '/piston/packages/node/15.10.0/bin/node',\n  '/piston/jobs/9501b09d-0105-496b-b61a-e5148cf66384/my_cool_code.js',\n  '1',\n  '2',\n  '3'\n]\n",
+    "stderr": "",
+    "code": 0,
+    "signal": null
+  }
 }
 ```
 If a problem exists with the request, a `400` status code is returned and the reason in the `message` key.
@@ -196,43 +247,35 @@ HTTP/1.1 400 Bad Request
 Content-Type: application/json
 
 {
-    "message": "Supplied language is not supported by Piston"
+    "message": "html-5.0.0 runtime is unknown"
 }
 ```
 
 <br>
 
-# Supported Languages
-
-`python`,`php`,`node`
-
-
-<br>
-<!--
 # Principle of Operation
-Piston utilizes LXC as the primary mechanism for sandboxing. There is a small API written in Node which takes
-in execution requests and executes them in the container. High level, the API writes
-a temporary source and args file to `/tmp` and that gets mounted read-only along with the execution scripts into the container.
+
+Piston uses Docker as the primary mechanism for sandboxing. There is an API within the container written in Node
+which takes in execution requests and executees them within the container safely.
+High level, the API writes any source code to a temporary directory in `/piston/jobs`.
 The source file is either ran or compiled and ran (in the case of languages like c, c++, c#, go, etc.).
 
 <br>
-<!--
+
 # Security
-LXC provides a great deal of security out of the box in that it's separate from the system.
+Docker provides a great deal of security out of the box in that it's separate from the system.
 Piston takes additional steps to make it resistant to
 various privilege escalation, denial-of-service, and resource saturation threats. These steps include:
 - Disabling outgoing network interaction
-- Capping max processes at 64 (resists `:(){ :|: &}:;`, `while True: os.fork()`, etc.)
+- Capping max processes at 256 by default (resists `:(){ :|: &}:;`, `while True: os.fork()`, etc.)
 - Capping max files at 2048 (resists various file based attacks)
-- Mounting all resources read-only (resists `sudo rm -rf --no-preserve-root /`)
 - Cleaning up all temp space after each execution (resists out of drive space attacks)
 - Running as a variety of unprivileged users
 - Capping runtime execution at 3 seconds
 - Capping stdout to 65536 characters (resists yes/no bombs and runaway output)
 - SIGKILLing misbehaving code
--->
+
 <br>
-<!--  Someone please do this -->
 
 # License
 Piston is licensed under the MIT license.

repo/Dockerfile

@@ -1,6 +1,6 @@
 FROM debian:buster-slim
 
-RUN apt-get update && apt-get install -y unzip autoconf build-essential libssl-dev pkg-config zlib1g-dev libargon2-dev libsodium-dev libcurl4-openssl-dev sqlite3 libsqlite3-dev libonig-dev libxml2 libxml2-dev bc curl git linux-headers-amd64 perl xz-utils python3 python3-pip gnupg jq zlib1g-dev cmake cmake-doc extra-cmake-modules build-essential gcc binutils bash coreutils util-linux pciutils usbutils coreutils binutils findutils grep libncurses5-dev libncursesw5-dev python3-pip && \
+RUN apt-get update && apt-get install -y unzip autoconf build-essential libssl-dev pkg-config zlib1g-dev libargon2-dev libsodium-dev libcurl4-openssl-dev sqlite3 libsqlite3-dev libonig-dev libxml2 libxml2-dev bc curl git linux-headers-amd64 perl xz-utils python3 python3-pip gnupg jq zlib1g-dev cmake cmake-doc extra-cmake-modules build-essential gcc binutils bash coreutils util-linux pciutils usbutils coreutils binutils findutils grep libncurses5-dev libncursesw5-dev python3-pip libgmp-dev libmpfr-dev python2 && \
         ln -sf /bin/bash /bin/sh && \
         rm -rf /var/lib/apt/lists/*