cli: add ppman install and list

api-client/index.cjs

@@ -1,5 +1,10 @@
 const fetch = require('node-fetch')
 
+function url_join(base, endpoint){
+    return base + endpoint
+    //return new URL(endpoint, base).href;
+}
+
 class APIWrapper {
     #base;
     constructor(base_url){
@@ -7,7 +12,7 @@ class APIWrapper {
     }
 
     async query(endpoint, options={}){
-        const url = new URL(endpoint, this.#base).href;
+        const url = url_join(this.#base, endpoint);
         return await fetch(url, options)
             .then(res=>res.json())
             .then(res=>{if(res.data)return res.data; throw new Error(res.message)});
@@ -44,7 +49,7 @@ class APIWrapper {
 
 class PistonEngineRepositoryPackage extends APIWrapper {
     constructor(repo, {language, language_version, author, buildfile, size, dependencies, installed}){
-        super(new URL(`/packages/${language}/${language_version}`,repo.url_base))
+        super(url_join(repo.url_base, `/packages/${language}/${language_version}`))
 
         this.language = language;
         this.language_version = language_version;
@@ -67,7 +72,7 @@ class PistonEngineRepositoryPackage extends APIWrapper {
 class PistonEngineRepository extends APIWrapper {
     
     constructor(engine, {slug, url, packages}){
-        super(new URL(`/repos/${slug}`, engine.url_base,))
+        super(url_join(engine.url_base,`/repos/${slug}`))
 
         this.slug = slug;
         this.url = url;

api/package.json

@@ -6,6 +6,7 @@
   "dependencies": {
     "body-parser": "^1.19.0",
     "express": "^4.17.1",
+    "express-validator": "^6.10.0",
     "is-docker": "^2.1.1",
     "js-yaml": "^4.0.0",
     "logplease": "^1.2.15",

api/yarn.lock

@@ -531,6 +531,14 @@ etag@~1.8.1:
   resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887"
   integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=
 
+express-validator@^6.10.0:
+  version "6.10.0"
+  resolved "https://registry.yarnpkg.com/express-validator/-/express-validator-6.10.0.tgz#66f70f73d04fb55c227401c75fe3713879c9cb70"
+  integrity sha512-gDtepU94EpUzgFvKO/8JzjZ4uqIF4xHekjYtcNgFDiBK6Hob3MQhPU8s/c3NaWd1xi5e5nA0oVmOJ0b0ZBO36Q==
+  dependencies:
+    lodash "^4.17.20"
+    validator "^13.5.2"
+
 express@^4.17.1:
   version "4.17.1"
   resolved "https://registry.yarnpkg.com/express/-/express-4.17.1.tgz#4491fc38605cf51f8629d39c2b5d026f98a4c134"
@@ -1270,6 +1278,11 @@ v8-compile-cache@^2.0.3:
   resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.2.0.tgz#9471efa3ef9128d2f7c6a7ca39c4dd6b5055b132"
   integrity sha512-gTpR5XQNKFwOd4clxfnhaqvfqMpqEwr4tOtCyz4MtYZX2JYhfr1JvBFKdS+7K/9rfpZR3VLX+YWBbKoxCgS43Q==
 
+validator@^13.5.2:
+  version "13.5.2"
+  resolved "https://registry.yarnpkg.com/validator/-/validator-13.5.2.tgz#c97ae63ed4224999fb6f42c91eaca9567fe69a46"
+  integrity sha512-mD45p0rvHVBlY2Zuy3F3ESIe1h5X58GPfAtslBjY7EtTqGquZTj+VX/J4RnHWN8FKq0C9WRVt1oWAcytWRuYLQ==
+
 vary@~1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc"

cli/commands/ppman.js

@@ -0,0 +1,7 @@
+exports.command = 'ppman'
+exports.aliases = ['pkg']
+exports.describe = 'Package Manager'
+
+exports.builder = yargs => yargs
+    .commandDir('ppman_commands')
+    .demandCommand()

cli/commands/ppman_commands/install.js

@@ -0,0 +1,35 @@
+const {PistonEngine} = require('piston-api-client');
+const chalk = require('chalk');
+
+exports.command = ['install <language> <language-version>']
+exports.aliases = ['i']
+exports.describe = 'Installs the named package'
+
+
+const msg_format = {
+    'color': p => `${p.success ? chalk.green.bold('✓') : chalk.red.bold('❌')} Installation ${p.success ? "succeeded" : "failed: " + p.message}`,
+    'monochrome': p => `Installation ${p.success ? "succeeded" : "failed: " + p.message}`,
+    'json': JSON.stringify
+
+}
+
+exports.handler = async function(argv){
+    const api = new PistonEngine(argv['piston-url']);
+    
+    const repos = await api.list_repos();
+    const repos_obj = await Promise.all(repos.repos.map(({slug}) => api.get_repo(slug)));
+    const repo_pkgs = await Promise.all(repos_obj.map(
+        async repo => ({
+            repo: repo,
+            packages: await repo.list_packages().catch(x=>[])
+        })
+    ))
+
+    const repo = repo_pkgs.find(r => r.packages.find(p=>p.language == argv['language'] && p.language_version == argv['language-version']))
+    if(!repo) throw Error("Package could not be located")
+
+    const package = await repo.repo.get_package(argv['language'], argv['language-version'])
+    const install = await package.install().catch(x=>x)
+    
+    console.log(msg_format.color(install));
+}

cli/commands/ppman_commands/list.js

@@ -0,0 +1,31 @@
+const {PistonEngine} = require('piston-api-client');
+const chalk = require('chalk');
+
+exports.command = ['list']
+exports.aliases = ['l']
+exports.describe = 'Lists all available packages'
+
+
+const msg_format = {
+    'color': p => `${chalk[p.installed ? "green":"red"]("•")} ${p.language} ${p.language_version}`,
+    'monochrome': p => `${p.language} ${p.language_version} ${p.installed ? "(INSTALLED)": ""}`,
+    'json': JSON.stringify
+
+}
+
+exports.handler = async function(argv){
+    const api = new PistonEngine(argv['piston-url']);
+    
+    const repos = await api.list_repos();
+    const repos_obj = await Promise.all(repos.repos.map(({slug}) => api.get_repo(slug)));
+    const packages = await repos_obj.reduce(async (a, c) => [
+        ...await a,
+        ...await c.list_packages().catch(x=>{console.log(x); return []})
+    ], []);
+    
+    const pkg_msg = packages
+        .map(msg_format.color)
+        .join('\n');
+
+    console.log(pkg_msg);
+}

packages/.gitignore

@@ -1 +1,2 @@
-*/*/
+build/
+*.pkg.tar.gz

packages/CONTRIBUTING.MD

@@ -1,43 +1,65 @@
 # Contributing packages to the Piston Repository
 
+## Naming Lanaguages
+
+Some languages have multiple different wide-spread interpreters (node/deno for example) which need to be accounted for.
+
+If the given language has multiple incompatiable interpreters (e.g. deno/node/v8), these should be named as `[language]-[interpreter]`.
+For cases where the language has many compatable interpreters (e.g. gcc/llvm), pick one and name it `[language]`.
+For languages with only 1 wide-spread interpreter (e.g. Kotlin), name it `[language]`
+
+## File Naming Rules
+
+Different versions of interpreters require different steps to build, and in this case they should be given 2 different files, but keep the same language name.
+
+Use `[language-name].mk` (see naming languages) for languages with a common build script.
+When the build steps become obsolete, create a new file named `[language-name]-[version].mk`, with the first version where the new build steps were required.
+
 ## Creating new languages
 
-1. Create a new branch on your fork of engineer-man/piston.
+See [python.mk](python.mk) or any other `.mk` file (except `common.mk`) for an example.
-2. Create a directory (all lowercase) with the name of the language.
-3. Create a Makefile containing the following, filling in the fields marked with %
-Versions can be a list of version numbers, separated by spaces.
-Quotes are not required.
-```makefile
-LANGUAGE=%
-VERSIONS=%
 
-include ../secondary.mk
+Please note that this is a regular Makefile, with a few extra varibles added by `common.mk`, any additional variables not listed here can be located in there.
-```
-4. Create a `base.mk` file, this is the file which will hold the correct targets to make each build.
-It should download any sources it requires uring `curl` and use as much version templating as possible.
-You should include 3 variables: `AUTHOR`, `DEPS` and `COMPILED`.
 
-`AUTHOR` is simply your name, and email address formatted as `Your Name <your@email>`.
+1. Create a new branch on your fork of engineer-man/piston
-`DEPS` is a list (separated by spaces) of packages which are required with this one for it to work correctly.
-This is mainly used in the case of golfing languages, which use other languages (e.g. python and eilxar) to interpret them. Give a package name and a SemVer selector (e.g. `python==3.x.x` for any python3 build).
-`COMPILED` is simply a true or false field indicating if the package contains a `compile` target (see below).
 
-You should also include 3/4 targets: `run`, `${NAME}-${VERSION}`, `${NAME}-${VERSION}/environment` and optionally `compile` (only if the language is a compiled language, e.g. cpp/c)
+2. Create a new file for your language, following the naming rules as listed above
 
-The `run` file is one which is passed the main source file (first argument) and the arguments for the process (remaining arguments). STDIN data is also piped in. Generate this by using either a `cat` or an `echo` statement(s) and piping them into `$@` (it means the target name in Makefiles). You have to escape `$` symbols with `$$`.
+3. Add `NAME=[language name]` to this file, replacing `[language name]` with the language name all in lowercase, removing any punctuation and numbers (e.g. 05AB1E becomes osabie).
 
-The `${NAME}-${VERSION}` directory should contain all the binaries required to run the language.
+4. Add `AUTHOR=[author]` to this file, replacing `[author]` with your name and email address in the format `Full Name <your@email.address>` (standard git format).
-You should use this target to compile all the sources, and installing them into the `$@` location
 
-`${NAME}-${VERSION}/environment` should contain a bunch of environment variables which will be passed into the `run` and `compile` scripts of not only this language, but any which depend on it.
+5. Add `DEPENDENCIES=[dependencies list]` to this file, replacing `[dependencies list]` with a list of dependencies, seperated by spaces in the format `[language]==[version]`. If there are none, simply leave this as `DEPENDENCIES=`
-If your package depends on any others, their environment variables are automatically sourced, then your packages ones are sourced after.
 
-The `compile` file contains instructions to compile source files into a final binary which should be run by `run`. It's first argument is the main source file, with other source files being provided as other arguments. STDIN is left blank, but both STDOUT and STDERR are returned. The `run` file will be skipped if this file returns a non-zero error code.
+6. Add `COMPILED=[true/false]` to this file, set this to true if the language requires the compile stage, and false if it only requires run.
 
-5. Commit your new package with the following message format: `pkg([language name] [version(s)]): [description including versions]`
+7. Add `VERSIONS=[version list]` to this file, replacing `[version list]` with a list of [SemVer](https://semver.org/) compilant version numbers for the language which this build file can build. This value will be passed in as `${VERSION}` for you to access
 
-Examples:
+8. Add `include common.mk`, to include all the common Makefile rules and variables.
-* `pkg(python 3.9.1): new version`
-* `pkg(python 3.9.1 2.7.1): refactor`
 
-6. Create a pull request, referencing the issue number of the language
+9. Create the target for the run file by adding `${RUN_FILE}:`, followed by steps on new lines, indented by a tab (`\t`) to create a bash script, which is run for every job (`/execute` endpoint) for this language and returns on STDOUT/STDERR the output of the run.
+The script is expected to take in the first argument (`$1`) as the main code file, with subsequent arguments being passed in from the execute endpoint.
+The script is passed STDIN as specified in the job, and expects output on STDOUT/STDERR. It should also pass through the exit code from the interpreter.
+It is recommended to use `echo` statements, redirecting output to `$@` (a special Makefile variable expanding to the target file, in this case RUN_FILE).
+Make sure to escape any `$` with `$$`, as `$` will expand make variables.
+
+10. (optional, only use if language requires compliation)
+Create the target for the compile file by adding `${COMPILE_FILE}:`, followed by steps on new lines, indented by a tab (`\t`) to create a bash script, which is run for every job (`/execute` endpoint) for this language, if it requires compilation.
+This script is expected to take all code files in as arguements, and output binary files. STDERR/STDOUT are captured and passed out, along with error code. The job STDIN and args are not passed in to this script.
+This script should compile source code into a binary, and has a seperate time limit from run, and thus should split the stages.
+Follow all priciples from step 9, using echo statements and escaping `$`.
+
+11. Create the target for the environment file by adding `${ENV_FILE}:`, followed by steps on new lines, indented by a tab (`\t`) to create a bash script, which modifies environment variables exactly how you would a `.profile` or `.bashrc` file. Note that this file is only run at install time, and thus cannot dynamicly adjust to the arguements passed into it.
+The environment file is also appended with all dependencies before being cached.
+You should add in the path to any binaries. The current working directory for the script is contained within the `${BIN_DIR}` folder.
+
+12. Create the target for the binaries by adding `${BIN_DIR}:`, followed by steps on new lines, indented by a tab (`\t`) which will download sources, compile and output binaries in to the `${BIN_DIR}` (`$@`)
+
+13. Locally test your Makefile builds with `make build-[language]-[version]`.
+If everything went well, you should now have a `[language]-[version].pkg.tar.gz` file sitting in the root.
+If not, read through your error logs, and if in doubt ask for help in #emkc-felix-piston in Discord.
+
+14. Commit your changes, using message format of `pkg([language]): Added [language] [version]`
+Any additional commits regarding this package should start with `pkg([language]): `
+
+15. Create a pull request (currently to v3), referencing an Issue number (if there is one associated).

packages/Makefile

@@ -0,0 +1,30 @@
+# Wraps all other Makefiles 
+
+# Variables
+PKG_FILES=$(filter-out common.mk,$(wildcard *.mk))
+PKG_SLUGS=$(foreach pkg, ${PKG_FILES}, $(addprefix $(shell make -f ${pkg} name)-, $(shell make -f ${pkg} versions)))
+
+# Functions
+define pkg_info
+	$(eval PKG_SLUG=$(patsubst $1-%,%,$2))
+	$(eval PKG_PARTS=$(subst -, ,${PKG_SLUG}))
+	$(eval PKG_NAME=$(word 1,${PKG_PARTS}))
+	$(eval PKG_VERSION=$(word 2,${PKG_PARTS}))
+	$(eval PKG_FILE=$(shell grep '^VERSIONS\s*=.*${PKG_VERSION}' $(shell grep "NAME\s*=\s*${PKG_NAME}" ${PKG_FILES} -l) -l))
+endef
+
+# Targets
+
+build: $(foreach pkg, ${PKG_FILES}, $(addprefix build-$(shell make -f ${pkg} name)-, $(lastword $(shell make -f ${pkg} versions))))
+
+$(addprefix build-, ${PKG_SLUGS}):
+	$(call pkg_info,build,$@)
+	$(MAKE) -f ${PKG_FILE} VERSION=${PKG_VERSION} build
+
+
+
+clean: $(foreach pkg, ${PKG_FILES}, $(addprefix clean-$(shell make -f ${pkg} name)-, $(shell make -f ${pkg} versions)))
+	rm -rf build/
+$(addprefix clean-, ${PKG_SLUGS}):
+	$(call pkg_info,clean,$@)
+	$(MAKE) -f ${PKG_FILE} VERSION=${PKG_VERSION} clean

packages/README.MD

@@ -0,0 +1,7 @@
+# Piston Package Build Scripts
+
+## Building
+
+```bash
+make build-[name]-[version]
+```

packages/common.mk

@@ -1,46 +1,70 @@
-LANG_NAME=$(or ${NAME},none)
+# Variables
-LANG_VERSION=$(or ${VERSION},0.0.0)
+PKG_SLUG=${NAME}-${VERSION}
-LANG_AUTHOR=$(or ${AUTHOR},HexF <thomas@hexf.me>)
+BUILD_DIR=build/${PKG_SLUG}/
-LANG_DEPS=$(or ${DEPS})
-LANG_COMPILED=$(or ${COMPILED}, false)
 
-LANG_PKG_TARGETS=pkg-info.json ${LANG_NAME}-${LANG_VERSION}/ ${LANG_NAME}-${LANG_VERSION}/environment run
+BIN_DIR=${BUILD_DIR}${PKG_SLUG}/
+RUN_FILE=${BUILD_DIR}run
+COMPILE_FILE=${BUILD_DIR}compile
+ENV_FILE=${BIN_DIR}environment
+INFO_FILE=${BUILD_DIR}pkg-info.jq
 
-BUILD_PLATFORM=$(or ${PLATFORM}, baremetal-$(shell grep -oP "^ID=\K\w+" /etc/os-release ))
+PKG_FILE=${PKG_SLUG}.pkg.tar.gz
 
-ifeq (${LANG_COMPILED}, true)
+VERSION_MINOR=$(shell grep -oP "\d+.\d+"<<<${VERSION})
-${LANG_NAME}-${LANG_VERSION}.pkg.tar.gz: $(LANG_PKG_TARGETS) compile
+VERSION_MAJOR=$(shell grep -oP "\d+"<<<${VERSION})
+
+PKG_TARGETS=${BIN_DIR} ${ENV_FILE} ${RUN_FILE} ${INFO_FILE}
+
+
+# Command Targets
+
+.PHONY: catch versions name build clean
+catch:
+	# Catch manual calling
+	# This is done to make sure people don't call without ${VERSION}, which can cause problems
+	@echo "Don't directly call individual scripts, instead call the common Makefile"
+	@exit 1
+
+versions:
+	@echo ${VERSIONS}
+
+name:
+	@echo ${NAME}
+
+build: ${BUILD_DIR} ${PKG_FILE}
+clean: 
+	rm -rf ${BUILD_DIR}
+	rm -f ${PKG_FILE}
+
+# mkdir
+${BUILD_DIR}:
+	mkdir -p ${BUILD_DIR}
+
+
+# Generated files
+
+ifeq (${COMPILED}, true)
+${PKG_FILE}: ${PKG_TARGETS} ${COMPILE_FILE}
 endif
-${LANG_NAME}-${LANG_VERSION}.pkg.tar.gz: $(LANG_PKG_TARGETS)
+${PKG_FILE}: ${PKG_TARGETS}
-	tar czf $@ $?
+	tar -czC ${BUILD_DIR} -f $@ ${patsubst ${BUILD_DIR}%,%,$?}
+	
+${INFO_FILE}:
+	echo '.language="${NAME}"' > $@
+	echo '.version="${VERSION}"' >> $@
+	echo '.author="${AUTHOR}"' >> $@
+	echo '.dependencies={}' >> $@
+	echo '.build_platform="$(or ${PLATFORM}, baremetal-$(shell grep -oP "^ID=\K\w+" /etc/os-release ))"' >> $@
+	$(foreach dep, ${DEPENDENCIES}, echo '.dependencies.$(word 1,$(subst =, ,${dep}))="$(word 2,$(subst =, ,${dep}))"' >> $@)
+
+
+
+# Helpers
+
+%/: %.tgz
+	cd ${BUILD_DIR} && tar xzf $(patsubst ${BUILD_DIR}%,%,$<)
+%/: %.tar.gz
+	cd ${BUILD_DIR} && tar xzf $(patsubst ${BUILD_DIR}%,%,$<)
 
 %.json: %.jq
 	jq '$(shell tr '\n' '|' < $<).' <<< "{}" > $@
-	
-pkg-info.jq:
-	echo '.language="${LANG_NAME}"' > pkg-info.jq
-	echo '.version="${LANG_VERSION}"' >> pkg-info.jq
-	echo '.author="${LANG_AUTHOR}"' >> pkg-info.jq
-	echo '.dependencies={}' >> pkg-info.jq
-	echo '.build_platform="${BUILD_PLATFORM}"' >> pkg-info.jq
-	$(foreach dep, ${LANG_DEPS}, echo '.dependencies.$(word 1,$(subst =, ,${dep}))="$(word 2,$(subst =, ,${dep}))"' >> pkg-info.jq)
-
-%.asc: %
-	gpg --detach-sig --armor --batch --output $@ $< 
-
-%/: %.tgz
-	tar xzf $<
-
-%/: %.tar.gz
-	tar xzf $<
-
-.PHONY: clean
-clean: 
-	rm -rf $(filter-out Makefile, $(wildcard *))
-
-,PHONY: cleanup
-cleanup:
-	rm -rf $(filter-out ${LANG_NAME}-${LANG_VERSION}.pkg.tar.gz.asc, $(filter-out ${LANG_NAME}-${LANG_VERSION}.pkg.tar.gz, $(filter-out Makefile, $(wildcard *))))
-	
-.PHONY: sign
-sign: ${LANG_NAME}-${LANG_VERSION}.pkg.tar.gz.asc

packages/python.mk

@@ -0,0 +1,22 @@
+NAME=python
+AUTHOR=Thomas Hobson <thomas@hexf.me>
+DEPENDENCIES=
+COMPILED=false
+VERSIONS=2.7.1 3.5.1 3.9.1
+
+include common.mk
+
+
+${RUN_FILE}:
+	echo 'python${VERSION_MINOR} $$*' > $@
+
+${ENV_FILE}:
+	echo 'export PATH=$$PWD/bin:$$PATH' > $@
+
+${BIN_DIR}: ${BUILD_DIR}Python-${VERSION}/
+	cd $< && ./configure --prefix /
+	$(MAKE) -j64 -C $<
+	DESTDIR=../${PKG_SLUG} $(MAKE) -j64 -C $< altinstall || true
+
+${BUILD_DIR}Python-${VERSION}.tgz: 
+	curl "https://www.python.org/ftp/python/${VERSION}/Python-${VERSION}.tgz" -o $@

packages/python/Makefile

@@ -1,4 +0,0 @@
-LANGUAGE=python
-VERSIONS=2.7.1 3.9.1
-
-include ../secondary.mk

packages/python/base.mk

@@ -1,19 +0,0 @@
-AUTHOR=Thomas Hobson <thomas@hexf.me>
-DEPS=
-COMPILED=false
-
-include ../../common.mk
-
-run:
-	echo 'python$(shell grep -oP "\d+.\d+"<<<${VERSION}) $$*' > run
-
-${NAME}-${VERSION}/environment:
-	echo 'export PATH=$$PWD/bin:$$PATH' > $@
-
-${NAME}-${VERSION}/: Python-${VERSION}/
-	cd $< && ./configure --prefix /
-	$(MAKE) -j$(or ${MAKE_JOBS},64) -C $<
-	DESTDIR=../$@ $(MAKE) -j$(or ${MAKE_JOBS},64) -C $< altinstall || true
-
-Python-${VERSION}.tgz: 
-	curl "https://www.python.org/ftp/python/${VERSION}/$@" -o $@

packages/secondary.mk

@@ -1,23 +0,0 @@
-.PHONY: build sign cleanup clean
-build: $(patsubst %,%/${LANGUAGE}-%.pkg.tar.gz,${VERSIONS})
-sign: $(patsubst %,%/${LANGUAGE}-%.pkg.tar.gz.asc,${VERSIONS})
-clean:
-	rm -rf ${VERSIONS}
-cleanup: $(patsubst %,%/cleanup,${VERSIONS})
-	
-
-%/cleanup: %/Makefile
-	$(MAKE) -C $(shell dirname $<) cleanup
-	rm $(shell dirname $<)/Makefile
-
-%/${LANGUAGE}-%.pkg.tar.gz.asc: %/Makefile
-	$(MAKE) -C $(shell dirname $<) sign
-%/${LANGUAGE}-%.pkg.tar.gz: %/Makefile
-	$(MAKE) -C $(shell dirname $<)
-
-
-%/Makefile: 
-	@mkdir -p $(shell dirname $@)
-	@echo 'VERSION=$(patsubst %/Makefile,%,$@)' > $@
-	@echo 'NAME=${LANGUAGE}' >> $@
-	@echo 'include ../base.mk' >> $@