cli: fix issue where negate removes too many packages

* Command to build packages locally
* Help message
* Start/Stop/Restart commands
* Select production/development environements
* clean pkgs/repo commands

.gitignore

@@ -1 +1,2 @@
-data/
+data/
+.piston_env

cli/commands/ppman_commands/spec.js

@@ -0,0 +1,160 @@
+const chalk = require('chalk');
+const fs = require('fs/promises');
+const minimatch = require("minimatch");
+const semver = require('semver');
+
+exports.command = ['spec <specfile>'];
+exports.aliases = ['s'];
+exports.describe = 'Install the packages described in the spec file, uninstalling packages which aren\'t in the list'
+
+function does_match(package, rule){
+    const nameMatch = minimatch(package.language, rule.package_selector);
+    const versionMatch = semver.satisfies(package.language_version, rule.version_selector)
+
+    return nameMatch && versionMatch;
+}
+
+exports.handler = async ({axios, specfile}) => {
+    const spec_contents = await fs.readFile(specfile);
+    const spec_lines = spec_contents.toString().split("\n");
+
+    const rules = [];
+
+    for(const line of spec_lines){
+        const rule = {
+            _raw: line.trim(),
+            comment: false,
+            package_selector: null,
+            version_selector: null,
+            negate: false
+        };
+
+        if(line.starts_with("#")){
+            rule.comment = true;
+        }else {
+            let l = line.trim();
+            if(line.starts_with("!")){
+                rule.negate = true;
+                l = line.slice(1).trim();
+            }
+
+            const [pkg, ver] = l.split(" ", 2);
+            rule.package_selector = pkg;
+            rule.version_selector = ver;
+        }
+
+        if(rule._raw.length != 0) rules.push(rule);
+    }
+
+    const packages_req = await axios.get('/api/v2/packages');
+    const packages = packages_req.data;
+
+    const installed = packages.filter(pkg => pkg.installed);
+
+    let ensure_packages = [];
+
+    for(const rule of rules){
+        if(rule.comment) continue;
+
+        const matches = [];
+
+        if(!rule.negate){
+            for(const package of packages){
+                if(does_match(package, rule))
+                    matches.push(package)
+            }
+
+            const latest_matches = matches.filter(
+                pkg => {
+                    const versions = matches
+                        .filter(x=>x.language == pkg.language)
+                        .map(x=>x.language_version).sort(semver.rcompare);
+                    return versions[0] == pkg.language_version
+                }
+            );
+
+            for(const match of latest_matches){
+                if(!ensure_packages.find(pkg => pkg.language == match.language && pkg.language_version == match.language_version))
+                    ensure_packages.push(match)
+            }
+        }else{
+            ensure_packages = ensure_packages.filter(
+                pkg => !does_match(pkg, rule)
+            )
+        }
+
+        
+    }
+
+    const operations = [];
+
+    for(const package of ensure_packages){
+        if(!package.installed)
+            operations.push({
+                type: "install",
+                package: package.language,
+                version: package.language_version
+            });
+    }
+
+    for(const installed_package of installed){
+        if(!ensure_packages.find(
+            pkg => pkg.language == installed_package.language &&
+            pkg.language_version == installed_package.language_version
+        ))
+            operations.push({
+                type: "uninstall",
+                package: installed_package.language,
+                version: installed_package.language_version
+            })
+    }
+
+    console.log(chalk.bold.yellow("Actions"))
+    for(const op of operations){
+        console.log((op.type == "install" ? chalk.green("Install") : chalk.red("Uninstall")) + ` ${op.package} ${op.version}`)
+    }
+
+    if(operations.length == 0){
+        console.log(chalk.gray("None"))
+    }
+
+    for(const op of operations){
+        if(op.type == "install"){
+            try{
+                const install = await axios.post(`/api/v2/packages`, {
+                    language: op.package,
+                    version: op.version
+                });
+
+                if(!install.data.language)
+                    throw new Error(install.data.message); // Go to exception handler
+
+                console.log(chalk.bold.green("Installed"), op.package, op.version)
+
+            }catch(e){
+                console.log(chalk.bold.red("Failed to install") + ` ${op.package} ${op.version}:`, e.message)
+            }
+        }
+        else if(op.type == "uninstall"){
+            try{
+                const install = await axios.delete(`/api/v2/packages`, {
+                    data: {
+                        language: op.package,
+                        version: op.version
+                    }
+                });
+
+                if(!install.data.language)
+                    throw new Error(install.data.message); // Go to exception handler
+
+                console.log(chalk.bold.green("Uninstalled"), op.package, op.version)
+
+            }catch(e){
+                console.log(chalk.bold.red("Failed to uninstall") + ` ${op.package} ${op.version}:`, e.message)
+            }
+        }
+    }
+    
+
+
+}

cli/index.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-
+require('nocamel');
 const axios = require('axios').default;
 
 const axios_instance = argv => {

cli/package-lock.json

@@ -11,6 +11,9 @@
             "dependencies": {
                 "axios": "^0.21.1",
                 "chalk": "^4.1.0",
+                "minimatch": "^3.0.4",
+                "nocamel": "^1.0.2",
+                "semver": "^7.3.5",
                 "yargs": "^16.2.0"
             }
         },
@@ -41,6 +44,20 @@
                 "follow-redirects": "^1.10.0"
             }
         },
+        "node_modules/balanced-match": {
+            "version": "1.0.2",
+            "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
+            "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
+        },
+        "node_modules/brace-expansion": {
+            "version": "1.1.11",
+            "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+            "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+            "dependencies": {
+                "balanced-match": "^1.0.0",
+                "concat-map": "0.0.1"
+            }
+        },
         "node_modules/chalk": {
             "version": "4.1.0",
             "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
@@ -79,6 +96,11 @@
             "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
             "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
         },
+        "node_modules/concat-map": {
+            "version": "0.0.1",
+            "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
+            "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
+        },
         "node_modules/emoji-regex": {
             "version": "8.0.0",
             "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
@@ -124,6 +146,33 @@
                 "node": ">=8"
             }
         },
+        "node_modules/lru-cache": {
+            "version": "6.0.0",
+            "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+            "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+            "dependencies": {
+                "yallist": "^4.0.0"
+            },
+            "engines": {
+                "node": ">=10"
+            }
+        },
+        "node_modules/minimatch": {
+            "version": "3.0.4",
+            "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
+            "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
+            "dependencies": {
+                "brace-expansion": "^1.1.7"
+            },
+            "engines": {
+                "node": "*"
+            }
+        },
+        "node_modules/nocamel": {
+            "version": "1.0.2",
+            "resolved": "https://registry.npmjs.org/nocamel/-/nocamel-1.0.2.tgz",
+            "integrity": "sha512-CRkRSRLChj+H6e4lHS851QS6YGCoTETnSG/z+XGanxLSsTbBkvEeIWaIYMKzuBznFwWM0YcLGXsFyXg4xWYnWA=="
+        },
         "node_modules/require-directory": {
             "version": "2.1.1",
             "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@@ -132,6 +181,20 @@
                 "node": ">=0.10.0"
             }
         },
+        "node_modules/semver": {
+            "version": "7.3.5",
+            "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
+            "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==",
+            "dependencies": {
+                "lru-cache": "^6.0.0"
+            },
+            "bin": {
+                "semver": "bin/semver.js"
+            },
+            "engines": {
+                "node": ">=10"
+            }
+        },
         "node_modules/string-width": {
             "version": "4.2.2",
             "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz",
@@ -188,6 +251,11 @@
                 "node": ">=10"
             }
         },
+        "node_modules/yallist": {
+            "version": "4.0.0",
+            "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+            "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
+        },
         "node_modules/yargs": {
             "version": "16.2.0",
             "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz",
@@ -236,6 +304,20 @@
                 "follow-redirects": "^1.10.0"
             }
         },
+        "balanced-match": {
+            "version": "1.0.2",
+            "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
+            "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
+        },
+        "brace-expansion": {
+            "version": "1.1.11",
+            "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+            "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+            "requires": {
+                "balanced-match": "^1.0.0",
+                "concat-map": "0.0.1"
+            }
+        },
         "chalk": {
             "version": "4.1.0",
             "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
@@ -268,6 +350,11 @@
             "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
             "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
         },
+        "concat-map": {
+            "version": "0.0.1",
+            "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
+            "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
+        },
         "emoji-regex": {
             "version": "8.0.0",
             "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
@@ -298,11 +385,40 @@
             "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
             "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="
         },
+        "lru-cache": {
+            "version": "6.0.0",
+            "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
+            "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
+            "requires": {
+                "yallist": "^4.0.0"
+            }
+        },
+        "minimatch": {
+            "version": "3.0.4",
+            "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
+            "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
+            "requires": {
+                "brace-expansion": "^1.1.7"
+            }
+        },
+        "nocamel": {
+            "version": "1.0.2",
+            "resolved": "https://registry.npmjs.org/nocamel/-/nocamel-1.0.2.tgz",
+            "integrity": "sha512-CRkRSRLChj+H6e4lHS851QS6YGCoTETnSG/z+XGanxLSsTbBkvEeIWaIYMKzuBznFwWM0YcLGXsFyXg4xWYnWA=="
+        },
         "require-directory": {
             "version": "2.1.1",
             "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
             "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I="
         },
+        "semver": {
+            "version": "7.3.5",
+            "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
+            "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==",
+            "requires": {
+                "lru-cache": "^6.0.0"
+            }
+        },
         "string-width": {
             "version": "4.2.2",
             "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz",
@@ -344,6 +460,11 @@
             "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.5.tgz",
             "integrity": "sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg=="
         },
+        "yallist": {
+            "version": "4.0.0",
+            "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+            "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
+        },
         "yargs": {
             "version": "16.2.0",
             "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz",

cli/package.json

@@ -7,6 +7,9 @@
     "dependencies": {
         "axios": "^0.21.1",
         "chalk": "^4.1.0",
+        "minimatch": "^3.0.4",
+        "nocamel": "^1.0.2",
+        "semver": "^7.3.5",
         "yargs": "^16.2.0"
     }
 }

dev.pps

@@ -0,0 +1,8 @@
+#!/usr/bin/env -S piston ppman spec
+
+# Development Piston Packages
+# Defines packages to be installed by developers
+
+# All packages, latest version
+# Don't use this when connected to public repo, in excess of 10GB
+* *

docker-compose.dev.yaml

@@ -1,24 +1,24 @@
-version: '3.2'
+version: "3.2"
 
 services:
-    api:
+  api:
-        build: api
+    build: api
-        container_name: piston_api
+    container_name: piston_api
-        cap_add:
+    cap_add:
-            - CAP_SYS_ADMIN
+      - CAP_SYS_ADMIN
-        restart: always
+    restart: always
-        ports:
+    ports:
-            - 2000:2000
+      - 2000:2000
-        volumes:
+    volumes:
-            - ./data/piston:/piston
+      - ./data/piston:/piston
-        environment:
+    environment:
-            - PISTON_REPO_URL=http://repo:8000/index
+      - PISTON_REPO_URL=http://repo:8000/index
-        tmpfs:
+    tmpfs:
-            - /piston/jobs:exec
+      - /piston/jobs:exec
 
-    repo: # Local testing of packages
+  repo: # Local testing of packages
-        build: repo
+    build: repo
-        container_name: piston_repo
+    container_name: piston_repo
-        command: ['dart-2.12.1'] # Only build dart
+    command: ["--no-build"] # Don't build anything
-        volumes:
+    volumes:
-            - .:/piston
+      - .:/piston

piston

@@ -1,23 +1,77 @@
 #!/usr/bin/env bash
+cd "$(dirname "$0")"
+
+PISTON_ENV=$(cat .piston_env || echo dev)
+
+docker_compose(){
+    if [ -f "docker-compose.$PISTON_ENV.yaml" ]; then
+        docker-compose -f "docker-compose.$PISTON_ENV.yaml" "$@"
+    else
+        docker-compose "$@"
+    fi
+}
 
 case $1 in
-    dev)
+    help)
-        shift
+        echo "=== Piston Management ==="
-        docker-compose -f docker-compose.dev.yaml "$@"
+        echo "Current Environment: $PISTON_ENV"
-        ;;
+        echo
-    prod)
+        echo "Commands:"
-        shift
+        echo " select <environment>           Select the environment"
-        docker-compose -f docker-compose.yaml "$@"
+        echo " docker_compose <args...>       Interact directly with the docker-compose for the selected environment"
+        echo
+        echo " start                          Starts piston"
+        echo " stop                           Stops piston"
+        echo " restart                        Restarts piston"
+        echo
+        echo " update                         Fetches and applies latest updates"
+        echo
+        echo " <args..>                       Passthrough to piston cli tool"
+        echo 
+        echo "Development Commands:"
+        
+        if [ $PISTON_ENV == dev ]; then
+        
+        echo " clean-pkgs                     Clean any package build artifacts on disk"
+        echo " clean-repo                     Remove all packages from local repo"
+        echo " build-pkg <package> <version>  Build a package"
+        
+        else
+        
+        echo " Switch to developement environment for more info"
+        echo " > piston switch dev"
+        
+        fi
         ;;
+
+
+    select) echo "$2" > .piston_env ;;
+    docker_compose) shift; docker_compose "$@";;
+
+    restart) docker_compose restart ;;
+    start) docker_compose up -d ;;
+    stop) docker_compose down ;;
+
     update)
         git pull
-        docker-compose pull api
+        docker_compose pull
-        docker-compose up -d api
+        docker_compose up -d
         ;;
-    clean-pkgs)
+
-        git clean -fqXd packages
+    clean-pkgs) git clean -fqXd packages ;;
+    clean-repo) git clean -fqXd repo ;;
+
+    build-pkg)
+        PKGSLUG="$2-$3"
+        echo "Building $PKGSLUG"
+        echo "Ensuring latest builder image"
+        docker build repo -t piston-repo-builder
+        docker run -v "$(realpath $(dirname "$0")):/piston" piston-repo-builder --no-server $PKGSLUG
         ;;
     *)
+        cd cli
+        npm i > /dev/null
+        cd ../
         node cli/index.js "$@"
-        ;;
+    ;;
 esac

public.pps

@@ -0,0 +1,14 @@
+#!/usr/bin/env -S piston ppman spec
+
+# Public Piston Packages
+# Defines packages to be installed on the public piston installation
+
+# All packages, latest version
+* *
+
+# Except python
+!python *
+
+# Install python 3.* and 2.*
+python 3.*
+python 2.*

repo/.dockerignore

@@ -0,0 +1 @@
+*.pkg.tar.gz