Fixes nasm32 - adds 32bit arch to seccomp filter (#231)

* pkg(mono-6.12.0): filename fixes

* pkg(mono-6.12.0): compile cs files

* pkg(mono-6.12.0): add provides in metadata

.github/workflows/package-pr.yaml

@@ -88,16 +88,16 @@ jobs:
       - name: Run tests
         run: |
           ls -la
-          docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' -d --name piston_fs_repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build
-          docker run --network container:piston_fs_repo -v $(pwd)'/data:/piston' -d --name api docker.pkg.github.com/engineer-man/piston/api
+          docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' -d --name repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build
+          docker run --network container:repo -v $(pwd)'/data:/piston' -d --name api docker.pkg.github.com/engineer-man/piston/api
           echo Waiting for API to start..
           docker run --network container:api appropriate/curl -s --retry 10 --retry-connrefused http://localhost:2000/runtimes
 
           echo Waiting for Index to start..
-          docker run --network container:piston_fs_repo appropriate/curl -s --retry 999 --retry-max-time 0 --retry-connrefused http://localhost:8000/index
+          docker run --network container:repo appropriate/curl -s --retry 999 --retry-max-time 0 --retry-connrefused http://localhost:8000/index
 
           echo Adjusting index
-          sed -i 's/piston_fs_repo/localhost/g' repo/index
+          sed -i 's/repo/localhost/g' repo/index
 
           echo Listing Packages
           PACKAGES_JSON=$(docker run --network container:api appropriate/curl -s http://localhost:2000/packages)

api/src/executor/job.js

@@ -17,7 +17,7 @@ let gid = 0;
 
 class Job {
 
-    constructor({ runtime, files, args, stdin, timeouts, alias }) {
+    constructor({ runtime, files, args, stdin, timeouts }) {
         this.uuid =  uuidv4();
         this.runtime = runtime;
         this.files = files.map((file,i) => ({
@@ -28,7 +28,6 @@ class Job {
         this.args = args;
         this.stdin = stdin;
         this.timeouts = timeouts;
-        this.alias = alias;
 
         this.uid = config.runner_uid_min + uid;
         this.gid = config.runner_gid_min + gid;
@@ -90,7 +89,7 @@ class Job {
             const proc = cp.spawn(proc_call[0], proc_call.splice(1) ,{
                 env: { 
                     ...this.runtime.env_vars,
-                    PISTON_ALIAS: this.alias
+                    PISTON_LANGUAGE: this.runtime.language
                 },
                 stdio: 'pipe',
                 cwd: this.dir,

api/src/index.js

@@ -50,7 +50,7 @@ const app = express();
         .flat()
         .filter(pkg => fss.exists_sync(path.join(pkg, globals.pkg_installed_file)));
 
-    installed_languages.forEach(pkg => new runtime.Runtime(pkg));
+    installed_languages.forEach(pkg => runtime.load_package(pkg));
 
     logger.info('Starting API Server');
     logger.debug('Constructing Express App');
@@ -100,7 +100,8 @@ const app = express();
                 return {
                     language: rt.language,
                     version: rt.version.raw,
-                    aliases: rt.aliases
+                    aliases: rt.aliases,
+                    runtime: rt.runtime
                 };
             });
 

api/src/nosocket/nosocket.c

@@ -24,6 +24,22 @@ int main(int argc, char *argv[])
         return 1;
     }
 
+    // Add 32 bit and 64 bit architectures to seccomp filter
+    int rc;
+    uint32_t arch[] = {SCMP_ARCH_X86_64, SCMP_ARCH_X86, SCMP_ARCH_X32};
+    // We first remove the existing arch, otherwise our subsequent call to add
+    // it will fail
+    seccomp_arch_remove(ctx, seccomp_arch_native());
+    for (int i = 0; i < sizeof(arch) / sizeof(arch[0]); i++)
+    {
+        rc = seccomp_arch_add(ctx, arch[i]);
+        if (rc != 0)
+        {
+            fprintf(stderr, "Unable to add arch: %d\n", arch[i]);
+            return 1;
+        }
+    }
+
     // Add a seccomp rule to the syscall blacklist - blacklist the socket syscall
     if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EACCES), SCMP_SYS(socket), 0) < 0)
     {

api/src/ppman/package.js

@@ -85,7 +85,7 @@ class Package {
         });
 
         logger.debug('Registering runtime');
-        new runtime.Runtime(this.install_path);
+        runtime.load_package(this.install_path);
 
         logger.debug('Caching environment');
         const get_env_command = `cd ${this.install_path}; source environment; env`;
@@ -136,7 +136,7 @@ class Package {
         logger.info(`Uninstalling ${this.language}-${this.version.raw}`);
 
         logger.debug("Finding runtime")
-        const found_runtime = runtime.get_latest_runtime_matching_language_version(this.language, this.version.raw);
+        const found_runtime = runtime.get_runtime_by_name_and_version(this.language, this.version.raw);
 
         if(!found_runtime){
             logger.error(`Uninstalling ${this.language}-${this.version.raw} failed: Not installed`)

api/src/runtime.js

@@ -9,17 +9,21 @@ const runtimes = [];
 
 class Runtime {
 
-    constructor(package_dir){
+    constructor({language, version, aliases, pkgdir, runtime}){
+        this.language = language;
+        this.version = version;
+        this.aliases = aliases || [];
+        this.pkgdir = pkgdir;
+        this.runtime = runtime;
+    }
+
+    static load_package(package_dir){
         let info = JSON.parse(
             fss.read_file_sync(path.join(package_dir, 'pkg-info.json'))
         );
 
-        const { language, version, build_platform, aliases } = info;
-
-        this.pkgdir = package_dir;
-        this.language = language;
-        this.version = semver.parse(version);
-        this.aliases = aliases;
+        let { language, version, build_platform, aliases, provides } = info;
+        version = semver.parse(version);
 
         if (build_platform !== globals.platform) {
             logger.warn(
@@ -28,9 +32,29 @@ class Runtime {
             );
         }
 
+        if(provides){
+            // Multiple languages in 1 package
+            provides.forEach(lang => {
+                runtimes.push(new Runtime({
+                    language: lang.language,
+                    aliases: lang.aliases,
+                    version,
+                    pkgdir: package_dir,
+                    runtime: language
+                }));
+            });
+        }else{
+            runtimes.push(new Runtime({
+                language,
+                version,
+                aliases,
+                pkgdir: package_dir
+            }))
+        }
+
         logger.debug(`Package ${language}-${version} was loaded`);
 
-        runtimes.push(this);
+        
     }
 
     get compiled() {
@@ -79,3 +103,9 @@ module.exports.get_latest_runtime_matching_language_version = function(lang, ver
     return module.exports.get_runtimes_matching_language_version(lang, ver)
         .sort((a,b) => semver.rcompare(a.version, b.version))[0];
 };
+
+module.exports.get_runtime_by_name_and_version = function(runtime, ver){
+    return runtimes.find(rt => (rt.runtime == runtime || (rt.runtime === undefined && rt.language == runtime)) && semver.satisfies(rt.version, ver));
+}
+
+module.exports.load_package = Runtime.load_package;

packages/gawk/5.1.0/compile

@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-
-# Put instructions to compile source code, remove this file if the language does not require this stage

packages/gawk/5.1.0/metadata.json

@@ -1,5 +1,10 @@
 {
     "language": "gawk",
     "version": "5.1.0",
-    "aliases": ["awk"]
+    "provides": [
+        {
+            "language": "awk",
+            "aliases": ["gawk"]
+        }
+    ]
 }

packages/gcc/10.2.0/compile

@@ -3,21 +3,21 @@
 # Put instructions to compile source code, remove this file if the language does not require this stage
 
 
-case "${PISTON_ALIAS}" in
-    gcc | c)
+case "${PISTON_LANGUAGE}" in
+    c)
         rename 's/$/\.c/' "$@" # Add .c extension
         gcc -std=c11 *.c -lm
         ;;
-    g++ | c++ | cpp)
+    c++)
         rename 's/$/\.cpp/' "$@" # Add .cpp extension
         g++ -std=c++17 *.cpp
         ;;
-    gdc | d)
-        rename 's/$/\.d/' "$@" # Add .d extension
+    d)
+        rename 's/.code$/\.d/' "$@" # Add .d extension
         gdc *.d
         ;;
     *)
-        echo "How did you get here? (${PISTON_ALIAS})"
+        echo "How did you get here? (${PISTON_LANGUAGE})"
         exit 1
         ;;
 esac

packages/gcc/10.2.0/metadata.json

@@ -1,5 +1,18 @@
 {
     "language": "gcc",
     "version": "10.2.0",
-    "aliases": ["c", "g++", "c++", "cpp", "gdc", "d"]
+    "provides": [
+        {
+            "language":"c",
+            "aliases": ["gcc"]
+        },
+        {
+            "language": "c++",
+            "aliases": ["cpp", "g++"]
+        },
+        {
+            "language": "d",
+            "aliases": ["gdc"]
+        }
+    ]
 }

packages/go/1.16.2/run

@@ -1,3 +1,6 @@
 #!/usr/bin/env bash
 
-GOCACHE=$PWD go run "$@"
+mv $1 $1.go
+filename=$1.go
+shift
+GOCACHE=$PWD go run $filename "$@"

packages/mono/6.12.0/compile

@@ -1,3 +1,4 @@
 #!/bin/bash
 
-csc "$@"
+rename 's/$/\.cs/' "$@" # Add .cs extension
+csc -out:out *.cs

packages/mono/6.12.0/environment

@@ -1 +1 @@
-export PATH=$PWD/mono-6.12.0:$PATH
+export PATH=$PWD/bin:$PATH

packages/mono/6.12.0/metadata.json

@@ -1,5 +1,10 @@
 {
     "language": "mono",
     "version": "6.12.0",
-    "aliases": ["csharp", "cs"]
+    "provides": [
+        {
+            "language": "csharp",
+            "aliases": ["mono", "mono-csharp", "mono-c#", "mono-cs", "c#", "cs"]
+        }
+    ]
 }

packages/mono/6.12.0/run

@@ -1,5 +1,4 @@
 #!/bin/bash
 
-CODE=${1/cs/exe}
 shift
-mono $CODE "$@"
+mono out "$@"

packages/nasm/2.15.5/compile

@@ -3,7 +3,7 @@
 # Put instructions to compile source code, remove this file if the language does not require this stage
 
 
-case "${PISTON_ALIAS}" in
+case "${PISTON_LANGUAGE}" in
     nasm)
         nasm -f elf32 -o binary.o "$@"
         ld -m elf_i386 binary.o -o binary
@@ -13,7 +13,7 @@ case "${PISTON_ALIAS}" in
         ld -m elf_x86_64 binary.o -o binary
         ;;
     *)
-        echo "How did you get here? (${PISTON_ALIAS})"
+        echo "How did you get here? (${PISTON_LANGUAGE})"
         exit 1
         ;;
 esac

packages/nasm/2.15.5/metadata.json

@@ -1,5 +1,14 @@
 {
     "language": "nasm",
     "version": "2.15.5",
-    "aliases": ["nasm64"]
+    "provides": [
+        {
+            "language": "nasm",
+            "aliases": ["asm", "nasm32"]
+        },
+        {
+            "language": "nasm64",
+            "aliases": ["asm64"]
+        }
+    ]
 }

packages/node/15.10.0/metadata.json

@@ -1,5 +1,10 @@
 {
     "language": "node",
     "version": "15.10.0",
-    "aliases": ["node-javascript", "node-js", "javascript", "js"]
+    "provides": [
+        {
+            "language": "javascript",
+            "aliases": ["node-javascript", "node-js", "javascript", "js"]
+        }
+    ]
 }

packages/ponylang/0.39.0/compile

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 
 # Compile pony file(s)
-rename 's/$/\.pong/' "$@" # Add .pony extension
+rename 's/$/\.pony/' "$@" # Add .pony extension
 ponyc -b out

packages/scala/3.0.0/run

@@ -1,4 +1,7 @@
 #!/usr/bin/env bash
 
 # Put instructions to run the runtime
-scala "$@"
+mv $1 $1.scala
+filename=$1.scala
+shift
+scala $filename "$@"

packages/test.sh

@@ -5,7 +5,8 @@ AUTH_HEADER="Authorization: $API_KEY"
 for test_file in */*/test.*
 do
 	IFS='/' read -ra test_parts <<< "$test_file"
-	language=${test_parts[0]}
+	IFS='.' read -ra file_parts <<< "$(basename $test_file)"
+	language=${file_parts[1]}
 	lang_ver=${test_parts[1]}
 
 	test_src=$(python3 -c "import json; print(json.dumps(open('$test_file').read()))")

piston

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+case $1 in
+    dev)
+        shift
+        docker-compose -f docker-compose.dev.yaml "$@"
+        ;;
+    prod)
+        shift
+        docker-compose -f docker-compose.yaml "$@"
+        ;;
+    update)
+        git pull
+        docker-compose pull api
+        docker-compose up -d api
+        ;;
+    clean-pkgs)
+        git clean -fqXd packages
+        ;;
+    *)
+        node cli/index.js "$@"
+        ;;
+esac

repo/mkindex.sh

@@ -1,4 +1,4 @@
-BASEURL=http://piston_fs_repo:8000/
+BASEURL=http://repo:8000/
 
 i=0