Mascali
/
piston
mirror of https://github.com/engineer-man/piston.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Compare commits

base: Mascali:fb102ebe8347d705bfe75d248e2d75d3cc7ee9de
Branches Tags
Mascali:master
Mascali:realtux-patch-1
Mascali:dependabot/npm_and_yarn/cli/semver-7.5.2
Mascali:dependabot/npm_and_yarn/api/semver-7.5.2
Mascali:dependabot/npm_and_yarn/api/node-fetch-2.6.7
Mascali:dependabot/npm_and_yarn/cli/minimatch-3.0.5
Mascali:nix-packages
Mascali:binary
Mascali:websocket
Mascali:HexF-patch-2
Mascali:HexF-patch-1
Mascali:pkgs
...
compare: Mascali:0da1cd586747d845f6b380828127b65fe8d9e310
Branches Tags
Mascali:master
Mascali:realtux-patch-1
Mascali:dependabot/npm_and_yarn/cli/semver-7.5.2
Mascali:dependabot/npm_and_yarn/api/semver-7.5.2
Mascali:dependabot/npm_and_yarn/api/node-fetch-2.6.7
Mascali:dependabot/npm_and_yarn/cli/minimatch-3.0.5
Mascali:nix-packages
Mascali:binary
Mascali:websocket
Mascali:HexF-patch-2
Mascali:HexF-patch-1
Mascali:pkgs

2 Commits
fb102ebe83 ... 0da1cd5867

Author SHA1 Message Date
Thomas Hobson 0da1cd5867
use real uid for killing processes instead of using euid (#206) 2021-05-07 22:11:28 +12:00
Thomas Hobson 972228412a
switch to taking config by env 
To help with ease of deployment, configuration now is administered through the use of environment variables.
This allows for easy configuration directly from the docker-compose file and is a much more robust solution.
 2021-05-07 21:11:36 +12:00
7 changed files with 91 additions and 469 deletions
Show Stats Expand all files Collapse all files

23
.github/workflows/package-pr.yaml vendored
View File

@ -58,27 +58,6 @@ jobs:
- name: Relocate downloaded packages - name: Relocate downloaded packages
run: mv *.pkg.tar.gz packages/ run: mv *.pkg.tar.gz packages/
- name: Write test config file
uses: DamianReeves/write-file-action@v1.0
with:
path: data/config.yaml
contents: |
log_level: DEBUG
bind_address: 0.0.0.0:2000
data_directory: /piston
runner_uid_min: 1100
runner_uid_max: 1500
runner_gid_min: 1100
runner_gid_max: 1500
disable_networking: false
output_max_size: 1024
max_process_count: 64
max_open_files: 2048
max_file_size: 1000000
repo_url: http://localhost:8000/index
write-mode: overwrite
- name: Login to GitHub registry - name: Login to GitHub registry
uses: docker/login-action@v1 uses: docker/login-action@v1
with: with:
@ -90,7 +69,7 @@ jobs:
run: | run: |
ls -la ls -la
docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' -d --name repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' -d --name repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build
docker run --network container:repo -v $(pwd)'/data:/piston' -d --name api docker.pkg.github.com/engineer-man/piston/api docker run --network container:repo -v $(pwd)'/data:/piston' -e PISTON_LOG_LEVEL=DEBUG -e 'PISTON_REPO_URL=http://localhost:8000/index' -d --name api docker.pkg.github.com/engineer-man/piston/api
echo Waiting for API to start.. echo Waiting for API to start..
docker run --network container:api appropriate/curl -s --retry 10 --retry-connrefused http://localhost:2000/api/v1/runtimes docker run --network container:api appropriate/curl -s --retry 10 --retry-connrefused http://localhost:2000/api/v1/runtimes

2
api/Dockerfile
View File

@ -25,6 +25,6 @@ COPY ./src ./src
RUN make -C ./src/nosocket/ all && make -C ./src/nosocket/ install RUN make -C ./src/nosocket/ all && make -C ./src/nosocket/ install
CMD [ "node", "src", "-m", "-c", "/piston/config.yaml"] CMD [ "node", "src"]
EXPOSE 2000/tcp EXPOSE 2000/tcp

342
api/package-lock.json generated
View File

@ -13,16 +13,12 @@
"chownr": "^2.0.0", "chownr": "^2.0.0",
"express": "^4.17.1", "express": "^4.17.1",
"is-docker": "^2.1.1", "is-docker": "^2.1.1",
"js-yaml": "^4.0.0",
"libsys": "^3.0.0",
"logplease": "^1.2.15", "logplease": "^1.2.15",
"nocamel": "HexF/nocamel#patch-1", "nocamel": "HexF/nocamel#patch-1",
"node-fetch": "^2.6.1", "node-fetch": "^2.6.1",
"ps-list": "^7.2.0",
"semver": "^7.3.4", "semver": "^7.3.4",
"uuid": "^8.3.2", "uuid": "^8.3.2",
"waitpid": "git+https://github.com/HexF/node-waitpid.git", "waitpid": "git+https://github.com/HexF/node-waitpid.git"
"yargs": "^16.2.0"
} }
}, },
"node_modules/accepts": { "node_modules/accepts": {
@ -37,30 +33,6 @@
"node": ">= 0.6" "node": ">= 0.6"
} }
}, },
"node_modules/ansi-regex": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
"integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
"engines": {
"node": ">=8"
}
},
"node_modules/ansi-styles": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
"dependencies": {
"color-convert": "^2.0.1"
},
"engines": {
"node": ">=8"
}
},
"node_modules/argparse": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="
},
"node_modules/array-flatten": { "node_modules/array-flatten": {
"version": "1.1.1", "version": "1.1.1",
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
@ -102,32 +74,6 @@
"node": ">=10" "node": ">=10"
} }
}, },
"node_modules/cliui": {
"version": "7.0.4",
"resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz",
"integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==",
"dependencies": {
"string-width": "^4.2.0",
"strip-ansi": "^6.0.0",
"wrap-ansi": "^7.0.0"
}
},
"node_modules/color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
"dependencies": {
"color-name": "~1.1.4"
},
"engines": {
"node": ">=7.0.0"
}
},
"node_modules/color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
},
"node_modules/content-disposition": { "node_modules/content-disposition": {
"version": "0.5.3", "version": "0.5.3",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz",
@ -186,11 +132,6 @@
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
"integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0="
}, },
"node_modules/emoji-regex": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
},
"node_modules/encodeurl": { "node_modules/encodeurl": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
@ -199,14 +140,6 @@
"node": ">= 0.8" "node": ">= 0.8"
} }
}, },
"node_modules/escalade": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
"integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==",
"engines": {
"node": ">=6"
}
},
"node_modules/escape-html": { "node_modules/escape-html": {
"version": "1.0.3", "version": "1.0.3",
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
@ -293,14 +226,6 @@
"node": ">= 0.6" "node": ">= 0.6"
} }
}, },
"node_modules/get-caller-file": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
"engines": {
"node": "6.* || 8.* || >= 10.*"
}
},
"node_modules/http-errors": { "node_modules/http-errors": {
"version": "1.7.2", "version": "1.7.2",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
@ -351,36 +276,6 @@
"node": ">=8" "node": ">=8"
} }
}, },
"node_modules/is-fullwidth-code-point": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
"engines": {
"node": ">=8"
}
},
"node_modules/js-yaml": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz",
"integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==",
"dependencies": {
"argparse": "^2.0.1"
},
"bin": {
"js-yaml": "bin/js-yaml.js"
}
},
"node_modules/libsys": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/libsys/-/libsys-3.0.0.tgz",
"integrity": "sha512-+c1HMLbV9dIGHfVcjs0PDme9di1CopZ+oL4xqi7LGjeHJfvTsSUAlJtvcfqTkN9wak4WU4cjhLiG1iZwPY1IBw==",
"dependencies": {
"nan": "^2.14.0"
},
"engines": {
"node": ">= 4.4.3"
}
},
"node_modules/logplease": { "node_modules/logplease": {
"version": "1.2.15", "version": "1.2.15",
"resolved": "https://registry.npmjs.org/logplease/-/logplease-1.2.15.tgz", "resolved": "https://registry.npmjs.org/logplease/-/logplease-1.2.15.tgz",
@ -453,11 +348,6 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
}, },
"node_modules/nan": {
"version": "2.14.2",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz",
"integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ=="
},
"node_modules/negotiator": { "node_modules/negotiator": {
"version": "0.6.2", "version": "0.6.2",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
@ -513,14 +403,6 @@
"node": ">= 0.10" "node": ">= 0.10"
} }
}, },
"node_modules/ps-list": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/ps-list/-/ps-list-7.2.0.tgz",
"integrity": "sha512-v4Bl6I3f2kJfr5o80ShABNHAokIgY+wFDTQfE+X3zWYgSGQOCBeYptLZUpoOALBqO5EawmDN/tjTldJesd0ujQ==",
"engines": {
"node": ">=10"
}
},
"node_modules/qs": { "node_modules/qs": {
"version": "6.7.0", "version": "6.7.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
@ -551,14 +433,6 @@
"node": ">= 0.8" "node": ">= 0.8"
} }
}, },
"node_modules/require-directory": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
"integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/safe-buffer": { "node_modules/safe-buffer": {
"version": "5.1.2", "version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
@ -638,30 +512,6 @@
"node": ">= 0.6" "node": ">= 0.6"
} }
}, },
"node_modules/string-width": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz",
"integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==",
"dependencies": {
"emoji-regex": "^8.0.0",
"is-fullwidth-code-point": "^3.0.0",
"strip-ansi": "^6.0.0"
},
"engines": {
"node": ">=8"
}
},
"node_modules/strip-ansi": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
"integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
"dependencies": {
"ansi-regex": "^5.0.0"
},
"engines": {
"node": ">=8"
}
},
"node_modules/toidentifier": { "node_modules/toidentifier": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
@ -717,56 +567,10 @@
"node_modules/waitpid": { "node_modules/waitpid": {
"resolved": "git+ssh://git@github.com/HexF/node-waitpid.git#a08d116a5d993a747624fe72ff890167be8c34aa" "resolved": "git+ssh://git@github.com/HexF/node-waitpid.git#a08d116a5d993a747624fe72ff890167be8c34aa"
}, },
"node_modules/wrap-ansi": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
"integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
"dependencies": {
"ansi-styles": "^4.0.0",
"string-width": "^4.1.0",
"strip-ansi": "^6.0.0"
},
"engines": {
"node": ">=10"
}
},
"node_modules/y18n": {
"version": "5.0.5",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.5.tgz",
"integrity": "sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg==",
"engines": {
"node": ">=10"
}
},
"node_modules/yallist": { "node_modules/yallist": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
},
"node_modules/yargs": {
"version": "16.2.0",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz",
"integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==",
"dependencies": {
"cliui": "^7.0.2",
"escalade": "^3.1.1",
"get-caller-file": "^2.0.5",
"require-directory": "^2.1.1",
"string-width": "^4.2.0",
"y18n": "^5.0.5",
"yargs-parser": "^20.2.2"
},
"engines": {
"node": ">=10"
}
},
"node_modules/yargs-parser": {
"version": "20.2.7",
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.7.tgz",
"integrity": "sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw==",
"engines": {
"node": ">=10"
}
} }
}, },
"dependencies": { "dependencies": {
@ -779,24 +583,6 @@
"negotiator": "0.6.2" "negotiator": "0.6.2"
} }
}, },
"ansi-regex": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
"integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg=="
},
"ansi-styles": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
"requires": {
"color-convert": "^2.0.1"
}
},
"argparse": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="
},
"array-flatten": { "array-flatten": {
"version": "1.1.1", "version": "1.1.1",
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
@ -829,29 +615,6 @@
"resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz",
"integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==" "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ=="
}, },
"cliui": {
"version": "7.0.4",
"resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz",
"integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==",
"requires": {
"string-width": "^4.2.0",
"strip-ansi": "^6.0.0",
"wrap-ansi": "^7.0.0"
}
},
"color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
"requires": {
"color-name": "~1.1.4"
}
},
"color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
},
"content-disposition": { "content-disposition": {
"version": "0.5.3", "version": "0.5.3",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz",
@ -898,21 +661,11 @@
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
"integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0="
}, },
"emoji-regex": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
},
"encodeurl": { "encodeurl": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
"integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k="
}, },
"escalade": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
"integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw=="
},
"escape-html": { "escape-html": {
"version": "1.0.3", "version": "1.0.3",
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
@ -984,11 +737,6 @@
"resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
"integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac="
}, },
"get-caller-file": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="
},
"http-errors": { "http-errors": {
"version": "1.7.2", "version": "1.7.2",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
@ -1024,27 +772,6 @@
"resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.1.1.tgz", "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.1.1.tgz",
"integrity": "sha512-ZOoqiXfEwtGknTiuDEy8pN2CfE3TxMHprvNer1mXiqwkOT77Rw3YVrUQ52EqAOU3QAWDQ+bQdx7HJzrv7LS2Hw==" "integrity": "sha512-ZOoqiXfEwtGknTiuDEy8pN2CfE3TxMHprvNer1mXiqwkOT77Rw3YVrUQ52EqAOU3QAWDQ+bQdx7HJzrv7LS2Hw=="
}, },
"is-fullwidth-code-point": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="
},
"js-yaml": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz",
"integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==",
"requires": {
"argparse": "^2.0.1"
}
},
"libsys": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/libsys/-/libsys-3.0.0.tgz",
"integrity": "sha512-+c1HMLbV9dIGHfVcjs0PDme9di1CopZ+oL4xqi7LGjeHJfvTsSUAlJtvcfqTkN9wak4WU4cjhLiG1iZwPY1IBw==",
"requires": {
"nan": "^2.14.0"
}
},
"logplease": { "logplease": {
"version": "1.2.15", "version": "1.2.15",
"resolved": "https://registry.npmjs.org/logplease/-/logplease-1.2.15.tgz", "resolved": "https://registry.npmjs.org/logplease/-/logplease-1.2.15.tgz",
@ -1096,11 +823,6 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
}, },
"nan": {
"version": "2.14.2",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz",
"integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ=="
},
"negotiator": { "negotiator": {
"version": "0.6.2", "version": "0.6.2",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
@ -1142,11 +864,6 @@
"ipaddr.js": "1.9.1" "ipaddr.js": "1.9.1"
} }
}, },
"ps-list": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/ps-list/-/ps-list-7.2.0.tgz",
"integrity": "sha512-v4Bl6I3f2kJfr5o80ShABNHAokIgY+wFDTQfE+X3zWYgSGQOCBeYptLZUpoOALBqO5EawmDN/tjTldJesd0ujQ=="
},
"qs": { "qs": {
"version": "6.7.0", "version": "6.7.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
@ -1168,11 +885,6 @@
"unpipe": "1.0.0" "unpipe": "1.0.0"
} }
}, },
"require-directory": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
"integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I="
},
"safe-buffer": { "safe-buffer": {
"version": "5.1.2", "version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
@ -1239,24 +951,6 @@
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
"integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow="
}, },
"string-width": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz",
"integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==",
"requires": {
"emoji-regex": "^8.0.0",
"is-fullwidth-code-point": "^3.0.0",
"strip-ansi": "^6.0.0"
}
},
"strip-ansi": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
"integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
"requires": {
"ansi-regex": "^5.0.0"
}
},
"toidentifier": { "toidentifier": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
@ -1295,44 +989,10 @@
"version": "git+ssh://git@github.com/HexF/node-waitpid.git#a08d116a5d993a747624fe72ff890167be8c34aa", "version": "git+ssh://git@github.com/HexF/node-waitpid.git#a08d116a5d993a747624fe72ff890167be8c34aa",
"from": "waitpid@git+https://github.com/HexF/node-waitpid.git" "from": "waitpid@git+https://github.com/HexF/node-waitpid.git"
}, },
"wrap-ansi": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
"integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
"requires": {
"ansi-styles": "^4.0.0",
"string-width": "^4.1.0",
"strip-ansi": "^6.0.0"
}
},
"y18n": {
"version": "5.0.5",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.5.tgz",
"integrity": "sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg=="
},
"yallist": { "yallist": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
},
"yargs": {
"version": "16.2.0",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz",
"integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==",
"requires": {
"cliui": "^7.0.2",
"escalade": "^3.1.1",
"get-caller-file": "^2.0.5",
"require-directory": "^2.1.1",
"string-width": "^4.2.0",
"y18n": "^5.0.5",
"yargs-parser": "^20.2.2"
}
},
"yargs-parser": {
"version": "20.2.7",
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.7.tgz",
"integrity": "sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw=="
} }
} }
} }

6
api/package.json
View File

@ -8,16 +8,12 @@
"chownr": "^2.0.0", "chownr": "^2.0.0",
"express": "^4.17.1", "express": "^4.17.1",
"is-docker": "^2.1.1", "is-docker": "^2.1.1",
"js-yaml": "^4.0.0",
"libsys": "^3.0.0",
"logplease": "^1.2.15", "logplease": "^1.2.15",
"nocamel": "HexF/nocamel#patch-1", "nocamel": "HexF/nocamel#patch-1",
"node-fetch": "^2.6.1", "node-fetch": "^2.6.1",
"ps-list": "^7.2.0",
"semver": "^7.3.4", "semver": "^7.3.4",
"uuid": "^8.3.2", "uuid": "^8.3.2",
"waitpid": "git+https://github.com/HexF/node-waitpid.git", "waitpid": "git+https://github.com/HexF/node-waitpid.git"
"yargs": "^16.2.0"
}, },
"license": "MIT" "license": "MIT"
} }

163
api/src/config.js
View File

@ -1,36 +1,7 @@
const fss = require('fs'); const fss = require('fs');
const yargs = require('yargs');
const hide_bin = require('yargs/helpers').hideBin;
const Logger = require('logplease'); const Logger = require('logplease');
const logger = Logger.create('config'); const logger = Logger.create('config');
const yaml = require('js-yaml');
const header = `#
# ____ _ _
# | _ \\(_)___| |_ ___ _ __
# | |_) | / __| __/ _ \\| '_ \\
# | __/| \\__ \\ || (_) | | | |
# |_| |_|___/\\__\\___/|_| |_|
#
# A High performance code execution engine
# github.com/engineer-man/piston
#
`;
const argv = yargs(hide_bin(process.argv))
.usage('Usage: $0 -c [config]')
.demandOption('c')
.option('config', {
alias: 'c',
describe: 'config file to load from',
default: '/piston/config.yaml'
})
.option('make-config', {
alias: 'm',
type: 'boolean',
describe: 'create config file and populate defaults if it does not already exist'
})
.argv;
const options = [ const options = [
{ {
@ -58,67 +29,100 @@ const options = [
key: 'runner_uid_min', key: 'runner_uid_min',
desc: 'Minimum uid to use for runner', desc: 'Minimum uid to use for runner',
default: 1001, default: 1001,
validators: [] parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'runner_uid_max', key: 'runner_uid_max',
desc: 'Maximum uid to use for runner', desc: 'Maximum uid to use for runner',
default: 1500, default: 1500,
validators: [] parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'runner_gid_min', key: 'runner_gid_min',
desc: 'Minimum gid to use for runner', desc: 'Minimum gid to use for runner',
default: 1001, default: 1001,
validators: [] parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'runner_gid_max', key: 'runner_gid_max',
desc: 'Maximum gid to use for runner', desc: 'Maximum gid to use for runner',
default: 1500, default: 1500,
validators: [] parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'disable_networking', key: 'disable_networking',
desc: 'Set to true to disable networking', desc: 'Set to true to disable networking',
default: true, default: true,
validators: [] parser: x => x === "true",
validators: [
x => typeof x === "boolean" || `${x} is not a boolean`
]
}, },
{ {
key: 'output_max_size', key: 'output_max_size',
desc: 'Max size of each stdio buffer', desc: 'Max size of each stdio buffer',
default: 1024, default: 1024,
validators: [] parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'max_process_count', key: 'max_process_count',
desc: 'Max number of processes per job', desc: 'Max number of processes per job',
default: 64, default: 64,
validators: [] parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'max_open_files', key: 'max_open_files',
desc: 'Max number of open files per job', desc: 'Max number of open files per job',
default: 2048, default: 2048,
validators: [] parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'max_file_size', key: 'max_file_size',
desc: 'Max file size in bytes for a file', desc: 'Max file size in bytes for a file',
default: 1000000, //1MB default: 1000000, //1MB
validators: [] parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'compile_memory_limit', key: 'compile_memory_limit',
desc: 'Max memory usage for compile stage in bytes (set to -1 for no limit)', desc: 'Max memory usage for compile stage in bytes (set to -1 for no limit)',
default: -1, // no limit default: -1, // no limit
validators: [] parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'run_memory_limit', key: 'run_memory_limit',
desc: 'Max memory usage for run stage in bytes (set to -1 for no limit)', desc: 'Max memory usage for run stage in bytes (set to -1 for no limit)',
default: -1, // no limit default: -1, // no limit
validators: [] parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'repo_url', key: 'repo_url',
@ -128,76 +132,40 @@ const options = [
} }
]; ];
const make_default_config = () => { logger.info(`Loading Configuration from environment`);
let content = header.split('\n');
options.forEach(option => {
content = content.concat(option.desc.split('\n').map(x=>`# ${x}`));
if (option.options) {
content.push('# Options: ' + option.options.join(', '));
}
content.push(`${option.key}: ${option.default}`);
content.push(''); // New line between
});
return content.join('\n');
};
logger.info(`Loading Configuration from ${argv.config}`);
if (argv['make-config']) {
logger.debug('Make configuration flag is set');
}
if (!!argv['make-config'] && !fss.exists_sync(argv.config)) {
logger.info('Writing default configuration...');
try {
fss.write_file_sync(argv.config, make_default_config());
} catch (e) {
logger.error('Error writing default configuration:', e.message);
process.exit(1);
}
}
let config = {};
logger.debug('Reading config file');
try {
const cfg_content = fss.read_file_sync(argv.config);
config = yaml.load(cfg_content);
} catch(err) {
logger.error('Error reading configuration file:', err.message);
process.exit(1);
}
logger.debug('Validating config entries');
let errored = false; let errored = false;
options.for_each(option => { let config = {};
logger.debug('Checking option', option.key);
let cfg_val = config[option.key]; options.forEach(option => {
const env_key = "PISTON_" + option.key.to_upper_case();
const parser = option.parser || (x=>x);
const env_val = process.env[env_key];
const parsed_val = parser(env_val);
const value = env_val || option.default;
if (cfg_val === undefined) {
errored = true;
logger.error(`Config key ${option.key} does not exist on currently loaded configuration`);
return;
}
option.validators.for_each(validator => { option.validators.for_each(validator => {
let response = validator(cfg_val); let response = null;
if(env_val)
response = validator(parsed_val, env_val);
else
response = validator(value, value);
if (!response) { if (response !== true) {
errored = true; errored = true;
logger.error(`Config option ${option.key} failed validation:`, response); logger.error(`Config option ${option.key} failed validation:`, response);
return; return;
} }
}); });
config[option.key] = value;
}); });
if (errored) { if (errored) {
@ -206,4 +174,5 @@ if (errored) {
logger.info('Configuration successfully loaded'); logger.info('Configuration successfully loaded');
module.exports = config; module.exports = config;

22
api/src/job.js
View File

@ -5,7 +5,6 @@ const path = require('path');
const config = require('./config'); const config = require('./config');
const globals = require('./globals'); const globals = require('./globals');
const fs = require('fs/promises'); const fs = require('fs/promises');
const ps_list = require('ps-list');
const wait_pid = require('waitpid'); const wait_pid = require('waitpid');
const job_states = { const job_states = {
@ -194,8 +193,25 @@ class Job {
async cleanup_processes(){ async cleanup_processes(){
let processes = [1]; let processes = [1];
while(processes.length > 0){ while(processes.length > 0){
processes = await ps_list(); processes = await new Promise((resolve, reject) => cp.execFile('ps', ['awwxo', 'pid,ruid'], function(err, stdout) {
processes = processes.filter(proc => proc.uid == this.uid); if(err === null){
const lines = stdout.split('\n').slice(1); //Remove header with slice
const procs = lines.map(line => {
const [pid, ruid] = line
.trim()
.split(/\s+/)
.map(n => parseInt(n));
return { pid, ruid }
})
resolve(procs)
}
else{
reject(error)
}
}));
processes = processes.filter(proc => proc.ruid == this.uid);
for(const proc of processes){ for(const proc of processes){
// First stop the processes, but keep their resources allocated so they cant re-fork // First stop the processes, but keep their resources allocated so they cant re-fork

2
docker-compose.dev.yaml
View File

@ -11,6 +11,8 @@ services:
- 2000:2000 - 2000:2000
volumes: volumes:
- ./data/piston:/piston - ./data/piston:/piston
environment:
- PISTON_REPO_URL=http://repo:8000/index
tmpfs: tmpfs:
- /piston/jobs:exec - /piston/jobs:exec