diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index 5c935c5..d5e2595 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -58,6 +58,27 @@ jobs: - name: Relocate downloaded packages run: mv *.pkg.tar.gz packages/ + - name: Write test config file + uses: DamianReeves/write-file-action@v1.0 + with: + path: data/config.yaml + contents: | + log_level: DEBUG + bind_address: 0.0.0.0:2000 + data_directory: /piston + runner_uid_min: 1100 + runner_uid_max: 1500 + runner_gid_min: 1100 + runner_gid_max: 1500 + disable_networking: false + output_max_size: 1024 + max_process_count: 64 + max_open_files: 2048 + max_file_size: 1000000 + repo_url: http://localhost:8000/index + + write-mode: overwrite + - name: Login to GitHub registry uses: docker/login-action@v1 with: @@ -69,7 +90,7 @@ jobs: run: | ls -la docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' -d --name repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build - docker run --network container:repo -v $(pwd)'/data:/piston' -e PISTON_LOG_LEVEL=DEBUG -e 'PISTON_REPO_URL=http://localhost:8000/index' -d --name api docker.pkg.github.com/engineer-man/piston/api + docker run --network container:repo -v $(pwd)'/data:/piston' -d --name api docker.pkg.github.com/engineer-man/piston/api echo Waiting for API to start.. docker run --network container:api appropriate/curl -s --retry 10 --retry-connrefused http://localhost:2000/api/v1/runtimes diff --git a/api/Dockerfile b/api/Dockerfile index cc2edf8..686ff02 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -25,6 +25,6 @@ COPY ./src ./src RUN make -C ./src/nosocket/ all && make -C ./src/nosocket/ install -CMD [ "node", "src"] +CMD [ "node", "src", "-m", "-c", "/piston/config.yaml"] EXPOSE 2000/tcp diff --git a/api/package-lock.json b/api/package-lock.json index c066efc..c523069 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -13,12 +13,16 @@ "chownr": "^2.0.0", "express": "^4.17.1", "is-docker": "^2.1.1", + "js-yaml": "^4.0.0", + "libsys": "^3.0.0", "logplease": "^1.2.15", "nocamel": "HexF/nocamel#patch-1", "node-fetch": "^2.6.1", + "ps-list": "^7.2.0", "semver": "^7.3.4", "uuid": "^8.3.2", - "waitpid": "git+https://github.com/HexF/node-waitpid.git" + "waitpid": "git+https://github.com/HexF/node-waitpid.git", + "yargs": "^16.2.0" } }, "node_modules/accepts": { @@ -33,6 +37,30 @@ "node": ">= 0.6" } }, + "node_modules/ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -74,6 +102,32 @@ "node": ">=10" } }, + "node_modules/cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, "node_modules/content-disposition": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", @@ -132,6 +186,11 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, "node_modules/encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -140,6 +199,14 @@ "node": ">= 0.8" } }, + "node_modules/escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "engines": { + "node": ">=6" + } + }, "node_modules/escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", @@ -226,6 +293,14 @@ "node": ">= 0.6" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, "node_modules/http-errors": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", @@ -276,6 +351,36 @@ "node": ">=8" } }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/js-yaml": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", + "integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/libsys": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/libsys/-/libsys-3.0.0.tgz", + "integrity": "sha512-+c1HMLbV9dIGHfVcjs0PDme9di1CopZ+oL4xqi7LGjeHJfvTsSUAlJtvcfqTkN9wak4WU4cjhLiG1iZwPY1IBw==", + "dependencies": { + "nan": "^2.14.0" + }, + "engines": { + "node": ">= 4.4.3" + } + }, "node_modules/logplease": { "version": "1.2.15", "resolved": "https://registry.npmjs.org/logplease/-/logplease-1.2.15.tgz", @@ -348,6 +453,11 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, + "node_modules/nan": { + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz", + "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==" + }, "node_modules/negotiator": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", @@ -403,6 +513,14 @@ "node": ">= 0.10" } }, + "node_modules/ps-list": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/ps-list/-/ps-list-7.2.0.tgz", + "integrity": "sha512-v4Bl6I3f2kJfr5o80ShABNHAokIgY+wFDTQfE+X3zWYgSGQOCBeYptLZUpoOALBqO5EawmDN/tjTldJesd0ujQ==", + "engines": { + "node": ">=10" + } + }, "node_modules/qs": { "version": "6.7.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", @@ -433,6 +551,14 @@ "node": ">= 0.8" } }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", @@ -512,6 +638,30 @@ "node": ">= 0.6" } }, + "node_modules/string-width": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", + "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dependencies": { + "ansi-regex": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/toidentifier": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", @@ -567,10 +717,56 @@ "node_modules/waitpid": { "resolved": "git+ssh://git@github.com/HexF/node-waitpid.git#a08d116a5d993a747624fe72ff890167be8c34aa" }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/y18n": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.5.tgz", + "integrity": "sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg==", + "engines": { + "node": ">=10" + } + }, "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, + "node_modules/yargs": { + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", + "dependencies": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs-parser": { + "version": "20.2.7", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.7.tgz", + "integrity": "sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw==", + "engines": { + "node": ">=10" + } } }, "dependencies": { @@ -583,6 +779,24 @@ "negotiator": "0.6.2" } }, + "ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==" + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "requires": { + "color-convert": "^2.0.1" + } + }, + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -615,6 +829,29 @@ "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==" }, + "cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, "content-disposition": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", @@ -661,11 +898,21 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" }, + "escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" + }, "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", @@ -737,6 +984,11 @@ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" + }, "http-errors": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", @@ -772,6 +1024,27 @@ "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.1.1.tgz", "integrity": "sha512-ZOoqiXfEwtGknTiuDEy8pN2CfE3TxMHprvNer1mXiqwkOT77Rw3YVrUQ52EqAOU3QAWDQ+bQdx7HJzrv7LS2Hw==" }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + }, + "js-yaml": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", + "integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==", + "requires": { + "argparse": "^2.0.1" + } + }, + "libsys": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/libsys/-/libsys-3.0.0.tgz", + "integrity": "sha512-+c1HMLbV9dIGHfVcjs0PDme9di1CopZ+oL4xqi7LGjeHJfvTsSUAlJtvcfqTkN9wak4WU4cjhLiG1iZwPY1IBw==", + "requires": { + "nan": "^2.14.0" + } + }, "logplease": { "version": "1.2.15", "resolved": "https://registry.npmjs.org/logplease/-/logplease-1.2.15.tgz", @@ -823,6 +1096,11 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, + "nan": { + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz", + "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==" + }, "negotiator": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", @@ -864,6 +1142,11 @@ "ipaddr.js": "1.9.1" } }, + "ps-list": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/ps-list/-/ps-list-7.2.0.tgz", + "integrity": "sha512-v4Bl6I3f2kJfr5o80ShABNHAokIgY+wFDTQfE+X3zWYgSGQOCBeYptLZUpoOALBqO5EawmDN/tjTldJesd0ujQ==" + }, "qs": { "version": "6.7.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", @@ -885,6 +1168,11 @@ "unpipe": "1.0.0" } }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" + }, "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", @@ -951,6 +1239,24 @@ "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" }, + "string-width": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", + "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + } + }, + "strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "requires": { + "ansi-regex": "^5.0.0" + } + }, "toidentifier": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", @@ -989,10 +1295,44 @@ "version": "git+ssh://git@github.com/HexF/node-waitpid.git#a08d116a5d993a747624fe72ff890167be8c34aa", "from": "waitpid@git+https://github.com/HexF/node-waitpid.git" }, + "wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + } + }, + "y18n": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.5.tgz", + "integrity": "sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg==" + }, "yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, + "yargs": { + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", + "requires": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + } + }, + "yargs-parser": { + "version": "20.2.7", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.7.tgz", + "integrity": "sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw==" } } } diff --git a/api/package.json b/api/package.json index cefd900..6787393 100644 --- a/api/package.json +++ b/api/package.json @@ -8,12 +8,16 @@ "chownr": "^2.0.0", "express": "^4.17.1", "is-docker": "^2.1.1", + "js-yaml": "^4.0.0", + "libsys": "^3.0.0", "logplease": "^1.2.15", "nocamel": "HexF/nocamel#patch-1", "node-fetch": "^2.6.1", + "ps-list": "^7.2.0", "semver": "^7.3.4", "uuid": "^8.3.2", - "waitpid": "git+https://github.com/HexF/node-waitpid.git" + "waitpid": "git+https://github.com/HexF/node-waitpid.git", + "yargs": "^16.2.0" }, "license": "MIT" } diff --git a/api/src/config.js b/api/src/config.js index 8cc1122..3a5cd72 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -1,7 +1,36 @@ const fss = require('fs'); +const yargs = require('yargs'); +const hide_bin = require('yargs/helpers').hideBin; const Logger = require('logplease'); const logger = Logger.create('config'); +const yaml = require('js-yaml'); +const header = `# +# ____ _ _ +# | _ \\(_)___| |_ ___ _ __ +# | |_) | / __| __/ _ \\| '_ \\ +# | __/| \\__ \\ || (_) | | | | +# |_| |_|___/\\__\\___/|_| |_| +# +# A High performance code execution engine +# github.com/engineer-man/piston +# + +`; +const argv = yargs(hide_bin(process.argv)) + .usage('Usage: $0 -c [config]') + .demandOption('c') + .option('config', { + alias: 'c', + describe: 'config file to load from', + default: '/piston/config.yaml' + }) + .option('make-config', { + alias: 'm', + type: 'boolean', + describe: 'create config file and populate defaults if it does not already exist' + }) + .argv; const options = [ { @@ -29,100 +58,67 @@ const options = [ key: 'runner_uid_min', desc: 'Minimum uid to use for runner', default: 1001, - parser: parse_int, - validators: [ - (x,raw) => !isNaN(x) || `${raw} is not a number`, - ] + validators: [] }, { key: 'runner_uid_max', desc: 'Maximum uid to use for runner', default: 1500, - parser: parse_int, - validators: [ - (x,raw) => !isNaN(x) || `${raw} is not a number`, - ] + validators: [] }, { key: 'runner_gid_min', desc: 'Minimum gid to use for runner', default: 1001, - parser: parse_int, - validators: [ - (x,raw) => !isNaN(x) || `${raw} is not a number`, - ] + validators: [] }, { key: 'runner_gid_max', desc: 'Maximum gid to use for runner', default: 1500, - parser: parse_int, - validators: [ - (x,raw) => !isNaN(x) || `${raw} is not a number`, - ] + validators: [] }, { key: 'disable_networking', desc: 'Set to true to disable networking', default: true, - parser: x => x === "true", - validators: [ - x => typeof x === "boolean" || `${x} is not a boolean` - ] + validators: [] }, { key: 'output_max_size', desc: 'Max size of each stdio buffer', default: 1024, - parser: parse_int, - validators: [ - (x,raw) => !isNaN(x) || `${raw} is not a number`, - ] + validators: [] }, { key: 'max_process_count', desc: 'Max number of processes per job', default: 64, - parser: parse_int, - validators: [ - (x,raw) => !isNaN(x) || `${raw} is not a number`, - ] + validators: [] }, { key: 'max_open_files', desc: 'Max number of open files per job', default: 2048, - parser: parse_int, - validators: [ - (x,raw) => !isNaN(x) || `${raw} is not a number`, - ] + validators: [] }, { key: 'max_file_size', desc: 'Max file size in bytes for a file', default: 1000000, //1MB - parser: parse_int, - validators: [ - (x,raw) => !isNaN(x) || `${raw} is not a number`, - ] + validators: [] }, { key: 'compile_memory_limit', desc: 'Max memory usage for compile stage in bytes (set to -1 for no limit)', default: -1, // no limit - parser: parse_int, - validators: [ - (x,raw) => !isNaN(x) || `${raw} is not a number`, - ] + validators: [] }, { key: 'run_memory_limit', desc: 'Max memory usage for run stage in bytes (set to -1 for no limit)', default: -1, // no limit - parser: parse_int, - validators: [ - (x,raw) => !isNaN(x) || `${raw} is not a number`, - ] + validators: [] }, { key: 'repo_url', @@ -132,40 +128,76 @@ const options = [ } ]; -logger.info(`Loading Configuration from environment`); +const make_default_config = () => { + let content = header.split('\n'); -let errored = false; + options.forEach(option => { + content = content.concat(option.desc.split('\n').map(x=>`# ${x}`)); + + if (option.options) { + content.push('# Options: ' + option.options.join(', ')); + } + + content.push(`${option.key}: ${option.default}`); + + content.push(''); // New line between + }); + + return content.join('\n'); +}; + +logger.info(`Loading Configuration from ${argv.config}`); + +if (argv['make-config']) { + logger.debug('Make configuration flag is set'); +} + +if (!!argv['make-config'] && !fss.exists_sync(argv.config)) { + logger.info('Writing default configuration...'); + try { + fss.write_file_sync(argv.config, make_default_config()); + } catch (e) { + logger.error('Error writing default configuration:', e.message); + process.exit(1); + } +} let config = {}; -options.forEach(option => { - const env_key = "PISTON_" + option.key.to_upper_case(); +logger.debug('Reading config file'); - const parser = option.parser || (x=>x); +try { + const cfg_content = fss.read_file_sync(argv.config); + config = yaml.load(cfg_content); +} catch(err) { + logger.error('Error reading configuration file:', err.message); + process.exit(1); +} - const env_val = process.env[env_key]; +logger.debug('Validating config entries'); - const parsed_val = parser(env_val); +let errored = false; - - const value = env_val || option.default; +options.for_each(option => { + logger.debug('Checking option', option.key); + let cfg_val = config[option.key]; + + if (cfg_val === undefined) { + errored = true; + logger.error(`Config key ${option.key} does not exist on currently loaded configuration`); + return; + } option.validators.for_each(validator => { - let response = null; - if(env_val) - response = validator(parsed_val, env_val); - else - response = validator(value, value); + let response = validator(cfg_val); - if (response !== true) { + if (!response) { errored = true; logger.error(`Config option ${option.key} failed validation:`, response); return; } }); - - config[option.key] = value; }); if (errored) { @@ -174,5 +206,4 @@ if (errored) { logger.info('Configuration successfully loaded'); - module.exports = config; diff --git a/api/src/job.js b/api/src/job.js index 9484974..fcbfb23 100644 --- a/api/src/job.js +++ b/api/src/job.js @@ -5,6 +5,7 @@ const path = require('path'); const config = require('./config'); const globals = require('./globals'); const fs = require('fs/promises'); +const ps_list = require('ps-list'); const wait_pid = require('waitpid'); const job_states = { @@ -193,25 +194,8 @@ class Job { async cleanup_processes(){ let processes = [1]; while(processes.length > 0){ - processes = await new Promise((resolve, reject) => cp.execFile('ps', ['awwxo', 'pid,ruid'], function(err, stdout) { - if(err === null){ - const lines = stdout.split('\n').slice(1); //Remove header with slice - const procs = lines.map(line => { - const [pid, ruid] = line - .trim() - .split(/\s+/) - .map(n => parseInt(n)); - - return { pid, ruid } - }) - resolve(procs) - } - else{ - reject(error) - } - })); - - processes = processes.filter(proc => proc.ruid == this.uid); + processes = await ps_list(); + processes = processes.filter(proc => proc.uid == this.uid); for(const proc of processes){ // First stop the processes, but keep their resources allocated so they cant re-fork diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 31772fe..f2b3a14 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -11,8 +11,6 @@ services: - 2000:2000 volumes: - ./data/piston:/piston - environment: - - PISTON_REPO_URL=http://repo:8000/index tmpfs: - /piston/jobs:exec