From 972228412ae2d9381e19c5f015d9b9d13533593d Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Fri, 7 May 2021 21:11:36 +1200 Subject: [PATCH] switch to taking config by env To help with ease of deployment, configuration now is administered through the use of environment variables. This allows for easy configuration directly from the docker-compose file and is a much more robust solution. --- .github/workflows/package-pr.yaml | 23 +-- api/Dockerfile | 2 +- api/package-lock.json | 328 +----------------------------- api/package.json | 5 +- api/src/config.js | 163 ++++++--------- docker-compose.dev.yaml | 2 + 6 files changed, 72 insertions(+), 451 deletions(-) diff --git a/.github/workflows/package-pr.yaml b/.github/workflows/package-pr.yaml index d5e2595..5c935c5 100644 --- a/.github/workflows/package-pr.yaml +++ b/.github/workflows/package-pr.yaml @@ -58,27 +58,6 @@ jobs: - name: Relocate downloaded packages run: mv *.pkg.tar.gz packages/ - - name: Write test config file - uses: DamianReeves/write-file-action@v1.0 - with: - path: data/config.yaml - contents: | - log_level: DEBUG - bind_address: 0.0.0.0:2000 - data_directory: /piston - runner_uid_min: 1100 - runner_uid_max: 1500 - runner_gid_min: 1100 - runner_gid_max: 1500 - disable_networking: false - output_max_size: 1024 - max_process_count: 64 - max_open_files: 2048 - max_file_size: 1000000 - repo_url: http://localhost:8000/index - - write-mode: overwrite - - name: Login to GitHub registry uses: docker/login-action@v1 with: @@ -90,7 +69,7 @@ jobs: run: | ls -la docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' -d --name repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build - docker run --network container:repo -v $(pwd)'/data:/piston' -d --name api docker.pkg.github.com/engineer-man/piston/api + docker run --network container:repo -v $(pwd)'/data:/piston' -e PISTON_LOG_LEVEL=DEBUG -e 'PISTON_REPO_URL=http://localhost:8000/index' -d --name api docker.pkg.github.com/engineer-man/piston/api echo Waiting for API to start.. docker run --network container:api appropriate/curl -s --retry 10 --retry-connrefused http://localhost:2000/api/v1/runtimes diff --git a/api/Dockerfile b/api/Dockerfile index 686ff02..cc2edf8 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -25,6 +25,6 @@ COPY ./src ./src RUN make -C ./src/nosocket/ all && make -C ./src/nosocket/ install -CMD [ "node", "src", "-m", "-c", "/piston/config.yaml"] +CMD [ "node", "src"] EXPOSE 2000/tcp diff --git a/api/package-lock.json b/api/package-lock.json index c523069..b462747 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -13,16 +13,13 @@ "chownr": "^2.0.0", "express": "^4.17.1", "is-docker": "^2.1.1", - "js-yaml": "^4.0.0", - "libsys": "^3.0.0", "logplease": "^1.2.15", "nocamel": "HexF/nocamel#patch-1", "node-fetch": "^2.6.1", "ps-list": "^7.2.0", "semver": "^7.3.4", "uuid": "^8.3.2", - "waitpid": "git+https://github.com/HexF/node-waitpid.git", - "yargs": "^16.2.0" + "waitpid": "git+https://github.com/HexF/node-waitpid.git" } }, "node_modules/accepts": { @@ -37,30 +34,6 @@ "node": ">= 0.6" } }, - "node_modules/ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", - "engines": { - "node": ">=8" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" - }, "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -102,32 +75,6 @@ "node": ">=10" } }, - "node_modules/cliui": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", - "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^7.0.0" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, "node_modules/content-disposition": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", @@ -186,11 +133,6 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, "node_modules/encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -199,14 +141,6 @@ "node": ">= 0.8" } }, - "node_modules/escalade": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", - "engines": { - "node": ">=6" - } - }, "node_modules/escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", @@ -293,14 +227,6 @@ "node": ">= 0.6" } }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, "node_modules/http-errors": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", @@ -351,36 +277,6 @@ "node": ">=8" } }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "engines": { - "node": ">=8" - } - }, - "node_modules/js-yaml": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", - "integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/libsys": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/libsys/-/libsys-3.0.0.tgz", - "integrity": "sha512-+c1HMLbV9dIGHfVcjs0PDme9di1CopZ+oL4xqi7LGjeHJfvTsSUAlJtvcfqTkN9wak4WU4cjhLiG1iZwPY1IBw==", - "dependencies": { - "nan": "^2.14.0" - }, - "engines": { - "node": ">= 4.4.3" - } - }, "node_modules/logplease": { "version": "1.2.15", "resolved": "https://registry.npmjs.org/logplease/-/logplease-1.2.15.tgz", @@ -453,11 +349,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, - "node_modules/nan": { - "version": "2.14.2", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz", - "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==" - }, "node_modules/negotiator": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", @@ -551,14 +442,6 @@ "node": ">= 0.8" } }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", @@ -638,30 +521,6 @@ "node": ">= 0.6" } }, - "node_modules/string-width": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", - "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", - "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", - "dependencies": { - "ansi-regex": "^5.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/toidentifier": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", @@ -717,56 +576,10 @@ "node_modules/waitpid": { "resolved": "git+ssh://git@github.com/HexF/node-waitpid.git#a08d116a5d993a747624fe72ff890167be8c34aa" }, - "node_modules/wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/y18n": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.5.tgz", - "integrity": "sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg==", - "engines": { - "node": ">=10" - } - }, "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, - "node_modules/yargs": { - "version": "16.2.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", - "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", - "dependencies": { - "cliui": "^7.0.2", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^20.2.2" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/yargs-parser": { - "version": "20.2.7", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.7.tgz", - "integrity": "sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw==", - "engines": { - "node": ">=10" - } } }, "dependencies": { @@ -779,24 +592,6 @@ "negotiator": "0.6.2" } }, - "ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==" - }, - "ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "requires": { - "color-convert": "^2.0.1" - } - }, - "argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" - }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -829,29 +624,6 @@ "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==" }, - "cliui": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", - "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", - "requires": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^7.0.0" - } - }, - "color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "requires": { - "color-name": "~1.1.4" - } - }, - "color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, "content-disposition": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", @@ -898,21 +670,11 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, - "emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" }, - "escalade": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" - }, "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", @@ -984,11 +746,6 @@ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, - "get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" - }, "http-errors": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", @@ -1024,27 +781,6 @@ "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.1.1.tgz", "integrity": "sha512-ZOoqiXfEwtGknTiuDEy8pN2CfE3TxMHprvNer1mXiqwkOT77Rw3YVrUQ52EqAOU3QAWDQ+bQdx7HJzrv7LS2Hw==" }, - "is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" - }, - "js-yaml": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", - "integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==", - "requires": { - "argparse": "^2.0.1" - } - }, - "libsys": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/libsys/-/libsys-3.0.0.tgz", - "integrity": "sha512-+c1HMLbV9dIGHfVcjs0PDme9di1CopZ+oL4xqi7LGjeHJfvTsSUAlJtvcfqTkN9wak4WU4cjhLiG1iZwPY1IBw==", - "requires": { - "nan": "^2.14.0" - } - }, "logplease": { "version": "1.2.15", "resolved": "https://registry.npmjs.org/logplease/-/logplease-1.2.15.tgz", @@ -1096,11 +832,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, - "nan": { - "version": "2.14.2", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz", - "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==" - }, "negotiator": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", @@ -1168,11 +899,6 @@ "unpipe": "1.0.0" } }, - "require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" - }, "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", @@ -1239,24 +965,6 @@ "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" }, - "string-width": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", - "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.0" - } - }, - "strip-ansi": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", - "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", - "requires": { - "ansi-regex": "^5.0.0" - } - }, "toidentifier": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", @@ -1295,44 +1003,10 @@ "version": "git+ssh://git@github.com/HexF/node-waitpid.git#a08d116a5d993a747624fe72ff890167be8c34aa", "from": "waitpid@git+https://github.com/HexF/node-waitpid.git" }, - "wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } - }, - "y18n": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.5.tgz", - "integrity": "sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg==" - }, "yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, - "yargs": { - "version": "16.2.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", - "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", - "requires": { - "cliui": "^7.0.2", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^20.2.2" - } - }, - "yargs-parser": { - "version": "20.2.7", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.7.tgz", - "integrity": "sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw==" } } } diff --git a/api/package.json b/api/package.json index 6787393..3517fd3 100644 --- a/api/package.json +++ b/api/package.json @@ -8,16 +8,13 @@ "chownr": "^2.0.0", "express": "^4.17.1", "is-docker": "^2.1.1", - "js-yaml": "^4.0.0", - "libsys": "^3.0.0", "logplease": "^1.2.15", "nocamel": "HexF/nocamel#patch-1", "node-fetch": "^2.6.1", "ps-list": "^7.2.0", "semver": "^7.3.4", "uuid": "^8.3.2", - "waitpid": "git+https://github.com/HexF/node-waitpid.git", - "yargs": "^16.2.0" + "waitpid": "git+https://github.com/HexF/node-waitpid.git" }, "license": "MIT" } diff --git a/api/src/config.js b/api/src/config.js index 3a5cd72..8cc1122 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -1,36 +1,7 @@ const fss = require('fs'); -const yargs = require('yargs'); -const hide_bin = require('yargs/helpers').hideBin; const Logger = require('logplease'); const logger = Logger.create('config'); -const yaml = require('js-yaml'); -const header = `# -# ____ _ _ -# | _ \\(_)___| |_ ___ _ __ -# | |_) | / __| __/ _ \\| '_ \\ -# | __/| \\__ \\ || (_) | | | | -# |_| |_|___/\\__\\___/|_| |_| -# -# A High performance code execution engine -# github.com/engineer-man/piston -# - -`; -const argv = yargs(hide_bin(process.argv)) - .usage('Usage: $0 -c [config]') - .demandOption('c') - .option('config', { - alias: 'c', - describe: 'config file to load from', - default: '/piston/config.yaml' - }) - .option('make-config', { - alias: 'm', - type: 'boolean', - describe: 'create config file and populate defaults if it does not already exist' - }) - .argv; const options = [ { @@ -58,67 +29,100 @@ const options = [ key: 'runner_uid_min', desc: 'Minimum uid to use for runner', default: 1001, - validators: [] + parser: parse_int, + validators: [ + (x,raw) => !isNaN(x) || `${raw} is not a number`, + ] }, { key: 'runner_uid_max', desc: 'Maximum uid to use for runner', default: 1500, - validators: [] + parser: parse_int, + validators: [ + (x,raw) => !isNaN(x) || `${raw} is not a number`, + ] }, { key: 'runner_gid_min', desc: 'Minimum gid to use for runner', default: 1001, - validators: [] + parser: parse_int, + validators: [ + (x,raw) => !isNaN(x) || `${raw} is not a number`, + ] }, { key: 'runner_gid_max', desc: 'Maximum gid to use for runner', default: 1500, - validators: [] + parser: parse_int, + validators: [ + (x,raw) => !isNaN(x) || `${raw} is not a number`, + ] }, { key: 'disable_networking', desc: 'Set to true to disable networking', default: true, - validators: [] + parser: x => x === "true", + validators: [ + x => typeof x === "boolean" || `${x} is not a boolean` + ] }, { key: 'output_max_size', desc: 'Max size of each stdio buffer', default: 1024, - validators: [] + parser: parse_int, + validators: [ + (x,raw) => !isNaN(x) || `${raw} is not a number`, + ] }, { key: 'max_process_count', desc: 'Max number of processes per job', default: 64, - validators: [] + parser: parse_int, + validators: [ + (x,raw) => !isNaN(x) || `${raw} is not a number`, + ] }, { key: 'max_open_files', desc: 'Max number of open files per job', default: 2048, - validators: [] + parser: parse_int, + validators: [ + (x,raw) => !isNaN(x) || `${raw} is not a number`, + ] }, { key: 'max_file_size', desc: 'Max file size in bytes for a file', default: 1000000, //1MB - validators: [] + parser: parse_int, + validators: [ + (x,raw) => !isNaN(x) || `${raw} is not a number`, + ] }, { key: 'compile_memory_limit', desc: 'Max memory usage for compile stage in bytes (set to -1 for no limit)', default: -1, // no limit - validators: [] + parser: parse_int, + validators: [ + (x,raw) => !isNaN(x) || `${raw} is not a number`, + ] }, { key: 'run_memory_limit', desc: 'Max memory usage for run stage in bytes (set to -1 for no limit)', default: -1, // no limit - validators: [] + parser: parse_int, + validators: [ + (x,raw) => !isNaN(x) || `${raw} is not a number`, + ] }, { key: 'repo_url', @@ -128,76 +132,40 @@ const options = [ } ]; -const make_default_config = () => { - let content = header.split('\n'); - - options.forEach(option => { - content = content.concat(option.desc.split('\n').map(x=>`# ${x}`)); - - if (option.options) { - content.push('# Options: ' + option.options.join(', ')); - } - - content.push(`${option.key}: ${option.default}`); - - content.push(''); // New line between - }); - - return content.join('\n'); -}; - -logger.info(`Loading Configuration from ${argv.config}`); - -if (argv['make-config']) { - logger.debug('Make configuration flag is set'); -} - -if (!!argv['make-config'] && !fss.exists_sync(argv.config)) { - logger.info('Writing default configuration...'); - try { - fss.write_file_sync(argv.config, make_default_config()); - } catch (e) { - logger.error('Error writing default configuration:', e.message); - process.exit(1); - } -} - -let config = {}; - -logger.debug('Reading config file'); - -try { - const cfg_content = fss.read_file_sync(argv.config); - config = yaml.load(cfg_content); -} catch(err) { - logger.error('Error reading configuration file:', err.message); - process.exit(1); -} - -logger.debug('Validating config entries'); +logger.info(`Loading Configuration from environment`); let errored = false; -options.for_each(option => { - logger.debug('Checking option', option.key); +let config = {}; - let cfg_val = config[option.key]; +options.forEach(option => { + const env_key = "PISTON_" + option.key.to_upper_case(); + + const parser = option.parser || (x=>x); + + const env_val = process.env[env_key]; + + const parsed_val = parser(env_val); + + + const value = env_val || option.default; - if (cfg_val === undefined) { - errored = true; - logger.error(`Config key ${option.key} does not exist on currently loaded configuration`); - return; - } option.validators.for_each(validator => { - let response = validator(cfg_val); + let response = null; + if(env_val) + response = validator(parsed_val, env_val); + else + response = validator(value, value); - if (!response) { + if (response !== true) { errored = true; logger.error(`Config option ${option.key} failed validation:`, response); return; } }); + + config[option.key] = value; }); if (errored) { @@ -206,4 +174,5 @@ if (errored) { logger.info('Configuration successfully loaded'); + module.exports = config; diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index f2b3a14..31772fe 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -11,6 +11,8 @@ services: - 2000:2000 volumes: - ./data/piston:/piston + environment: + - PISTON_REPO_URL=http://repo:8000/index tmpfs: - /piston/jobs:exec