Mascali
/
piston
mirror of https://github.com/engineer-man/piston.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

switch to taking config by env 

To help with ease of deployment, configuration now is administered through the use of environment variables.
This allows for easy configuration directly from the docker-compose file and is a much more robust solution.
This commit is contained in:
Thomas Hobson 2021-05-07 21:11:36 +12:00
parent fb102ebe83
commit 972228412a
No known key found for this signature in database
GPG Key ID: 9F1FD9D87950DB6F
6 changed files with 72 additions and 451 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
.github/workflows/package-pr.yaml vendored
View File

@ -58,27 +58,6 @@ jobs:
- name: Relocate downloaded packages - name: Relocate downloaded packages
run: mv *.pkg.tar.gz packages/ run: mv *.pkg.tar.gz packages/
- name: Write test config file
uses: DamianReeves/write-file-action@v1.0
with:
path: data/config.yaml
contents: |
log_level: DEBUG
bind_address: 0.0.0.0:2000
data_directory: /piston
runner_uid_min: 1100
runner_uid_max: 1500
runner_gid_min: 1100
runner_gid_max: 1500
disable_networking: false
output_max_size: 1024
max_process_count: 64
max_open_files: 2048
max_file_size: 1000000
repo_url: http://localhost:8000/index
write-mode: overwrite
- name: Login to GitHub registry - name: Login to GitHub registry
uses: docker/login-action@v1 uses: docker/login-action@v1
with: with:
@ -90,7 +69,7 @@ jobs:
run: | run: |
ls -la ls -la
docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' -d --name repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build docker run -v $(pwd)'/repo:/piston/repo' -v $(pwd)'/packages:/piston/packages' -d --name repo docker.pkg.github.com/engineer-man/piston/repo-builder --no-build
docker run --network container:repo -v $(pwd)'/data:/piston' -d --name api docker.pkg.github.com/engineer-man/piston/api docker run --network container:repo -v $(pwd)'/data:/piston' -e PISTON_LOG_LEVEL=DEBUG -e 'PISTON_REPO_URL=http://localhost:8000/index' -d --name api docker.pkg.github.com/engineer-man/piston/api
echo Waiting for API to start.. echo Waiting for API to start..
docker run --network container:api appropriate/curl -s --retry 10 --retry-connrefused http://localhost:2000/api/v1/runtimes docker run --network container:api appropriate/curl -s --retry 10 --retry-connrefused http://localhost:2000/api/v1/runtimes

2
api/Dockerfile
View File

@ -25,6 +25,6 @@ COPY ./src ./src
RUN make -C ./src/nosocket/ all && make -C ./src/nosocket/ install RUN make -C ./src/nosocket/ all && make -C ./src/nosocket/ install
CMD [ "node", "src", "-m", "-c", "/piston/config.yaml"] CMD [ "node", "src"]
EXPOSE 2000/tcp EXPOSE 2000/tcp

328
api/package-lock.json generated
View File

@ -13,16 +13,13 @@
"chownr": "^2.0.0", "chownr": "^2.0.0",
"express": "^4.17.1", "express": "^4.17.1",
"is-docker": "^2.1.1", "is-docker": "^2.1.1",
"js-yaml": "^4.0.0",
"libsys": "^3.0.0",
"logplease": "^1.2.15", "logplease": "^1.2.15",
"nocamel": "HexF/nocamel#patch-1", "nocamel": "HexF/nocamel#patch-1",
"node-fetch": "^2.6.1", "node-fetch": "^2.6.1",
"ps-list": "^7.2.0", "ps-list": "^7.2.0",
"semver": "^7.3.4", "semver": "^7.3.4",
"uuid": "^8.3.2", "uuid": "^8.3.2",
"waitpid": "git+https://github.com/HexF/node-waitpid.git", "waitpid": "git+https://github.com/HexF/node-waitpid.git"
"yargs": "^16.2.0"
} }
}, },
"node_modules/accepts": { "node_modules/accepts": {
@ -37,30 +34,6 @@
"node": ">= 0.6" "node": ">= 0.6"
} }
}, },
"node_modules/ansi-regex": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
"integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
"engines": {
"node": ">=8"
}
},
"node_modules/ansi-styles": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
"dependencies": {
"color-convert": "^2.0.1"
},
"engines": {
"node": ">=8"
}
},
"node_modules/argparse": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="
},
"node_modules/array-flatten": { "node_modules/array-flatten": {
"version": "1.1.1", "version": "1.1.1",
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
@ -102,32 +75,6 @@
"node": ">=10" "node": ">=10"
} }
}, },
"node_modules/cliui": {
"version": "7.0.4",
"resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz",
"integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==",
"dependencies": {
"string-width": "^4.2.0",
"strip-ansi": "^6.0.0",
"wrap-ansi": "^7.0.0"
}
},
"node_modules/color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
"dependencies": {
"color-name": "~1.1.4"
},
"engines": {
"node": ">=7.0.0"
}
},
"node_modules/color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
},
"node_modules/content-disposition": { "node_modules/content-disposition": {
"version": "0.5.3", "version": "0.5.3",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz",
@ -186,11 +133,6 @@
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
"integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0="
}, },
"node_modules/emoji-regex": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
},
"node_modules/encodeurl": { "node_modules/encodeurl": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
@ -199,14 +141,6 @@
"node": ">= 0.8" "node": ">= 0.8"
} }
}, },
"node_modules/escalade": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
"integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==",
"engines": {
"node": ">=6"
}
},
"node_modules/escape-html": { "node_modules/escape-html": {
"version": "1.0.3", "version": "1.0.3",
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
@ -293,14 +227,6 @@
"node": ">= 0.6" "node": ">= 0.6"
} }
}, },
"node_modules/get-caller-file": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
"engines": {
"node": "6.* || 8.* || >= 10.*"
}
},
"node_modules/http-errors": { "node_modules/http-errors": {
"version": "1.7.2", "version": "1.7.2",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
@ -351,36 +277,6 @@
"node": ">=8" "node": ">=8"
} }
}, },
"node_modules/is-fullwidth-code-point": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
"engines": {
"node": ">=8"
}
},
"node_modules/js-yaml": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz",
"integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==",
"dependencies": {
"argparse": "^2.0.1"
},
"bin": {
"js-yaml": "bin/js-yaml.js"
}
},
"node_modules/libsys": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/libsys/-/libsys-3.0.0.tgz",
"integrity": "sha512-+c1HMLbV9dIGHfVcjs0PDme9di1CopZ+oL4xqi7LGjeHJfvTsSUAlJtvcfqTkN9wak4WU4cjhLiG1iZwPY1IBw==",
"dependencies": {
"nan": "^2.14.0"
},
"engines": {
"node": ">= 4.4.3"
}
},
"node_modules/logplease": { "node_modules/logplease": {
"version": "1.2.15", "version": "1.2.15",
"resolved": "https://registry.npmjs.org/logplease/-/logplease-1.2.15.tgz", "resolved": "https://registry.npmjs.org/logplease/-/logplease-1.2.15.tgz",
@ -453,11 +349,6 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
}, },
"node_modules/nan": {
"version": "2.14.2",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz",
"integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ=="
},
"node_modules/negotiator": { "node_modules/negotiator": {
"version": "0.6.2", "version": "0.6.2",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
@ -551,14 +442,6 @@
"node": ">= 0.8" "node": ">= 0.8"
} }
}, },
"node_modules/require-directory": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
"integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/safe-buffer": { "node_modules/safe-buffer": {
"version": "5.1.2", "version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
@ -638,30 +521,6 @@
"node": ">= 0.6" "node": ">= 0.6"
} }
}, },
"node_modules/string-width": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz",
"integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==",
"dependencies": {
"emoji-regex": "^8.0.0",
"is-fullwidth-code-point": "^3.0.0",
"strip-ansi": "^6.0.0"
},
"engines": {
"node": ">=8"
}
},
"node_modules/strip-ansi": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
"integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
"dependencies": {
"ansi-regex": "^5.0.0"
},
"engines": {
"node": ">=8"
}
},
"node_modules/toidentifier": { "node_modules/toidentifier": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
@ -717,56 +576,10 @@
"node_modules/waitpid": { "node_modules/waitpid": {
"resolved": "git+ssh://git@github.com/HexF/node-waitpid.git#a08d116a5d993a747624fe72ff890167be8c34aa" "resolved": "git+ssh://git@github.com/HexF/node-waitpid.git#a08d116a5d993a747624fe72ff890167be8c34aa"
}, },
"node_modules/wrap-ansi": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
"integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
"dependencies": {
"ansi-styles": "^4.0.0",
"string-width": "^4.1.0",
"strip-ansi": "^6.0.0"
},
"engines": {
"node": ">=10"
}
},
"node_modules/y18n": {
"version": "5.0.5",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.5.tgz",
"integrity": "sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg==",
"engines": {
"node": ">=10"
}
},
"node_modules/yallist": { "node_modules/yallist": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
},
"node_modules/yargs": {
"version": "16.2.0",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz",
"integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==",
"dependencies": {
"cliui": "^7.0.2",
"escalade": "^3.1.1",
"get-caller-file": "^2.0.5",
"require-directory": "^2.1.1",
"string-width": "^4.2.0",
"y18n": "^5.0.5",
"yargs-parser": "^20.2.2"
},
"engines": {
"node": ">=10"
}
},
"node_modules/yargs-parser": {
"version": "20.2.7",
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.7.tgz",
"integrity": "sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw==",
"engines": {
"node": ">=10"
}
} }
}, },
"dependencies": { "dependencies": {
@ -779,24 +592,6 @@
"negotiator": "0.6.2" "negotiator": "0.6.2"
} }
}, },
"ansi-regex": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
"integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg=="
},
"ansi-styles": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
"requires": {
"color-convert": "^2.0.1"
}
},
"argparse": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="
},
"array-flatten": { "array-flatten": {
"version": "1.1.1", "version": "1.1.1",
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
@ -829,29 +624,6 @@
"resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz",
"integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==" "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ=="
}, },
"cliui": {
"version": "7.0.4",
"resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz",
"integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==",
"requires": {
"string-width": "^4.2.0",
"strip-ansi": "^6.0.0",
"wrap-ansi": "^7.0.0"
}
},
"color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
"requires": {
"color-name": "~1.1.4"
}
},
"color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
},
"content-disposition": { "content-disposition": {
"version": "0.5.3", "version": "0.5.3",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz",
@ -898,21 +670,11 @@
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
"integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0="
}, },
"emoji-regex": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
},
"encodeurl": { "encodeurl": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
"integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k="
}, },
"escalade": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
"integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw=="
},
"escape-html": { "escape-html": {
"version": "1.0.3", "version": "1.0.3",
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
@ -984,11 +746,6 @@
"resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
"integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac="
}, },
"get-caller-file": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="
},
"http-errors": { "http-errors": {
"version": "1.7.2", "version": "1.7.2",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
@ -1024,27 +781,6 @@
"resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.1.1.tgz", "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.1.1.tgz",
"integrity": "sha512-ZOoqiXfEwtGknTiuDEy8pN2CfE3TxMHprvNer1mXiqwkOT77Rw3YVrUQ52EqAOU3QAWDQ+bQdx7HJzrv7LS2Hw==" "integrity": "sha512-ZOoqiXfEwtGknTiuDEy8pN2CfE3TxMHprvNer1mXiqwkOT77Rw3YVrUQ52EqAOU3QAWDQ+bQdx7HJzrv7LS2Hw=="
}, },
"is-fullwidth-code-point": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="
},
"js-yaml": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz",
"integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==",
"requires": {
"argparse": "^2.0.1"
}
},
"libsys": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/libsys/-/libsys-3.0.0.tgz",
"integrity": "sha512-+c1HMLbV9dIGHfVcjs0PDme9di1CopZ+oL4xqi7LGjeHJfvTsSUAlJtvcfqTkN9wak4WU4cjhLiG1iZwPY1IBw==",
"requires": {
"nan": "^2.14.0"
}
},
"logplease": { "logplease": {
"version": "1.2.15", "version": "1.2.15",
"resolved": "https://registry.npmjs.org/logplease/-/logplease-1.2.15.tgz", "resolved": "https://registry.npmjs.org/logplease/-/logplease-1.2.15.tgz",
@ -1096,11 +832,6 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
}, },
"nan": {
"version": "2.14.2",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz",
"integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ=="
},
"negotiator": { "negotiator": {
"version": "0.6.2", "version": "0.6.2",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
@ -1168,11 +899,6 @@
"unpipe": "1.0.0" "unpipe": "1.0.0"
} }
}, },
"require-directory": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
"integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I="
},
"safe-buffer": { "safe-buffer": {
"version": "5.1.2", "version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
@ -1239,24 +965,6 @@
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
"integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow="
}, },
"string-width": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz",
"integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==",
"requires": {
"emoji-regex": "^8.0.0",
"is-fullwidth-code-point": "^3.0.0",
"strip-ansi": "^6.0.0"
}
},
"strip-ansi": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
"integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
"requires": {
"ansi-regex": "^5.0.0"
}
},
"toidentifier": { "toidentifier": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
@ -1295,44 +1003,10 @@
"version": "git+ssh://git@github.com/HexF/node-waitpid.git#a08d116a5d993a747624fe72ff890167be8c34aa", "version": "git+ssh://git@github.com/HexF/node-waitpid.git#a08d116a5d993a747624fe72ff890167be8c34aa",
"from": "waitpid@git+https://github.com/HexF/node-waitpid.git" "from": "waitpid@git+https://github.com/HexF/node-waitpid.git"
}, },
"wrap-ansi": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
"integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
"requires": {
"ansi-styles": "^4.0.0",
"string-width": "^4.1.0",
"strip-ansi": "^6.0.0"
}
},
"y18n": {
"version": "5.0.5",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.5.tgz",
"integrity": "sha512-hsRUr4FFrvhhRH12wOdfs38Gy7k2FFzB9qgN9v3aLykRq0dRcdcpz5C9FxdS2NuhOrI/628b/KSTJ3rwHysYSg=="
},
"yallist": { "yallist": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
},
"yargs": {
"version": "16.2.0",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz",
"integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==",
"requires": {
"cliui": "^7.0.2",
"escalade": "^3.1.1",
"get-caller-file": "^2.0.5",
"require-directory": "^2.1.1",
"string-width": "^4.2.0",
"y18n": "^5.0.5",
"yargs-parser": "^20.2.2"
}
},
"yargs-parser": {
"version": "20.2.7",
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.7.tgz",
"integrity": "sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw=="
} }
} }
} }

5
api/package.json
View File

@ -8,16 +8,13 @@
"chownr": "^2.0.0", "chownr": "^2.0.0",
"express": "^4.17.1", "express": "^4.17.1",
"is-docker": "^2.1.1", "is-docker": "^2.1.1",
"js-yaml": "^4.0.0",
"libsys": "^3.0.0",
"logplease": "^1.2.15", "logplease": "^1.2.15",
"nocamel": "HexF/nocamel#patch-1", "nocamel": "HexF/nocamel#patch-1",
"node-fetch": "^2.6.1", "node-fetch": "^2.6.1",
"ps-list": "^7.2.0", "ps-list": "^7.2.0",
"semver": "^7.3.4", "semver": "^7.3.4",
"uuid": "^8.3.2", "uuid": "^8.3.2",
"waitpid": "git+https://github.com/HexF/node-waitpid.git", "waitpid": "git+https://github.com/HexF/node-waitpid.git"
"yargs": "^16.2.0"
}, },
"license": "MIT" "license": "MIT"
} }

163
api/src/config.js
View File

@ -1,36 +1,7 @@
const fss = require('fs'); const fss = require('fs');
const yargs = require('yargs');
const hide_bin = require('yargs/helpers').hideBin;
const Logger = require('logplease'); const Logger = require('logplease');
const logger = Logger.create('config'); const logger = Logger.create('config');
const yaml = require('js-yaml');
const header = `#
# ____ _ _
# | _ \\(_)___| |_ ___ _ __
# | |_) | / __| __/ _ \\| '_ \\
# | __/| \\__ \\ || (_) | | | |
# |_| |_|___/\\__\\___/|_| |_|
#
# A High performance code execution engine
# github.com/engineer-man/piston
#
`;
const argv = yargs(hide_bin(process.argv))
.usage('Usage: $0 -c [config]')
.demandOption('c')
.option('config', {
alias: 'c',
describe: 'config file to load from',
default: '/piston/config.yaml'
})
.option('make-config', {
alias: 'm',
type: 'boolean',
describe: 'create config file and populate defaults if it does not already exist'
})
.argv;
const options = [ const options = [
{ {
@ -58,67 +29,100 @@ const options = [
key: 'runner_uid_min', key: 'runner_uid_min',
desc: 'Minimum uid to use for runner', desc: 'Minimum uid to use for runner',
default: 1001, default: 1001,
validators: [] parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'runner_uid_max', key: 'runner_uid_max',
desc: 'Maximum uid to use for runner', desc: 'Maximum uid to use for runner',
default: 1500, default: 1500,
validators: [] parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'runner_gid_min', key: 'runner_gid_min',
desc: 'Minimum gid to use for runner', desc: 'Minimum gid to use for runner',
default: 1001, default: 1001,
validators: [] parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'runner_gid_max', key: 'runner_gid_max',
desc: 'Maximum gid to use for runner', desc: 'Maximum gid to use for runner',
default: 1500, default: 1500,
validators: [] parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'disable_networking', key: 'disable_networking',
desc: 'Set to true to disable networking', desc: 'Set to true to disable networking',
default: true, default: true,
validators: [] parser: x => x === "true",
validators: [
x => typeof x === "boolean" || `${x} is not a boolean`
]
}, },
{ {
key: 'output_max_size', key: 'output_max_size',
desc: 'Max size of each stdio buffer', desc: 'Max size of each stdio buffer',
default: 1024, default: 1024,
validators: [] parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'max_process_count', key: 'max_process_count',
desc: 'Max number of processes per job', desc: 'Max number of processes per job',
default: 64, default: 64,
validators: [] parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'max_open_files', key: 'max_open_files',
desc: 'Max number of open files per job', desc: 'Max number of open files per job',
default: 2048, default: 2048,
validators: [] parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'max_file_size', key: 'max_file_size',
desc: 'Max file size in bytes for a file', desc: 'Max file size in bytes for a file',
default: 1000000, //1MB default: 1000000, //1MB
validators: [] parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'compile_memory_limit', key: 'compile_memory_limit',
desc: 'Max memory usage for compile stage in bytes (set to -1 for no limit)', desc: 'Max memory usage for compile stage in bytes (set to -1 for no limit)',
default: -1, // no limit default: -1, // no limit
validators: [] parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'run_memory_limit', key: 'run_memory_limit',
desc: 'Max memory usage for run stage in bytes (set to -1 for no limit)', desc: 'Max memory usage for run stage in bytes (set to -1 for no limit)',
default: -1, // no limit default: -1, // no limit
validators: [] parser: parse_int,
validators: [
(x,raw) => !isNaN(x) || `${raw} is not a number`,
]
}, },
{ {
key: 'repo_url', key: 'repo_url',
@ -128,76 +132,40 @@ const options = [
} }
]; ];
const make_default_config = () => { logger.info(`Loading Configuration from environment`);
let content = header.split('\n');
options.forEach(option => {
content = content.concat(option.desc.split('\n').map(x=>`# ${x}`));
if (option.options) {
content.push('# Options: ' + option.options.join(', '));
}
content.push(`${option.key}: ${option.default}`);
content.push(''); // New line between
});
return content.join('\n');
};
logger.info(`Loading Configuration from ${argv.config}`);
if (argv['make-config']) {
logger.debug('Make configuration flag is set');
}
if (!!argv['make-config'] && !fss.exists_sync(argv.config)) {
logger.info('Writing default configuration...');
try {
fss.write_file_sync(argv.config, make_default_config());
} catch (e) {
logger.error('Error writing default configuration:', e.message);
process.exit(1);
}
}
let config = {};
logger.debug('Reading config file');
try {
const cfg_content = fss.read_file_sync(argv.config);
config = yaml.load(cfg_content);
} catch(err) {
logger.error('Error reading configuration file:', err.message);
process.exit(1);
}
logger.debug('Validating config entries');
let errored = false; let errored = false;
options.for_each(option => { let config = {};
logger.debug('Checking option', option.key);
let cfg_val = config[option.key]; options.forEach(option => {
const env_key = "PISTON_" + option.key.to_upper_case();
const parser = option.parser || (x=>x);
const env_val = process.env[env_key];
const parsed_val = parser(env_val);
const value = env_val || option.default;
if (cfg_val === undefined) {
errored = true;
logger.error(`Config key ${option.key} does not exist on currently loaded configuration`);
return;
}
option.validators.for_each(validator => { option.validators.for_each(validator => {
let response = validator(cfg_val); let response = null;
if(env_val)
response = validator(parsed_val, env_val);
else
response = validator(value, value);
if (!response) { if (response !== true) {
errored = true; errored = true;
logger.error(`Config option ${option.key} failed validation:`, response); logger.error(`Config option ${option.key} failed validation:`, response);
return; return;
} }
}); });
config[option.key] = value;
}); });
if (errored) { if (errored) {
@ -206,4 +174,5 @@ if (errored) {
logger.info('Configuration successfully loaded'); logger.info('Configuration successfully loaded');
module.exports = config; module.exports = config;

2
docker-compose.dev.yaml
View File

@ -11,6 +11,8 @@ services:
- 2000:2000 - 2000:2000
volumes: volumes:
- ./data/piston:/piston - ./data/piston:/piston
environment:
- PISTON_REPO_URL=http://repo:8000/index
tmpfs: tmpfs:
- /piston/jobs:exec - /piston/jobs:exec