From 51b06018c11aff2a053ecfe89c4ef728dac296f9 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Fri, 19 Feb 2021 18:06:07 +1300 Subject: [PATCH 1/7] Add signing to common --- packages/Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/Makefile b/packages/Makefile index 51218a8..3ef4592 100644 --- a/packages/Makefile +++ b/packages/Makefile @@ -22,7 +22,12 @@ pkg-info.jq: echo '.dependencies={}' >> pkg-info.jq $(foreach dep, ${LANG_DEPS}, echo '.dependencies.$(word 1,$(subst =, ,${dep}))="$(word 2,$(subst =, ,${dep}))"' >> pkg-info.jq) +%.asc: % + gpg --detach-sig --armor --output $@ $< + .PHONY: clean clean: rm -rf $(filter-out Makefile, $(wildcard *)) - \ No newline at end of file + +.PHONY: sign +sign: ${LANG_NAME}-${LANG_VERSION}.pkg.tar.gz.asc \ No newline at end of file From 4f3e7a536d7680efc2fa896338c191c0456f3095 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Fri, 19 Feb 2021 18:53:26 +1300 Subject: [PATCH 2/7] Better python packaging --- packages/.gitignore | 3 --- packages/{Makefile => common.mk} | 9 ++++++++- packages/python/.gitignore | 1 + packages/python/3.9.1/Makefile | 28 ---------------------------- packages/python/Makefile | 24 ++++++++++++++++++++++++ packages/python/base.mk | 21 +++++++++++++++++++++ 6 files changed, 54 insertions(+), 32 deletions(-) delete mode 100644 packages/.gitignore rename packages/{Makefile => common.mk} (82%) create mode 100644 packages/python/.gitignore delete mode 100644 packages/python/3.9.1/Makefile create mode 100644 packages/python/Makefile create mode 100644 packages/python/base.mk diff --git a/packages/.gitignore b/packages/.gitignore deleted file mode 100644 index a9f5c08..0000000 --- a/packages/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!**/Makefile -!.gitignore \ No newline at end of file diff --git a/packages/Makefile b/packages/common.mk similarity index 82% rename from packages/Makefile rename to packages/common.mk index 3ef4592..8305faf 100644 --- a/packages/Makefile +++ b/packages/common.mk @@ -10,7 +10,7 @@ ifeq (${LANG_COMPILED}, true) ${LANG_NAME}-${LANG_VERSION}.pkg.tar.gz: $(LANG_PKG_TARGETS) compile endif ${LANG_NAME}-${LANG_VERSION}.pkg.tar.gz: $(LANG_PKG_TARGETS) - tar czvf $@ $? + tar czf $@ $? %.json: %.jq jq '$(shell tr '\n' '|' < $<).' <<< "{}" > $@ @@ -25,9 +25,16 @@ pkg-info.jq: %.asc: % gpg --detach-sig --armor --output $@ $< +%/: %.tgz + tar xzf $< + .PHONY: clean clean: rm -rf $(filter-out Makefile, $(wildcard *)) + +,PHONY: cleanup +cleanup: + rm -rf $(filter-out ${LANG_NAME}-${LANG_VERSION}.pkg.tar.gz.asc, $(filter-out ${LANG_NAME}-${LANG_VERSION}.pkg.tar.gz, $(filter-out Makefile, $(wildcard *)))) .PHONY: sign sign: ${LANG_NAME}-${LANG_VERSION}.pkg.tar.gz.asc \ No newline at end of file diff --git a/packages/python/.gitignore b/packages/python/.gitignore new file mode 100644 index 0000000..0a00d70 --- /dev/null +++ b/packages/python/.gitignore @@ -0,0 +1 @@ +*/ \ No newline at end of file diff --git a/packages/python/3.9.1/Makefile b/packages/python/3.9.1/Makefile deleted file mode 100644 index 6854683..0000000 --- a/packages/python/3.9.1/Makefile +++ /dev/null @@ -1,28 +0,0 @@ -NAME=python -VERSION=3.9.1 -AUTHOR=Thomas Hobson -DEPS= -COMPILED=false - -MAKE_JOBS=64 - -include ../../Makefile - -run: - echo "python3.9 $*" > run - -python-${VERSION}/environment: - echo 'export PATH=$$PWD/python-${VERSION}/bin:$$PATH' > $@ - -python-${VERSION}/: Python-${VERSION}/ - cd $< && ./configure --prefix / - make -j${MAKE_JOBS} -C $< - DESTDIR=../$@ make -j${MAKE_JOBS} -C $< altinstall || true - -Python-${VERSION}.tgz: - curl "https://www.python.org/ftp/python/${VERSION}/$@" -o $@ - -%/: %.tgz - tar xzvf $< - - diff --git a/packages/python/Makefile b/packages/python/Makefile new file mode 100644 index 0000000..4e1ab09 --- /dev/null +++ b/packages/python/Makefile @@ -0,0 +1,24 @@ +LANGUAGE=python +VERSIONS=2.7.1 3.9.1 + +.PHONY: build sign cleanup clean +build: $(patsubst %,%/${LANGUAGE}-%.pkg.tar.gz,${VERSIONS}) +sign: $(patsubst %,%/${LANGUAGE}-%.pkg.tar.gz.asc,${VERSIONS}) +clean: + rm -rf ${VERSIONS} +cleanup: $(patsubst %,%/cleanup,${VERSIONS}) + + +%/cleanup: %/Makefile + $(MAKE) -C $(shell dirname $<) cleanup + rm $(shell dirname $<)/Makefile + +%/${LANGUAGE}-%.pkg.tar.gz.asc: %/Makefile + $(MAKE) -C $(shell dirname $<) sign +%/${LANGUAGE}-%.pkg.tar.gz: %/Makefile + $(MAKE) -C $(shell dirname $<) + +%/Makefile: + @mkdir -p $(shell dirname $@) + @echo 'VERSION=$(patsubst %/Makefile,%,$@)' > $@ + @echo 'include ../base.mk' >> $@ diff --git a/packages/python/base.mk b/packages/python/base.mk new file mode 100644 index 0000000..bcb11d1 --- /dev/null +++ b/packages/python/base.mk @@ -0,0 +1,21 @@ +NAME=python +AUTHOR=Thomas Hobson +DEPS= +#VERSION= +COMPILED=false + +include ../../common.mk + +run: + echo 'python$(shell grep -oP "\d+.\d+"<<<${VERSION}) $$*' > run + +python-${VERSION}/environment: + echo 'export PATH=$$PWD/${NAME}-${VERSION}/bin:$$PATH' > $@ + +python-${VERSION}/: Python-${VERSION}/ + cd $< && ./configure --prefix / + $(MAKE) -j$(or ${MAKE_JOBS},64) -C $< + DESTDIR=../$@ $(MAKE) -j$(or ${MAKE_JOBS},64) -C $< altinstall || true + +Python-${VERSION}.tgz: + curl "https://www.python.org/ftp/python/${VERSION}/$@" -o $@ \ No newline at end of file From e03c7c50f36e5f21e944cb2667b5e73615d98780 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Fri, 19 Feb 2021 20:34:30 +1300 Subject: [PATCH 3/7] store build distro with package --- packages/common.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/common.mk b/packages/common.mk index 8305faf..9cb6bbd 100644 --- a/packages/common.mk +++ b/packages/common.mk @@ -20,6 +20,7 @@ pkg-info.jq: echo '.version="${LANG_VERSION}"' >> pkg-info.jq echo '.author="${LANG_AUTHOR}"' >> pkg-info.jq echo '.dependencies={}' >> pkg-info.jq + echo '.distro="$(shell grep -oP "^ID=\K\w+" /etc/os-release )"' $(foreach dep, ${LANG_DEPS}, echo '.dependencies.$(word 1,$(subst =, ,${dep}))="$(word 2,$(subst =, ,${dep}))"' >> pkg-info.jq) %.asc: % From 47c31e3792ca0c28eebf05166f5e07f009fa8ff3 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Fri, 19 Feb 2021 21:05:37 +1300 Subject: [PATCH 4/7] Allow build platform to be directly specified --- packages/common.mk | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/common.mk b/packages/common.mk index 9cb6bbd..f045d34 100644 --- a/packages/common.mk +++ b/packages/common.mk @@ -6,6 +6,8 @@ LANG_COMPILED=$(or ${COMPILED}, false) LANG_PKG_TARGETS=pkg-info.json ${LANG_NAME}-${LANG_VERSION}/ ${LANG_NAME}-${LANG_VERSION}/environment run +BUILD_PLATFORM=$(or ${PLATFORM}, baremetal-$(shell grep -oP "^ID=\K\w+" /etc/os-release )) + ifeq (${LANG_COMPILED}, true) ${LANG_NAME}-${LANG_VERSION}.pkg.tar.gz: $(LANG_PKG_TARGETS) compile endif @@ -20,7 +22,7 @@ pkg-info.jq: echo '.version="${LANG_VERSION}"' >> pkg-info.jq echo '.author="${LANG_AUTHOR}"' >> pkg-info.jq echo '.dependencies={}' >> pkg-info.jq - echo '.distro="$(shell grep -oP "^ID=\K\w+" /etc/os-release )"' + echo '.build_platform="${BUILD_PLATFORM}"' $(foreach dep, ${LANG_DEPS}, echo '.dependencies.$(word 1,$(subst =, ,${dep}))="$(word 2,$(subst =, ,${dep}))"' >> pkg-info.jq) %.asc: % From 32632926de75a8d95a3b0849ea40a6b0b730bf6b Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Fri, 19 Feb 2021 21:28:59 +1300 Subject: [PATCH 5/7] add the build platform to json --- packages/common.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/common.mk b/packages/common.mk index f045d34..08fc488 100644 --- a/packages/common.mk +++ b/packages/common.mk @@ -22,7 +22,7 @@ pkg-info.jq: echo '.version="${LANG_VERSION}"' >> pkg-info.jq echo '.author="${LANG_AUTHOR}"' >> pkg-info.jq echo '.dependencies={}' >> pkg-info.jq - echo '.build_platform="${BUILD_PLATFORM}"' + echo '.build_platform="${BUILD_PLATFORM}"' >> pkg-info.jq $(foreach dep, ${LANG_DEPS}, echo '.dependencies.$(word 1,$(subst =, ,${dep}))="$(word 2,$(subst =, ,${dep}))"' >> pkg-info.jq) %.asc: % From bda5129da70ec6742b4386429386e531fb059065 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Fri, 19 Feb 2021 21:58:31 +1300 Subject: [PATCH 6/7] more documentation --- design/fs.txt | 18 ++++++++++++ design/ppman.txt | 74 ++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 86 insertions(+), 6 deletions(-) create mode 100644 design/fs.txt diff --git a/design/fs.txt b/design/fs.txt new file mode 100644 index 0000000..030d563 --- /dev/null +++ b/design/fs.txt @@ -0,0 +1,18 @@ +== File System Layout == [ Piston ] + +All of pistons files are installed in the `/piston` directory. This directory +contains all runtimes, config, packages and cache that piston uses. + +Each package gets its own directory, where it its prebuilt binaries are +unpacked into. This is contained within `/piston/packages` + +The binaries folder contained within this is then symlinked into the runtimes +directory. This is where all the different runtimes available are placed. This +is contained within the `/piston/runtimes` directory. + +The cache directory a directory containing all the different cache files. It is +recommended to either sym-link this into a folder withing /tmp/ or directly +mount it as a tmpfs filesystem. + +Configuration is stored in a single file - piston.yaml and contains all +documentation required for configuring the piston API \ No newline at end of file diff --git a/design/ppman.txt b/design/ppman.txt index 7721ebd..bd5da28 100644 --- a/design/ppman.txt +++ b/design/ppman.txt @@ -19,8 +19,8 @@ containing the right content - as long as its accessable by a URL, its considered a valid repository by piston. A repository URL is simply a URL pointing to a repository index file, as set out by the following information. -A repository index file is a YAML file containing the keys: `schema`, `baseurl` -and `packages`. +A repository index file is a YAML file containing the keys: `schema`, +`baseurl`, `keys` and `packages`. The schema key simply should have a value of `ppman-repo-1`. This indicates the version and file format for the client to recieve. @@ -30,9 +30,12 @@ this doesn't need to be related to the url that the repository index is hosted at, only the downloadable files, which are possible to split over many domains by using absolute paths. +The keys key contains a list of GPG key ids which should be used when +verifying. + The packages key contains a list of packages, which contain the keys: `author`, -`language`, `version`, `checksums`, `dependencies`, `size`, `buildfile` and -`download`. +`language`, `version`, `checksums`, `dependencies`, `size`, `buildfile`, +`download` and `signature`. The author field is self explainatory, it is simply the authors name and email, formatted similar to git's default format: `Full Name `. If the @@ -63,12 +66,71 @@ means of reproducing the build. This field is purely so people can understand how the image was built, and to make sure you aren't packing any mallicious code into it. -The final field is download, this points to a URL of which the package file can +The download field is a URL pointing to a place of which the package file can be obtained from. If this is a relative url, the baseurl will be appended to it. This is particularly useful if everything is stored within 1 s3 bucket, or you have a repository in a folder. +The signature field is an armored signature + == Package File == -TODO \ No newline at end of file +Within a repository, many package files are contained. These files are +downloaded and installed into `/piston`. They need to all follow the format +as listed below for the API to properly utilize them. + +A package file is a gzipped tar archive, containing 4/5 things - `run`, +`compile` (optional), `pkg-info.json`, `lang-ver` and `lang-ver/environment`. +Replacing lang-ver with the language name and the version respectively. + + +The `pkg-info.json` file contains 5 different keys: `language`, `version`, +`author`, `dependencies` and `build_platform`. + +The language field is simply the name of the language, all lowercase and not +containing any version number. This is important in the case of python +specifically as python3 and python2 come under the same language, just +different versions. + +The version field is a sem-ver compliant version number for the interpreter +contained wthin the package. It should be a string. + +The author field contains the author name, and should be formatted exactly like +shown previously in the repo index spec. + +The dependencies field is simply a map containing packages which this package +depends on. This should only be used when one language is a requirement for +another, like how typescript is dependent on node. The key should be the name +of the package, with the value being the version selector. + +The build_platform field is used for defining which type of system the package +was built on, this helps resolve system compatability errors. It consists of 2 +parts - environment and disto. The environment is simply one of `baremetal`, +`docker`, `lxc` or any other type of environment you can run piston on. The +distro is the ID of the distro as contained in /etc/os-release on the system +which built the package. This is done to ensure system compatability, +especially inside the offically supported Docker container. + + +The `run` and `compile` files are used in the execution of a job, being used +to both compile and run the source code provided. They are both treated the +same inputs-wise but can have timeouts independently set per stage. The +arguments fed both these files are the same, with the first argument being a +path to the code file, and the rest being the arguments to passed in. These +files are run in a temporary directory contained within the cache folder. +Depending on the cache control level the code and associated files will either +be disposed of or kept. By default only files named `binary` will be kept. +STDIN is only passed into the `run` file, but both files have their output +captured and returned through the API + +The `lang-ver` folder should contain any interpreter specific files, such as +the binary to execute and any other files that may be required to run the +interpreter/compiler contained within the package. + +The `environment` file contained within `lang-ver` should contain export +commands like a ~/.bashrc file should, as this is its intended purpose. Firstly +the language which is being executed has its environment file sources, then it +walks down the dependency tree sourcing files. The environment variables are +eventually cached to speed up the execution process. + From 0242c993883d4abaabfd9e430ea418d4bb9641d8 Mon Sep 17 00:00:00 2001 From: Thomas Hobson Date: Fri, 19 Feb 2021 22:05:00 +1300 Subject: [PATCH 7/7] clarify exit code returning --- design/ppman.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design/ppman.txt b/design/ppman.txt index bd5da28..b4920db 100644 --- a/design/ppman.txt +++ b/design/ppman.txt @@ -122,7 +122,7 @@ files are run in a temporary directory contained within the cache folder. Depending on the cache control level the code and associated files will either be disposed of or kept. By default only files named `binary` will be kept. STDIN is only passed into the `run` file, but both files have their output -captured and returned through the API +captured and returned along with their exit codes through the API The `lang-ver` folder should contain any interpreter specific files, such as the binary to execute and any other files that may be required to run the