Merge pull request #63 from HexF/container-generation

Container generation via distrobuilder
This commit is contained in:
Brian Seymour 2021-02-05 16:13:26 -06:00 committed by GitHub
commit 716cf04e62
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 675 additions and 3 deletions

2
.gitignore vendored
View File

@ -2,3 +2,5 @@ api/api
api/package-lock.json
lxc/i
lxc/lockfile
container/build.yaml
container/*.tar.xz

6
container/build.sh Executable file
View File

@ -0,0 +1,6 @@
#!/bin/bash
python3 -m pip install pyyaml
python3 configure.py
distrobuilder build-lxc build.yaml

13
container/configure.py Normal file
View File

@ -0,0 +1,13 @@
import yaml
with open('piston.yaml') as dbc:
with open('install_script.sh') as install_script_file:
with open('build.yaml' , 'w+') as distrobuilder_config_file_new:
distrobuilder_config = yaml.safe_load(dbc)
distrobuilder_config['actions'].append({
'trigger': 'post-packages',
'action': install_script_file.read(),
})
yaml.dump(distrobuilder_config, distrobuilder_config_file_new)

284
container/install_script.sh Normal file
View File

@ -0,0 +1,284 @@
#!/bin/bash
#echo "Don't run this on your system!" && exit 0
# install all necessary piston dependencies
echo 'source /opt/.profile' >> /opt/.bashrc
echo 'export HOME=/opt' >> /opt/.profile
echo 'export TERM=linux' >> /opt/.profile
echo 'export PATH=$PATH:/opt/.local/bin' >> /opt/.profile
export HOME=/opt
export TERM=linux
sed -i 's/\/root/\/opt/' /etc/passwd
sed -i \
's/http:\/\/archive.ubuntu.com\/ubuntu/http:\/\/mirror.math.princeton.edu\/pub\/ubuntu/' \
/etc/apt/sources.list
apt-get update
apt-get install -y \
nano wget build-essential pkg-config libxml2-dev \
libsqlite3-dev mono-complete curl cmake libpython2.7-dev \
ruby libtinfo-dev unzip git openssl libssl-dev sbcl libevent-dev
# install python2
# final binary: /opt/python2/Python-2.7.17/python
# get version: /opt/python2/Python-2.7.17/python -V
cd /opt && mkdir python2 && cd python2
wget https://www.python.org/ftp/python/2.7.17/Python-2.7.17.tar.xz
unxz Python-2.7.17.tar.xz
tar -xf Python-2.7.17.tar
cd Python-2.7.17
./configure
# open Modules/Setup and uncomment zlib line
make
echo 'export PATH=$PATH:/opt/python2/Python-2.7.17' >> /opt/.profile
. /opt/.profile
# install python3
# final binary: /opt/python3/Python-3.8.2/python
# get version: /opt/python3/Python-3.8.2/python -V
cd /opt && mkdir python3 && cd python3
wget https://www.python.org/ftp/python/3.8.2/Python-3.8.2.tar.xz
unxz Python-3.8.2.tar.xz
tar -xf Python-3.8.2.tar
cd Python-3.8.2
./configure
make
ln -s python python3.8
echo 'export PATH=$PATH:/opt/python3/Python-3.8.2' >> /opt/.profile
. /opt/.profile
# install paradoc
# this is not a binary, it is a python module
# therefore it cannot be run directly as it requires python3 to be installed
cd /opt && mkdir paradoc && cd paradoc
git clone https://github.com/betaveros/paradoc.git
# install node.js
# final binary: /opt/nodejs/node-v12.16.1-linux-x64/bin/node
# get version: /opt/nodejs/node-v12.16.1-linux-x64/bin/node -v
cd /opt && mkdir nodejs && cd nodejs
wget https://nodejs.org/dist/v12.16.1/node-v12.16.1-linux-x64.tar.xz
unxz node-v12.16.1-linux-x64.tar.xz
tar -xf node-v12.16.1-linux-x64.tar
echo 'export PATH=$PATH:/opt/nodejs/node-v12.16.1-linux-x64/bin' >> /opt/.profile
. /opt/.profile
# install typescript
# final binary: /opt/nodejs/node-v12.16.1-linux-x64/bin/tsc
# get version: /opt/nodejs/node-v12.16.1-linux-x64/bin/tsc -v
/opt/nodejs/node-v12.16.1-linux-x64/bin/npm i -g typescript
# install golang
# final binary: /opt/go/go/bin/go
# get version: /opt/go/go/bin/go version
cd /opt && mkdir go && cd go
wget https://dl.google.com/go/go1.14.1.linux-amd64.tar.gz
tar -xzf go1.14.1.linux-amd64.tar.gz
echo 'export PATH=$PATH:/opt/go/go/bin' >> /opt/.profile
echo 'export GOROOT=/opt/go/go' >> /opt/.profile
echo 'export GOCACHE=/tmp' >> /opt/.profile
. /opt/.profile
# install php
# final binary: /usr/local/bin/php
# get version: /usr/local/bin/php -v
cd /opt && mkdir php && cd php
wget https://www.php.net/distributions/php-8.0.0.tar.gz
tar -xzf php-8.0.0.tar.gz
cd php-8.0.0
./configure
make
make install
# install rust
# final binary: /usr/local/bin/rustc
# get version: /usr/local/bin/rustc --version
cd /opt && mkdir rust && cd rust
wget https://static.rust-lang.org/dist/rust-1.49.0-x86_64-unknown-linux-gnu.tar.gz
tar -xzf rust-1.49.0-x86_64-unknown-linux-gnu.tar.gz
cd rust-1.49.0-x86_64-unknown-linux-gnu
./install.sh
# install swift
# final binary: /opt/swift/swift-5.1.5-RELEASE-ubuntu18.04/usr/bin/swift
# get version: /opt/swift/swift-5.1.5-RELEASE-ubuntu18.04/usr/bin/swift --version
cd /opt && mkdir swift && cd swift
wget https://swift.org/builds/swift-5.1.5-release/ubuntu1804/swift-5.1.5-RELEASE/swift-5.1.5-RELEASE-ubuntu18.04.tar.gz
tar -xzf swift-5.1.5-RELEASE-ubuntu18.04.tar.gz
echo 'export PATH=$PATH:/opt/swift/swift-5.1.5-RELEASE-ubuntu18.04/usr/bin' >> /opt/.profile
. /opt/.profile
# install nasm
# final binary: /opt/nasm/nasm-2.14.02/nasm
# get version: /opt/nasm/nasm-2.14.02/nasm -v
cd /opt && mkdir nasm && cd nasm
wget https://www.nasm.us/pub/nasm/releasebuilds/2.14.02/nasm-2.14.02.tar.gz
tar -xzf nasm-2.14.02.tar.gz
cd nasm-2.14.02
./configure
make
echo 'export PATH=$PATH:/opt/nasm/nasm-2.14.02' >> /opt/.profile
. /opt/.profile
# install java
# final binary: /opt/java/jdk-14/bin/java
# get version: /opt/java/jdk-14/bin/java -version
cd /opt && mkdir java && cd java
wget https://download.java.net/java/GA/jdk14/076bab302c7b4508975440c56f6cc26a/36/GPL/openjdk-14_linux-x64_bin.tar.gz
tar -xzf openjdk-14_linux-x64_bin.tar.gz
echo 'export PATH=$PATH:/opt/java/jdk-14/bin' >> /opt/.profile
. /opt/.profile
# install jelly
cd /opt && mkdir jelly && cd jelly
wget https://github.com/DennisMitchell/jellylanguage/archive/master.zip
unzip master.zip
cd jellylanguage-master
python3.8 -m pip install .
sed -i 's/\/usr\/local\/bin\/python3.8/\/opt\/python3\/Python-3.8.2\/python3.8/' /usr/local/bin/jelly
# install julia
# final binary: /opt/julia/julia-1.5.0/bin/julia
# get version: /opt/julia/julia-1.5.0/bin/julia --version
cd /opt && mkdir julia && cd julia
wget https://julialang-s3.julialang.org/bin/linux/x64/1.5/julia-1.5.0-linux-x86_64.tar.gz
tar -xzf julia-1.5.0-linux-x86_64.tar.gz
echo 'export PATH=$PATH:/opt/julia/julia-1.5.0/bin' >> /opt/.profile
. /opt/.profile
# install kotlin
# final binary: /opt/kotlinc/bin/kotlinc
# get version: /opt/kotlinc/bin/kotlinc -version
cd /opt
wget https://github.com/JetBrains/kotlin/releases/download/v1.4.10/kotlin-compiler-1.4.10.zip
unzip kotlin-compiler-1.4.10.zip
rm kotlin-compiler-1.4.10.zip
echo 'export PATH=$PATH:/opt/kotlinc/bin' >> /opt/.profile
. /opt/.profile
# install elixir and erlang
# final binary: /opt/elixir/bin/elixir
# get version: /opt/elixir/bin/elixir --version
# erlang
cd /opt && mkdir erlang && cd erlang
wget http://erlang.org/download/otp_src_23.0.tar.gz
gunzip -c otp_src_23.0.tar.gz | tar xf -
cd otp_src_23.0 && ./configure
make
echo 'export PATH=$PATH:/opt/erlang/otp_src_23.0/bin' >> /opt/.profile
. /opt/.profile
# elixir
cd /opt && mkdir elixir && cd elixir
wget https://github.com/elixir-lang/elixir/releases/download/v1.10.3/Precompiled.zip
mkdir elixir-1.10.3 && unzip Precompiled.zip -d elixir-1.10.3/
echo 'export PATH=$PATH:/opt/elixir/elixir-1.10.3/bin' >> /opt/.profile
. /opt/.profile
# install emacs
# final binary: /opt/emacs/emacs-26.3/src/emacs
# get version: /opt/emacs/emacs-26.3/src/emacs --version
cd /opt && mkdir emacs && cd emacs
wget https://mirrors.ocf.berkeley.edu/gnu/emacs/emacs-26.3.tar.xz
tar -xf emacs-26.3.tar.xz
rm emacs-26.3.tar.xz
cd emacs-26.3
./configure --with-gnutls=no
make
echo 'export PATH=$PATH:/opt/emacs/emacs-26.3/src' >> /opt/.profile
. /opt/.profile
# install lua
# final binary: /opt/lua/lua54/src/lua
# get version: /opt/lua/lua54/src/lua -v
cd /opt && mkdir lua && cd lua
wget https://sourceforge.net/projects/luabinaries/files/5.4.0/Docs%20and%20Sources/lua-5.4.0_Sources.tar.gz/download
tar -xzf download
cd lua54
make
echo 'export PATH=$PATH:/opt/lua/lua54/src' >> /opt/.profile
. /opt/.profile
# install haskell
# final binary: /usr/bin/ghc
# get version: /usr/bin/ghc --version
apt install -y ghc
# install deno
# final binary: /opt/.deno/bin/deno
# get version: /opt/.deno/bin/deno --version
cd /opt && mkdir deno && cd deno
curl -fsSL https://deno.land/x/install/install.sh | sh
echo 'export DENO_INSTALL="/opt/.deno"' >> /opt/.profile
echo 'export PATH="$DENO_INSTALL/bin:$PATH"' >> /opt/.profile
. /opt/.profile
# install brainfuck
cd /opt && mkdir bf && cd bf
git clone https://github.com/texus/Brainfuck-interpreter
cd Brainfuck-interpreter
echo 'export PATH=$PATH:/opt/bf/Brainfuck-interpreter' >> /opt/.profile
. /opt/.profile
# install crystal
# final binary: /opt/crystal/crystal-0.35.1-1/bin/crystal
# get version: /opt/crystal/crystal-0.35.1-1/bin/crystal -v
cd /opt && mkdir crystal && cd crystal
wget https://github.com/crystal-lang/crystal/releases/download/0.35.1/crystal-0.35.1-1-linux-x86_64.tar.gz
tar -xzf crystal-0.35.1-1-linux-x86_64.tar.gz
echo 'export PATH="$PATH:/opt/crystal/crystal-0.35.1-1/bin:$PATH"' >> /opt/.profile
. /opt/.profile
# install d
# final binary: /opt/d/dmd2/linux/bin64/dmd
# get version: /opt/d/dmd2/linux/bin64/dmd --version
cd /opt && mkdir d && cd d
wget http://downloads.dlang.org/releases/2.x/2.095.0/dmd.2.095.0.linux.tar.xz
unxz dmd.2.095.0.linux.tar.xz
tar -xf dmd.2.095.0.linux.tar
echo 'export PATH=$PATH:/opt/d/dmd2/linux/bin64' >> /opt/.profile
. /opt/.profile
# install zig
# final binary: /opt/zig/zig
# get version: /opt/zig/zig version
cd /opt && mkdir zig && cd zig
wget https://ziglang.org/download/0.7.1/zig-linux-x86_64-0.7.1.tar.xz
tar -xf zig-linux-x86_64-0.7.1.tar.xz
mv zig-linux-x86_64-0.7.1 zig
rm zig-linux-x86_64-0.7.1.tar.xz
echo 'export PATH=$PATH:/opt/zig/zig' >> /opt/.profile
. /opt/.profile
# install nim
# final binary: /opt/nim/bin/nim
# get version: /opt/nim/bin/nim -v
cd /opt && mkdir nim && cd nim
wget https://nim-lang.org/download/nim-1.4.0-linux_x64.tar.xz
unxz nim-1.4.0-linux_x64.tar.xz
tar -xf nim-1.4.0-linux_x64.tar
cd nim-1.4.0
./install.sh /opt
echo 'export PATH=$PATH:/opt/nim/bin' >> /opt/.profile
. /opt/.profile
# create runnable users and apply limits
for i in {1..150}; do
useradd -M runner$i
usermod -d /tmp runner$i
echo "runner$i soft nproc 64" >> /etc/security/limits.conf
echo "runner$i hard nproc 64" >> /etc/security/limits.conf
echo "runner$i soft nofile 2048" >> /etc/security/limits.conf
echo "runner$i hard nofile 2048" >> /etc/security/limits.conf
done
# remove any lingering write access to others
cd /opt
chown -R root: *
chmod -R o-w *
# cleanup
rm -rf /home/ubuntu
chmod 777 /tmp
# disable cron
systemctl stop cron
systemctl disable cron

355
container/piston.yaml Normal file
View File

@ -0,0 +1,355 @@
image:
name: ubuntu-bionic-x86_64-piston
distribution: ubuntu
release: bionic
description: |-
Ubuntu {{ image.release }} preconfigured for Piston
architecture: x86_64
source:
downloader: debootstrap
same_as: bionic
url: http://archive.ubuntu.com/ubuntu
keyserver: keyserver.ubuntu.com
keys:
- '0x790BC7277767219C42C86F933B4FE6ACC0B21F32'
- '0xf6ecb3762474eda9d21b7022871920d1991bc93c'
targets:
lxc:
create-message: |-
You just created an {{ image.description }} container.
To enable SSH, run: apt install openssh-server
No default root or user password are set by LXC.
config:
- type: all
before: 5
content: |-
lxc.include = LXC_TEMPLATE_CONFIG/ubuntu.common.conf
- type: user
before: 5
content: |-
lxc.include = LXC_TEMPLATE_CONFIG/ubuntu.userns.conf
- type: all
after: 4
content: |-
lxc.include = LXC_TEMPLATE_CONFIG/common.conf
# For Ubuntu 14.04
lxc.mount.entry = /sys/kernel/debug sys/kernel/debug none bind,optional 0 0
lxc.mount.entry = /sys/kernel/security sys/kernel/security none bind,optional 0 0
lxc.mount.entry = /sys/fs/pstore sys/fs/pstore none bind,optional 0 0
lxc.mount.entry = mqueue dev/mqueue mqueue rw,relatime,create=dir,optional 0 0
- type: user
after: 4
content: |-
lxc.include = LXC_TEMPLATE_CONFIG/userns.conf
# For Ubuntu 14.04
lxc.mount.entry = /sys/firmware/efi/efivars sys/firmware/efi/efivars none bind,optional 0 0
lxc.mount.entry = /proc/sys/fs/binfmt_misc proc/sys/fs/binfmt_misc none bind,optional 0 0
- type: all
content: |-
lxc.arch = {{ image.architecture_personality }}
files:
- path: /etc/hostname
generator: hostname
- path: /etc/hosts
generator: hosts
- path: /etc/resolvconf/resolv.conf.d/original
generator: remove
- path: /etc/resolvconf/resolv.conf.d/tail
generator: remove
- path: /etc/machine-id
generator: dump
- path: /var/lib/dbus/machine-id
generator: remove
- path: /etc/netplan/10-lxc.yaml
generator: dump
content: |-
network:
version: 2
ethernets:
eth0:
dhcp4: true
dhcp-identifier: mac
releases:
- bionic
- eoan
- focal
- groovy
types:
- container
variants:
- default
- path: /etc/network/interfaces
generator: dump
content: |-
# This file describes the network interfaces available on your system
# and how to activate them. For more information, see interfaces(5).
# The loopback network interface
auto lo
iface lo inet loopback
auto eth0
iface eth0 inet dhcp
source /etc/network/interfaces.d/*.cfg
releases:
- trusty
- xenial
types:
- container
- path: /etc/netplan/10-lxc.yaml
generator: dump
content: |-
network:
version: 2
ethernets:
enp5s0:
dhcp4: true
dhcp-identifier: mac
releases:
- bionic
- eoan
- focal
- groovy
types:
- vm
variants:
- default
- path: /etc/network/interfaces
generator: dump
content: |-
# This file describes the network interfaces available on your system
# and how to activate them. For more information, see interfaces(5).
# The loopback network interface
auto lo
iface lo inet loopback
auto enp5s0
iface enp5s0 inet dhcp
source /etc/network/interfaces.d/*.cfg
releases:
- trusty
- xenial
types:
- vm
- path: /etc/init/lxc-tty.conf
generator: upstart-tty
releases:
- trusty
types:
- container
- name: meta-data
generator: cloud-init
variants:
- cloud
- name: network-config
generator: cloud-init
variants:
- cloud
- name: user-data
generator: cloud-init
variants:
- cloud
- name: vendor-data
generator: cloud-init
variants:
- cloud
- name: ext4
generator: fstab
types:
- vm
- name: lxd-agent
generator: lxd-agent
types:
- vm
- path: /etc/default/grub.d/50-lxd.cfg
generator: dump
content: |-
GRUB_RECORDFAIL_TIMEOUT=0
GRUB_TIMEOUT=0
GRUB_CMDLINE_LINUX_DEFAULT="${GRUB_CMDLINE_LINUX_DEFAULT} console=tty1 console=ttyS0"
GRUB_TERMINAL=console
types:
- vm
- path: /etc/sudoers.d/90-lxd
generator: dump
mode: '0440'
content: |-
# User rules for ubuntu
ubuntu ALL=(ALL) NOPASSWD:ALL
variants:
- default
packages:
manager: apt
update: true
cleanup: true
sets:
- packages:
- apt-transport-https
- fuse
- language-pack-en
- openssh-client
- vim
action: install
- packages:
- cloud-init
action: install
variants:
- cloud
- packages:
- acpid
action: install
architectures:
- amd64
- arm64
types:
- vm
- packages:
- grub-efi-amd64-signed
- shim-signed
action: install
architectures:
- amd64
types:
- vm
- packages:
- grub-efi-arm64-signed
action: install
architectures:
- arm64
types:
- vm
- packages:
- shim-signed
action: install
architectures:
- arm64
releases:
- disco
- eoan
- focal
- groovy
types:
- vm
- packages:
- linux-virtual-hwe-16.04
action: install
releases:
- xenial
types:
- vm
- packages:
- linux-virtual
action: install
releases:
- bionic
- eoan
- focal
- groovy
types:
- vm
- packages:
- os-prober
action: remove
types:
- vm
repositories:
- name: sources.list
url: |-
deb http://archive.ubuntu.com/ubuntu {{ image.release }} main restricted universe multiverse
deb http://archive.ubuntu.com/ubuntu {{ image.release }}-updates main restricted universe multiverse
deb http://security.ubuntu.com/ubuntu {{ image.release }}-security main restricted universe multiverse
architectures:
- amd64
- i386
- name: sources.list
url: |-
deb http://ports.ubuntu.com/ubuntu-ports {{ image.release }} main restricted universe multiverse
deb http://ports.ubuntu.com/ubuntu-ports {{ image.release }}-updates main restricted universe multiverse
deb http://ports.ubuntu.com/ubuntu-ports {{ image.release }}-security main restricted universe multiverse
architectures:
- armhf
- arm64
- powerpc
- powerpc64
- ppc64el
actions:
- trigger: post-update
action: |-
#!/bin/sh
set -eux
# Create the ubuntu user account
getent group sudo >/dev/null 2>&1 || groupadd --system sudo
useradd --create-home -s /bin/bash -G sudo -U ubuntu
variants:
- default
- trigger: post-packages
action: |-
#!/bin/sh
set -eux
# Enable systemd-networkd
systemctl enable systemd-networkd
releases:
- bionic
- eoan
- focal
- groovy
- trigger: post-packages
action: |-
#!/bin/sh
set -eux
# Make sure the locale is built and functional
locale-gen en_US.UTF-8
update-locale LANG=en_US.UTF-8
# Cleanup underlying /run
mount -o bind / /mnt
rm -rf /mnt/run/*
umount /mnt
# Cleanup temporary shadow paths
rm /etc/*-
- trigger: post-files
action: |-
#!/bin/sh
set -eux
TARGET="x86_64"
[ "$(uname -m)" = "aarch64" ] && TARGET="arm64"
update-grub
grub-install --uefi-secure-boot --target="${TARGET}-efi" --no-nvram --removable
update-grub
sed -i "s#root=[^ ]*#root=/dev/sda2#g" /boot/grub/grub.cfg
types:
- vm
mappings:
architecture_map: debian

7
container/readme.md Normal file
View File

@ -0,0 +1,7 @@
# LXC Container Build
Requires: `lxc`, `lxc-net`, `packer` (Hashicorp Packer)
To build: `packer build -var 'apt_mirror=[apt mirror]' -var 'make_threads=[-j flag]' piston.pkr.hcl`
After roughly 30 minutes (on an i7-4790k), you should have an image built

View File

@ -141,12 +141,17 @@ cd piston/lxc
#### Installation (simple)
- Coming soon.
- Install additional dependencies python3, pip and distrobuilder
- `cd ../container && ./build.sh`
- Wait, it may take up to an hour.
- `lxc-create -n piston -t local -- --metadata meta.tar.xz --fstree rootfs.tar.xz`
- `cd ../lxc && ./start`
- Good to go!
#### Installation (advanced)
- See `var/install.txt` for how to create a new LXC container and install all of the required
software.
- See `var/install.txt` for how to build the container manually
#### CLI Usage
- `cli/execute [language] [file path] [args]`